diff options
| -rw-r--r-- | Makefile | 12 | ||||
| -rw-r--r-- | fzopen.3.in | 15 | ||||
| -rw-r--r-- | zio.c | 417 | ||||
| -rw-r--r-- | zio.h.in | 3 | ||||
| -rw-r--r-- | zioP.h | 22 |
5 files changed, 461 insertions, 8 deletions
@@ -8,7 +8,7 @@ LARGE = $(shell getconf LFS_CFLAGS) CFLAGS = $(RPM_OPT_FLAGS) -pipe -Wall -D_GNU_SOURCE -D_REENTRANT -D_DEFAULT_SOURCE $(LARGE) CC = gcc MAJOR = 1 -MINOR = 06 +MINOR = 08 VERSION = $(MAJOR).$(MINOR) SONAME = libzio.so.$(MAJOR) LDMAP = -Wl,--version-script=zio.map @@ -47,6 +47,7 @@ CFLAGS += $(call cc-include,zlib.h,HAS_ZLIB_H) CFLAGS += $(call cc-include,bzlib.h,HAS_BZLIB_H) CFLAGS += $(call cc-include,lzmadec.h,HAS_LZMADEC_H) CFLAGS += $(call cc-include,lzma.h,HAS_LZMA_H) +CFLAGS += $(call cc-include,zstd.h,HAS_ZSTD_H) LIBS = -lz ifeq ($(call cc-library,libbz2),yes) @@ -59,6 +60,9 @@ ifeq ($(call cc-library,lzmadec),yes) LIBS += -llzmadec endif endif +ifeq ($(call cc-library,libzstd),yes) + LIBS += -lzstd +endif all: shared static noweak: CFLAGS += -DNO_WEAK @@ -117,14 +121,14 @@ install-data: zio.h fzopen.3 install -m 0644 fzopen.3 $(DESTDIR)$(mandir)/man3/ clean: - rm -f *.a *.so* testt tests zio.h + rm -f *.a *.so* testt tests testx zio.h rm -rf obj/ obs/ - rm -f libzio-$(VERSION).tar.gz + rm -f libzio-$(VERSION).tar.bz2 dest: clean mkdir libzio-$(VERSION) cp -p $(FILES) libzio-$(VERSION)/ - tar czf libzio-$(VERSION).tar.gz libzio-$(VERSION)/ + tar czf libzio-$(VERSION).tar.bz2 libzio-$(VERSION)/ rm -rf libzio-$(VERSION)/ testt: testt.c libzio.a diff --git a/fzopen.3.in b/fzopen.3.in index b4d8357..1b226a6 100644 --- a/fzopen.3.in +++ b/fzopen.3.in @@ -69,6 +69,9 @@ The file descriptor points to a lzma2ed file. .TP .BR Z The file descriptor points to a file in LZW format. +.TP +.BR s +The file descriptor points to a file in zst format. .PP Note that .B fzopen @@ -116,6 +119,7 @@ bzip2 yes yes yes .bz2 -lbz2 LZW yes no yes .Z builtin lzma yes yes(no) yes .lzma -llzma (-llzmadec) xz yes yes yes .xz -llzma +zstd yes yes no .zst -lzstd .TE .PP .PP @@ -184,6 +188,15 @@ if test "$am_cv_libzio" = yes; then ]) done ]) + AC_CHECK_HEADER(zstd.h, [ + for lib in lzstd; do + AC_CHECK_LIB($lib, ZSTD_createCStream, [ + LIBS="$LIBS -l$lib" + am_cv_libzio="$am_cv_libzio lib$lib" + break + ]) + done + ]) AC_MSG_NOTICE([libzio is used [$]am_cv_libzio]) fi .fi @@ -347,5 +360,7 @@ functions of the e.g. .br 2006,2008,2009 Werner Fink, 2006,2008,2009 SuSE Products GmbH, Germany. +.br +2021 Werner Fink, 2021 SUSE Software Solutions Germany GmbH, Germany. .SH AUTHOR Werner Fink <werner@suse.de> @@ -1,11 +1,12 @@ /* - * zio.c Provide an streamable interface to gziped/bzip2ed/LZW/LZMA files + * zio.c Provide an streamable interface to gziped/bzip2ed/LZW/LZMA/zst files * * Copyright 2004 Werner Fink, 2004 SuSE LINUX AG, Germany. * Copyright 2006 Werner Fink, 2006 SuSE Products GmbH, Germany. * Copyright 2009 Werner Fink, 2009 SuSE Products GmbH, Germany. * Copyright 2013 Werner Fink, 2013 SuSE Products GmbH, Germany. * Copyright 2017 Werner Fink, 2017 SUSE Linux GmbH, Germany. + * Copyright 2021 Werner Fink, 2021 SUSE Software Solutions Germany GmbH, Germany. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -526,6 +527,261 @@ static cookie_io_functions_t iolzma = { # endif /* !HAS_LZMADEC_H */ #endif /* !HAS_LZMA_H */ +#if defined(HAS_ZSTD_H) + +#ifndef MIN +# define MIN(x,y) ((x) < (y) ? (x) : (y)) +#endif + +typedef struct zstdfile_s { + ZSTD_CStream* cstream; + ZSTD_DStream* dstream; + ZSTD_inBuffer in; + ZSTD_outBuffer out; + void *buf_in, *buf_out; + size_t init; + size_t insize; +#if 0 + size_t prevwrite; +#endif + FILE *file; + FILE *stdio; + int level; +} zstdfile_t; + +static int zstdopen(void *cookie) +{ + int ret = -1; + + zstdfile_t *zstd = (zstdfile_t*)cookie; + if (zstd->cstream) { + zstd->init = ZSTD_initCStream(zstd->cstream, zstd->level); + if (ZSTD_isError(zstd->init)) { + fprintf(stderr, "ZSTD_initCStream error: %s\n", ZSTD_getErrorName(zstd->init)); + goto out; + } + zstd->in.size = ZSTD_CStreamInSize(); + zstd->out.size = ZSTD_CStreamOutSize(); + zstd->buf_in = malloc(zstd->in.size); + if (!zstd->buf_in) + goto out; + zstd->in.src = zstd->buf_in; + zstd->buf_out = malloc(zstd->out.size); + if (!zstd->buf_out) { + free(zstd->buf_in); + goto out; + } + zstd->out.dst = zstd->buf_out; + zstd->in.pos = zstd->out.pos = 0; + zstd->insize = zstd->in.size; +#if 0 + zstd->prevwrite = 0; +#endif + ret = 0; + } else if (zstd->dstream) { + zstd->init = ZSTD_initDStream(zstd->dstream); + if (ZSTD_isError(zstd->init)) { + fprintf(stderr, "ZSTD_initDStream error: %s\n", ZSTD_getErrorName(zstd->init)); + goto out; + } + zstd->in.size = ZSTD_DStreamInSize(); + zstd->out.size = ZSTD_DStreamOutSize(); + zstd->buf_in = malloc(zstd->in.size); + if (!zstd->buf_in) + goto out; + zstd->in.src = zstd->buf_in; + zstd->buf_out = malloc(zstd->out.size); + if (!zstd->buf_out) { + free(zstd->buf_in); + goto out; + } + zstd->out.dst = zstd->buf_out; + zstd->in.pos = zstd->out.pos = 0; + zstd->insize = zstd->in.size; + ret = 0; + } +out: + return ret; +} + +static ssize_t zstdread(void *cookie, char *buf, size_t count) +{ + ssize_t ret = -1; + zstdfile_t *zstd = (zstdfile_t*)cookie; + size_t rlen, rtodo; + + if (zstd->cstream || !zstd->dstream) { + errno = ENOTSUP; + goto out; + } + + ret = 0; + if (zstd->out.pos > 0) { + rlen = zstd->out.pos; + if (rlen > count) + rlen = count; + memcpy(buf, zstd->out.dst, rlen); /* Copy into stdio buffer ... */ + zstd->out.dst += rlen; + zstd->out.pos -= rlen; + ret = rlen; + goto out; /* ... and escape here */ + } + zstd->out.dst = zstd->buf_out; + zstd->out.pos = 0; + + rtodo = zstd->init; + while ((rlen = fread(zstd->buf_in, 1, rtodo, zstd->file))) { + zstd->in.src = zstd->buf_in; + zstd->in.size = rlen; + zstd->in.pos = 0; + + while (zstd->in.pos < zstd->in.size) { + + rtodo = ZSTD_decompressStream(zstd->dstream, &zstd->out, &zstd->in); + if (ZSTD_isError(rtodo)) { + fprintf(stderr, "ZSTD_decompressStream error: %s\n", ZSTD_getErrorName(rtodo)); + ret = -1; + goto out; + } + + if (zstd->out.pos > 0) { + rlen = zstd->out.pos; + if (rlen > count) + rlen = count; + memcpy(buf, zstd->out.dst, rlen); /* Copy into stdio buffer ... */ + zstd->out.dst += rlen; + zstd->out.pos -= rlen; + ret = rlen; + goto out; /* ... and escape here */ + } + + zstd->out.dst = zstd->buf_out; + zstd->out.pos = 0; + + if (rtodo == 0) { + zstd->init = ZSTD_initDStream(zstd->dstream); + if (ZSTD_isError(zstd->init)) { + fprintf(stderr, "ZSTD_decompressStream error: %s\n", ZSTD_getErrorName(rtodo)); + ret = -1; + goto out; + } + rtodo = zstd->init; + ret = 0; + } + } + } +out: + return ret; +} + +static ssize_t zstdwrite(void *cookie, const char *buf, size_t count) +{ + ssize_t ret = -1; + zstdfile_t *zstd = (zstdfile_t*)cookie; + size_t wlen, wtodo; + void *src; + +#if 0 + if (zstd->prevwrite == 0) + zstd->prevwrite = count; +#endif + + ret = 0; + src = zstd->buf_in+zstd->in.pos; + if (zstd->in.pos < zstd->insize) { + wlen = zstd->insize - zstd->in.pos; + if (wlen > count) + wlen = count; + memcpy(src, buf, wlen); /* Copy from stdio buffer ... */ + src += wlen; + zstd->in.pos += wlen; + ret = wlen; + } + +#if 0 + fprintf(stderr, "Harry %lu %lu %lu %lu\n", zstd->prevwrite, count, zstd->in.pos, zstd->insize); + if (zstd->prevwrite == count && zstd->in.pos < zstd->insize) + goto out; /* ... and ask for more here */ +#endif + + zstd->in.src = zstd->buf_in; + zstd->in.size = zstd->in.pos; + zstd->in.pos = 0; + + while (zstd->in.pos < zstd->in.size) { + + wtodo = ZSTD_compressStream(zstd->cstream, &zstd->out, &zstd->in); + if (ZSTD_isError(wtodo)) { + fprintf(stderr, "ZSTD_compressStream error: %s\n", ZSTD_getErrorName(wtodo)); + ret = -1; + goto out; + } + + while (zstd->out.pos > 0) { + wlen = fwrite(zstd->out.dst, 1, zstd->out.pos, zstd->file); + if (wlen == zstd->out.pos || wlen == 0) + break; + zstd->out.dst += wlen; + zstd->out.pos -= wlen; + } + zstd->out.dst = zstd->buf_out; + zstd->out.pos = 0; + } + zstd->in.pos = 0; +out: + return ret; +} + +static zio_int_t zstdseek(void *cookie, zio_off_t *poffset, int whence) +{ + errno = ENOTSUP; + return -1; +} + +static int zstdclose(void *cookie) +{ + zstdfile_t *zstd = (zstdfile_t*)cookie; + int fret; + if (zstd->cstream) { + size_t wlen, end_res; + do { + zstd->out.dst = zstd->buf_out; + zstd->out.pos = 0; + end_res = ZSTD_endStream(zstd->cstream, &zstd->out); + if (ZSTD_isError(end_res)) + fprintf(stderr, "ZSTD_endStream error: %s\n", ZSTD_getErrorName(end_res)); + while (zstd->out.pos > 0) { + wlen = fwrite(zstd->out.dst, 1, zstd->out.pos, zstd->file); + if (wlen == zstd->out.pos || wlen == 0) + break; + zstd->out.dst += wlen; + zstd->out.pos -= wlen; + } + } while (end_res > 0); + ZSTD_freeCStream(zstd->cstream); + free(zstd->buf_in); + free(zstd->buf_out); + } else if (zstd->dstream) { + ZSTD_freeDStream(zstd->dstream); + free(zstd->buf_in); + free(zstd->buf_out); + } + fret = fclose(zstd->file); + free(cookie); + return fret; +} + +__extension__ +static cookie_io_functions_t iozstd = { + .read = (cookie_read_function_t*) zstdread, + .write = (cookie_write_function_t*)zstdwrite, + .seek = (cookie_seek_function_t*) zstdseek, + .close = (cookie_close_function_t*)zstdclose, +}; +#else /* !HAS_ZSTD_H */ +# warning No support for .zst(d) format +#endif /* !HAS_ZSTD_H */ + #ifndef MIN # define MIN(x,y) ((x) < (y) ? (x) : (y)) #endif @@ -685,6 +941,10 @@ static inline char autodetect(char **__restrict path, const char *__restrict che what = 'l'; else if (strcmp(suff, "xz" ) == 0) what = 'x'; + else if (strcmp(suff, "zst" ) == 0) + what = 's'; + else if (strcmp(suff, "zstd" ) == 0) + what = 's'; } if (what == 'n' && *check == 'r') { @@ -729,6 +989,17 @@ static inline char autodetect(char **__restrict path, const char *__restrict che } *suff = '\0'; + if (stat(strcat(ext, ".zst"), &st) == 0) { + what = 's'; + goto skip; + } + *suff = '\0'; + if (stat(strcat(ext, ".zstd"), &st) == 0) { + what = 's'; + goto skip; + } + *suff = '\0'; + if ((fd = open(ext, O_RDONLY|O_NOCTTY)) < 0) goto skip; if (read(fd, m, sizeof(m)) == sizeof(m)) { @@ -746,6 +1017,8 @@ static inline char autodetect(char **__restrict path, const char *__restrict che what = 'l'; else if (m[0] == '\375' && m[1] == '7' && m[2] == 'z' && m[3] == 'X' && m[4] == 'Z') what = 'x'; + else if (m[0] >= 34 && m[0] <= 40 && m[1] == '\265' && m[2] == '\057' && m[3] == '\375') + what = 's'; } close(fd); skip: @@ -1020,6 +1293,77 @@ FILE * fzopen(const char * path, const char * mode) # endif /* !HAS_LZMADEC_H */ #endif /* !HAS_LZMA_H */ break; + case 's': +#if defined(HAS_ZSTD_H) + { + zstdfile_t *__restrict cookie; + int level = 7; + int zret; +# ifndef NO_WEAK + if ((ZSTD_createCStream) == 0 || (ZSTD_createDStream) == 0) { + errno = ENOSYS; + goto out; + } +# endif + if (posix_memalign((void*)&cookie, sizeof(void*), alignof(zstdfile_t)) != 0) + goto out; + memset(cookie, 0, alignof(zstdfile_t)); + + if ((cookie->file = fopen(ext, check)) == NULL) { + free(cookie); + goto out; + } + + n = strlen(mode); + for (i = 1; i < n; i++) { + if (mode[i] >= '1' && mode[i] <= '9') { + level = (int)mode[i]; + break; + } + } + + if (check[0] == 'w') { + cookie->cstream = ZSTD_createCStream(); + if (!cookie->cstream) { + fclose(cookie->file); + free(cookie); + errno = EINVAL; + goto out; + } + } else { + cookie->dstream = ZSTD_createDStream(); + if (!cookie->dstream) { + fclose(cookie->file); + free(cookie); + errno = EINVAL; + goto out; + } + } + cookie->level = level; + zret = zstdopen(cookie); + + if (zret < 0) { + fclose(cookie->file); + free(cookie); + errno = EINVAL; + goto out; + } + if (!(ret = fopencookie((void*)cookie, check, iozstd))) { + fclose(cookie->file); + free(cookie); + errno = EINVAL; + goto out; + } +# ifndef LIBIO_IS_FIXED + if (ret->_fileno < 0) + ret->_fileno = 0; +# endif + cookie->stdio = ret; + } +#else /* !HAS_ZSTD_H */ + errno = ENOTSUP; +#endif /* !HAS_ZSTD_H */ + break; default: ret = fopen(ext, mode); break; @@ -1289,6 +1633,77 @@ FILE * fdzopen(int fildes, const char * mode, const char *what) # endif /* !HAS_LZMADEC_H */ #endif /* !HAS_LZMA_H */ break; + case 's': +#if defined(HAS_ZSTD_H) + { + zstdfile_t *__restrict cookie; + int level = 7; + int zret; +# ifndef NO_WEAK + if ((ZSTD_createCStream) == 0 || (ZSTD_createDStream) == 0) { + errno = ENOSYS; + goto out; + } +# endif + if (posix_memalign((void*)&cookie, sizeof(void*), alignof(zstdfile_t)) != 0) + goto out; + memset(cookie, 0, alignof(zstdfile_t)); + + if ((cookie->file = fdopen(fildes, check)) == NULL) { + free(cookie); + goto out; + } + + n = strlen(mode); + for (i = 1; i < n; i++) { + if (mode[i] >= '1' && mode[i] <= '9') { + level = (int)mode[i]; + break; + } + } + + if (check[0] == 'w') { + cookie->cstream = ZSTD_createCStream(); + if (!cookie->cstream) { + fclose(cookie->file); + free(cookie); + errno = EINVAL; + goto out; + } + } else { + cookie->dstream = ZSTD_createDStream(); + if (!cookie->dstream) { + fclose(cookie->file); + free(cookie); + errno = EINVAL; + goto out; + } + } + cookie->level = level; + zret = zstdopen(cookie); + + if (zret < 0) { + fclose(cookie->file); + free(cookie); + errno = EINVAL; + goto out; + } + if (!(ret = fopencookie((void*)cookie, check, iozstd))) { + fclose(cookie->file); + free(cookie); + errno = EINVAL; + goto out; + } +# ifndef LIBIO_IS_FIXED + if (ret->_fileno < 0) + ret->_fileno = 0; +# endif + cookie->stdio = ret; + } +#else /* !HAS_ZSTD_H */ + errno = ENOTSUP; +#endif /* !HAS_ZSTD_H */ + break; default: ret = fdopen(fildes, mode); break; @@ -1,8 +1,9 @@ /* - * zio.h Provide an streamable interface to gziped/bzip2ed/LZW/LZMA files + * zio.h Provide an streamable interface to gziped/bzip2ed/LZW/LZMA/zst files * * Copyright 2004 Werner Fink, 2004 SuSE LINUX AG, Germany. * Copyright 2013 Werner Fink, 2013 SuSE LINUX AG, Germany. + * Copyright 2021 Werner Fink, 2021 SUSE Software Solutions Germany GmbH, Germany. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -1,11 +1,12 @@ /* * zioP.h Internal header for libzio, including required standard glibc - * header, zlib.h, lzma.h or lzmadec.h, and bzlib.h. - * Making the used libz, liblzma, and bzlib functions weak symbols. + * header, zlib.h, lzma.h or lzmadec.h, zstd.h, and bzlib.h. + * Making the used libz, liblzma, libzstd, and bzlib functions weak symbols. * * Copyright 2004 Werner Fink, 2004 SuSE LINUX AG, Germany. * Copyright 2006 Werner Fink, 2006 SuSE Products GmbH, Germany. * Copyright 2013 Werner Fink, 2013 SuSE Products GmbH, Germany. + * Copyright 2021 Werner Fink, 2021 SUSE Software Solutions Germany GmbH, Germany. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -135,6 +136,23 @@ weak_symbol(lzmadec_close); # endif #endif /* !HAS_LZMA_H */ +#if defined(HAS_ZSTD_H) +# include <zstd.h> +# ifndef NO_WEAK +weak_symbol(ZSTD_initCStream); +weak_symbol(ZSTD_initDStream); +weak_symbol(ZSTD_createCStream); +weak_symbol(ZSTD_createDStream); +weak_symbol(ZSTD_compressStream); +weak_symbol(ZSTD_decompressStream); +weak_symbol(ZSTD_endStream); +weak_symbol(ZSTD_freeCStream); +weak_symbol(ZSTD_freeDStream); +weak_symbol(ZSTD_isError); +weak_symbol(ZSTD_getErrorName); +# endif +#endif + #if defined(HAVE_FOPENCOOKIE) # undef HAVE_FUNOPEN __extension__ typedef off_t zio_off_t; |