From dedc2e63ed6e5ef1c36821aa62ccce03a1d81b17 Mon Sep 17 00:00:00 2001 From: Paul McCarthy Date: Mon, 7 Nov 2022 18:39:48 +0000 Subject: [PATCH 01/10] MNT: move c sources into separate dir --- setup.py | 13 +++++++------ {indexed_gzip => zran}/zran.c | 0 {indexed_gzip => zran}/zran.h | 0 {indexed_gzip => zran}/zran_file_util.c | 0 {indexed_gzip => zran}/zran_file_util.h | 0 5 files changed, 7 insertions(+), 6 deletions(-) rename {indexed_gzip => zran}/zran.c (100%) rename {indexed_gzip => zran}/zran.h (100%) rename {indexed_gzip => zran}/zran_file_util.c (100%) rename {indexed_gzip => zran}/zran_file_util.h (100%) diff --git a/setup.py b/setup.py index fbcea2ee..5f02c7bb 100644 --- a/setup.py +++ b/setup.py @@ -40,6 +40,7 @@ def finalize_options(self): def run(self): base = op.dirname(__file__) + zrbase = op.join(base, 'zran') igzbase = op.join(base, 'indexed_gzip') shutil.rmtree(op.join(base, 'build'), @@ -60,8 +61,8 @@ def run(self): files = [ '*.so', op.join(igzbase, 'indexed_gzip.c'), - op.join(igzbase, 'zran.o'), - op.join(igzbase, 'zran_file_util.o'), + op.join(zrbase, 'zran.o'), + op.join(zrbase, 'zran_file_util.o'), op.join(igzbase, '*.pyc'), op.join(igzbase, '*.so'), op.join(igzbase, 'tests', '*.so'), @@ -176,8 +177,8 @@ def run(self): igzip_ext = Extension( 'indexed_gzip.indexed_gzip', [op.join('indexed_gzip', 'indexed_gzip.{}'.format(pyx_ext)), - op.join('indexed_gzip', 'zran.c'), - op.join('indexed_gzip', 'zran_file_util.c')] + extra_srcs, + op.join('zran', 'zran.c'), + op.join('zran', 'zran_file_util.c')] + extra_srcs, libraries=libs, library_dirs=lib_dirs, include_dirs=include_dirs, @@ -202,8 +203,8 @@ def run(self): test_exts.append(Extension( 'indexed_gzip.tests.ctest_zran', [op.join('indexed_gzip', 'tests', 'ctest_zran.{}'.format(pyx_ext)), - op.join('indexed_gzip', 'zran.c'), - op.join('indexed_gzip', 'zran_file_util.c')] + extra_srcs, + op.join('zran', 'zran.c'), + op.join('zran', 'zran_file_util.c')] + extra_srcs, libraries=libs, library_dirs=lib_dirs, include_dirs=include_dirs, diff --git a/indexed_gzip/zran.c b/zran/zran.c similarity index 100% rename from indexed_gzip/zran.c rename to zran/zran.c diff --git a/indexed_gzip/zran.h b/zran/zran.h similarity index 100% rename from indexed_gzip/zran.h rename to zran/zran.h diff --git a/indexed_gzip/zran_file_util.c b/zran/zran_file_util.c similarity index 100% rename from indexed_gzip/zran_file_util.c rename to zran/zran_file_util.c diff --git a/indexed_gzip/zran_file_util.h b/zran/zran_file_util.h similarity index 100% rename from indexed_gzip/zran_file_util.h rename to zran/zran_file_util.h From cbcbe29e0e05f0501de2435a1c2a919868ba0a31 Mon Sep 17 00:00:00 2001 From: Paul McCarthy Date: Mon, 7 Nov 2022 19:18:49 +0000 Subject: [PATCH 02/10] ENH: compile-time option to use zlib-ng --- zran/zran.c | 76 ++++++++++++++++++++++++++++++++++------------------- 1 file changed, 49 insertions(+), 27 deletions(-) diff --git a/zran/zran.c b/zran/zran.c index 73e749f5..47228c6b 100644 --- a/zran/zran.c +++ b/zran/zran.c @@ -12,7 +12,12 @@ #include #include #include + +#ifdef ZRAN_USE_ZLIB_NG +#include "zlib-ng.h" +#else #include "zlib.h" +#endif #define PY_SSIZE_T_CLEAN #include @@ -72,6 +77,25 @@ static double round(double val) #define zran_log(...) #endif +/* Use equivalent symbols from zlib or zlib-ng */ +#ifdef ZRAN_USE_ZLIB_NG +#define Z_STREAM zng_stream +#define INFLATEINIT2 zng_inflateInit2 +#define INFLATE zng_inflate +#define INFLATEEND zng_inflateEnd +#define INFLATEPRIME zng_inflatePrime +#define INFLATESETDICTIONARY zng_inflateSetDictionary +#define CRC32 zng_crc32 +#else +#define Z_STREAM z_stream +#define INFLATEINIT2 inflateInit2 +#define INFLATE inflate +#define INFLATEEND inflateEnd +#define INFLATEPRIME inflatePrime +#define INFLATESETDICTIONARY inflateSetDictionary +#define CRC32 crc32 +#endif + /* * Identifier and version number for index files created by zran_export_index. @@ -233,9 +257,7 @@ static uint64_t _zran_estimate_offset( */ static int _zran_init_zlib_inflate( zran_index_t *index, /* The index */ - - z_stream *stream, /* Pointer to a z_stream struct */ - + Z_STREAM *stream, /* Pointer to a z_stream struct */ zran_point_t *point /* Pass in NULL to initialise for inflation from the current location in the input file. Or pass a pointer to the index point @@ -328,7 +350,7 @@ int ZRAN_READ_DATA_ERROR = -2; */ static int _zran_read_data_from_file( zran_index_t *index, /* The index */ - z_stream *stream, /* The z_stream struct */ + Z_STREAM *stream, /* The z_stream struct */ uint64_t cmp_offset, /* Current offset in the compressed data */ uint64_t uncmp_offset, /* Current offset in the uncompressed data */ uint32_t need_atleast /* Skip read if the read buffer already has @@ -361,7 +383,7 @@ int ZRAN_FIND_STREAM_NOT_FOUND = -1; */ static int _zran_find_next_stream( zran_index_t *index, /* The index */ - z_stream *stream, /* The z_stream struct */ + Z_STREAM *stream, /* The z_stream struct */ int *offset /* Used to store the number of bytes skipped over */ ); @@ -388,7 +410,7 @@ int ZRAN_VALIDATE_STREAM_INVALID = -1; */ static int _zran_validate_stream( zran_index_t *index, /* The index */ - z_stream *stream, /* The z_stream struct */ + Z_STREAM *stream, /* The z_stream struct */ int *offset /* Used to store the number of bytes skipped over */ ); @@ -558,7 +580,7 @@ uint32_t ZRAN_INFLATE_STOP_AT_BLOCK = 64; static int _zran_inflate( zran_index_t *index, /* Pointer to the index. */ - z_stream *strm, /* Pointer to a z_stream struct. */ + Z_STREAM *strm, /* Pointer to a z_stream struct. */ uint64_t offset, /* Compressed data offset to start inflation from. */ @@ -1201,7 +1223,7 @@ int _zran_add_point(zran_index_t *index, /* Initialise the given z_stream struct for decompression/inflation. */ int _zran_init_zlib_inflate(zran_index_t *index, - z_stream *strm, + Z_STREAM *strm, zran_point_t *point) { int ret; @@ -1227,9 +1249,9 @@ int _zran_init_zlib_inflate(zran_index_t *index, zran_log("_zran_init_zlib_inflate from current " "seek location (expecting GZIP header)\n"); - if (inflateInit2(strm, window + 32) != Z_OK) { goto fail; } - if (inflate(strm, Z_BLOCK) != Z_OK) { goto fail_free_strm; } - if (inflateEnd(strm) != Z_OK) { goto fail; } + if (INFLATEINIT2(strm, window + 32) != Z_OK) { goto fail; } + if (INFLATE(strm, Z_BLOCK) != Z_OK) { goto fail_free_strm; } + if (INFLATEEND(strm) != Z_OK) { goto fail; } } /* @@ -1262,7 +1284,7 @@ int _zran_init_zlib_inflate(zran_index_t *index, * in _zran_inflate). */ - if (inflateInit2(strm, -window) != Z_OK) { + if (INFLATEINIT2(strm, -window) != Z_OK) { goto fail; } @@ -1287,7 +1309,7 @@ int _zran_init_zlib_inflate(zran_index_t *index, goto fail_free_strm; } - if (inflatePrime(strm, + if (INFLATEPRIME(strm, point->bits, ret >> (8 - point->bits)) != Z_OK) goto fail_free_strm; } @@ -1296,7 +1318,7 @@ int _zran_init_zlib_inflate(zran_index_t *index, * Initialise the inflate stream * with the index point data. */ - if (inflateSetDictionary(strm, + if (INFLATESETDICTIONARY(strm, point->data, index->window_size) != Z_OK) goto fail_free_strm; @@ -1327,7 +1349,7 @@ int _zran_init_zlib_inflate(zran_index_t *index, * clause. */ fail_free_strm: - inflateEnd(strm); + INFLATEEND(strm); /* Something has gone wrong */ fail: return -1; @@ -1339,7 +1361,7 @@ int _zran_init_zlib_inflate(zran_index_t *index, * decompression. */ static int _zran_read_data_from_file(zran_index_t *index, - z_stream *stream, + Z_STREAM *stream, uint64_t cmp_offset, uint64_t uncmp_offset, uint32_t need_atleast) { @@ -1467,7 +1489,7 @@ static int _zran_read_data_from_file(zran_index_t *index, * contains concatenated streams). */ int _zran_find_next_stream(zran_index_t *index, - z_stream *stream, + Z_STREAM *stream, int *offset) { int ret; @@ -1511,7 +1533,7 @@ int _zran_find_next_stream(zran_index_t *index, * Re-configure for inflation * from the new stream. */ - if (inflateEnd(stream) != Z_OK) { + if (INFLATEEND(stream) != Z_OK) { goto fail; } @@ -1535,7 +1557,7 @@ int _zran_find_next_stream(zran_index_t *index, /* Validate the CRC32 and size of a GZIP stream. */ static int _zran_validate_stream(zran_index_t *index, - z_stream *stream, + Z_STREAM *stream, int *offset) { uint32_t crc; @@ -1581,7 +1603,7 @@ static int _zran_validate_stream(zran_index_t *index, /* The workhorse. Inflate/decompress data from the file. */ static int _zran_inflate(zran_index_t *index, - z_stream *strm, + Z_STREAM *strm, uint64_t offset, uint16_t flags, uint32_t *total_consumed, @@ -1633,7 +1655,7 @@ static int _zran_inflate(zran_index_t *index, * if we are initialising. */ if (inflate_init_stream(flags)) { - memset(strm, 0, sizeof(z_stream)); + memset(strm, 0, sizeof(Z_STREAM)); } /* @@ -1898,10 +1920,10 @@ static int _zran_inflate(zran_index_t *index, * stream, or it runs out of input or output. */ if (inflate_stop_at_block(flags)) { - z_ret = inflate(strm, Z_BLOCK); + z_ret = INFLATE(strm, Z_BLOCK); } else { - z_ret = inflate(strm, Z_NO_FLUSH); + z_ret = INFLATE(strm, Z_NO_FLUSH); } /* @@ -1964,7 +1986,7 @@ static int _zran_inflate(zran_index_t *index, index->validating && !(index->flags & ZRAN_SKIP_CRC_CHECK)) { index->stream_size += bytes_output; - index->stream_crc32 = crc32(index->stream_crc32, + index->stream_crc32 = CRC32(index->stream_crc32, strm->next_out - bytes_output, bytes_output); } @@ -2182,7 +2204,7 @@ static int _zran_inflate(zran_index_t *index, * is active, do just that. */ if (inflate_free_stream(flags)) { - if (inflateEnd(strm) != Z_OK) + if (INFLATEEND(strm) != Z_OK) goto fail; } @@ -2240,7 +2262,7 @@ int _zran_expand_index(zran_index_t *index, uint64_t until) { int z_ret; /* Zlib stream struct */ - z_stream strm; + Z_STREAM strm; /* * Number of bytes read/decompressed @@ -2701,7 +2723,7 @@ int64_t zran_read(zran_index_t *index, * Zlib stream struct and starting * index point for the read.. */ - z_stream strm; + Z_STREAM strm; zran_point_t *start = NULL; /* From 369b8224f0abd56818961473bf28e903dffa1c07 Mon Sep 17 00:00:00 2001 From: Paul McCarthy Date: Mon, 7 Nov 2022 19:26:19 +0000 Subject: [PATCH 03/10] MNT: build time option to statically link against local zlib-ng build --- setup.py | 65 +++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 45 insertions(+), 20 deletions(-) diff --git a/setup.py b/setup.py index 5f02c7bb..eec55c43 100644 --- a/setup.py +++ b/setup.py @@ -77,13 +77,24 @@ def run(self): # Platform information -python2 = sys.version_info[0] == 2 -noc99 = python2 or (sys.version_info[0] == 3 and sys.version_info[1] <= 4) -windows = sys.platform.startswith("win") -testing = 'INDEXED_GZIP_TESTING' in os.environ +python2 = sys.version_info[0] == 2 +noc99 = python2 or (sys.version_info[0] == 3 and sys.version_info[1] <= 4) +windows = sys.platform.startswith("win") +testing = 'INDEXED_GZIP_TESTING' in os.environ + +# We can use zlib or zlib-ng via one +# of the following mechanisms: +# +# - Default: Dynamically link against system zlib +# - ZLIB_HOME: Path to zlib build directory - statically +# link against built zlib library. Takes +# precedence over ZLIB_NG_HOME +# - ZLIB_NG_HOME: Path to zlib-ng build directory - statically +# link against built zlib-ng library # compile ZLIB source? -ZLIB_HOME = os.environ.get("ZLIB_HOME", None) +ZLIB_HOME = os.environ.get('ZLIB_HOME', None) +ZLIB_NG_HOME = os.environ.get('ZLIB_NG_HOME', None) # Load README description readme = op.join(op.dirname(__file__), 'README.md') @@ -113,39 +124,54 @@ def run(self): have_numpy = False print('indexed_gzip setup') -print(' have_cython: {} (if True, modules will be cythonized, ' +print(' have_cython: {} (if True, modules will be cythonized, ' 'otherwise pre-cythonized C files are assumed to be ' 'present)'.format(have_cython)) -print(' have_numpy: {} (if True, test modules will ' +print(' have_numpy: {} (if True, test modules will ' 'be compiled)'.format(have_numpy)) -print(' ZLIB_HOME: {} (if set, ZLIB sources are compiled into ' - 'the indexed_gzip extension)'.format(ZLIB_HOME)) -print(' testing: {} (if True, code will be compiled with line ' +print(' ZLIB_HOME: {} (if set, indexed_gzip is linked against ' + 'zlib)'.format(ZLIB_HOME)) +print(' ZLIB_NG_HOME: {} (if set, indexed_gzip is linked against ' + 'zlib-ng (unless ZLIB_HOME is also set)'.format(ZLIB_NG_HOME)) +print(' testing: {} (if True, code will be compiled with line ' 'tracing enabled)'.format(testing)) # compile flags -include_dirs = ['indexed_gzip'] +include_dirs = ['zran'] lib_dirs = [] libs = [] extra_srcs = [] +extra_objects = [] extra_compile_args = [] compiler_directives = {'language_level' : 2} define_macros = [] +# Link against zlib built in ZLIB_HOME if ZLIB_HOME is not None: include_dirs.append(ZLIB_HOME) - extra_srcs.extend(glob.glob(op.join(ZLIB_HOME, '*.c'))) + if windows: extra_objects.append(op.join(ZLIB_HOME, 'zlibstatic.lib')) + else: extra_objects.append(op.join(ZLIB_HOME, 'libz.a')) + +# Link against zlib built in ZLIB_NG_HOME +elif ZLIB_NG_HOME is not None: + include_dirs .append(ZLIB_NG_HOME) + define_macros.append(('ZRAN_USE_ZLIB_NG', '1')) + if windows: extra_objects.append(op.join(ZLIB_NG_HOME, 'zlib-ngstatic.lib')) + else: extra_objects.append(op.join(ZLIB_NG_HOME, 'libz-ng.a')) + +# Link against system zlib +else: + if windows: libs.append('zlib') + else: libs.append('z') # If numpy is present, we need # to include the headers if have_numpy: include_dirs.append(np.get_include()) +# Windows specific flags if windows: - if ZLIB_HOME is None: - libs.append('zlib') - # For stdint.h which is not included in the old Visual C # compiler used for Python 2 if python2: @@ -156,12 +182,8 @@ def run(self): if noc99: extra_compile_args += ['-DNO_C99'] -# linux / macOS +# linux / macOS specific flags else: - # if ZLIB_HOME is set, statically link, - # rather than use system-provided zlib - if ZLIB_HOME is None: - libs.append('z') extra_compile_args += ['-Wall', '-Wno-unused-function'] if testing: @@ -183,6 +205,7 @@ def run(self): library_dirs=lib_dirs, include_dirs=include_dirs, extra_compile_args=extra_compile_args, + extra_objects=extra_objects, define_macros=define_macros) # Optional test modules @@ -194,6 +217,7 @@ def run(self): libraries=libs, library_dirs=lib_dirs, include_dirs=include_dirs, + extra_objects=extra_objects, extra_compile_args=extra_compile_args, define_macros=define_macros) ] @@ -208,6 +232,7 @@ def run(self): libraries=libs, library_dirs=lib_dirs, include_dirs=include_dirs, + extra_objects=extra_objects, extra_compile_args=extra_compile_args, define_macros=define_macros)) From afc0c1737d70a80c7d2f470ce7d03ef7d8a1bb00 Mon Sep 17 00:00:00 2001 From: Paul McCarthy Date: Wed, 9 Nov 2022 11:46:04 +0000 Subject: [PATCH 04/10] ENH: simplified interface for reading from gzipped or non-gzipped files. Makefile for compiling libcindexed_gzip.[so|a] --- cindexed_gzip/Makefile | 58 ++++++++ cindexed_gzip/cindexed_gzip.c | 174 +++++++++++++++++++++++ cindexed_gzip/cindexed_gzip.h | 53 +++++++ {zran => cindexed_gzip}/zran.c | 0 {zran => cindexed_gzip}/zran.h | 10 ++ {zran => cindexed_gzip}/zran_file_util.c | 0 {zran => cindexed_gzip}/zran_file_util.h | 0 7 files changed, 295 insertions(+) create mode 100644 cindexed_gzip/Makefile create mode 100644 cindexed_gzip/cindexed_gzip.c create mode 100644 cindexed_gzip/cindexed_gzip.h rename {zran => cindexed_gzip}/zran.c (100%) rename {zran => cindexed_gzip}/zran.h (99%) rename {zran => cindexed_gzip}/zran_file_util.c (100%) rename {zran => cindexed_gzip}/zran_file_util.h (100%) diff --git a/cindexed_gzip/Makefile b/cindexed_gzip/Makefile new file mode 100644 index 00000000..5ebd9ede --- /dev/null +++ b/cindexed_gzip/Makefile @@ -0,0 +1,58 @@ +# Build the cindexed_gzip C library. +# +# The library can be built in one of the following ways: +# - Dynamically linked against the system zlib installation (default) +# - Statically linked against a zlib build at $ZLIB_HOME +# - Statically linked against a zlib-ng build at $ZLIB_NG_HOME + +all: libcindexed_gzip.so libcindexed_gzip_static.a + +AR ?= ar +CC ?= gcc +PYTHON ?= python +LIBS = +OBJFILES = zran.o zran_file_util.o cindexed_gzip.o +PYTHON_INCLUDE_DIRECTORY = $(shell ${PYTHON} -c "from sysconfig import get_paths; print(get_paths()['include'])") +CFLAGS += -I${PYTHON_INCLUDE_DIRECTORY} + +# statically link against built zlib +ifdef ZLIB_HOME + OBJFILES += ${ZLIB_HOME}/libz.a + CFLAGS += -I${ZLIB_HOME} +# statically link against built zlib-ng +else ifdef ZLIB_NG_HOME + OBJFILES += ${ZLIB_NG_HOME}/libz-ng.a + CFLAGS += -I${ZLIB_NG_HOME} -DZRAN_USE_ZLIB_NG=1 +# dynamically link against system zlib +else + LIBS += -lz +endif + + +clean: + rm -f *.o *.so *.a + + +install: +ifndef PREFIX + @echo "PREFIX not set - aborting" + exit 1 +endif + mkdir -p ${PREFIX}/lib + mkdir -p ${PREFIX}/include/cindexed_gzip + cp libcindexed_gzip.so ${PREFIX}/lib/ + cp libcindexed_gzip_static.a ${PREFIX}/lib/ + cp cindexed_gzip.h ${PREFIX}/include/cindexed_gzip/ + cp zran.h ${PREFIX}/include/cindexed_gzip/ + + +%.o: %.c + ${CC} ${CFLAGS} -c -o $@ $^ + + +libcindexed_gzip.so: ${OBJFILES} + ${CC} ${CFLAGS} -shared -o $@ $^ ${LDFLAGS} ${LIBS} + + +libcindexed_gzip_static.a: ${OBJFILES} + ${AR} -r $@ $^ ${LIBS} diff --git a/cindexed_gzip/cindexed_gzip.c b/cindexed_gzip/cindexed_gzip.c new file mode 100644 index 00000000..3c8b0fe7 --- /dev/null +++ b/cindexed_gzip/cindexed_gzip.c @@ -0,0 +1,174 @@ +/* + * Simplified C interface for reading GZIP files with indexed_gzip. + * + * Both GZIP and other files may be loaded via this interface; GZIP files will + * be read via the zran module, and other files will read normally. + */ +#include + +#include "zran.h" +#include "cindexed_gzip.h" + + +/* + * igz_file struct, for reading from a GZIP or other file. File handle is + * opened/closed on-demand. + */ +struct _igz_file { + zran_index_t index; + int compressed; + char *filepath; +}; + + +/* + * Returns 1 if the given file is a GZIP file, 0 if it is not, -1 if an error + * occurs. + * + * https://www.rfc-editor.org/rfc/rfc1952#section-2.3.1 + */ + +static int is_gzip_file(FILE *f) { + + uint8_t magic[2]; + + if (fseek(f, 0, SEEK_SET) != 0) { goto fail; } + if (fread(magic, 1, 2, f) != 2) { goto fail; } + if (fseek(f, 0, SEEK_SET) != 0) { goto fail; } + + if (magic[0] == 0x1f && magic[1] == 0x8b) { return 1; } + else { return 0; } +fail: + return -1; +}; + + +/* + * Returns the size of the given file. + */ +static uint64_t file_size(FILE *f) { + + size_t s; + if (fseek(f, 0, SEEK_END) != 0) { goto fail; } + + s = ftello(f); + + if (s < 0) { goto fail; } + if (fseek(f, 0, SEEK_SET) != 0) { goto fail; } + + return (uint64_t)s; +fail: + return -1; +} + + +/* + * Open a file for reading. GZIP files will be loaded via the zran module. + * Other files will be read normally (via fseek/fread). The igz_file must be + * passed to igz_close when it is no longer needed. + */ +igz_file * igz_open(char *filepath) { + + igz_file *gzf = NULL; + FILE *f = NULL; + size_t namelen = 0; + off_t size = 0; + + f = fopen(filepath, "rb"); + if (f == NULL) { + goto fail; + } + + gzf = calloc(1, sizeof(igz_file)); + if (gzf == NULL) { + goto fail; + } + + namelen = strlen(filepath); + gzf->filepath = malloc(namelen + 1); + strcpy(gzf->filepath, filepath); + + gzf->compressed = is_gzip_file(f); + if (gzf->compressed == -1) { + goto fail; + } + + if (gzf->compressed) { + if (zran_init(&(gzf->index), f, NULL, 4194304, + 32768, 1048576, ZRAN_AUTO_BUILD) != 0) { + goto fail; + } + gzf->index.fd = NULL; + } + + fclose(f); + + return gzf; + + fail: + if (gzf != NULL) { free(gzf); } + if (f != NULL) { fclose(f); } + return NULL; +} + + +/* + * Free resources associated with the igz_file. Must only be called once for + * a given igz_file. + */ +int igz_close(igz_file *gzf) { + if (gzf->compressed) { + zran_free(&(gzf->index)); + } + free(gzf->filepath); + free(gzf); +} + +/* + * Read up to len bytes from the given igz_file, starting from off. The bytes + * are copied into buf. + * + * Returns th number of bytes that were read, 0 if off == EOF, or a negative + * value if an error occurred. + */ +int64_t igz_read(igz_file *gzf, void *buf, uint64_t len, uint64_t off) { + + FILE *f = NULL; + int64_t bytes_read = 0; + + if (len > INT64_MAX) { + goto fail; + } + + f = fopen(gzf->filepath, "rb"); + if (f == NULL) { + goto fail; + } + + if (gzf->compressed) { + gzf->index.fd = f; + if (zran_seek(&(gzf->index), off, SEEK_SET, NULL) != ZRAN_SEEK_OK) { + goto fail; + } + + bytes_read = zran_read(&(gzf->index), buf, len); + if (bytes_read == ZRAN_READ_EOF) { + goto fail; + } + } + else { + fseek(f, off, SEEK_SET); + fread(buf, len, 1, f); + } + + gzf->index.fd = NULL; + fclose(f); + + return bytes_read; + +fail: + if (f != NULL) { + fclose(f); + } + return -1; +} diff --git a/cindexed_gzip/cindexed_gzip.h b/cindexed_gzip/cindexed_gzip.h new file mode 100644 index 00000000..608cdad4 --- /dev/null +++ b/cindexed_gzip/cindexed_gzip.h @@ -0,0 +1,53 @@ +/* + * Simplified C interface for reading GZIP files with indexed_gzip. + * + * Both GZIP and other files may be loaded via this interface; GZIP files will + * be read via the zran module, and other files will read normally. + */ +#ifndef __CINDEXED_GZIP_H__ +#define __CINDEXED_GZIP_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +#include "zran.h" + +/* + * Struct representing a file which has been loaded via igz_open. + */ +struct _igz_file; +typedef struct _igz_file igz_file; + + +/* + * Open a file for reading. GZIP files will be loaded via the zran module. + * Other files will be read normally (via fseek/fread). The igz_file must be + * passed to igz_close when it is no longer needed. + */ +igz_file * igz_open(char *filepath); + + +/* + * Free resources associated with the igz_file. Must only be called once for + * a given igz_file. + */ +int igz_close(igz_file *gzf); + + +/* + * Read up to len bytes from the given igz_file, starting from off. The bytes + * are copied into buf. + * + * Returns th number of bytes that were read, 0 if off == EOF, or a negative + * value if an error occurred. + */ +int64_t igz_read(igz_file *gzf, void *buf, uint64_t len, uint64_t off); + +#ifdef __cplusplus +} +#endif + +#endif /* __CINDEXED_GZIP_H__ */ diff --git a/zran/zran.c b/cindexed_gzip/zran.c similarity index 100% rename from zran/zran.c rename to cindexed_gzip/zran.c diff --git a/zran/zran.h b/cindexed_gzip/zran.h similarity index 99% rename from zran/zran.h rename to cindexed_gzip/zran.h index 094c68ae..d7b4e05a 100644 --- a/zran/zran.h +++ b/cindexed_gzip/zran.h @@ -1,6 +1,12 @@ #ifndef __ZRAN_H__ #define __ZRAN_H__ + +#ifdef __cplusplus +extern "C" { +#endif + + /* * The zran module is an adaptation of the zran example, written by Mark * Alder, which ships with the zlib source code. It allows the creation @@ -502,4 +508,8 @@ int zran_import_index( PyObject *f /* Open handle to import file object */ ); +#ifdef __cplusplus +} +#endif + #endif /* __ZRAN_H__ */ diff --git a/zran/zran_file_util.c b/cindexed_gzip/zran_file_util.c similarity index 100% rename from zran/zran_file_util.c rename to cindexed_gzip/zran_file_util.c diff --git a/zran/zran_file_util.h b/cindexed_gzip/zran_file_util.h similarity index 100% rename from zran/zran_file_util.h rename to cindexed_gzip/zran_file_util.h From bbde66e3da8f9eabefb13d86d2d4f0dd40f667cf Mon Sep 17 00:00:00 2001 From: Paul McCarthy Date: Wed, 9 Nov 2022 12:12:45 +0000 Subject: [PATCH 05/10] DOC: Changelog --- CHANGELOG.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index be307e54..102c181e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,15 @@ # `indexed_gzip` changelog +## 1.8.0 (Under development) + + +* Changes to allow linking against + [zlib-ng](https://github.com/zlib-ng/zlib-ng/) (#107). +* Simplified C interface `cindexed_gzip`, and machinery for compiling as a + static or shared C library (#107). + + ## 1.7.1 (March 31st 2023) @@ -8,7 +17,6 @@ which do not implement `fileno()` (#118). - ## 1.7.0 (September 12th 2022) From 6309225ae586d9781d471354639c42cc905b890e Mon Sep 17 00:00:00 2001 From: Paul McCarthy Date: Wed, 9 Nov 2022 13:04:25 +0000 Subject: [PATCH 06/10] MNT: inject igzip version number into C lib --- cindexed_gzip/Makefile | 31 +++++++++++-------- .../{cindexed_gzip.h => cindexed_gzip.h.in} | 7 +++++ 2 files changed, 25 insertions(+), 13 deletions(-) rename cindexed_gzip/{cindexed_gzip.h => cindexed_gzip.h.in} (89%) diff --git a/cindexed_gzip/Makefile b/cindexed_gzip/Makefile index 5ebd9ede..d392f53f 100644 --- a/cindexed_gzip/Makefile +++ b/cindexed_gzip/Makefile @@ -14,6 +14,7 @@ LIBS = OBJFILES = zran.o zran_file_util.o cindexed_gzip.o PYTHON_INCLUDE_DIRECTORY = $(shell ${PYTHON} -c "from sysconfig import get_paths; print(get_paths()['include'])") CFLAGS += -I${PYTHON_INCLUDE_DIRECTORY} +INDEXED_GZIP_VERSION = $(shell cat ../indexed_gzip/__init__.py | grep __version__ | cut -d ' ' -f 3 | tr -d "'") # statically link against built zlib ifdef ZLIB_HOME @@ -30,7 +31,23 @@ endif clean: - rm -f *.o *.so *.a + rm -f *.o *.so *.a cindexed_gzip.h + + +cindexed_gzip.h: cindexed_gzip.h.in + sed -e "s/@INDEXED_GZIP_VERSION@/${INDEXED_GZIP_VERSION}/" $< > $@ + + +%.o: %.c cindexed_gzip.h + ${CC} ${CFLAGS} -c -o $@ $< + + +libcindexed_gzip.so: ${OBJFILES} + ${CC} ${CFLAGS} -shared -o $@ $^ ${LDFLAGS} ${LIBS} + + +libcindexed_gzip_static.a: ${OBJFILES} + ${AR} -r $@ $^ ${LIBS} install: @@ -44,15 +61,3 @@ endif cp libcindexed_gzip_static.a ${PREFIX}/lib/ cp cindexed_gzip.h ${PREFIX}/include/cindexed_gzip/ cp zran.h ${PREFIX}/include/cindexed_gzip/ - - -%.o: %.c - ${CC} ${CFLAGS} -c -o $@ $^ - - -libcindexed_gzip.so: ${OBJFILES} - ${CC} ${CFLAGS} -shared -o $@ $^ ${LDFLAGS} ${LIBS} - - -libcindexed_gzip_static.a: ${OBJFILES} - ${AR} -r $@ $^ ${LIBS} diff --git a/cindexed_gzip/cindexed_gzip.h b/cindexed_gzip/cindexed_gzip.h.in similarity index 89% rename from cindexed_gzip/cindexed_gzip.h rename to cindexed_gzip/cindexed_gzip.h.in index 608cdad4..d9a66b93 100644 --- a/cindexed_gzip/cindexed_gzip.h +++ b/cindexed_gzip/cindexed_gzip.h.in @@ -15,6 +15,13 @@ extern "C" { #include "zran.h" +/* + * Version string, defined in indexed_gzip/__init__.py (see + * cindexed_gzip/Makefile). + */ +#define CINDEXED_GZIP_VERSION "@INDEXED_GZIP_VERSION@" + + /* * Struct representing a file which has been loaded via igz_open. */ From 438ba1008843ec55a1ddb14e7d56028e4c4efd3b Mon Sep 17 00:00:00 2001 From: Paul McCarthy Date: Thu, 10 Nov 2022 15:43:22 +0000 Subject: [PATCH 07/10] MNT: add ability to zran to be compiled without using Python API --- cindexed_gzip/zran.c | 15 +++++----- cindexed_gzip/zran.h | 5 ++++ cindexed_gzip/zran_file_util.c | 55 +++++++++++++++++++++++++++++++--- cindexed_gzip/zran_file_util.h | 5 ++++ 4 files changed, 68 insertions(+), 12 deletions(-) diff --git a/cindexed_gzip/zran.c b/cindexed_gzip/zran.c index 47228c6b..0df65d88 100644 --- a/cindexed_gzip/zran.c +++ b/cindexed_gzip/zran.c @@ -19,26 +19,25 @@ #include "zlib.h" #endif +#ifdef ZRAN_SUPPORT_PYTHON #define PY_SSIZE_T_CLEAN #include +#endif #ifdef _WIN32 #include "windows.h" #include "io.h" -static int is_readonly(FILE *fd, PyObject *f) +static int is_readonly(FILE *fd) { /* Can't find a way to do this correctly under - Windows and the check is not required anyway - since the underlying Python module checks it - already */ + Windows */ return 1; } #else #include /* Check if file is read-only */ -static int is_readonly(FILE *fd, PyObject *f) +static int is_readonly(FILE *fd) { - /* Skip the test for python file-likes */ return fd != NULL ? (fcntl(fileno(fd), F_GETFL) & O_ACCMODE) == O_RDONLY : 1; @@ -661,7 +660,7 @@ int zran_init(zran_index_t *index, goto fail; /* The file must be opened in read-only mode */ - if (!is_readonly(fd, f)) + if (!is_readonly(fd)) goto fail; /* @@ -3236,7 +3235,7 @@ int zran_import_index(zran_index_t *index, index->flags |= ZRAN_SKIP_CRC_CHECK; /* Check if file is read only. */ - if (!is_readonly(fd, f)) goto fail; + if (!is_readonly(fd)) goto fail; /* Read ID, and check for file errors and EOF. */ f_ret = fread_(file_id, sizeof(file_id), 1, fd, f); diff --git a/cindexed_gzip/zran.h b/cindexed_gzip/zran.h index d7b4e05a..c8cf6b09 100644 --- a/cindexed_gzip/zran.h +++ b/cindexed_gzip/zran.h @@ -14,11 +14,16 @@ extern "C" { * of random seek/read access to the uncompressed data. */ +#include #include #include +#ifdef ZRAN_SUPPORT_PYTHON #define PY_SSIZE_T_CLEAN #include +#else +#define PyObject void +#endif struct _zran_index; struct _zran_point; diff --git a/cindexed_gzip/zran_file_util.c b/cindexed_gzip/zran_file_util.c index cb01dde1..7c1fcbb4 100644 --- a/cindexed_gzip/zran_file_util.c +++ b/cindexed_gzip/zran_file_util.c @@ -12,8 +12,10 @@ #include #include +#ifdef ZRAN_SUPPORT_PYTHON #define PY_SSIZE_T_CLEAN #include +#endif #include "zran_file_util.h" @@ -27,6 +29,9 @@ #define FTELL ftello #endif + +#ifdef ZRAN_SUPPORT_PYTHON + /* * The zran functions are typically called with the GIL released. These * macros are used to temporarily (re-)acquire and release the GIL when @@ -39,12 +44,14 @@ #define _ZRAN_FILE_UTIL_RELEASE_GIL \ PyGILState_Release(s); - /* * Implements a method analogous to fread that is performed on Python * file-like objects. */ -size_t _fread_python(void *ptr, size_t size, size_t nmemb, PyObject *f) { +size_t _fread_python(void *ptr, + size_t size, + size_t nmemb, + PyObject *f) { PyObject *data = NULL; char *buf; @@ -273,27 +280,42 @@ int _seekable_python2(PyObject *f) { return ret >= 0; } + +#endif /* ZRAN_SUPPORT_PYTHON */ + /* * Calls ferror on fd if specified, otherwise the Python-specific method on f. */ int ferror_(FILE *fd, PyObject *f) { + #ifdef ZRAN_SUPPORT_PYTHON return fd != NULL ? ferror(fd) : _ferror_python(f); + #else + return ferror(fd); + #endif } /* * Calls fseek on fd if specified, otherwise the Python-specific method on f. */ int fseek_(FILE *fd, PyObject *f, int64_t offset, int whence) { + #ifdef ZRAN_SUPPORT_PYTHON return fd != NULL ? FSEEK(fd, offset, whence) : _fseek_python(f, offset, whence); + #else + return FSEEK(fd, offset, whence); + #endif } /* * Calls ftell on fd if specified, otherwise the Python-specific method on f. */ int64_t ftell_(FILE *fd, PyObject *f) { + #ifdef ZRAN_SUPPORT_PYTHON return fd != NULL ? FTELL(fd) : _ftell_python(f); + #else + return FTELL(fd); + #endif } /* @@ -301,9 +323,13 @@ int64_t ftell_(FILE *fd, PyObject *f) { */ size_t fread_(void *ptr, size_t size, size_t nmemb, FILE *fd, PyObject *f) { + #ifdef ZRAN_SUPPORT_PYTHON return fd != NULL ? fread(ptr, size, nmemb, fd) : _fread_python(ptr, size, nmemb, f); + #else + return fread(ptr, size, nmemb, fd); + #endif } /* @@ -312,14 +338,22 @@ size_t fread_(void *ptr, size_t size, size_t nmemb, FILE *fd, PyObject *f) { * read, to determine if the file is at EOF. */ int feof_(FILE *fd, PyObject *f, size_t f_ret) { + #ifdef ZRAN_SUPPORT_PYTHON return fd != NULL ? feof(fd): _feof_python(f, f_ret); + #else + return feof(fd); + #endif } /* * Calls fflush on fd if specified, otherwise the Python-specific method on f. */ int fflush_(FILE *fd, PyObject *f) { + #ifdef ZRAN_SUPPORT_PYTHON return fd != NULL ? fflush(fd): _fflush_python(f); + #else + return fflush(fd); + #endif } /* @@ -330,26 +364,39 @@ size_t fwrite_(const void *ptr, size_t nmemb, FILE *fd, PyObject *f) { + #ifdef ZRAN_SUPPORT_PYTHON return fd != NULL ? fwrite(ptr, size, nmemb, fd) : _fwrite_python(ptr, size, nmemb, f); + #else + return fwrite(ptr, size, nmemb, fd); + #endif } /* * Calls getc on fd if specified, otherwise the Python-specific method on f. */ int getc_(FILE *fd, PyObject *f) { + #ifdef ZRAN_SUPPORT_PYTHON return fd != NULL ? getc(fd): _getc_python(f); + #else + return getc(fd); + #endif } /* - * Returns whether the given file is seekable. If fd is specified, assumes it's always seekable. - * If f is specified, calls f.seekable() to see if the Python file object is seekable. + * Returns whether the given file is seekable. If fd is specified, assumes + * it's always seekable. If f is specified, calls f.seekable() to see if the + * Python file object is seekable. */ int seekable_(FILE *fd, PyObject *f) { + #ifdef ZRAN_SUPPORT_PYTHON #if PY_MAJOR_VERSION > 2 return fd != NULL ? 1: _seekable_python(f); #else return fd != NULL ? 1: _seekable_python2(f); #endif + #else + return 1; + #endif } diff --git a/cindexed_gzip/zran_file_util.h b/cindexed_gzip/zran_file_util.h index 28747096..4fbac1f5 100644 --- a/cindexed_gzip/zran_file_util.h +++ b/cindexed_gzip/zran_file_util.h @@ -9,6 +9,9 @@ #include #include +#ifndef ZRAN_SUPPORT_PYTHON +#define PyObject void +#else #define PY_SSIZE_T_CLEAN #include @@ -65,6 +68,8 @@ int _getc_python(PyObject *f); */ int _seekable_python(PyObject *f); +#endif /* ZRAN_SUPPORT_PYTHON */ + /* * Calls ferror on fd if specified, otherwise the Python-specific method on f. */ From 1493124f481f870fdf8d87f5b4e27202deeb8b19 Mon Sep 17 00:00:00 2001 From: Paul McCarthy Date: Thu, 10 Nov 2022 16:03:28 +0000 Subject: [PATCH 08/10] RF: Don't over-specify cindexed_gzip linking options. New namespaced ccindexed_gzip.h header file for using in c++. Use const char * in igz_open --- cindexed_gzip/Makefile | 30 +++++++++++++++--------------- cindexed_gzip/ccindexed_gzip.h | 8 ++++++++ cindexed_gzip/cindexed_gzip.c | 7 ++++--- cindexed_gzip/cindexed_gzip.h.in | 2 +- setup.py | 16 ++++++++-------- 5 files changed, 36 insertions(+), 27 deletions(-) create mode 100644 cindexed_gzip/ccindexed_gzip.h diff --git a/cindexed_gzip/Makefile b/cindexed_gzip/Makefile index d392f53f..a61a7883 100644 --- a/cindexed_gzip/Makefile +++ b/cindexed_gzip/Makefile @@ -1,32 +1,31 @@ # Build the cindexed_gzip C library. # # The library can be built in one of the following ways: -# - Dynamically linked against the system zlib installation (default) -# - Statically linked against a zlib build at $ZLIB_HOME -# - Statically linked against a zlib-ng build at $ZLIB_NG_HOME +# - Linked against the system zlib installation (default) +# - Linked against a zlib build at $ZLIB_HOME +# - Linked against a zlib-ng build at $ZLIB_NG_HOME all: libcindexed_gzip.so libcindexed_gzip_static.a AR ?= ar CC ?= gcc PYTHON ?= python -LIBS = OBJFILES = zran.o zran_file_util.o cindexed_gzip.o PYTHON_INCLUDE_DIRECTORY = $(shell ${PYTHON} -c "from sysconfig import get_paths; print(get_paths()['include'])") -CFLAGS += -I${PYTHON_INCLUDE_DIRECTORY} +CFLAGS += -I${PYTHON_INCLUDE_DIRECTORY} -I.. INDEXED_GZIP_VERSION = $(shell cat ../indexed_gzip/__init__.py | grep __version__ | cut -d ' ' -f 3 | tr -d "'") -# statically link against built zlib +# link against built zlib ifdef ZLIB_HOME - OBJFILES += ${ZLIB_HOME}/libz.a - CFLAGS += -I${ZLIB_HOME} -# statically link against built zlib-ng + LDFLAGS += -L${ZLIB_HOME} -lz + CFLAGS += -I${ZLIB_HOME} +# link against built zlib-ng else ifdef ZLIB_NG_HOME - OBJFILES += ${ZLIB_NG_HOME}/libz-ng.a - CFLAGS += -I${ZLIB_NG_HOME} -DZRAN_USE_ZLIB_NG=1 -# dynamically link against system zlib + LDFLAGS += -L${ZLIB_NG_HOME} -lz-ng + CFLAGS += -I${ZLIB_NG_HOME} -DZRAN_USE_ZLIB_NG=1 +# link against system zlib else - LIBS += -lz + LDFLAGS += -lz endif @@ -43,11 +42,11 @@ cindexed_gzip.h: cindexed_gzip.h.in libcindexed_gzip.so: ${OBJFILES} - ${CC} ${CFLAGS} -shared -o $@ $^ ${LDFLAGS} ${LIBS} + ${CC} ${CFLAGS} -shared -o $@ $^ ${LDFLAGS} libcindexed_gzip_static.a: ${OBJFILES} - ${AR} -r $@ $^ ${LIBS} + ${AR} -r $@ $^ install: @@ -60,4 +59,5 @@ endif cp libcindexed_gzip.so ${PREFIX}/lib/ cp libcindexed_gzip_static.a ${PREFIX}/lib/ cp cindexed_gzip.h ${PREFIX}/include/cindexed_gzip/ + cp ccindexed_gzip.h ${PREFIX}/include/cindexed_gzip/ cp zran.h ${PREFIX}/include/cindexed_gzip/ diff --git a/cindexed_gzip/ccindexed_gzip.h b/cindexed_gzip/ccindexed_gzip.h new file mode 100644 index 00000000..50fd9e06 --- /dev/null +++ b/cindexed_gzip/ccindexed_gzip.h @@ -0,0 +1,8 @@ +#ifndef CCINDEXED_GZIP_H +#define CCINDEXED_GZIP_H + +namespace cindexed_gzip { +#include "cindexed_gzip/cindexed_gzip.h" +} + +#endif diff --git a/cindexed_gzip/cindexed_gzip.c b/cindexed_gzip/cindexed_gzip.c index 3c8b0fe7..a742781d 100644 --- a/cindexed_gzip/cindexed_gzip.c +++ b/cindexed_gzip/cindexed_gzip.c @@ -4,10 +4,11 @@ * Both GZIP and other files may be loaded via this interface; GZIP files will * be read via the zran module, and other files will read normally. */ +#include #include -#include "zran.h" -#include "cindexed_gzip.h" +#include "cindexed_gzip/zran.h" +#include "cindexed_gzip/cindexed_gzip.h" /* @@ -67,7 +68,7 @@ static uint64_t file_size(FILE *f) { * Other files will be read normally (via fseek/fread). The igz_file must be * passed to igz_close when it is no longer needed. */ -igz_file * igz_open(char *filepath) { +igz_file * igz_open(const char *filepath) { igz_file *gzf = NULL; FILE *f = NULL; diff --git a/cindexed_gzip/cindexed_gzip.h.in b/cindexed_gzip/cindexed_gzip.h.in index d9a66b93..98ac93c9 100644 --- a/cindexed_gzip/cindexed_gzip.h.in +++ b/cindexed_gzip/cindexed_gzip.h.in @@ -34,7 +34,7 @@ typedef struct _igz_file igz_file; * Other files will be read normally (via fseek/fread). The igz_file must be * passed to igz_close when it is no longer needed. */ -igz_file * igz_open(char *filepath); +igz_file * igz_open(const char *filepath); /* diff --git a/setup.py b/setup.py index eec55c43..5fb5b128 100644 --- a/setup.py +++ b/setup.py @@ -138,14 +138,14 @@ def run(self): # compile flags -include_dirs = ['zran'] +include_dirs = ['cindexed_gzip'] lib_dirs = [] libs = [] extra_srcs = [] extra_objects = [] extra_compile_args = [] compiler_directives = {'language_level' : 2} -define_macros = [] +define_macros = [('ZRAN_SUPPORT_PYTHON', '1')] # Link against zlib built in ZLIB_HOME if ZLIB_HOME is not None: @@ -198,9 +198,9 @@ def run(self): # The indexed_gzip module igzip_ext = Extension( 'indexed_gzip.indexed_gzip', - [op.join('indexed_gzip', 'indexed_gzip.{}'.format(pyx_ext)), - op.join('zran', 'zran.c'), - op.join('zran', 'zran_file_util.c')] + extra_srcs, + [op.join('indexed_gzip', 'indexed_gzip.{}'.format(pyx_ext)), + op.join('cindexed_gzip', 'zran.c'), + op.join('cindexed_gzip', 'zran_file_util.c')] + extra_srcs, libraries=libs, library_dirs=lib_dirs, include_dirs=include_dirs, @@ -226,9 +226,9 @@ def run(self): # Uses POSIX memmap API so won't work on Windows test_exts.append(Extension( 'indexed_gzip.tests.ctest_zran', - [op.join('indexed_gzip', 'tests', 'ctest_zran.{}'.format(pyx_ext)), - op.join('zran', 'zran.c'), - op.join('zran', 'zran_file_util.c')] + extra_srcs, + [op.join('indexed_gzip', 'tests', 'ctest_zran.{}'.format(pyx_ext)), + op.join('cindexed_gzip', 'zran.c'), + op.join('cindexed_gzip', 'zran_file_util.c')] + extra_srcs, libraries=libs, library_dirs=lib_dirs, include_dirs=include_dirs, From 19204075701bb4f5a0fd46bc973574a22717c859 Mon Sep 17 00:00:00 2001 From: Paul McCarthy Date: Thu, 10 Nov 2022 17:49:16 +0000 Subject: [PATCH 09/10] BF: check fseek result, set bytes_read for uncompresesd reads --- cindexed_gzip/cindexed_gzip.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/cindexed_gzip/cindexed_gzip.c b/cindexed_gzip/cindexed_gzip.c index a742781d..a23c6c92 100644 --- a/cindexed_gzip/cindexed_gzip.c +++ b/cindexed_gzip/cindexed_gzip.c @@ -158,11 +158,12 @@ int64_t igz_read(igz_file *gzf, void *buf, uint64_t len, uint64_t off) { } } else { - fseek(f, off, SEEK_SET); - fread(buf, len, 1, f); + if (fseek(f, off, SEEK_SET) != 0) { + goto fail; + } + bytes_read = fread(buf, 1, len, f); } - gzf->index.fd = NULL; fclose(f); return bytes_read; @@ -171,5 +172,7 @@ int64_t igz_read(igz_file *gzf, void *buf, uint64_t len, uint64_t off) { if (f != NULL) { fclose(f); } + gzf->index.fd = NULL; + return -1; } From d2012c5815728b4e5cd18a44ee6c6118c381e171 Mon Sep 17 00:00:00 2001 From: Paul McCarthy Date: Fri, 16 Jun 2023 16:22:45 +0100 Subject: [PATCH 10/10] CI: Make sure pip is up to date --- .ci/create_test_env.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.ci/create_test_env.sh b/.ci/create_test_env.sh index 791b08c8..babca60f 100755 --- a/.ci/create_test_env.sh +++ b/.ci/create_test_env.sh @@ -7,6 +7,8 @@ set -e set -x +pip install --upgrade pip + envdir="$1" thisdir=$(cd $(dirname "$0") && pwd)