diff --git a/.ci/create_test_env.sh b/.ci/create_test_env.sh index 791b08c8..babca60f 100755 --- a/.ci/create_test_env.sh +++ b/.ci/create_test_env.sh @@ -7,6 +7,8 @@ set -e set -x +pip install --upgrade pip + envdir="$1" thisdir=$(cd $(dirname "$0") && pwd) diff --git a/CHANGELOG.md b/CHANGELOG.md index be307e54..102c181e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,15 @@ # `indexed_gzip` changelog +## 1.8.0 (Under development) + + +* Changes to allow linking against + [zlib-ng](https://github.com/zlib-ng/zlib-ng/) (#107). +* Simplified C interface `cindexed_gzip`, and machinery for compiling as a + static or shared C library (#107). + + ## 1.7.1 (March 31st 2023) @@ -8,7 +17,6 @@ which do not implement `fileno()` (#118). - ## 1.7.0 (September 12th 2022) diff --git a/cindexed_gzip/Makefile b/cindexed_gzip/Makefile new file mode 100644 index 00000000..a61a7883 --- /dev/null +++ b/cindexed_gzip/Makefile @@ -0,0 +1,63 @@ +# Build the cindexed_gzip C library. +# +# The library can be built in one of the following ways: +# - Linked against the system zlib installation (default) +# - Linked against a zlib build at $ZLIB_HOME +# - Linked against a zlib-ng build at $ZLIB_NG_HOME + +all: libcindexed_gzip.so libcindexed_gzip_static.a + +AR ?= ar +CC ?= gcc +PYTHON ?= python +OBJFILES = zran.o zran_file_util.o cindexed_gzip.o +PYTHON_INCLUDE_DIRECTORY = $(shell ${PYTHON} -c "from sysconfig import get_paths; print(get_paths()['include'])") +CFLAGS += -I${PYTHON_INCLUDE_DIRECTORY} -I.. +INDEXED_GZIP_VERSION = $(shell cat ../indexed_gzip/__init__.py | grep __version__ | cut -d ' ' -f 3 | tr -d "'") + +# link against built zlib +ifdef ZLIB_HOME + LDFLAGS += -L${ZLIB_HOME} -lz + CFLAGS += -I${ZLIB_HOME} +# link against built zlib-ng +else ifdef ZLIB_NG_HOME + LDFLAGS += -L${ZLIB_NG_HOME} -lz-ng + CFLAGS += -I${ZLIB_NG_HOME} -DZRAN_USE_ZLIB_NG=1 +# link against system zlib +else + LDFLAGS += -lz +endif + + +clean: + rm -f *.o *.so *.a cindexed_gzip.h + + +cindexed_gzip.h: cindexed_gzip.h.in + sed -e "s/@INDEXED_GZIP_VERSION@/${INDEXED_GZIP_VERSION}/" $< > $@ + + +%.o: %.c cindexed_gzip.h + ${CC} ${CFLAGS} -c -o $@ $< + + +libcindexed_gzip.so: ${OBJFILES} + ${CC} ${CFLAGS} -shared -o $@ $^ ${LDFLAGS} + + +libcindexed_gzip_static.a: ${OBJFILES} + ${AR} -r $@ $^ + + +install: +ifndef PREFIX + @echo "PREFIX not set - aborting" + exit 1 +endif + mkdir -p ${PREFIX}/lib + mkdir -p ${PREFIX}/include/cindexed_gzip + cp libcindexed_gzip.so ${PREFIX}/lib/ + cp libcindexed_gzip_static.a ${PREFIX}/lib/ + cp cindexed_gzip.h ${PREFIX}/include/cindexed_gzip/ + cp ccindexed_gzip.h ${PREFIX}/include/cindexed_gzip/ + cp zran.h ${PREFIX}/include/cindexed_gzip/ diff --git a/cindexed_gzip/ccindexed_gzip.h b/cindexed_gzip/ccindexed_gzip.h new file mode 100644 index 00000000..50fd9e06 --- /dev/null +++ b/cindexed_gzip/ccindexed_gzip.h @@ -0,0 +1,8 @@ +#ifndef CCINDEXED_GZIP_H +#define CCINDEXED_GZIP_H + +namespace cindexed_gzip { +#include "cindexed_gzip/cindexed_gzip.h" +} + +#endif diff --git a/cindexed_gzip/cindexed_gzip.c b/cindexed_gzip/cindexed_gzip.c new file mode 100644 index 00000000..a23c6c92 --- /dev/null +++ b/cindexed_gzip/cindexed_gzip.c @@ -0,0 +1,178 @@ +/* + * Simplified C interface for reading GZIP files with indexed_gzip. + * + * Both GZIP and other files may be loaded via this interface; GZIP files will + * be read via the zran module, and other files will read normally. + */ +#include +#include + +#include "cindexed_gzip/zran.h" +#include "cindexed_gzip/cindexed_gzip.h" + + +/* + * igz_file struct, for reading from a GZIP or other file. File handle is + * opened/closed on-demand. + */ +struct _igz_file { + zran_index_t index; + int compressed; + char *filepath; +}; + + +/* + * Returns 1 if the given file is a GZIP file, 0 if it is not, -1 if an error + * occurs. + * + * https://www.rfc-editor.org/rfc/rfc1952#section-2.3.1 + */ + +static int is_gzip_file(FILE *f) { + + uint8_t magic[2]; + + if (fseek(f, 0, SEEK_SET) != 0) { goto fail; } + if (fread(magic, 1, 2, f) != 2) { goto fail; } + if (fseek(f, 0, SEEK_SET) != 0) { goto fail; } + + if (magic[0] == 0x1f && magic[1] == 0x8b) { return 1; } + else { return 0; } +fail: + return -1; +}; + + +/* + * Returns the size of the given file. + */ +static uint64_t file_size(FILE *f) { + + size_t s; + if (fseek(f, 0, SEEK_END) != 0) { goto fail; } + + s = ftello(f); + + if (s < 0) { goto fail; } + if (fseek(f, 0, SEEK_SET) != 0) { goto fail; } + + return (uint64_t)s; +fail: + return -1; +} + + +/* + * Open a file for reading. GZIP files will be loaded via the zran module. + * Other files will be read normally (via fseek/fread). The igz_file must be + * passed to igz_close when it is no longer needed. + */ +igz_file * igz_open(const char *filepath) { + + igz_file *gzf = NULL; + FILE *f = NULL; + size_t namelen = 0; + off_t size = 0; + + f = fopen(filepath, "rb"); + if (f == NULL) { + goto fail; + } + + gzf = calloc(1, sizeof(igz_file)); + if (gzf == NULL) { + goto fail; + } + + namelen = strlen(filepath); + gzf->filepath = malloc(namelen + 1); + strcpy(gzf->filepath, filepath); + + gzf->compressed = is_gzip_file(f); + if (gzf->compressed == -1) { + goto fail; + } + + if (gzf->compressed) { + if (zran_init(&(gzf->index), f, NULL, 4194304, + 32768, 1048576, ZRAN_AUTO_BUILD) != 0) { + goto fail; + } + gzf->index.fd = NULL; + } + + fclose(f); + + return gzf; + + fail: + if (gzf != NULL) { free(gzf); } + if (f != NULL) { fclose(f); } + return NULL; +} + + +/* + * Free resources associated with the igz_file. Must only be called once for + * a given igz_file. + */ +int igz_close(igz_file *gzf) { + if (gzf->compressed) { + zran_free(&(gzf->index)); + } + free(gzf->filepath); + free(gzf); +} + +/* + * Read up to len bytes from the given igz_file, starting from off. The bytes + * are copied into buf. + * + * Returns th number of bytes that were read, 0 if off == EOF, or a negative + * value if an error occurred. + */ +int64_t igz_read(igz_file *gzf, void *buf, uint64_t len, uint64_t off) { + + FILE *f = NULL; + int64_t bytes_read = 0; + + if (len > INT64_MAX) { + goto fail; + } + + f = fopen(gzf->filepath, "rb"); + if (f == NULL) { + goto fail; + } + + if (gzf->compressed) { + gzf->index.fd = f; + if (zran_seek(&(gzf->index), off, SEEK_SET, NULL) != ZRAN_SEEK_OK) { + goto fail; + } + + bytes_read = zran_read(&(gzf->index), buf, len); + if (bytes_read == ZRAN_READ_EOF) { + goto fail; + } + } + else { + if (fseek(f, off, SEEK_SET) != 0) { + goto fail; + } + bytes_read = fread(buf, 1, len, f); + } + + fclose(f); + + return bytes_read; + +fail: + if (f != NULL) { + fclose(f); + } + gzf->index.fd = NULL; + + return -1; +} diff --git a/cindexed_gzip/cindexed_gzip.h.in b/cindexed_gzip/cindexed_gzip.h.in new file mode 100644 index 00000000..98ac93c9 --- /dev/null +++ b/cindexed_gzip/cindexed_gzip.h.in @@ -0,0 +1,60 @@ +/* + * Simplified C interface for reading GZIP files with indexed_gzip. + * + * Both GZIP and other files may be loaded via this interface; GZIP files will + * be read via the zran module, and other files will read normally. + */ +#ifndef __CINDEXED_GZIP_H__ +#define __CINDEXED_GZIP_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +#include "zran.h" + +/* + * Version string, defined in indexed_gzip/__init__.py (see + * cindexed_gzip/Makefile). + */ +#define CINDEXED_GZIP_VERSION "@INDEXED_GZIP_VERSION@" + + +/* + * Struct representing a file which has been loaded via igz_open. + */ +struct _igz_file; +typedef struct _igz_file igz_file; + + +/* + * Open a file for reading. GZIP files will be loaded via the zran module. + * Other files will be read normally (via fseek/fread). The igz_file must be + * passed to igz_close when it is no longer needed. + */ +igz_file * igz_open(const char *filepath); + + +/* + * Free resources associated with the igz_file. Must only be called once for + * a given igz_file. + */ +int igz_close(igz_file *gzf); + + +/* + * Read up to len bytes from the given igz_file, starting from off. The bytes + * are copied into buf. + * + * Returns th number of bytes that were read, 0 if off == EOF, or a negative + * value if an error occurred. + */ +int64_t igz_read(igz_file *gzf, void *buf, uint64_t len, uint64_t off); + +#ifdef __cplusplus +} +#endif + +#endif /* __CINDEXED_GZIP_H__ */ diff --git a/indexed_gzip/zran.c b/cindexed_gzip/zran.c similarity index 98% rename from indexed_gzip/zran.c rename to cindexed_gzip/zran.c index 73e749f5..0df65d88 100644 --- a/indexed_gzip/zran.c +++ b/cindexed_gzip/zran.c @@ -12,28 +12,32 @@ #include #include #include + +#ifdef ZRAN_USE_ZLIB_NG +#include "zlib-ng.h" +#else #include "zlib.h" +#endif +#ifdef ZRAN_SUPPORT_PYTHON #define PY_SSIZE_T_CLEAN #include +#endif #ifdef _WIN32 #include "windows.h" #include "io.h" -static int is_readonly(FILE *fd, PyObject *f) +static int is_readonly(FILE *fd) { /* Can't find a way to do this correctly under - Windows and the check is not required anyway - since the underlying Python module checks it - already */ + Windows */ return 1; } #else #include /* Check if file is read-only */ -static int is_readonly(FILE *fd, PyObject *f) +static int is_readonly(FILE *fd) { - /* Skip the test for python file-likes */ return fd != NULL ? (fcntl(fileno(fd), F_GETFL) & O_ACCMODE) == O_RDONLY : 1; @@ -72,6 +76,25 @@ static double round(double val) #define zran_log(...) #endif +/* Use equivalent symbols from zlib or zlib-ng */ +#ifdef ZRAN_USE_ZLIB_NG +#define Z_STREAM zng_stream +#define INFLATEINIT2 zng_inflateInit2 +#define INFLATE zng_inflate +#define INFLATEEND zng_inflateEnd +#define INFLATEPRIME zng_inflatePrime +#define INFLATESETDICTIONARY zng_inflateSetDictionary +#define CRC32 zng_crc32 +#else +#define Z_STREAM z_stream +#define INFLATEINIT2 inflateInit2 +#define INFLATE inflate +#define INFLATEEND inflateEnd +#define INFLATEPRIME inflatePrime +#define INFLATESETDICTIONARY inflateSetDictionary +#define CRC32 crc32 +#endif + /* * Identifier and version number for index files created by zran_export_index. @@ -233,9 +256,7 @@ static uint64_t _zran_estimate_offset( */ static int _zran_init_zlib_inflate( zran_index_t *index, /* The index */ - - z_stream *stream, /* Pointer to a z_stream struct */ - + Z_STREAM *stream, /* Pointer to a z_stream struct */ zran_point_t *point /* Pass in NULL to initialise for inflation from the current location in the input file. Or pass a pointer to the index point @@ -328,7 +349,7 @@ int ZRAN_READ_DATA_ERROR = -2; */ static int _zran_read_data_from_file( zran_index_t *index, /* The index */ - z_stream *stream, /* The z_stream struct */ + Z_STREAM *stream, /* The z_stream struct */ uint64_t cmp_offset, /* Current offset in the compressed data */ uint64_t uncmp_offset, /* Current offset in the uncompressed data */ uint32_t need_atleast /* Skip read if the read buffer already has @@ -361,7 +382,7 @@ int ZRAN_FIND_STREAM_NOT_FOUND = -1; */ static int _zran_find_next_stream( zran_index_t *index, /* The index */ - z_stream *stream, /* The z_stream struct */ + Z_STREAM *stream, /* The z_stream struct */ int *offset /* Used to store the number of bytes skipped over */ ); @@ -388,7 +409,7 @@ int ZRAN_VALIDATE_STREAM_INVALID = -1; */ static int _zran_validate_stream( zran_index_t *index, /* The index */ - z_stream *stream, /* The z_stream struct */ + Z_STREAM *stream, /* The z_stream struct */ int *offset /* Used to store the number of bytes skipped over */ ); @@ -558,7 +579,7 @@ uint32_t ZRAN_INFLATE_STOP_AT_BLOCK = 64; static int _zran_inflate( zran_index_t *index, /* Pointer to the index. */ - z_stream *strm, /* Pointer to a z_stream struct. */ + Z_STREAM *strm, /* Pointer to a z_stream struct. */ uint64_t offset, /* Compressed data offset to start inflation from. */ @@ -639,7 +660,7 @@ int zran_init(zran_index_t *index, goto fail; /* The file must be opened in read-only mode */ - if (!is_readonly(fd, f)) + if (!is_readonly(fd)) goto fail; /* @@ -1201,7 +1222,7 @@ int _zran_add_point(zran_index_t *index, /* Initialise the given z_stream struct for decompression/inflation. */ int _zran_init_zlib_inflate(zran_index_t *index, - z_stream *strm, + Z_STREAM *strm, zran_point_t *point) { int ret; @@ -1227,9 +1248,9 @@ int _zran_init_zlib_inflate(zran_index_t *index, zran_log("_zran_init_zlib_inflate from current " "seek location (expecting GZIP header)\n"); - if (inflateInit2(strm, window + 32) != Z_OK) { goto fail; } - if (inflate(strm, Z_BLOCK) != Z_OK) { goto fail_free_strm; } - if (inflateEnd(strm) != Z_OK) { goto fail; } + if (INFLATEINIT2(strm, window + 32) != Z_OK) { goto fail; } + if (INFLATE(strm, Z_BLOCK) != Z_OK) { goto fail_free_strm; } + if (INFLATEEND(strm) != Z_OK) { goto fail; } } /* @@ -1262,7 +1283,7 @@ int _zran_init_zlib_inflate(zran_index_t *index, * in _zran_inflate). */ - if (inflateInit2(strm, -window) != Z_OK) { + if (INFLATEINIT2(strm, -window) != Z_OK) { goto fail; } @@ -1287,7 +1308,7 @@ int _zran_init_zlib_inflate(zran_index_t *index, goto fail_free_strm; } - if (inflatePrime(strm, + if (INFLATEPRIME(strm, point->bits, ret >> (8 - point->bits)) != Z_OK) goto fail_free_strm; } @@ -1296,7 +1317,7 @@ int _zran_init_zlib_inflate(zran_index_t *index, * Initialise the inflate stream * with the index point data. */ - if (inflateSetDictionary(strm, + if (INFLATESETDICTIONARY(strm, point->data, index->window_size) != Z_OK) goto fail_free_strm; @@ -1327,7 +1348,7 @@ int _zran_init_zlib_inflate(zran_index_t *index, * clause. */ fail_free_strm: - inflateEnd(strm); + INFLATEEND(strm); /* Something has gone wrong */ fail: return -1; @@ -1339,7 +1360,7 @@ int _zran_init_zlib_inflate(zran_index_t *index, * decompression. */ static int _zran_read_data_from_file(zran_index_t *index, - z_stream *stream, + Z_STREAM *stream, uint64_t cmp_offset, uint64_t uncmp_offset, uint32_t need_atleast) { @@ -1467,7 +1488,7 @@ static int _zran_read_data_from_file(zran_index_t *index, * contains concatenated streams). */ int _zran_find_next_stream(zran_index_t *index, - z_stream *stream, + Z_STREAM *stream, int *offset) { int ret; @@ -1511,7 +1532,7 @@ int _zran_find_next_stream(zran_index_t *index, * Re-configure for inflation * from the new stream. */ - if (inflateEnd(stream) != Z_OK) { + if (INFLATEEND(stream) != Z_OK) { goto fail; } @@ -1535,7 +1556,7 @@ int _zran_find_next_stream(zran_index_t *index, /* Validate the CRC32 and size of a GZIP stream. */ static int _zran_validate_stream(zran_index_t *index, - z_stream *stream, + Z_STREAM *stream, int *offset) { uint32_t crc; @@ -1581,7 +1602,7 @@ static int _zran_validate_stream(zran_index_t *index, /* The workhorse. Inflate/decompress data from the file. */ static int _zran_inflate(zran_index_t *index, - z_stream *strm, + Z_STREAM *strm, uint64_t offset, uint16_t flags, uint32_t *total_consumed, @@ -1633,7 +1654,7 @@ static int _zran_inflate(zran_index_t *index, * if we are initialising. */ if (inflate_init_stream(flags)) { - memset(strm, 0, sizeof(z_stream)); + memset(strm, 0, sizeof(Z_STREAM)); } /* @@ -1898,10 +1919,10 @@ static int _zran_inflate(zran_index_t *index, * stream, or it runs out of input or output. */ if (inflate_stop_at_block(flags)) { - z_ret = inflate(strm, Z_BLOCK); + z_ret = INFLATE(strm, Z_BLOCK); } else { - z_ret = inflate(strm, Z_NO_FLUSH); + z_ret = INFLATE(strm, Z_NO_FLUSH); } /* @@ -1964,7 +1985,7 @@ static int _zran_inflate(zran_index_t *index, index->validating && !(index->flags & ZRAN_SKIP_CRC_CHECK)) { index->stream_size += bytes_output; - index->stream_crc32 = crc32(index->stream_crc32, + index->stream_crc32 = CRC32(index->stream_crc32, strm->next_out - bytes_output, bytes_output); } @@ -2182,7 +2203,7 @@ static int _zran_inflate(zran_index_t *index, * is active, do just that. */ if (inflate_free_stream(flags)) { - if (inflateEnd(strm) != Z_OK) + if (INFLATEEND(strm) != Z_OK) goto fail; } @@ -2240,7 +2261,7 @@ int _zran_expand_index(zran_index_t *index, uint64_t until) { int z_ret; /* Zlib stream struct */ - z_stream strm; + Z_STREAM strm; /* * Number of bytes read/decompressed @@ -2701,7 +2722,7 @@ int64_t zran_read(zran_index_t *index, * Zlib stream struct and starting * index point for the read.. */ - z_stream strm; + Z_STREAM strm; zran_point_t *start = NULL; /* @@ -3214,7 +3235,7 @@ int zran_import_index(zran_index_t *index, index->flags |= ZRAN_SKIP_CRC_CHECK; /* Check if file is read only. */ - if (!is_readonly(fd, f)) goto fail; + if (!is_readonly(fd)) goto fail; /* Read ID, and check for file errors and EOF. */ f_ret = fread_(file_id, sizeof(file_id), 1, fd, f); diff --git a/indexed_gzip/zran.h b/cindexed_gzip/zran.h similarity index 99% rename from indexed_gzip/zran.h rename to cindexed_gzip/zran.h index 094c68ae..c8cf6b09 100644 --- a/indexed_gzip/zran.h +++ b/cindexed_gzip/zran.h @@ -1,6 +1,12 @@ #ifndef __ZRAN_H__ #define __ZRAN_H__ + +#ifdef __cplusplus +extern "C" { +#endif + + /* * The zran module is an adaptation of the zran example, written by Mark * Alder, which ships with the zlib source code. It allows the creation @@ -8,11 +14,16 @@ * of random seek/read access to the uncompressed data. */ +#include #include #include +#ifdef ZRAN_SUPPORT_PYTHON #define PY_SSIZE_T_CLEAN #include +#else +#define PyObject void +#endif struct _zran_index; struct _zran_point; @@ -502,4 +513,8 @@ int zran_import_index( PyObject *f /* Open handle to import file object */ ); +#ifdef __cplusplus +} +#endif + #endif /* __ZRAN_H__ */ diff --git a/indexed_gzip/zran_file_util.c b/cindexed_gzip/zran_file_util.c similarity index 88% rename from indexed_gzip/zran_file_util.c rename to cindexed_gzip/zran_file_util.c index cb01dde1..7c1fcbb4 100644 --- a/indexed_gzip/zran_file_util.c +++ b/cindexed_gzip/zran_file_util.c @@ -12,8 +12,10 @@ #include #include +#ifdef ZRAN_SUPPORT_PYTHON #define PY_SSIZE_T_CLEAN #include +#endif #include "zran_file_util.h" @@ -27,6 +29,9 @@ #define FTELL ftello #endif + +#ifdef ZRAN_SUPPORT_PYTHON + /* * The zran functions are typically called with the GIL released. These * macros are used to temporarily (re-)acquire and release the GIL when @@ -39,12 +44,14 @@ #define _ZRAN_FILE_UTIL_RELEASE_GIL \ PyGILState_Release(s); - /* * Implements a method analogous to fread that is performed on Python * file-like objects. */ -size_t _fread_python(void *ptr, size_t size, size_t nmemb, PyObject *f) { +size_t _fread_python(void *ptr, + size_t size, + size_t nmemb, + PyObject *f) { PyObject *data = NULL; char *buf; @@ -273,27 +280,42 @@ int _seekable_python2(PyObject *f) { return ret >= 0; } + +#endif /* ZRAN_SUPPORT_PYTHON */ + /* * Calls ferror on fd if specified, otherwise the Python-specific method on f. */ int ferror_(FILE *fd, PyObject *f) { + #ifdef ZRAN_SUPPORT_PYTHON return fd != NULL ? ferror(fd) : _ferror_python(f); + #else + return ferror(fd); + #endif } /* * Calls fseek on fd if specified, otherwise the Python-specific method on f. */ int fseek_(FILE *fd, PyObject *f, int64_t offset, int whence) { + #ifdef ZRAN_SUPPORT_PYTHON return fd != NULL ? FSEEK(fd, offset, whence) : _fseek_python(f, offset, whence); + #else + return FSEEK(fd, offset, whence); + #endif } /* * Calls ftell on fd if specified, otherwise the Python-specific method on f. */ int64_t ftell_(FILE *fd, PyObject *f) { + #ifdef ZRAN_SUPPORT_PYTHON return fd != NULL ? FTELL(fd) : _ftell_python(f); + #else + return FTELL(fd); + #endif } /* @@ -301,9 +323,13 @@ int64_t ftell_(FILE *fd, PyObject *f) { */ size_t fread_(void *ptr, size_t size, size_t nmemb, FILE *fd, PyObject *f) { + #ifdef ZRAN_SUPPORT_PYTHON return fd != NULL ? fread(ptr, size, nmemb, fd) : _fread_python(ptr, size, nmemb, f); + #else + return fread(ptr, size, nmemb, fd); + #endif } /* @@ -312,14 +338,22 @@ size_t fread_(void *ptr, size_t size, size_t nmemb, FILE *fd, PyObject *f) { * read, to determine if the file is at EOF. */ int feof_(FILE *fd, PyObject *f, size_t f_ret) { + #ifdef ZRAN_SUPPORT_PYTHON return fd != NULL ? feof(fd): _feof_python(f, f_ret); + #else + return feof(fd); + #endif } /* * Calls fflush on fd if specified, otherwise the Python-specific method on f. */ int fflush_(FILE *fd, PyObject *f) { + #ifdef ZRAN_SUPPORT_PYTHON return fd != NULL ? fflush(fd): _fflush_python(f); + #else + return fflush(fd); + #endif } /* @@ -330,26 +364,39 @@ size_t fwrite_(const void *ptr, size_t nmemb, FILE *fd, PyObject *f) { + #ifdef ZRAN_SUPPORT_PYTHON return fd != NULL ? fwrite(ptr, size, nmemb, fd) : _fwrite_python(ptr, size, nmemb, f); + #else + return fwrite(ptr, size, nmemb, fd); + #endif } /* * Calls getc on fd if specified, otherwise the Python-specific method on f. */ int getc_(FILE *fd, PyObject *f) { + #ifdef ZRAN_SUPPORT_PYTHON return fd != NULL ? getc(fd): _getc_python(f); + #else + return getc(fd); + #endif } /* - * Returns whether the given file is seekable. If fd is specified, assumes it's always seekable. - * If f is specified, calls f.seekable() to see if the Python file object is seekable. + * Returns whether the given file is seekable. If fd is specified, assumes + * it's always seekable. If f is specified, calls f.seekable() to see if the + * Python file object is seekable. */ int seekable_(FILE *fd, PyObject *f) { + #ifdef ZRAN_SUPPORT_PYTHON #if PY_MAJOR_VERSION > 2 return fd != NULL ? 1: _seekable_python(f); #else return fd != NULL ? 1: _seekable_python2(f); #endif + #else + return 1; + #endif } diff --git a/indexed_gzip/zran_file_util.h b/cindexed_gzip/zran_file_util.h similarity index 97% rename from indexed_gzip/zran_file_util.h rename to cindexed_gzip/zran_file_util.h index 28747096..4fbac1f5 100644 --- a/indexed_gzip/zran_file_util.h +++ b/cindexed_gzip/zran_file_util.h @@ -9,6 +9,9 @@ #include #include +#ifndef ZRAN_SUPPORT_PYTHON +#define PyObject void +#else #define PY_SSIZE_T_CLEAN #include @@ -65,6 +68,8 @@ int _getc_python(PyObject *f); */ int _seekable_python(PyObject *f); +#endif /* ZRAN_SUPPORT_PYTHON */ + /* * Calls ferror on fd if specified, otherwise the Python-specific method on f. */ diff --git a/setup.py b/setup.py index fbcea2ee..5fb5b128 100644 --- a/setup.py +++ b/setup.py @@ -40,6 +40,7 @@ def finalize_options(self): def run(self): base = op.dirname(__file__) + zrbase = op.join(base, 'zran') igzbase = op.join(base, 'indexed_gzip') shutil.rmtree(op.join(base, 'build'), @@ -60,8 +61,8 @@ def run(self): files = [ '*.so', op.join(igzbase, 'indexed_gzip.c'), - op.join(igzbase, 'zran.o'), - op.join(igzbase, 'zran_file_util.o'), + op.join(zrbase, 'zran.o'), + op.join(zrbase, 'zran_file_util.o'), op.join(igzbase, '*.pyc'), op.join(igzbase, '*.so'), op.join(igzbase, 'tests', '*.so'), @@ -76,13 +77,24 @@ def run(self): # Platform information -python2 = sys.version_info[0] == 2 -noc99 = python2 or (sys.version_info[0] == 3 and sys.version_info[1] <= 4) -windows = sys.platform.startswith("win") -testing = 'INDEXED_GZIP_TESTING' in os.environ +python2 = sys.version_info[0] == 2 +noc99 = python2 or (sys.version_info[0] == 3 and sys.version_info[1] <= 4) +windows = sys.platform.startswith("win") +testing = 'INDEXED_GZIP_TESTING' in os.environ + +# We can use zlib or zlib-ng via one +# of the following mechanisms: +# +# - Default: Dynamically link against system zlib +# - ZLIB_HOME: Path to zlib build directory - statically +# link against built zlib library. Takes +# precedence over ZLIB_NG_HOME +# - ZLIB_NG_HOME: Path to zlib-ng build directory - statically +# link against built zlib-ng library # compile ZLIB source? -ZLIB_HOME = os.environ.get("ZLIB_HOME", None) +ZLIB_HOME = os.environ.get('ZLIB_HOME', None) +ZLIB_NG_HOME = os.environ.get('ZLIB_NG_HOME', None) # Load README description readme = op.join(op.dirname(__file__), 'README.md') @@ -112,39 +124,54 @@ def run(self): have_numpy = False print('indexed_gzip setup') -print(' have_cython: {} (if True, modules will be cythonized, ' +print(' have_cython: {} (if True, modules will be cythonized, ' 'otherwise pre-cythonized C files are assumed to be ' 'present)'.format(have_cython)) -print(' have_numpy: {} (if True, test modules will ' +print(' have_numpy: {} (if True, test modules will ' 'be compiled)'.format(have_numpy)) -print(' ZLIB_HOME: {} (if set, ZLIB sources are compiled into ' - 'the indexed_gzip extension)'.format(ZLIB_HOME)) -print(' testing: {} (if True, code will be compiled with line ' +print(' ZLIB_HOME: {} (if set, indexed_gzip is linked against ' + 'zlib)'.format(ZLIB_HOME)) +print(' ZLIB_NG_HOME: {} (if set, indexed_gzip is linked against ' + 'zlib-ng (unless ZLIB_HOME is also set)'.format(ZLIB_NG_HOME)) +print(' testing: {} (if True, code will be compiled with line ' 'tracing enabled)'.format(testing)) # compile flags -include_dirs = ['indexed_gzip'] +include_dirs = ['cindexed_gzip'] lib_dirs = [] libs = [] extra_srcs = [] +extra_objects = [] extra_compile_args = [] compiler_directives = {'language_level' : 2} -define_macros = [] +define_macros = [('ZRAN_SUPPORT_PYTHON', '1')] +# Link against zlib built in ZLIB_HOME if ZLIB_HOME is not None: include_dirs.append(ZLIB_HOME) - extra_srcs.extend(glob.glob(op.join(ZLIB_HOME, '*.c'))) + if windows: extra_objects.append(op.join(ZLIB_HOME, 'zlibstatic.lib')) + else: extra_objects.append(op.join(ZLIB_HOME, 'libz.a')) + +# Link against zlib built in ZLIB_NG_HOME +elif ZLIB_NG_HOME is not None: + include_dirs .append(ZLIB_NG_HOME) + define_macros.append(('ZRAN_USE_ZLIB_NG', '1')) + if windows: extra_objects.append(op.join(ZLIB_NG_HOME, 'zlib-ngstatic.lib')) + else: extra_objects.append(op.join(ZLIB_NG_HOME, 'libz-ng.a')) + +# Link against system zlib +else: + if windows: libs.append('zlib') + else: libs.append('z') # If numpy is present, we need # to include the headers if have_numpy: include_dirs.append(np.get_include()) +# Windows specific flags if windows: - if ZLIB_HOME is None: - libs.append('zlib') - # For stdint.h which is not included in the old Visual C # compiler used for Python 2 if python2: @@ -155,12 +182,8 @@ def run(self): if noc99: extra_compile_args += ['-DNO_C99'] -# linux / macOS +# linux / macOS specific flags else: - # if ZLIB_HOME is set, statically link, - # rather than use system-provided zlib - if ZLIB_HOME is None: - libs.append('z') extra_compile_args += ['-Wall', '-Wno-unused-function'] if testing: @@ -175,13 +198,14 @@ def run(self): # The indexed_gzip module igzip_ext = Extension( 'indexed_gzip.indexed_gzip', - [op.join('indexed_gzip', 'indexed_gzip.{}'.format(pyx_ext)), - op.join('indexed_gzip', 'zran.c'), - op.join('indexed_gzip', 'zran_file_util.c')] + extra_srcs, + [op.join('indexed_gzip', 'indexed_gzip.{}'.format(pyx_ext)), + op.join('cindexed_gzip', 'zran.c'), + op.join('cindexed_gzip', 'zran_file_util.c')] + extra_srcs, libraries=libs, library_dirs=lib_dirs, include_dirs=include_dirs, extra_compile_args=extra_compile_args, + extra_objects=extra_objects, define_macros=define_macros) # Optional test modules @@ -193,6 +217,7 @@ def run(self): libraries=libs, library_dirs=lib_dirs, include_dirs=include_dirs, + extra_objects=extra_objects, extra_compile_args=extra_compile_args, define_macros=define_macros) ] @@ -201,12 +226,13 @@ def run(self): # Uses POSIX memmap API so won't work on Windows test_exts.append(Extension( 'indexed_gzip.tests.ctest_zran', - [op.join('indexed_gzip', 'tests', 'ctest_zran.{}'.format(pyx_ext)), - op.join('indexed_gzip', 'zran.c'), - op.join('indexed_gzip', 'zran_file_util.c')] + extra_srcs, + [op.join('indexed_gzip', 'tests', 'ctest_zran.{}'.format(pyx_ext)), + op.join('cindexed_gzip', 'zran.c'), + op.join('cindexed_gzip', 'zran_file_util.c')] + extra_srcs, libraries=libs, library_dirs=lib_dirs, include_dirs=include_dirs, + extra_objects=extra_objects, extra_compile_args=extra_compile_args, define_macros=define_macros))