diff --git a/.github/workflows/clang-format-check.yml b/.github/workflows/clang-format-check.yml index 0eb8971..dbac87b 100644 --- a/.github/workflows/clang-format-check.yml +++ b/.github/workflows/clang-format-check.yml @@ -11,10 +11,14 @@ jobs: formatting-check: runs-on: ubuntu-latest name: Formatting Check + strategy: + matrix: + path: + - 'src' steps: - uses: actions/checkout@v4 - name: Run clang-format style check uses: jidicula/clang-format-action@v4.15.0 with: clang-format-version: '20' - check-path: 'src' + check-path: ${{ matrix.path }} diff --git a/.github/workflows/test-build-c.yml b/.github/workflows/test-build-c.yml new file mode 100644 index 0000000..458c045 --- /dev/null +++ b/.github/workflows/test-build-c.yml @@ -0,0 +1,200 @@ +name: C Build Tests + +on: + push: + branches: [main, dev] + pull_request: + workflow_dispatch: + +jobs: + # build different hdf5 artefacts + build-hdf5: + runs-on: ubuntu-latest + timeout-minutes: 30 + strategy: + matrix: + include: + - name: system + prefix: /usr/local + artefact-name: hdf5-system + - name: custom + prefix: /opt/hdf5 + artefact-name: hdf5-custom + container: gcc:7 + name: Build HDF5 - ${{ matrix.name }} + steps: + - name: Cache HDF5 build + id: cache-hdf5 + uses: actions/cache@v4 + with: + path: /tmp/${{ matrix.artefact-name }} + key: hdf5-1.14.6-${{ matrix.name }}-gcc7-${{ runner.os }}-v1 + restore-keys: | + hdf5-1.14.6-${{ matrix.name }}-gcc7-${{ runner.os }}- + hdf5-1.14.6-${{ matrix.name }}-gcc7- + + - name: Download and build HDF5 + if: steps.cache-hdf5.outputs.cache-hit != 'true' + run: | + # use archive repositories for Debian Buster + sed -i 's|deb.debian.org|archive.debian.org|g' /etc/apt/sources.list + sed -i 's|security.debian.org|archive.debian.org|g' /etc/apt/sources.list + sed -i '/buster-updates/d' /etc/apt/sources.list + apt-get update && apt-get install -y wget + + # use HDF5 1.14.x + wget https://github.com/HDFGroup/hdf5/releases/download/hdf5_1.14.6/hdf5-1.14.6.tar.gz + tar -xzf hdf5-1.14.6.tar.gz + cd hdf5-1.14.6 + + # configure for specified prefix + ./configure --prefix=${{ matrix.prefix }} + make -j$(nproc) + + # staged for artefact + make install DESTDIR=/tmp/${{ matrix.artefact-name }} + + - name: Create tarball artefact + run: | + cd /tmp/${{ matrix.artefact-name }} + tar -czf /tmp/${{ matrix.artefact-name }}.tar.gz . + + - name: Upload HDF5 artefact + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.artefact-name }} + path: /tmp/${{ matrix.artefact-name }}.tar.gz + compression-level: 0 + + # test gcc 7-14 + test-gcc-versions: + needs: build-hdf5 + runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + gcc-version: [7, 8, 9, 10, 11, 12, 13, 14] + container: gcc:${{ matrix.gcc-version }} + name: GCC ${{ matrix.gcc-version }} - system-wide installation + steps: + - name: Checkout repository + uses: actions/checkout@v5 + + - name: Download HDF5 system artefact + uses: actions/download-artifact@v4 + with: + name: hdf5-system + + - name: Install HDF5 system-wide + run: | + tar -xzf hdf5-system.tar.gz -C / + ldconfig + + - name: Use archive repositories for Debian Buster + if: matrix.gcc-version == 7 || matrix.gcc-version == 8 + run: | + # gcc 7, 8 use debian buster which reached EOL + sed -i 's|deb.debian.org|archive.debian.org|g' /etc/apt/sources.list + sed -i 's|security.debian.org|archive.debian.org|g' /etc/apt/sources.list + sed -i '/buster-updates/d' /etc/apt/sources.list + + - name: Install build dependencies + run: | + apt-get update + apt-get install -y autoconf automake libtool pkg-config + + - name: Generate configure script + run: autoreconf -i + + - name: Run build test + run: | + ./tests/test_build/test_build_c.sh --only system + + - name: Upload test logs on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: logs-gcc-${{ matrix.gcc-version }}-system + path: tests/test_build/logs/ + retention-days: 7 + + # test different methods of specifying HDF5 + test-hdf5-methods: + needs: build-hdf5 + runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + include: + # system-wide + - method: system + artefact: hdf5-system + test-name: system + setup: "" + lib-path: "" + description: "system-wide HDF5 at /usr/local" + + # explicit path + - method: explicit + artefact: hdf5-custom + test-name: explicit + setup: "export HDF5_TEST_PATH=/opt/hdf5" + lib-path: /opt/hdf5/lib + description: "explicit --with-hdf5=/opt/hdf5" + + # via environment variable + - method: env_var + artefact: hdf5-custom + test-name: hdf5_root + setup: "export HDF5_ROOT=/opt/hdf5" + lib-path: /opt/hdf5/lib + description: "environment variable HDF5_ROOT" + + container: gcc:11 + name: HDF5 ${{ matrix.method }} + env: + LD_LIBRARY_PATH: ${{ matrix.lib-path }} + steps: + - name: Checkout repository + uses: actions/checkout@v5 + + - name: Download HDF5 artefact + uses: actions/download-artifact@v4 + with: + name: ${{ matrix.artefact }} + + - name: Install HDF5 + run: | + echo "Installing HDF5 for: ${{ matrix.description }}" + tar -xzf ${{ matrix.artefact }}.tar.gz -C / + # update library cache for system-wide installation + if [ "${{ matrix.method }}" = "system" ]; then + ldconfig + fi + + - name: Install build dependencies + run: | + apt-get update + apt-get install -y autoconf automake libtool pkg-config + + - name: Generate configure script + run: autoreconf -i + + - name: Run build test + run: | + # set test-specific environment + ${{ matrix.setup }} + + # run test + echo "Testing: ${{ matrix.description }}" + ./tests/test_build/test_build_c.sh --only ${{ matrix.test-name }} + + - name: Upload test logs on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: logs-hdf5-${{ matrix.method }} + path: tests/test_build/logs/ + retention-days: 7 diff --git a/.github/workflows/test-build-python.yml b/.github/workflows/test-build-python.yml new file mode 100644 index 0000000..660ab9a --- /dev/null +++ b/.github/workflows/test-build-python.yml @@ -0,0 +1,116 @@ +name: Python Build Tests + +on: + push: + branches: [main, dev] + pull_request: + +jobs: + # build hdf5 library for python tests + build-hdf5: + runs-on: ubuntu-latest + timeout-minutes: 30 + container: gcc:7 + name: Build HDF5 + steps: + - name: Cache HDF5 build + id: cache-hdf5 + uses: actions/cache@v4 + with: + path: /tmp/hdf5-python + key: hdf5-1.14.6-python-gcc7-${{ runner.os }}-v1 + restore-keys: | + hdf5-1.14.6-python-gcc7-${{ runner.os }}- + hdf5-1.14.6-python-gcc7- + + - name: Download and build HDF5 + if: steps.cache-hdf5.outputs.cache-hit != 'true' + run: | + # use archive repositories for Debian Buster + sed -i 's|deb.debian.org|archive.debian.org|g' /etc/apt/sources.list + sed -i 's|security.debian.org|archive.debian.org|g' /etc/apt/sources.list + sed -i '/buster-updates/d' /etc/apt/sources.list + apt-get update && apt-get install -y wget + + # use HDF5 1.14.x + wget https://github.com/HDFGroup/hdf5/releases/download/hdf5_1.14.6/hdf5-1.14.6.tar.gz + tar -xzf hdf5-1.14.6.tar.gz + cd hdf5-1.14.6 + + # configure for /opt/hdf5 + ./configure --prefix=/opt/hdf5 + make -j$(nproc) + + # staged for artefact + make install DESTDIR=/tmp/hdf5-python + + - name: Create tarball artefact + run: | + cd /tmp/hdf5-python + tar -czf /tmp/hdf5-python.tar.gz . + + - name: Upload HDF5 artefact + uses: actions/upload-artifact@v4 + with: + name: hdf5-python + path: /tmp/hdf5-python.tar.gz + compression-level: 0 + + # test python installation methods + test-python: + needs: build-hdf5 + runs-on: ubuntu-latest + timeout-minutes: 15 + env: + HDF5_ROOT: /opt/hdf5 + LD_LIBRARY_PATH: /opt/hdf5/lib + strategy: + fail-fast: false + matrix: + python-version: ["3.10", "3.11", "3.12", "3.13"] + install-method: ["standard"] + include: + # add editable and pipx test for lowest-supported Python + - python-version: "3.10" + install-method: "editable" + - python-version: "3.10" + install-method: "pipx" + name: Python ${{ matrix.python-version }} - ${{ matrix.install-method }} + steps: + - name: Checkout repository + uses: actions/checkout@v5 + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y autoconf automake libtool + # install pipx only when needed + if [[ "${{ matrix.install-method }}" == "pipx" ]]; then + sudo apt-get install -y pipx + fi + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.python-version }} + + - name: Download HDF5 artefact + uses: actions/download-artifact@v4 + with: + name: hdf5-python + + - name: Install HDF5 + run: | + sudo tar -xzf hdf5-python.tar.gz -C / + + - name: Run build test + run: | + ./tests/test_build/test_build_python.sh --only ${{ matrix.install-method }} + + - name: Upload test logs on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: logs-py${{ matrix.python-version }}-${{ matrix.install-method }} + path: tests/test_build/logs/ + retention-days: 7 diff --git a/.gitignore b/.gitignore index 70c4d04..682aed5 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,24 @@ doc/.build/* *egg-info* *__pycache__* *sg_execution_times.rst +venv/* + +# autotools generated files +**/aclocal.m4 +**/autom4te.cache/ +**/compile +**/config.h +**/config.h.in +**/config.log +**/config.status +**/configure +**/depcomp +**/install-sh +**/missing +**/stamp-h1 +**/Makefile +**/Makefile.in +**/.deps/ + +# cython generated c files +**/_wrapper.c diff --git a/.markdownlint.yaml b/.markdownlint.yaml new file mode 100644 index 0000000..05c6bc3 --- /dev/null +++ b/.markdownlint.yaml @@ -0,0 +1,2 @@ +MD024: + siblings_only: true diff --git a/Makefile.am b/Makefile.am new file mode 100644 index 0000000..af437a6 --- /dev/null +++ b/Makefile.am @@ -0,0 +1 @@ +SUBDIRS = src diff --git a/README.md b/README.md index c3a2067..eea4de2 100644 --- a/README.md +++ b/README.md @@ -49,39 +49,71 @@ Below shows the hierarchy of the resulting HDF5 file: - GCC compiler (7.4 or newer, older versions may work) - HDF5 development libraries (1.10.4 or newer) - Make (3.82 or newer) +- GNU Autotools (autoconf >= 2.64, automake) - only required when building +the latest version #### Building from Source ```bash -# Clone the repository +# Download and extract the release tarball +tar xzf mib2h5-X.Y.Z.tar.gz +cd mib2h5-X.Y.Z +./configure --prefix=/path/to/install +make +make install +``` + +Without `--prefix`, the library will be installed to `/usr/local`. + +#### Building the Latest Version + +For the latest version: + +```bash git clone git@github.com:ePSIC-DLS/mib2h5.git cd mib2h5 -# Configure and build +# Generate configure script (requires GNU Autotools) +autoreconf -i + +# Then follow the standard build process ./configure --prefix=/path/to/install make make install ``` -Without `--prefix`, the library will be installed to `/usr/local`. +#### Configuration Options -### Python +The configure script supports several options: -#### Prerequisites +##### HDF5 Location -- Python (3.9 or newer) +- `--with-hdf5=/path/to/hdf5`: Specify HDF5 installation path +- If not specified, the configure script searches for HDF5 in the following +order: + 1. System paths (standard locations) + 2. `$HDF5_ROOT` + 3. `$HDF5_HOME` + 4. `$HDF5_DIR` -#### Via pip +##### Compression Support -```bash -python -m pip install mib2h5 -``` +- `--enable-compression`: Enable Blosc compression (requires +[c-blosc](https://github.com/Blosc/c-blosc) and +[hdf5-blosc](https://github.com/Blosc/hdf5-blosc)) +- `--with-blosc=/path/to/blosc` +- `--with-hdf5-blosc=/path/to/hdf5-blosc` -#### Via conda +##### Build Variants -```bash -conda install -c conda-forge mib2h5 -``` +- `--enable-debug`: Debug build with symbols and static analysis +- `--enable-asan`: For memory debugging + +Run `./configure --help` for all available options. + +### Python + +For Python installation and usage, please refer to the [Python wrapper documentation](python/README.md). ## Usage @@ -110,17 +142,48 @@ This will create `file1.h5`, `file2.h5` and `file3.h5` in the directory #### Advanced Options -Convert with compression, custom dataset key, and reshape dimensions: +Convert with compression, custom dataset key, and excluding metadata: ```bash -mib2h5 -c -d '/rawdata' -r '10x10' -t 300 input.mib +mib2h5 -c -d '/rawdata' -N -- input.mib ``` This will: + - enable Blosc compression - store the frames at the dataset key `/rawdata` in the HDF5 file -- reshape the data to `(10, 10, det_y, det_x)` if there are 100 frames with -dimensions of `(det_y, det_x)`. +- exclude metadata from the output (using `-N` or `--no-metadata`) + +#### Using Long Options + +Long options make commands more readable and self-documenting. You can find the +list of long options by `mib2h5 --help`. + +#### Metadata Control + +By default, metadata is included in the HDF5 output. You can control this +behavior: + +```bash +# Explicitly include metadata (default behavior) +mib2h5 -M input.mib +mib2h5 --with-metadata input.mib + +# Exclude metadata from output +mib2h5 -N input.mib +mib2h5 --no-metadata input.mib +``` + +#### Environment Variables + +When compression is enabled with `-c`, you can fine-tune the Blosc compression +settings: + +```bash +export MIB2H5_SHUFFLE=0 # Shuffle level (0-2, default: 2) +export MIB2H5_COMPRESSION_LEVEL=5 # Compression level (0-9, default: 9) +mib2h5 -c input.mib +``` ### C API Examples @@ -187,45 +250,6 @@ int main() { } ``` -### Python API Examples - -#### Basic Python Example - -```python -from mib2h5 import convert - -# Basic conversion -try: - convert("input.mib") -except (ValueError, RuntimeError): - print("Conversion failed.") -else: - print("Conversion successful!") -``` - -#### Advanced Python Example - -```python -from mib2h5 import convert - -try: - convert( - ["file1.mib", "file2.mib", "file3.mib"], - output_dir="/path/to/output", - include_metadata=True, - dataset_key="/rawdata", - metadata_key="/meta", - use_compression=True, - reshape_dims="10x10", - report_progress=True, - timeout_seconds=300 - ) -except (ValueError, RuntimeError): - print("Conversion failed.") -else: - print("Conversion successful!") -``` - ## API Reference ### C API @@ -247,24 +271,24 @@ int mib_to_h5( const char* mib_to_h5_last_error(void); ``` -### Python API - -```python -convert( - input_files, - output_dir=None, - include_metadata=True, - dataset_key="/data", - metadata_key="/metadata", - use_compression=False, - reshape_dims=None, - report_progress=True, - timeout_seconds=900 -) -``` +For the Python API reference and detailed parameter descriptions, see the +[Python wrapper documentation](python/README.md). + +## Contributing + +Contribution is very welcomed. Please use the [issue +page](https://github.com/ePSIC-DLS/mib2h5/issues) to report any bug and missing +feature. + +### Maintainers + +- Timothy Poon (@ptim0626) + +### Contributors -For detailed parameter descriptions, refer to the header file `mib2h5.h` or the -Python docstrings. +- Teo Ching (@teoching0705) +- Yousef Moazzam (@yousefmoazzam) +- Timothy Poon (@ptim0626) ## Licence diff --git a/configure.ac b/configure.ac new file mode 100644 index 0000000..b497bea --- /dev/null +++ b/configure.ac @@ -0,0 +1,223 @@ +# 2.64 is needed for AS_VAR_SET +# check https://lists.gnu.org/archive/html/autotools-announce/2009-07/msg00000.html for detail +AC_PREREQ([2.64]) +AC_INIT([mib2h5], [0.0.1], [https://github.com/ePSIC-DLS/mib2h5/issues], [mib2h5], [https://github.com/ePSIC-DLS/mib2h5]) +AM_INIT_AUTOMAKE([-Wall foreign -Werror tar-pax dist-xz]) +AC_USE_SYSTEM_EXTENSIONS +m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) +AC_CONFIG_SRCDIR([src/main.c]) +AC_CONFIG_HEADERS([config.h]) +AC_PROG_CC + +# --- Build Type Options -- +AC_ARG_ENABLE([debug], + AS_HELP_STRING([--enable-debug], [Enable debug build]), + [debug_build=$enableval], [debug_build=no]) + +AC_ARG_ENABLE([asan], + AS_HELP_STRING([--enable-asan], [Enable AddressSanitizer build]), + [asan_build=$enableval], [asan_build=no]) + +# --- Check for HDF5 --- + +AC_ARG_WITH([hdf5], + [AS_HELP_STRING([--with-hdf5=PATH], + [Path to HDF5 installation. If not specified, searches in order: system paths, $HDF5_ROOT, $HDF5_HOME, $HDF5_DIR])], + [HDFDIR="$withval"], + []) + +HDF5_OK=no + +AS_IF([test -n "$HDFDIR"], [ + AC_MSG_NOTICE([Testing HDF5 at $HDFDIR]) + CPPFLAGS="$CPPFLAGS -I$HDFDIR/include" + LDFLAGS="$LDFLAGS -L$HDFDIR/lib" + + AC_CHECK_HEADER([hdf5.h], [ + AC_CHECK_LIB([hdf5], [H5open], [ + AC_DEFINE([HAVE_HDF5], [1], [Define if HDF5 is available]) + HDF5_OK=yes + ], [ + AC_MSG_ERROR([libhdf5 not found at $HDFDIR/lib]) + ]) + ], [ + AC_MSG_ERROR([hdf5.h not found at $HDFDIR/include]) + ]) +]) + +AS_IF([test "x$HDF5_OK" != xyes], [ + AS_UNSET([ac_cv_header_hdf5_h]) + AC_MSG_NOTICE([Trying system-wide HDF5 installation]) + AC_CHECK_HEADER([hdf5.h], [ + AC_CHECK_LIB([hdf5], [H5open], [ + AC_DEFINE([HAVE_HDF5], [1], [Define if HDF5 is available]) + HDF5_OK=yes + AC_MSG_NOTICE([HDF5 found in system path]) + ], [ + AC_MSG_NOTICE([libhdf5 not found system-wide]) + HDF5_OK=no + ]) + ], [ + AC_MSG_NOTICE([hdf5.h not found system-wide]) + HDF5_OK=no + ]) +]) + +if test "x$HDF5_OK" != xyes; then + for var in HDF5_ROOT HDF5_HOME HDF5_DIR; do + eval dir=\$$var + if test -n "$dir"; then + AS_UNSET([ac_cv_header_hdf5_h]) + AC_MSG_NOTICE([Trying $var at $dir]) + CPPFLAGS="$CPPFLAGS -I$dir/include" + LDFLAGS="$LDFLAGS -L$dir/lib -Wl,-rpath,$dir/lib" + AC_CHECK_HEADER([hdf5.h], [ + AC_CHECK_LIB([hdf5], [H5open], [ + AC_DEFINE([HAVE_HDF5], [1], [Define if HDF5 is available]) + HDF5_OK=yes + HDFDIR="$dir" + AC_MSG_NOTICE([HDF5 found in $HDFDIR]) + ], [ + AC_MSG_NOTICE([libhdf5 not found at $dir/lib]) + AS_VAR_SET([CPPFLAGS], [`echo "$CPPFLAGS" | sed "s|-I$dir/include||g"`]) + AS_VAR_SET([LDFLAGS], [`echo "$LDFLAGS" | sed "s|-L$dir/lib||g; s|-Wl,-rpath,$dir/lib||g"`]) + HDF5_OK=no + ]) + ], [ + AC_MSG_NOTICE([hdf5.h not found at $dir/include]) + AS_VAR_SET([CPPFLAGS], [`echo "$CPPFLAGS" | sed "s|-I$dir/include||g"`]) + AS_VAR_SET([LDFLAGS], [`echo "$LDFLAGS" | sed "s|-L$dir/lib||g; s|-Wl,-rpath,$dir/lib||g"`]) + HDF5_OK=no + ]) + test "x$HDF5_OK" = xyes && break + fi + done +fi + +AS_IF([test "x$HDF5_OK" != xyes], [ + AC_MSG_ERROR([HDF5 not found. Use --with-hdf5=/path or set HDF5_ROOT, HDF5_HOME, or HDF5_DIR]) +]) + +# ---------------- Compression Option ---------------- +AC_ARG_ENABLE([compression], + [AS_HELP_STRING([--enable-compression], [Enable compression with blosc and hdf5-blosc])], + [], + [enable_compression=no]) + +AC_ARG_WITH([blosc], + [AS_HELP_STRING([--with-blosc=PATH], [Path to c-blosc installation])], + [BLOSCDIR="$withval"], + []) + +AC_ARG_WITH([hdf5-blosc], + [AS_HELP_STRING([--with-hdf5-blosc=PATH], [Path to hdf5-blosc installation])], + [HBDIR="$withval"], + []) + +AS_IF([test "x$enable_compression" != xno], [ + + AC_MSG_NOTICE([Checking for Blosc compression support...]) + # --- If user gave paths, use them directly --- + + BLOSC_OK=no + + AS_IF([test -n "$BLOSCDIR"], [ + AS_IF([test -d "$BLOSCDIR/lib64"], [ + BLOSC_LIBDIR="$BLOSCDIR/lib64" + ], [ + AS_IF([test -d "$BLOSCDIR/lib"], [ + BLOSC_LIBDIR="$BLOSCDIR/lib" + ], [ + AC_MSG_NOTICE([Neither lib64 nor lib directory exists in $BLOSCDIR]) + BLOSC_OK=no + ]) + ]) + ]) + + CPPFLAGS="$CPPFLAGS -I$BLOSCDIR/include" + LDFLAGS="$LDFLAGS -L$BLOSC_LIBDIR -Wl,-rpath,$BLOSC_LIBDIR" + + AC_CHECK_HEADER([blosc.h], [ + AC_CHECK_LIB([blosc], [blosc_init], [BLOSC_OK=yes], [ + AS_VAR_SET([CPPFLAGS], [`echo "$CPPFLAGS" | sed "s|-I$BLOSCDIR/include||g"`]) + AS_VAR_SET([LDFLAGS], [`echo "$LDFLAGS" | sed "s|-L$BLOSC_LIBDIR||g; s|-Wl,-rpath,$BLOSC_LIBDIR||g"`]) + AC_MSG_NOTICE([libblosc not found in $BLOSC_LIBDIR]) + BLOSC_OK=no + ]) + ], [ + AS_VAR_SET([CPPFLAGS], [`echo "$CPPFLAGS" | sed "s|-I$BLOSCDIR/include||g"`]) + AS_VAR_SET([LDFLAGS], [`echo "$LDFLAGS" | sed "s|-L$BLOSC_LIBDIR||g; s|-Wl,-rpath,$BLOSC_LIBDIR||g"`]) + AC_MSG_NOTICE([blosc.h not found in $BLOSCDIR/include]) + BLOSC_OK=no + ]) + + + AS_IF([test "x$BLOSC_OK" != xyes], [ + AS_UNSET([ac_cv_lib_blosc_blosc_init]) + AS_UNSET([ac_cv_header_blosc_h]) + AC_MSG_NOTICE([Falling back on system path for blosc]) + AC_CHECK_HEADER([blosc.h], [ + AC_CHECK_LIB([blosc], [blosc_init], [BLOSC_OK=yes], [BLOSC_OK=no]) + ], [BLOSC_OK=no]) + ]) + + AS_IF([test -n "$HBDIR"], [ + CPPFLAGS="$CPPFLAGS -I$HBDIR/src" + LDFLAGS="$LDFLAGS -L$HBDIR/build -Wl,-rpath,$HBDIR/build" + + AC_CHECK_HEADER([blosc_filter.h], [ + AC_CHECK_LIB([blosc_filter], [register_blosc], [HDF5_BLOSC_OK=yes], [ + AS_VAR_SET([CPPFLAGS], [`echo "$CPPFLAGS" | sed "s|-I$HBDIR/src||g"`]) + AS_VAR_SET([LDFLAGS], [`echo "$LDFLAGS" | sed "s|-$HBDIR/build||g; s|-Wl,-rpath,$HBDIR/build||g"`]) + AC_MSG_NOTICE([libblosc_filter not found in $HBDIR/build]) + HDF5_BLOSC_OK=no + ]) + ], [ + AS_VAR_SET([CPPFLAGS], [`echo "$CPPFLAGS" | sed "s|-I$HBDIR/src||g"`]) + AS_VAR_SET([LDFLAGS], [`echo "$LDFLAGS" | sed "s|-$HBDIR/build||g; s|-Wl,-rpath,$HBDIR/build||g"`]) + AC_MSG_NOTICE([blosc_filter.h not found in $HBDIR/src]) + HDF5_BLOSC_OK=no + ]) + ]) + + AS_IF([test "x$HDF5_BLOSC_OK" != xyes], [ + AS_UNSET([ac_cv_lib_blosc_filter_register_blosc]) + AS_UNSET([ac_cv_header_blosc_filter_h]) + AC_MSG_NOTICE([Falling back on system path for hdf5-blosc]) + AC_CHECK_HEADER([blosc_filter.h], [ + AC_CHECK_LIB([blosc_filter], [register_blosc], [HDF5_BLOSC_OK=yes], [HDF5_BLOSC_OK=no]) + ]) + ]) + + AS_IF([test "x$BLOSC_OK" = xyes && test "x$HDF5_BLOSC_OK" = xyes], [ + AC_DEFINE([HAVE_COMPRESSION], [1], [Define if compression is enabled]) + AC_SUBST([LIBCOMPRESSION], ["-lblosc -lblosc_filter"]) + ], [ + AS_IF([test "x$enable_compression" != xcheck], [ + AC_MSG_FAILURE([Compression requested but blosc and hdf5-blosc not found system-wide, and no --with-blosc and --with-hdf5-blosc given]) + ]) + ]) +]) + +# --- Set compiler flags based on build type --- + +AS_IF([test "x$asan_build" = "xyes"], [ + CFLAGS="$CFLAGS -Wall -Wextra -std=c11 -g -O0 -fsanitize=address" + LDFLAGS="$LDFLAGS -fsanitize=address" +], [test "x$debug_build" = "xyes"], [ + CFLAGS="$CFLAGS -Wall -Wextra -std=c11 -g -O0 -fanalyzer" +], [ + CFLAGS="$CFLAGS -Wall -Wextra -std=c11 -O2" +]) + +AC_CONFIG_FILES([Makefile src/Makefile]) +AC_OUTPUT + +# --- Configuration Summary --- +AS_IF([test "x$HDF5_OK" = xyes], [ + AS_IF([test -n "$HDFDIR"], [ + AC_MSG_NOTICE([HDF5 library found at: $HDFDIR]) + ], [ + AC_MSG_NOTICE([HDF5 library found in system paths]) + ]) +]) diff --git a/python/MANIFEST.in b/python/MANIFEST.in new file mode 100644 index 0000000..72b7464 --- /dev/null +++ b/python/MANIFEST.in @@ -0,0 +1,18 @@ +# include C source files from parent directory +include ../src/*.c +include ../src/*.h +include ../configure.ac +include ../Makefile.am +include ../Makefile.in +include ../aclocal.m4 + +# include Cython source +include src/mib2h5/_wrapper.pyx + +# exclude compiled/cached files +global-exclude *.o +global-exclude *.so +global-exclude __pycache__ +global-exclude *.pyc +global-exclude *.pyo +global-exclude *~ diff --git a/python/README.md b/python/README.md new file mode 100644 index 0000000..0e17a17 --- /dev/null +++ b/python/README.md @@ -0,0 +1,152 @@ +# mib2h5 Python Wrapper + +Python bindings for `mib2h5`, the `.mib` to HDF5 converter. + +## Introduction + +This package provides a Python interface to convert MerlinEM `.mib` files from +Quantum Detector to HDF5 format (`.h5`). + +## Installation + +### Prerequisites + +- Python >= 3.10 +- HDF5 library (1.10.4 or newer) +- Cython >= 3.0 - only required when building from source + +### Via pip + +```bash +python -m pip install mib2h5 +``` + +### Via conda + +```bash +conda install -c conda-forge mib2h5 +``` + +### Via pipx + +You can use `pipx` to install only the command-line tool `mib2h5`: + +```bash +pipx install mib2h5 +``` + +### Building from Source + +```bash +# Set HDF5 location (if not in standard system paths) +export HDF5_ROOT=/path/to/hdf5 + +# Install the package +python -m pip install . + +# For development, install in editable mode +python -m pip install -e . +``` + +## Usage + +### Command-line Interface + +This is the same as in `mib2h5`, see the usage [here](../README.md). + +### Python API + +#### Basic Example + +```python +from mib2h5 import convert + +# Convert a single file +convert("input.mib") + +# Convert with custom output directory +convert("input.mib", output_dir="/path/to/output") +``` + +#### Advanced Example + +```python +from mib2h5 import convert + +# Convert multiple files +files = ["file1.mib", "file2.mib", "file3.mib"] +convert( + files, + output_dir="/path/to/output", + include_metadata=True, + dataset_key="/rawdata", + use_compression=True +) + +# Handle errors for multiple files +try: + convert(files, output_dir="/path/to/output") +except RuntimeError as e: + print(f"Some files failed to convert:\n{e}") +``` + +#### Compression Control + +Compression is controlled via environment variables when +`use_compression=True`: + +```python +import os +from mib2h5 import convert + +# Set compression parameters +os.environ['MIB2H5_SHUFFLE'] = '2' # 0-2, default: 2 +os.environ['MIB2H5_COMPRESSION_LEVEL'] = '5' # 0-9, default: 9 + +# Convert with compression +convert("input.mib", use_compression=True) +``` + +## API Reference + +```python +convert( + input_files, + output_dir=None, + include_metadata=True, + dataset_key="/data", + metadata_key="/metadata", + use_compression=False, + reshape_dims=None, + report_progress=True, + timeout_seconds=900 +) +``` + +### Parameters + +- `input_files`: Path to input MIB file(s) (string or list of strings) +- `output_dir`: Directory for output HDF5 file(s) (default: current directory) +- `include_metadata`: Whether to include metadata in HDF5 file (default: True) +- `dataset_key`: HDF5 dataset path for frames (default: "/data") +- `metadata_key`: HDF5 group path for metadata (default: "/metadata") +*[Not yet implemented]* +- `use_compression`: Enable Blosc compression if available (default: False) +- `reshape_dims`: Reshape dimensions string like "10x10" (default: None) +*[Not yet implemented]* +- `report_progress`: Report conversion progress (default: True) +*[Not yet implemented]* +- `timeout_seconds`: Timeout in seconds, 0 for no limit (default: 900) +*[Not yet implemented]* + +### Notes + +- When converting multiple files, the function continues processing remaining +files even if some fail +- All errors are collected and reported together at the end +- Compression settings are controlled via environment variables +`MIB2H5_SHUFFLE` and `MIB2H5_COMPRESSION_LEVEL` + +## Licence + +MIT diff --git a/python/pyproject.toml b/python/pyproject.toml new file mode 100644 index 0000000..a14adfb --- /dev/null +++ b/python/pyproject.toml @@ -0,0 +1,61 @@ +[build-system] +requires = ["setuptools>=77.0", "cython>=3", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "mib2h5" +version = "0.0.1" +description = "Python wrapper for MIB to HDF5 conversion" +readme = "README.md" +requires-python = ">=3.10" +license = "MIT" +authors = [ + {name = "Timothy Poon", email = "timothy0626@gmail.com"}, +] +keywords = ["MIB", + "HDF5", + "MerlinEM", + "Quantum Detector", + "electron microscopy", + "data conversion", +] +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Science/Research", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Cython", + "Topic :: Scientific/Engineering", +] + +[project.optional-dependencies] +test = [ + "pytest", + "h5py>=3.0", +] + +[project.scripts] +mib2h5 = "mib2h5.cli:main" + +[tool.setuptools] +zip-safe = false + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.ruff] +line-length = 79 +target-version = "py310" + +[tool.ruff.lint] +select = [ + "E", + "F", + "UP", + "B", + "SIM", + "I", +] diff --git a/python/setup.py b/python/setup.py new file mode 100644 index 0000000..734fb6f --- /dev/null +++ b/python/setup.py @@ -0,0 +1,259 @@ +import os +import subprocess +from pathlib import Path +from subprocess import CalledProcessError + +from Cython.Build import cythonize +from setuptools import Extension, setup +from setuptools.command.build_ext import build_ext + + +class AutotoolsError(Exception): + """Exception raised when autotools configuration fails.""" + pass + + +class ConfigureBuildExt(build_ext): + """Custom build_ext that runs configure if needed.""" + + def run(self): + """Run build extension with autotools configuration if needed.""" + config_h_path = self.parent_dir / "config.h" + + if not config_h_path.exists(): + print("config.h not found. Running autotools configuration...") + print("This is only needed once for the initial setup.") + self._run_autotools() + + super().run() + + def _run_autotools(self): + """Run autotools to generate config.h.""" + configure_path = self.parent_dir / "configure" + + if not configure_path.exists(): + self._run_autoreconf() + + self._run_configure() + print("Configuration completed.") + + def _run_autoreconf(self): + """Run autoreconf to generate configure script.""" + print("Running autoreconf -i...") + try: + subprocess.run( + ["autoreconf", "-i"], + cwd=self.parent_dir, + capture_output=True, + text=True, + check=True, + ) + except CalledProcessError as err: + raise AutotoolsError( + f"Failed to run autoreconf: {err.stderr}" + ) from err + + def _run_configure(self): + """Run configure script.""" + print("Running ./configure...") + configure_args = self._get_configure_args() + + try: + subprocess.run( + configure_args, + cwd=self.parent_dir, + capture_output=True, + text=True, + env=os.environ.copy(), + check=True, + ) + except CalledProcessError as err: + raise AutotoolsError( + "Configure failed. This is usually because of missing HDF5 " + f"libraries: {err.stderr}" + ) from err + + def _get_configure_args(self): + """Get configure command arguments.""" + # check hdf5 environment variables in order of priority + for env_var in ("HDF5_ROOT", "HDF5_HOME", "HDF5_DIR"): + hdf5_path_str = os.environ.get(env_var) + if hdf5_path_str: + hdf5_path = Path(hdf5_path_str).resolve() + if not hdf5_path.exists(): + msg = f"{env_var} path does not exist: {hdf5_path}" + raise AutotoolsError(msg) + print(f"Using {env_var}={hdf5_path} for configure") + return ["./configure", f"--with-hdf5={hdf5_path}"] + return ["./configure"] + + @property + def parent_dir(self): + return Path(__file__).parent.parent + + +def _check_header_exists(include_dir, header_file): + """Check if a header file exists in the given directory.""" + header_path = Path(include_dir) / header_file + return header_path.exists() and header_path.is_file() + + +def _get_library_path(root_path, lib_name="lib"): + """Get library path, checking lib64 first, then lib.""" + lib64_path = root_path / "lib64" + lib_path = root_path / lib_name + + if lib64_path.exists(): + return lib64_path + elif lib_path.exists(): + return lib_path + else: + raise FileNotFoundError(f"No library directory found in {root_path}") + + +def _find_hdf5_paths(): + """Find HDF5 include and library paths.""" + # check hdf5 environment variables in order of priority + hdf5_root = None + for env_var in ("HDF5_ROOT", "HDF5_HOME", "HDF5_DIR"): + hdf5_path_str = os.environ.get(env_var) + if hdf5_path_str: + hdf5_root = Path(hdf5_path_str).resolve() + if hdf5_root.exists(): + print(f"Using {env_var}={hdf5_root} for HDF5") + break + else: + print(f"Warning: {env_var} is set but path doesn't exist: " + f"{hdf5_root}") + hdf5_root = None + + if not hdf5_root: + # try default paths + for default_path in ("/usr", "/usr/local"): + candidate = Path(default_path) + if _check_header_exists(candidate / "include", "hdf5.h"): + hdf5_root = candidate + break + else: + msg = ("HDF5 not found in default locations. Please set one of " + "HDF5_ROOT, HDF5_HOME, or HDF5_DIR environment variables " + "to your HDF5 installation.") + raise FileNotFoundError(msg) + + hdf5_include = hdf5_root / "include" + hdf5_lib = _get_library_path(hdf5_root) + + # verify hdf5 header exists + if not _check_header_exists(hdf5_include, "hdf5.h"): + msg = (f"HDF5 header not found at '{hdf5_include}'. Please verify " + "your HDF5 installation.") + raise FileNotFoundError(msg) + + return hdf5_include, hdf5_lib + + +def _check_compression_support(): + """Check for optional compression libraries.""" + blosc_root_env = os.environ.get("BLOSC_ROOT") + hdf5_blosc_root_env = os.environ.get("HDF5_BLOSC_ROOT") + + if not (blosc_root_env and hdf5_blosc_root_env): + return None + + try: + blosc_root = Path(blosc_root_env).resolve() + hdf5_blosc_root = Path(hdf5_blosc_root_env).resolve() + + blosc_include = blosc_root / "include" + blosc_lib = _get_library_path(blosc_root) + + hdf5_blosc_include = hdf5_blosc_root / "src" + hdf5_blosc_lib = hdf5_blosc_root / "build" + + # verify headers exist + if (_check_header_exists(blosc_include, "blosc.h") and + _check_header_exists(hdf5_blosc_include, "blosc_filter.h")): + return { + "include": blosc_include, + "lib": blosc_lib, + "hdf5_blosc_include": hdf5_blosc_include, + "hdf5_blosc_lib": hdf5_blosc_lib, + } + except FileNotFoundError: + pass + + return None + + +def _build_extension_config(): + """Build extension configuration.""" + # find hdf5 paths + hdf5_include, hdf5_lib = _find_hdf5_paths() + + # base configuration + include_dirs = ["..", "../src", hdf5_include] + library_dirs = [hdf5_lib] + libraries = ["hdf5"] + extra_link_args = [f"-Wl,-rpath,{hdf5_lib}"] + extra_compile_args = ["-std=c11"] + + # check for compression support + compression_config = _check_compression_support() + if compression_config: + include_dirs.extend([ + compression_config["include"], + compression_config["hdf5_blosc_include"], + ]) + library_dirs.extend([ + compression_config["lib"], + compression_config["hdf5_blosc_lib"], + ]) + libraries.extend(["blosc", "blosc_filter"]) + extra_link_args.extend([ + f"-Wl,-rpath,{compression_config['lib']}", + f"-Wl,-rpath,{compression_config['hdf5_blosc_lib']}", + ]) + extra_compile_args.append("-DENABLE_COMPRESSION") + print("Compression support enabled.") + else: + print("Compression support disabled (blosc libraries not found).") + + return { + "include_dirs": [str(path) for path in include_dirs], + "library_dirs": [str(path) for path in library_dirs], + "libraries": libraries, + "extra_compile_args": extra_compile_args, + "extra_link_args": extra_link_args, + } + +# source files for the extension +SOURCE_FILES = [ + "src/mib2h5/_wrapper.pyx", + "../src/mib_to_h5.c", + "../src/append.c", + "../src/compress.c", + "../src/framebuffer.c", + "../src/hdf5_init.c", + "../src/hdf5_init_meta.c", + "../src/io_header.c", + "../src/parser.c", + "../src/read.c", + "../src/utils.c", +] + +config = _build_extension_config() + +ext = Extension( + name="mib2h5._wrapper", + sources=SOURCE_FILES, + **config, +) + +setup( + name="mib2h5", + packages=["mib2h5"], + package_dir={"": "src"}, + ext_modules=cythonize(ext, language_level=3), + python_requires=">=3.10", + cmdclass={"build_ext": ConfigureBuildExt}, +) diff --git a/python/src/mib2h5/__init__.py b/python/src/mib2h5/__init__.py new file mode 100644 index 0000000..e73a676 --- /dev/null +++ b/python/src/mib2h5/__init__.py @@ -0,0 +1,16 @@ +"""mib2h5 - Python wrapper for MIB to HDF5 conversion. + +This package provides Python binding for converting MerlinEM MIB files +to HDF5 files. +""" + +from importlib.metadata import PackageNotFoundError, version + +from ._wrapper import convert + +try: + __version__ = version("mib2h5") +except PackageNotFoundError: + __version__ = "unknown" + +__all__ = ["convert", "__version__"] diff --git a/python/src/mib2h5/_wrapper.pyx b/python/src/mib2h5/_wrapper.pyx new file mode 100644 index 0000000..7a54542 --- /dev/null +++ b/python/src/mib2h5/_wrapper.pyx @@ -0,0 +1,150 @@ +# cython: language_level=3 + +from .constants import ( + DEFAULT_OUTPUT_DIRECTORY, + DEFAULT_DATASET_KEY, + DEFAULT_INCLUDE_METADATA, + DEFAULT_METADATA_KEY, + DEFAULT_USE_COMPRESSION, + DEFAULT_RESHAPE_DIMS, + DEFAULT_REPORT_PROGRESS, + DEFAULT_TIMEOUT_SECONDS, +) + +cdef extern from "mib_to_h5.h": + int mib_to_h5_single_file(const char *filename, + const char *output_directory, + const char *dataset_key, + bint include_metadata, + const char *compressor, + unsigned int shuffle, + unsigned int compression_level) + +def convert(input_files, + output_dir=DEFAULT_OUTPUT_DIRECTORY, + bint include_metadata=DEFAULT_INCLUDE_METADATA, + str dataset_key=DEFAULT_DATASET_KEY, + str metadata_key=DEFAULT_METADATA_KEY, + bint use_compression=DEFAULT_USE_COMPRESSION, + reshape_dims=DEFAULT_RESHAPE_DIMS, + bint report_progress=DEFAULT_REPORT_PROGRESS, + int timeout_seconds=DEFAULT_TIMEOUT_SECONDS): + """ + Convert MIB file(s) to HDF5 file(s). + + Parameters + ---------- + input_files : str or list of str + Path(s) to the input MIB file(s) + output_dir : str, optional + Directory where the output HDF5 file(s) will be saved (default: + current directory) + include_metadata : bool, optional + Whether to include metadata in the HDF5 file (default: True) + dataset_key : str, optional + HDF5 dataset key for frames (default: "/data") + metadata_key : str, optional + HDF5 group path for metadata (default: "/metadata") + use_compression : bool, optional + Enable Blosc compression if available (default: False) + Note: Blosc compression settings can be controlled via + environment variables: + - MIB2H5_SHUFFLE (0-2, default: 2) + - MIB2H5_COMPRESSION_LEVEL (0-9, default: 9) + reshape_dims : str, optional + Reshape dimensions string like "10x10" (default: None) + report_progress : bool, optional + Report conversion progress (default: True) + timeout_seconds : int, optional + Timeout in seconds, 0 for no limit (default: 900) + + Returns + ------- + None + + Raises + ------ + ValueError + If required parameters are missing or invalid + RuntimeError + If the conversion fails for any file + """ + # ensure input_files is a list + if isinstance(input_files, str): + files = [input_files] + else: + files = list(input_files) + + if not files: + raise ValueError("No input files provided") + + if not dataset_key: + raise ValueError("Dataset key cannot be empty") + + if timeout_seconds < 0: + raise ValueError("Timeout must be non-negative") + + # prepare output directory + cdef bytes b_output_dir = output_dir.encode() if output_dir else b"./" + cdef bytes b_dataset_key = dataset_key.encode() + + # determine compressor string based on use_compression + cdef bytes b_compressor = b"blosclz" if use_compression else b"" + + # use default values for shuffle and compression_level + # these may be overridden by environment variables if provided + cdef unsigned int shuffle = 2 if use_compression else 0 + cdef unsigned int compression_level = 9 if use_compression else 0 + + # track errors for reporting + errors = [] + successful = [] + + # process each file + cdef bytes b_filename + cdef int result + + for filename in files: + if not filename: + errors.append((filename, "Empty filename")) + continue + + # encode filename + b_filename = filename.encode() + + # call the C function for single file + result = mib_to_h5_single_file( + b_filename, + b_output_dir, + b_dataset_key, + include_metadata, + b_compressor, + shuffle, + compression_level + ) + + if result == 0: + successful.append(filename) + else: + errors.append((filename, + f"Conversion failed with error code: {result}") + ) + + + # report succeeded and failed files if there is error + if errors: + error_messages = [] + for filename, msg in errors: + error_messages.append(f"{filename}: {msg}") + + error_report = "\n".join(error_messages) + + if successful: + success_msg = f"Successfully converted {len(successful)} file(s)" + failure_msg = (f"Failed to convert {len(errors)} " + f"file(s):\n{error_report}") + raise RuntimeError(f"{success_msg}\n{failure_msg}") + else: + msg = (f"Failed to convert all {len(errors)} " + f"file(s):\n{error_report}") + raise RuntimeError(msg) diff --git a/python/src/mib2h5/cli.py b/python/src/mib2h5/cli.py new file mode 100644 index 0000000..6a79cbd --- /dev/null +++ b/python/src/mib2h5/cli.py @@ -0,0 +1,159 @@ +"""Command-line interface for mib2h5 package.""" + +import argparse +import sys +from importlib.metadata import PackageNotFoundError, version +from pathlib import Path + +from ._wrapper import convert +from .constants import ( + DEFAULT_DATASET_KEY, + DEFAULT_INCLUDE_METADATA, + DEFAULT_METADATA_KEY, + DEFAULT_OUTPUT_DIRECTORY, + DEFAULT_REPORT_PROGRESS, + DEFAULT_TIMEOUT_SECONDS, + DEFAULT_USE_COMPRESSION, +) + + +def create_parser() -> argparse.ArgumentParser: + """Create argument parser for mib2h5 CLI. + + Returns + ------- + argparse.ArgumentParser + the argument parser for mib2h5 CLI + """ + parser = argparse.ArgumentParser( + description="Convert MIB file(s) to HDF5 file(s)", + prog="mib2h5", + epilog="Environment variables:\n" + " MIB2H5_SHUFFLE Shuffle level for Blosc compression (0-2, default: 2)\n" + " MIB2H5_COMPRESSION_LEVEL Blosc compression level (0-9, default: 9)", + formatter_class=argparse.RawDescriptionHelpFormatter + ) + + # positional arguments for input files + parser.add_argument( + "input_files", + nargs="+", + help="input MIB file(s) to convert" + ) + + # optional arguments + parser.add_argument( + "-o", "--output-dir", + default=DEFAULT_OUTPUT_DIRECTORY, + help="output directory for HDF5 files (default: current directory)" + ) + + parser.add_argument( + "-d", "--dataset-key", + default=DEFAULT_DATASET_KEY, + help=f"HDF5 dataset path for frames (default: {DEFAULT_DATASET_KEY})" + ) + + parser.add_argument( + "-c", "--compression", + action="store_true", + default=DEFAULT_USE_COMPRESSION, + help="enable Blosc compression (settings via env vars)" + ) + + parser.add_argument( + "-N", "--no-metadata", + action="store_false", + dest="include_metadata", + default=DEFAULT_INCLUDE_METADATA, + help="exclude metadata from HDF5 output" + ) + + parser.add_argument( + "--metadata-key", + default=DEFAULT_METADATA_KEY, + help=f"HDF5 group path for metadata (default: {DEFAULT_METADATA_KEY})" + ) + + parser.add_argument( + "--no-progress", + action="store_false", + dest="report_progress", + default=DEFAULT_REPORT_PROGRESS, + help="disable progress reporting" + ) + + parser.add_argument( + "--timeout", + type=int, + default=DEFAULT_TIMEOUT_SECONDS, + help=("timeout in seconds, 0 for no limit " + f"(default: {DEFAULT_TIMEOUT_SECONDS})") + ) + + parser.add_argument( + "--version", + action="version", + version=_get_version() + ) + + return parser + + +def _get_version() -> str: + """Get package version. + + Returns + ------- + str + Package version string + """ + try: + return version("mib2h5") + except PackageNotFoundError: + return "unknown" + + +def main() -> int: + """Entry point for mib2h5 CLI. + + Returns + ------- + int + Exit code: 0 for success, non-zero for error + """ + parser = create_parser() + args = parser.parse_args() + + # validate input files exist + missing_files = [] + for filepath in args.input_files: + if not Path(filepath).exists(): + missing_files.append(filepath) + + if missing_files: + missing_files_msg = "\n".join(missing_files) + msg = f"The following input files do not exist: {missing_files_msg}" + raise FileNotFoundError(msg) + + if args.timeout < 0: + raise ValueError("Timeout must be non-negative") + + # call the conversion function + convert( + input_files=args.input_files, + output_dir=args.output_dir, + include_metadata=args.include_metadata, + dataset_key=args.dataset_key, + metadata_key=args.metadata_key, + use_compression=args.compression, + reshape_dims=None, + report_progress=args.report_progress, + timeout_seconds=args.timeout + ) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/python/src/mib2h5/constants.py b/python/src/mib2h5/constants.py new file mode 100644 index 0000000..2fa5de3 --- /dev/null +++ b/python/src/mib2h5/constants.py @@ -0,0 +1,11 @@ +"""Constants for mib2h5 package.""" + +# default values matching C implementation +DEFAULT_OUTPUT_DIRECTORY = None +DEFAULT_DATASET_KEY = "/data" +DEFAULT_INCLUDE_METADATA = True +DEFAULT_METADATA_KEY = "/metadata" +DEFAULT_USE_COMPRESSION = False +DEFAULT_RESHAPE_DIMS = None +DEFAULT_REPORT_PROGRESS = True +DEFAULT_TIMEOUT_SECONDS = 900 diff --git a/src/Makefile.am b/src/Makefile.am new file mode 100644 index 0000000..caed3b0 --- /dev/null +++ b/src/Makefile.am @@ -0,0 +1,9 @@ +# Target binary +bin_PROGRAMS = mib2h5 + +# Source files +mib2h5_SOURCES = main.c mib_to_h5.c utils.c io_header.c parser.c framebuffer.c hdf5_init.c hdf5_init_meta.c read.c append.c compress.c +mib2h5_LDADD = -lhdf5 @LIBCOMPRESSION@ + +# Install public header +include_HEADERS = mib2h5.h diff --git a/src/append.c b/src/append.c new file mode 100644 index 0000000..968483c --- /dev/null +++ b/src/append.c @@ -0,0 +1,279 @@ +#include "append.h" +#include "framebuffer.h" +#include "io_header.h" +#include "macros.h" + +#include +#include +#include + +void append_frame_to_dataset(hid_t dset, framebuffer *fb, int cbytes) +{ + if (!dset || !fb) { + fprintf(stderr, "Error in passing parameters in append_frame_to_dataset\n"); + return; + } + + unsigned int detx = *(fb->mq1_header->det_x); + unsigned int dety = *(fb->mq1_header->det_y); + int bufsize = (fb->mq1_header->pixel_depth[1] - '0') * 10 + + (fb->mq1_header->pixel_depth[2] - '0'); + bufsize = bufsize / 8; + + if (cbytes == 0) { + cbytes = dety * detx * bufsize; + printf("cbytes == 0\n"); + } + + hid_t filespace = H5Dget_space(dset); + if (filespace == H5I_INVALID_HID) { + fprintf(stderr, "Error in H5Dget_space in append_frame_to_dataset\n"); + return; + } + hsize_t dims[3]; + int rank = H5Sget_simple_extent_dims(filespace, dims, NULL); + if (rank < 0) { + fprintf(stderr, + "Error in H5Sget_simple_extent_dims in append_frame_to_dataset\n"); + H5Sclose(filespace); + return; + } + + hsize_t frame_index = dims[0]; + + hsize_t new_dims[3] = {frame_index + 1, dety, detx}; + if (H5Dset_extent(dset, new_dims) < 0) { + fprintf(stderr, "Error in extending dataset in append_frame_to_dataset\n"); + return; + } + + hsize_t offset_chunk[3] = {frame_index, 0, 0}; + uint32_t filter_mask = 0; + + if (H5Dwrite_chunk(dset, H5P_DEFAULT, filter_mask, offset_chunk, cbytes, + fb->data) < 0) { + fprintf(stderr, "Failed in writing chunk\n"); + return; + } + + H5Sclose(filespace); +} + +void append_meta_to_dataset(hid_t *meta_handle, framebuffer *fb) +{ + // meta_handle will be free outside this + if (meta_handle == NULL) { + fprintf(stderr, "meta_handle is NULL in append_meta_to_dataset\n"); + return; + } + if (fb == NULL) { + fprintf(stderr, "framebuffer is NULL in append_meta_to_dataset\n"); + return; + } + hid_t datatype, filespace, memspace; + info *mq1_iter = mq1_fields_info(fb->mq1_header); + + hsize_t dim[1]; + hsize_t frame_index; + hsize_t new_dim[1]; + hsize_t start[1]; + hsize_t count[1] = {1}; + hsize_t mem_dim[1] = {1}; + int rank = 0; + herr_t status; + + for (size_t i = 0; i < MQ1_FIELDS_NUM_FIELDS; i++) { + datatype = H5Dget_type(meta_handle[i]); + if (datatype < 0) { + fprintf(stderr, "Error in H5Dget_type in append_meta_to_dataset\n"); + return; + } + filespace = H5Dget_space(meta_handle[i]); + if (filespace == H5I_INVALID_HID) { + fprintf(stderr, "Error in H5Dget_space in append_meta_to_dataset\n"); + H5Tclose(datatype); + return; + } + rank = H5Sget_simple_extent_dims(filespace, dim, NULL); + if (rank < 0) { + fprintf(stderr, + "Error in H5Sget_simple_extent_dims in append_meta_to_dataset\n"); + H5Tclose(datatype); + return; + } + frame_index = dim[0]; + new_dim[0] = dim[0] + 1; + + if (H5Dset_extent(meta_handle[i], new_dim) < 0) { + fprintf(stderr, + "Error in extending dataset %s in append_meta_to_dataset\n", + mq1_iter[i].name); + H5Tclose(datatype); + H5Sclose(filespace); + return; + } + // update filespace as the dimension is extented + H5Sclose(filespace); + filespace = H5Dget_space(meta_handle[i]); + if (filespace == H5I_INVALID_HID) { + fprintf(stderr, "Error in H5Dget_space in append_meta_to_dataset\n"); + return; + } + + start[0] = frame_index; + status = + H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, NULL, count, NULL); + if (status < 0) { + fprintf(stderr, "Error H5Sselect_hyperslab in append_meta_to_dataset\n"); + return; + } + + memspace = H5Screate_simple(1, mem_dim, NULL); + if (memspace < 0) { + fprintf(stderr, "Error H5Screate_simple in append_meta_to_dataset\n"); + return; + } + + if (H5Dwrite(meta_handle[i], datatype, memspace, filespace, H5P_DEFAULT, + mq1_iter[i].data) < 0) { + fprintf(stderr, "Error writing %s to metadata dataset\n", + mq1_iter[i].name); + } + + H5Sclose(memspace); + H5Sclose(filespace); + H5Tclose(datatype); + } + + if (mq1_iter) { + free(mq1_iter); + } +} + +void append_dac_to_dataset(unsigned int num_chips, + hid_t *dac_handle, + framebuffer *fb) +{ + // dac_handle will be freed outside this + hid_t datatype, filespace, memspace; + info *d_array[4]; + size_t ind = 0; + + if (!fb) { + fprintf(stderr, "Null framebuffer in append_dac_to_dataset\n"); + return; + } + + switch (num_chips) { + case 1: { + if (!fb->dac0) { + fprintf(stderr, "Null dac0 pointer in append_dac_to_dataset\n"); + return; + } else { + d_array[0] = dac_info(fb->dac0); + d_array[1] = NULL; + d_array[2] = NULL; + d_array[3] = NULL; + } + break; + } + case 4: { + if (!fb->dac0 || !fb->dac1 || !fb->dac2 || !fb->dac3) { + fprintf(stderr, "Null dac pointer detected in append_dac_to_dataset\n"); + return; + } else { + d_array[0] = dac_info(fb->dac0); + d_array[1] = dac_info(fb->dac1); + d_array[2] = dac_info(fb->dac2); + d_array[3] = dac_info(fb->dac3); + } + break; + } + default: { + fprintf(stderr, "Num_chips should be 1 or 4 in append_dac_to_dataset\n"); + return; + } + } + + hsize_t dim[1]; + hsize_t frame_index; + hsize_t new_dim[1]; + hsize_t start[1]; + hsize_t count[1] = {1}; + hsize_t mem_dim[1] = {1}; + int rank = 0; + herr_t status; + + for (size_t i = 0; i < (size_t) num_chips; i++) { + for (size_t j = 0; j < DAC_NUM_FIELDS; j++) { + ind = i * DAC_NUM_FIELDS + j; + datatype = H5Dget_type(dac_handle[ind]); + if (datatype < 0) { + fprintf(stderr, "Error in H5Dget_type in append_meta_to_dataset\n"); + return; + } + filespace = H5Dget_space(dac_handle[ind]); + if (filespace == H5I_INVALID_HID) { + fprintf(stderr, "Error in H5Dget_space in append_meta_to_dataset\n"); + H5Tclose(datatype); + return; + } + rank = H5Sget_simple_extent_dims(filespace, dim, NULL); + if (rank < 0) { + fprintf( + stderr, + "Error in H5Sget_simple_extent_dims in append_meta_to_dataset\n"); + H5Tclose(datatype); + return; + } + frame_index = dim[0]; + new_dim[0] = dim[0] + 1; + + if (H5Dset_extent(dac_handle[ind], new_dim) < 0) { + fprintf(stderr, + "Error in extending dataset %s in append_dac_to_dataset\n", + d_array[i][j].name); + H5Tclose(datatype); + H5Sclose(filespace); + continue; + } + + // Close and update the filespace as the dimension has extended + H5Sclose(filespace); + filespace = H5Dget_space(dac_handle[ind]); + if (filespace == H5I_INVALID_HID) { + fprintf(stderr, "Error in H5Dget_space in append_meta_to_dataset\n"); + H5Tclose(datatype); + return; + } + + start[0] = frame_index; + status = H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, NULL, + count, NULL); + if (status < 0) { + fprintf(stderr, "Error H5Sselect_hyperslab in append_dac_to_dataset\n"); + return; + } + + memspace = H5Screate_simple(1, mem_dim, NULL); + if (memspace == H5I_INVALID_HID) { + fprintf(stderr, "Error H5Screate_simple in append_dac_to_dataset\n"); + return; + } + + if (H5Dwrite(dac_handle[ind], datatype, memspace, filespace, H5P_DEFAULT, + d_array[i][j].data) < 0) { + fprintf(stderr, "Error writing chip%ld, %s to metadata dac\n", i, + d_array[i][j].name); + return; + } + H5Sclose(memspace); + H5Sclose(filespace); + H5Tclose(datatype); + } + + if (d_array[i]) { + free(d_array[i]); + } + } +} diff --git a/src/append.h b/src/append.h new file mode 100644 index 0000000..891431f --- /dev/null +++ b/src/append.h @@ -0,0 +1,20 @@ +// clang-format Language: C +#ifndef APPEND_H +#define APPEND_H + +#include "framebuffer.h" +#include "io_header.h" +#include "parser.h" +#include "utils.h" +#include +#include + +void append_frame_to_dataset(hid_t dset, framebuffer *fb, int cbytes); + +void append_meta_to_dataset(hid_t *meta_handle, framebuffer *fb); + +void append_dac_to_dataset(unsigned int num_chips, + hid_t *dac_handle, + framebuffer *fb); + +#endif diff --git a/src/compress.c b/src/compress.c new file mode 100644 index 0000000..4fa9cde --- /dev/null +++ b/src/compress.c @@ -0,0 +1,136 @@ +#include "config.h" +#include "framebuffer.h" +#include "io_header.h" +#include "macros.h" +#include "mib_header.h" +#include "utils.h" +#include +#include +#include + +#ifdef HAVE_COMPRESSION +#include +#include +#endif + +hid_t dcpl_compress(size_t dim, + hsize_t *frame_dim, + unsigned int compression_level, + unsigned int shuffle, + char *compressor) +{ + hid_t dcpl = H5I_INVALID_HID; + unsigned int cd_values[7] = {0}; + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) == H5I_INVALID_HID) { + fprintf(stderr, "Error in creating dcpl\n"); + return H5I_INVALID_HID; + } else { + if (H5Pset_chunk(dcpl, dim, frame_dim) < 0) { + fprintf(stderr, "Error in H5Pset_chunk\n"); + H5Pclose(dcpl); + return H5I_INVALID_HID; + } + if (H5Pset_fill_time(dcpl, H5D_FILL_TIME_NEVER) < 0) { + fprintf(stderr, "Error in H5Pset_fill_time\n"); + H5Pclose(dcpl); + return H5I_INVALID_HID; + } +#ifdef HAVE_COMPRESSION + char *version = (char *) malloc(sizeof(char) * 512); + if (version == NULL) { + fprintf(stderr, "malloc for version failed\n"); + H5Pclose(dcpl); + return H5I_INVALID_HID; + } + char *date = (char *) malloc(sizeof(char) * 512); + if (date == NULL) { + fprintf(stderr, "malloc for date failed\n"); + H5Pclose(dcpl); + free(version); + return H5I_INVALID_HID; + } + cd_values[0] = 0; + cd_values[1] = 0; // unused + cd_values[2] = 0; // unused + cd_values[3] = 0; // blocksize + cd_values[4] = compression_level; + cd_values[5] = shuffle; + cd_values[6] = blosc_compname_to_compcode(compressor); + if (H5Pset_filter(dcpl, FILTER_BLOSC, H5Z_FLAG_OPTIONAL, 7, cd_values) < + 0) { + fprintf(stderr, "Error in H5Pset_filter\n"); + H5Pclose(dcpl); + return H5I_INVALID_HID; + } + if (register_blosc(&version, &date) < 0) { + fprintf(stderr, "Error in register_blosc\n"); + H5Pclose(dcpl); + free(version); + free(date); + return H5I_INVALID_HID; + } else { + printf("Blosc version info: %s (%s)\n", version, date); + } + free(version); + free(date); +#endif + } + return dcpl; +} + +int compress_frame(framebuffer *fb, + unsigned int compression_level, + unsigned int shuffle, + char *compressor, + size_t blocksize, + int numinternalthreads) +{ + int bufsize = (fb->mq1_header->pixel_depth[1] - '0') * 10 + + (fb->mq1_header->pixel_depth[2] - '0'); + bufsize = bufsize / 8; + + int detx = (int) *(fb->mq1_header->det_x); + int dety = (int) *(fb->mq1_header->det_y); + + size_t nbytes = dety * detx * bufsize; +#ifdef HAVE_COMPRESSION + size_t destsize = nbytes + BLOSC_MAX_OVERHEAD; + void *dest = malloc(destsize); + if (!dest) { + fprintf(stderr, "Error in malloc for dest\n"); + return -1; + } + + int cbytes = blosc_compress_ctx(compression_level, shuffle, bufsize, nbytes, + fb->data, dest, destsize, compressor, + blocksize, numinternalthreads); + + if (cbytes < 0) { + fprintf(stderr, "Error in blosc_compress\n"); + free(dest); + return -1; + } + if (cbytes == 0) { + fprintf(stderr, "Blosc returned 0 bytes (uncompressible?). Forcing " + "fallback to uncompressed write.\n"); + cbytes = nbytes; + free(dest); + return cbytes; + } + + void *temp = realloc(fb->data, destsize); + if (temp) { + fb->data = temp; + memcpy(fb->data, dest, cbytes); + } else { + fprintf(stderr, + "Error in realloc for fb->data, fall back on the original data\n"); + free(dest); + return nbytes; + } + free(dest); + return cbytes; +#else + return nbytes; +#endif +} diff --git a/src/compress.h b/src/compress.h new file mode 100644 index 0000000..c5ae65d --- /dev/null +++ b/src/compress.h @@ -0,0 +1,18 @@ +// clang-format Language: C +#ifndef COMPRESS_H +#define COMPRESS_H + +hid_t dcpl_compress(size_t dim, + hsize_t *frame_dim, + unsigned int compression_level, + unsigned int shuffle, + char *compressor); + +int compress_frame(framebuffer *fb, + unsigned int compression_level, + unsigned int shuffle, + char *compressor, + size_t blocksize, + int numinternalthreads); + +#endif diff --git a/src/framebuffer.c b/src/framebuffer.c new file mode 100644 index 0000000..89c76f8 --- /dev/null +++ b/src/framebuffer.c @@ -0,0 +1,173 @@ +/* These are functions of the struct framebuffer declared + * in framebuffer.h + * + * allocate_frame_header only allocates mq1_header and dacs in framebuffer + * allocate_frame_data only allocates rows and data in framebuffer + * + * Please check if both of them have been used before any usage on the struct + * + * Please also call deallocate_frame for freeing the memory. deallocate_frame + * frees both header and data + */ + +#include "framebuffer.h" +#include "io_header.h" +#include "macros.h" +#include "mib_header.h" +#include "utils.h" + +#include +#include +#include +#include + +void allocate_frame_header(framebuffer *fb) +{ + fb->mq1_header = malloc(sizeof(MQ1_fields)); + if (!fb->mq1_header) { + fprintf(stderr, + "Error in malloc for mq1_header in allocate_frame_header\n"); + return; + } + *(fb->mq1_header) = allocate_MQ1_fields(1); + fb->dac0 = (dac_rx *) malloc(sizeof(dac_rx)); + if (!fb->dac0) { + fprintf(stderr, "Error in malloc for dac0 in allocate_frame_header\n"); + return; + } + fb->dac1 = (dac_rx *) malloc(sizeof(dac_rx)); + if (!fb->dac1) { + fprintf(stderr, "Error in malloc for dac1 in allocate_frame_header\n"); + return; + } + fb->dac2 = (dac_rx *) malloc(sizeof(dac_rx)); + if (!fb->dac2) { + fprintf(stderr, "Error in malloc for dac2 in allocate_frame_header\n"); + return; + } + fb->dac3 = (dac_rx *) malloc(sizeof(dac_rx)); + if (!fb->dac3) { + fprintf(stderr, "Error in malloc for dac3 in allocate_frame_header\n"); + return; + } +} + +void allocate_frame_data(framebuffer *fb) +{ + int bufsize = (fb->mq1_header->pixel_depth[1] - '0') * 10 + + (fb->mq1_header->pixel_depth[2] - '0'); + bufsize = bufsize / 8; + + int detx = (int) *(fb->mq1_header->det_x); + int dety = (int) *(fb->mq1_header->det_y); + + void **buffer = malloc(sizeof(void *) * dety); + if (!buffer) { + fprintf(stderr, "Error in malloc for fb->rows in allocate_frame_data\n"); + fb->rows = NULL; + fb->data = NULL; + return; + } + void *data = NULL; + fb->rows = buffer; + + switch (bufsize) { + case 1: { + data = malloc(sizeof(uint8_t) * detx * dety); + if (!data) { + fprintf(stderr, "malloc failed for data in allocate_frame_data\n"); + free(buffer); + fb->rows = NULL; + fb->data = NULL; + return; + } + fb->data = data; + for (int i = 0; i < (int) dety; i++) { + buffer[i] = (uint8_t *) data + i * detx; + } + break; + } + case 2: { + data = malloc(sizeof(uint16_t) * detx * dety); + if (!data) { + fprintf(stderr, "malloc failed for data in allocate_frame_data\n"); + free(buffer); + fb->rows = NULL; + fb->data = NULL; + return; + } + fb->data = data; + for (int i = 0; i < (int) dety; i++) { + buffer[i] = (uint16_t *) data + i * detx; + } + break; + } + case 4: { + data = malloc(sizeof(uint32_t) * detx * dety); + if (!data) { + fprintf(stderr, "malloc failed for data in allocate_frame_data\n"); + free(buffer); + fb->rows = NULL; + fb->data = NULL; + return; + } + fb->data = data; + for (int i = 0; i < (int) dety; i++) { + buffer[i] = (uint32_t *) data + i * detx; + } + break; + } + case 8: { + data = malloc(sizeof(uint64_t) * detx * dety); + if (!data) { + fprintf(stderr, "malloc failed for data in allocate_frame_data\n"); + free(buffer); + fb->rows = NULL; + fb->data = NULL; + return; + } + fb->data = data; + for (int i = 0; i < (int) dety; i++) { + buffer[i] = (uint64_t *) data + i * detx; + } + break; + } + default: + fprintf(stderr, "Unsupported pixel depth in allocate_frame_data\n"); + free(buffer); + fb->rows = NULL; + fb->data = NULL; + return; + } +} + +void deallocate_frame(framebuffer *fb) +{ + if (fb == NULL) { + fprintf(stderr, "NULL framebuffer\n"); + return; + } + if (fb->rows) + free(fb->rows); + if (fb->data) + free(fb->data); + if (fb->dac0) + free(fb->dac0); + if (fb->dac1) + free(fb->dac1); + if (fb->dac2) + free(fb->dac2); + if (fb->dac3) + free(fb->dac3); + if (fb->mq1_header) { + deallocate_MQ1_fields(*(fb->mq1_header)); + free(fb->mq1_header); + fb->mq1_header = NULL; + } + fb->rows = NULL; + fb->data = NULL; + fb->dac0 = NULL; + fb->dac1 = NULL; + fb->dac2 = NULL; + fb->dac3 = NULL; +} diff --git a/src/framebuffer.h b/src/framebuffer.h new file mode 100644 index 0000000..084fe24 --- /dev/null +++ b/src/framebuffer.h @@ -0,0 +1,29 @@ +// clang-format Language: C +#include "io_header.h" +#include "mib_header.h" + +#ifndef FRAMEBUFFER_H +#define FRAMEBUFFER_H + +typedef struct { + /* This struct is to hold the data inside a frame */ + MQ1_fields *mq1_header; // metadata for the frame + dac_rx *dac0; // dac0 for both single and quad headers + dac_rx *dac1; // dac1 for quad(if the header is single then this will be NULL + dac_rx *dac2; // dac2 for quad(same as dac1) + dac_rx *dac3; // dac3 for quad(same as dac1) + void **rows; // just a holder for easy access of actual data, i.e. can be used + // like fb->rows[i][j] + void *data; // row-major array for storing converted binary data in the frame +} framebuffer; + +// allocate memory for header in framebuffer +void allocate_frame_header(framebuffer *fb); + +// allocate memory for data in framebuffer +void allocate_frame_data(framebuffer *fb); + +// this is responsible for freeing the whole struct +void deallocate_frame(framebuffer *fb); + +#endif diff --git a/src/hdf5_init.c b/src/hdf5_init.c new file mode 100644 index 0000000..024a157 --- /dev/null +++ b/src/hdf5_init.c @@ -0,0 +1,97 @@ +#include "hdf5_init.h" +#include "macros.h" +#include "utils.h" +#include +#include +#include + +void initialize_plist(char *output_dir, + hid_t *fapl_id, + hid_t *fcpl_id, + hid_t *lcpl_id) +{ + if ((*fcpl_id = H5Pcreate(H5P_FILE_CREATE)) == H5I_INVALID_HID) { + fprintf(stderr, "Error in creating fcpl in create_file\n"); + return; + } + unsigned long f_blocksize = get_filesystem_block_size(output_dir); + printf("block size of filesystem: %ld\n", f_blocksize); + if ((*fapl_id = H5Pcreate(H5P_FILE_ACCESS)) == H5I_INVALID_HID) { + fprintf(stderr, "Error in creating fapl in create_file\n"); + return; + } else { + if (H5Pset_alignment(*fapl_id, ALIGNMENT_THRESHOLD, f_blocksize) < 0) { + fprintf(stderr, "Error in H5Pset_alignment\n"); + return; + } + } + if ((*lcpl_id = H5Pcreate(H5P_LINK_CREATE)) == H5I_INVALID_HID) { + fprintf(stderr, "Error in creating lcpl in initialize_dataset_plist\n"); + return; + } +} + +void initialize_file(char *filename, hid_t *file_id, hid_t fapl, hid_t fcpl) +{ + if (filename == NULL) { + fprintf(stderr, "Empty or other error in filename, please check\n"); + return; + } + + if (!H5Iis_valid(fapl) || !H5Iis_valid(fcpl)) { + fprintf(stderr, "fapl or fcpl is invalid in initialize_file\n"); + return; + } + + if ((*file_id = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) == + H5I_INVALID_HID) { + fprintf(stderr, "Error in creating file_id in intialize_file\n"); + return; + } +} + +void create_merlin_dataset(hid_t *merlin_dataset_id, + hid_t file, + char *merlin_dataset_name, + hid_t dtype, + hid_t dcpl, + hid_t lcpl, + hsize_t *frame_dim) +{ + hid_t dapl = H5I_INVALID_HID; + + if ((dapl = H5Pcreate(H5P_DATASET_ACCESS)) == H5I_INVALID_HID) { + fprintf(stderr, "Error in creating dapl_id\n"); + goto cleanup; + } else { + // frame_dim is 2D with det_y and det_x + unsigned int y = frame_dim[0]; + unsigned int x = frame_dim[1]; + + size_t dtype_size = H5Tget_size(dtype); + if (H5Pset_chunk_cache(dapl, PRIME_FOR_HASH, + NUM_CHUNKS_IN_CACHE * dtype_size * y * x, 1.0) < 0) { + fprintf(stderr, "Error in H5Pset_chunk_cache\n"); + goto cleanup; + } + } + + // create 3D dataspace for the dataset (frames x height x width) + // frames start at 0 for expandable dataset + hsize_t dims[3] = {0, frame_dim[0], frame_dim[1]}; + hsize_t maxdims[3] = {H5S_UNLIMITED, frame_dim[0], frame_dim[1]}; + hid_t filespace = H5Screate_simple(3, dims, maxdims); + + if ((*merlin_dataset_id = H5Dcreate2(file, merlin_dataset_name, dtype, + filespace, lcpl, dcpl, dapl)) == + H5I_INVALID_HID) { + fprintf(stderr, "Error in creating merlin_dataset\n"); + H5Sclose(filespace); + goto cleanup; + } + H5Sclose(filespace); + +cleanup: + if (H5Iis_valid(dapl)) + H5Pclose(dapl); +} diff --git a/src/hdf5_init.h b/src/hdf5_init.h new file mode 100644 index 0000000..cb614ae --- /dev/null +++ b/src/hdf5_init.h @@ -0,0 +1,23 @@ +// clang-format Language: C +#include +#include + +#ifndef HDF5_INIT_H +#define HDF5_INIT_H + +void initialize_plist(char *output_dir, + hid_t *fapl_id, + hid_t *fcpl_id, + hid_t *lcpl_id); + +void initialize_file(char *filename, hid_t *file_id, hid_t fapl, hid_t fcpl); + +void create_merlin_dataset(hid_t *merlin_dataset_id, + hid_t file, + char *merlin_dataset_name, + hid_t dtype, + hid_t dcpl, + hid_t lcpl, + hsize_t *frame_dim); + +#endif diff --git a/src/hdf5_init_meta.c b/src/hdf5_init_meta.c new file mode 100644 index 0000000..55902b5 --- /dev/null +++ b/src/hdf5_init_meta.c @@ -0,0 +1,351 @@ +#include "hdf5_init_meta.h" +#include "macros.h" + +#include +#include +#include +#include + +void create_meta_mq1_fields_dataset(hid_t file, hid_t lcpl, hid_t *meta_handle) +{ + hid_t header_id_type = H5I_INVALID_HID; + hid_t pixel_depth_type = H5I_INVALID_HID; + hid_t sensor_layout_type = H5I_INVALID_HID; + hid_t chip_select_type = H5I_INVALID_HID; + hid_t timestamp_type = H5I_INVALID_HID; + hid_t header_extension_id_type = H5I_INVALID_HID; + hid_t extended_timestamp_type = H5I_INVALID_HID; + hid_t dataspace = H5I_INVALID_HID; + hid_t dcpl = H5I_INVALID_HID; + hid_t dapl = H5I_INVALID_HID; + hid_t meta_group = H5I_INVALID_HID; + if (meta_handle == NULL) { + fprintf(stderr, "meta_handle is NULL in create_meta_fields_dataset\n"); + return; + } + + char meta_group_path[64] = "metadata"; + meta_group = H5Gcreate(file, meta_group_path, lcpl, H5P_DEFAULT, H5P_DEFAULT); + if (meta_group < 0) { + fprintf(stderr, "Error creating group in create_meta_mq1_fields_dataset\n"); + goto cleanup; + } + + hsize_t dim[1] = {0}; + hsize_t max_dim[1] = {H5S_UNLIMITED}; + hsize_t chunk_dim[1] = {1}; + + dataspace = H5Screate_simple(1, dim, max_dim); + if (dataspace < 0) { + fprintf(stderr, + "Error creating dataspace in create_meta_mq1_fields_dataset\n"); + goto cleanup; + } + + dcpl = H5Pcreate(H5P_DATASET_CREATE); + if (dcpl < 0) { + fprintf(stderr, "Error creating dcpl in create_meta_mq1_fields_dataset\n"); + goto cleanup; + } else { + if (H5Pset_chunk(dcpl, 1, chunk_dim) < 0) { + fprintf(stderr, + "Error setting chunking in create_meta_mq1_fields_dataset\n"); + goto cleanup; + } + } + + dapl = H5Pcreate(H5P_DATASET_ACCESS); + if (dapl < 0) { + fprintf(stderr, + "Error in creating dapl in create_meta_mq1_fields_dataset\n"); + goto cleanup; + } else { + } + + header_id_type = H5Tcopy(H5T_C_S1); + if (header_id_type < 0) { + fprintf(stderr, "H5Tcopy failed for header_id_type\n"); + goto cleanup; + } + if (H5Tset_size(header_id_type, MQ1_CHAR_LEN_HEADER_ID) < 0) { + fprintf(stderr, "H5Tset_size failed for header_id_type\n"); + goto cleanup; + } + + pixel_depth_type = H5Tcopy(H5T_C_S1); + if (pixel_depth_type < 0) { + fprintf(stderr, "H5Tcopy failed for pixel_depth_type\n"); + goto cleanup; + } + if (H5Tset_size(pixel_depth_type, MQ1_CHAR_LEN_PIXEL_DEPTH) < 0) { + fprintf(stderr, "H5Tset_size failed for pixel_depth_type\n"); + goto cleanup; + } + + sensor_layout_type = H5Tcopy(H5T_C_S1); + if (sensor_layout_type < 0) { + fprintf(stderr, "H5Tcopy failed for sensor_layout_type\n"); + goto cleanup; + } + if (H5Tset_size(sensor_layout_type, MQ1_CHAR_LEN_SENSOR_LAYOUT) < 0) { + fprintf(stderr, "H5Tset_size failed for sensor_layout_type\n"); + goto cleanup; + } + + chip_select_type = H5Tcopy(H5T_C_S1); + if (chip_select_type < 0) { + fprintf(stderr, "H5Tcopy failed for chip_select_type\n"); + goto cleanup; + } + if (H5Tset_size(chip_select_type, MQ1_CHAR_LEN_CHIP_SELECT) < 0) { + fprintf(stderr, "H5Tset_size failed for chip_select_type\n"); + goto cleanup; + } + + timestamp_type = H5Tcopy(H5T_C_S1); + if (timestamp_type < 0) { + fprintf(stderr, "H5Tcopy failed for timestamp_type\n"); + goto cleanup; + } + if (H5Tset_size(timestamp_type, MQ1_CHAR_LEN_TIMESTAMP) < 0) { + fprintf(stderr, "H5Tset_size failed for timestamp_type\n"); + goto cleanup; + } + + header_extension_id_type = H5Tcopy(H5T_C_S1); + if (header_extension_id_type < 0) { + fprintf(stderr, "H5Tcopy failed for header_extension_id_type\n"); + goto cleanup; + } + if (H5Tset_size(header_extension_id_type, MQ1_CHAR_LEN_HEADER_EXTENSION_ID) < + 0) { + fprintf(stderr, "H5Tset_size failed for header_extension_id_type\n"); + goto cleanup; + } + + extended_timestamp_type = H5Tcopy(H5T_C_S1); + if (extended_timestamp_type < 0) { + fprintf(stderr, "H5Tcopy failed for extended_timestamp_type\n"); + goto cleanup; + } + if (H5Tset_size(extended_timestamp_type, MQ1_CHAR_LEN_EXTENDED_TIMESTAMP) < + 0) { + fprintf(stderr, "H5Tset_size failed for extended_timestamp_type\n"); + goto cleanup; + } + + struct { + const char *name; + hid_t type; + } fields[] = { + {"header_id", header_id_type}, + {"max_length", H5T_NATIVE_UINT}, + {"sequence_number", H5T_NATIVE_UINT}, + {"header_bytes", H5T_NATIVE_UINT}, + {"num_chips", H5T_NATIVE_UINT}, + {"det_x", H5T_NATIVE_UINT}, + {"det_y", H5T_NATIVE_UINT}, + {"pixel_depth", pixel_depth_type}, + {"sensor_layout", sensor_layout_type}, + {"chip_select", chip_select_type}, + {"timestamp", timestamp_type}, + {"exposure_time_s", H5T_NATIVE_DOUBLE}, + {"counter", H5T_NATIVE_UINT}, + {"colour_mode", H5T_NATIVE_UINT}, + {"gain_mode", H5T_NATIVE_UINT}, + {"threshold0", H5T_NATIVE_FLOAT}, + {"threshold1", H5T_NATIVE_FLOAT}, + {"threshold2", H5T_NATIVE_FLOAT}, + {"threshold3", H5T_NATIVE_FLOAT}, + {"threshold4", H5T_NATIVE_FLOAT}, + {"threshold5", H5T_NATIVE_FLOAT}, + {"threshold6", H5T_NATIVE_FLOAT}, + {"threshold7", H5T_NATIVE_FLOAT}, + {"header_extension_id", header_extension_id_type}, + {"extended_timestamp", extended_timestamp_type}, + {"exposure_time_ns", H5T_NATIVE_UINT}, + {"bit_depth", H5T_NATIVE_UINT}, + }; + + size_t num_datasets = sizeof(fields) / sizeof(fields[0]); + + if (num_datasets != MQ1_FIELDS_NUM_FIELDS) { + fprintf(stderr, + "Number of dataset not match in create_meta_mq1_fields_dataset\n"); + goto cleanup; + } + + for (size_t i = 0; i < num_datasets; i++) { + char dataset_path[256]; + + snprintf(dataset_path, sizeof(dataset_path), "%s", fields[i].name); + + hid_t dataset = H5Dcreate2(meta_group, dataset_path, fields[i].type, + dataspace, lcpl, dcpl, dapl); + if (dataset < 0) { + fprintf(stderr, "Error creating dataset : %s\n", dataset_path); + meta_handle[i] = -1; + continue; + } + + meta_handle[i] = dataset; + } + +cleanup: + if (H5Iis_valid(extended_timestamp_type)) + H5Tclose(extended_timestamp_type); + if (H5Iis_valid(header_extension_id_type)) + H5Tclose(header_extension_id_type); + if (H5Iis_valid(timestamp_type)) + H5Tclose(timestamp_type); + if (H5Iis_valid(chip_select_type)) + H5Tclose(chip_select_type); + if (H5Iis_valid(sensor_layout_type)) + H5Tclose(sensor_layout_type); + if (H5Iis_valid(pixel_depth_type)) + H5Tclose(pixel_depth_type); + if (H5Iis_valid(header_id_type)) + H5Tclose(header_id_type); + if (H5Iis_valid(dapl)) + H5Pclose(dapl); + if (H5Iis_valid(dcpl)) + H5Pclose(dcpl); + if (H5Iis_valid(dataspace)) + H5Sclose(dataspace); + if (H5Iis_valid(meta_group)) + H5Gclose(meta_group); +} + +void create_dac_dataset(unsigned int num_chips, + hid_t file, + hid_t lcpl, + hid_t *dac_handle) +{ + hid_t dcpl = H5I_INVALID_HID; + hid_t dapl = H5I_INVALID_HID; + hid_t chip_group = H5I_INVALID_HID; + hid_t dataspace = H5I_INVALID_HID; + hid_t dataset = H5I_INVALID_HID; + hid_t str_type = H5I_INVALID_HID; + + if (dac_handle == NULL) { + fprintf(stderr, "dac_handle is NULL in create_dac_meta_dataset\n"); + return; + } + + char chip_group_path[256]; + char dataset_path[512]; + hsize_t dim[1] = {0}; + hsize_t max_dim[1] = {H5S_UNLIMITED}; + hsize_t chunk_dim[1] = {1}; + + dcpl = H5Pcreate(H5P_DATASET_CREATE); + if (dcpl < 0) { + fprintf(stderr, "Error creating dcpl in create_dac_dataset\n"); + goto cleanup; + } else { + if (H5Pset_chunk(dcpl, 1, chunk_dim) < 0) { + fprintf(stderr, "Error setting chunking in create_dac_dataset\n"); + goto cleanup; + } + } + + dapl = H5Pcreate(H5P_DATASET_ACCESS); + if (dapl < 0) { + fprintf(stderr, "Error creating dapl in create_dac_dataset\n"); + goto cleanup; + } + + str_type = H5Tcopy(H5T_C_S1); + if (str_type < 0) { + fprintf(stderr, "H5Tcopy failed for str_type\n"); + goto cleanup; + } + if (H5Tset_size(str_type, 4) < 0) { + fprintf(stderr, "Error setting string type size in create_dac_dataset\n"); + goto cleanup; + } + + struct { + const char *name; + hid_t type; + } datasets[] = { + {"dac_format", str_type}, {"threshold0", H5T_NATIVE_UINT}, + {"threshold1", H5T_NATIVE_UINT}, {"threshold2", H5T_NATIVE_UINT}, + {"threshold3", H5T_NATIVE_UINT}, {"threshold4", H5T_NATIVE_UINT}, + {"threshold5", H5T_NATIVE_UINT}, {"threshold6", H5T_NATIVE_UINT}, + {"threshold7", H5T_NATIVE_UINT}, {"preamp", H5T_NATIVE_UINT}, + {"ikrum", H5T_NATIVE_UINT}, {"shaper", H5T_NATIVE_UINT}, + {"disc", H5T_NATIVE_UINT}, {"disc_LS", H5T_NATIVE_UINT}, + {"shaper_test", H5T_NATIVE_UINT}, {"dac_disc_L", H5T_NATIVE_UINT}, + {"dac_test", H5T_NATIVE_UINT}, {"dac_disc_H", H5T_NATIVE_UINT}, + {"delay", H5T_NATIVE_UINT}, {"TP_buff_in", H5T_NATIVE_UINT}, + {"TP_buff_out", H5T_NATIVE_UINT}, {"RPZ", H5T_NATIVE_UINT}, + {"GND", H5T_NATIVE_UINT}, {"TP_ref", H5T_NATIVE_UINT}, + {"FBK", H5T_NATIVE_UINT}, {"Cas", H5T_NATIVE_UINT}, + {"TP_ref_A", H5T_NATIVE_UINT}, {"TP_ref_B", H5T_NATIVE_UINT}}; + + size_t num_datasets = sizeof(datasets) / sizeof(datasets[0]); + int handle_pos = 0; + + for (unsigned int i = 0; i < num_chips; i++) { + snprintf(chip_group_path, sizeof(chip_group_path), "metadata/Chip%02d", i); + + chip_group = + H5Gcreate(file, chip_group_path, lcpl, H5P_DEFAULT, H5P_DEFAULT); + if (chip_group < 0) { + fprintf(stderr, "Error creating group chip%02d in create_dac_dataset\n", + i); + goto cleanup; + } + + dataspace = H5Screate_simple(1, dim, max_dim); + if (dataspace < 0) { + fprintf(stderr, "Error creating dataspace in create_dac_meta_dataset\n"); + goto cleanup; + } + + handle_pos = num_datasets * (size_t) i; + + for (size_t j = handle_pos; j < (handle_pos + num_datasets); j++) { + + snprintf(dataset_path, sizeof(dataset_path), "%s", + datasets[j - handle_pos].name); + + dataset = + H5Dcreate2(chip_group, dataset_path, datasets[j - handle_pos].type, + dataspace, lcpl, dcpl, dapl); + if (dataset < 0) { + fprintf(stderr, "Error creating dataset: %s\n", dataset_path); + dac_handle[j] = H5I_INVALID_HID; + continue; + } + dac_handle[j] = dataset; + } + H5Sclose(dataspace); + H5Gclose(chip_group); + } + +cleanup: + if (H5Iis_valid(str_type)) + H5Tclose(str_type); + if (H5Iis_valid(dcpl)) + H5Pclose(dcpl); + if (H5Iis_valid(dapl)) + H5Pclose(dapl); + if (H5Iis_valid(chip_group)) + H5Gclose(chip_group); +} + +void close_dataset_handle(hid_t *handle, size_t count) +{ + if (!handle) { + printf("No handle\n"); + return; + } + for (size_t i = 0; i < count; i++) { + if (H5Iis_valid(handle[i])) { + H5Dclose(handle[i]); + } + } +} diff --git a/src/hdf5_init_meta.h b/src/hdf5_init_meta.h new file mode 100644 index 0000000..6ea050b --- /dev/null +++ b/src/hdf5_init_meta.h @@ -0,0 +1,18 @@ +// clang-format Language: C +#ifndef HDF5_INIT_META_H +#define HDF5_INIT_META_H + +#include "macros.h" +#include +#include + +void create_meta_mq1_fields_dataset(hid_t file, hid_t lcpl, hid_t *meta_handle); + +void create_dac_dataset(unsigned int num_chips, + hid_t file, + hid_t lcpl, + hid_t *dac_handle); + +void close_dataset_handle(hid_t *handle, size_t count); + +#endif diff --git a/src/io_header.c b/src/io_header.c new file mode 100644 index 0000000..24e3af1 --- /dev/null +++ b/src/io_header.c @@ -0,0 +1,434 @@ +#include "io_header.h" +#include "macros.h" +#include "mib_header.h" +#include "parser.h" +#include "utils.h" +#include +#include +#include +#include + +unsigned int mq1_single_from_file(FILE *mib_ptr, + unsigned int nheaders, + unsigned int detector_frame_bytes, + mq1s *mq1s_h, + MQ1_fields *mq1_fields) +{ + unsigned int num_parsed = 0; + char header_buffer[MQ1_SINGLE_HEADER_BYTES + 1] = {0}; + + for (unsigned int i = 0; i < nheaders; ++i) { + /*at the current file position, read the size of a header and save in a + * buffer*/ + if (fgets(header_buffer, sizeof(header_buffer), mib_ptr) == NULL) { + const char *current_file = only_file_name(__FILE__); + fprintf(stderr, + "%s:%d: error: failed to read the header (%u) of the " + "MIB file into a buffer\n", + current_file, __LINE__, i); + return 0; + } + + /*parse the buffer and assign to the fields of a header struct*/ + parse_mq1_single(header_buffer, &mq1s_h[i]); + + /*upon completion, increased the count of parsed header*/ + num_parsed += 1; + + /*update the corresponding field array*/ + fill_MQ1_single_fields(mq1_fields, i, mq1s_h[i]); + + /*move the file by the size of the detector*/ + /*i.e. point to the next header*/ + fseek(mib_ptr, (long int) detector_frame_bytes, SEEK_CUR); + } + + /*go to the beginning after parsing*/ + if (fseek(mib_ptr, 0, SEEK_SET) != 0) { + const char *current_file = only_file_name(__FILE__); + fprintf(stderr, + "%s:%d: error: failed to seek to the beginning of the " + "MIB file\n", + current_file, __LINE__); + num_parsed = 0; + } + + return num_parsed; +} + +unsigned int mq1_quad_from_file(FILE *mib_ptr, + unsigned int nheaders, + unsigned int detector_frame_bytes, + mq1q *mq1q_h, + MQ1_fields *mq1_fields) +{ + unsigned int num_parsed = 0; + char header_buffer[MQ1_QUAD_HEADER_BYTES + 1] = {0}; + + for (unsigned int i = 0; i < nheaders; ++i) { + /*at the current file position, read the size of a header and save in a + * buffer*/ + if (fgets(header_buffer, sizeof(header_buffer), mib_ptr) == NULL) { + const char *current_file = only_file_name(__FILE__); + fprintf(stderr, + "%s:%d: error: failed to read the header (%u) of the " + "MIB file into a buffer\n", + current_file, __LINE__, i); + return 0; + } + + /*parse the buffer and assign to the fields of a header struct*/ + parse_mq1_quad(header_buffer, &mq1q_h[i]); + + /*upon completion, increased the count of parsed header*/ + num_parsed += 1; + + /*update the corresponding field array*/ + fill_MQ1_quad_fields(mq1_fields, i, mq1q_h[i]); + + /*move the file by the size of the detector*/ + /*i.e. point to the next header*/ + fseek(mib_ptr, (long int) detector_frame_bytes, SEEK_CUR); + } + + /*go to the beginning after parsing*/ + if (fseek(mib_ptr, 0, SEEK_SET) != 0) { + const char *current_file = only_file_name(__FILE__); + fprintf(stderr, + "%s:%d: error: failed to seek to the beginning of the " + "MIB file\n", + current_file, __LINE__); + num_parsed = 0; + } + + return num_parsed; +} + +/* ensure consistent memory allocation for all fields + * use deallocate_MQ1_fields to free all the allocated memory here + * */ +MQ1_fields allocate_MQ1_fields(unsigned int nheaders) +{ + MQ1_fields mq1_fields; + + mq1_fields.max_length = nheaders; + + mq1_fields.header_id = XMALLOC( + sizeof(char) * MQ1_CHAR_LEN_HEADER_ID * mq1_fields.max_length, "header_id"); + mq1_fields.sequence_number = + XMALLOC(sizeof(unsigned int) * mq1_fields.max_length, "sequence_number"); + mq1_fields.header_bytes = + XMALLOC(sizeof(unsigned int) * mq1_fields.max_length, "header_bytes"); + mq1_fields.num_chips = + XMALLOC(sizeof(unsigned int) * mq1_fields.max_length, "num_chips"); + mq1_fields.det_x = + XMALLOC(sizeof(unsigned int) * mq1_fields.max_length, "det_x"); + mq1_fields.det_y = + XMALLOC(sizeof(unsigned int) * mq1_fields.max_length, "det_y"); + mq1_fields.pixel_depth = + XMALLOC(sizeof(char) * MQ1_CHAR_LEN_PIXEL_DEPTH * mq1_fields.max_length, + "pixel_depth"); + mq1_fields.sensor_layout = + XMALLOC(sizeof(char) * MQ1_CHAR_LEN_SENSOR_LAYOUT * mq1_fields.max_length, + "sensor_layout"); + mq1_fields.chip_select = + XMALLOC(sizeof(char) * MQ1_CHAR_LEN_CHIP_SELECT * mq1_fields.max_length, + "chip_select"); + mq1_fields.timestamp = XMALLOC( + sizeof(char) * MQ1_CHAR_LEN_TIMESTAMP * mq1_fields.max_length, "timestamp"); + mq1_fields.exposure_time_s = + XMALLOC(sizeof(double) * mq1_fields.max_length, "exposure_time_s"); + mq1_fields.counter = + XMALLOC(sizeof(unsigned int) * mq1_fields.max_length, "counter"); + mq1_fields.colour_mode = + XMALLOC(sizeof(unsigned int) * mq1_fields.max_length, "colour_mode"); + mq1_fields.gain_mode = + XMALLOC(sizeof(unsigned int) * mq1_fields.max_length, "gain_mode"); + mq1_fields.threshold0 = + XMALLOC(sizeof(float) * mq1_fields.max_length, "threshold0"); + mq1_fields.threshold1 = + XMALLOC(sizeof(float) * mq1_fields.max_length, "threshold1"); + mq1_fields.threshold2 = + XMALLOC(sizeof(float) * mq1_fields.max_length, "threshold2"); + mq1_fields.threshold3 = + XMALLOC(sizeof(float) * mq1_fields.max_length, "threshold3"); + mq1_fields.threshold4 = + XMALLOC(sizeof(float) * mq1_fields.max_length, "threshold4"); + mq1_fields.threshold5 = + XMALLOC(sizeof(float) * mq1_fields.max_length, "threshold5"); + mq1_fields.threshold6 = + XMALLOC(sizeof(float) * mq1_fields.max_length, "threshold6"); + mq1_fields.threshold7 = + XMALLOC(sizeof(float) * mq1_fields.max_length, "threshold7"); + mq1_fields.header_extension_id = XMALLOC( + sizeof(char) * MQ1_CHAR_LEN_HEADER_EXTENSION_ID * mq1_fields.max_length, + "header_extension_id"); + mq1_fields.extended_timestamp = XMALLOC( + sizeof(char) * MQ1_CHAR_LEN_EXTENDED_TIMESTAMP * mq1_fields.max_length, + "extended_timestamp"); + mq1_fields.exposure_time_ns = + XMALLOC(sizeof(unsigned int) * mq1_fields.max_length, "exposure_time_ns"); + mq1_fields.bit_depth = + XMALLOC(sizeof(unsigned int) * mq1_fields.max_length, "bit_depth"); + + return mq1_fields; +} + +/* ensure all allocated memory is free */ +void deallocate_MQ1_fields(MQ1_fields mq1_fields) +{ + free(mq1_fields.header_id); + mq1_fields.header_id = NULL; + free(mq1_fields.sequence_number); + mq1_fields.sequence_number = NULL; + free(mq1_fields.header_bytes); + mq1_fields.header_bytes = NULL; + free(mq1_fields.num_chips); + mq1_fields.num_chips = NULL; + free(mq1_fields.det_x); + mq1_fields.det_x = NULL; + free(mq1_fields.det_y); + mq1_fields.det_y = NULL; + free(mq1_fields.pixel_depth); + mq1_fields.pixel_depth = NULL; + free(mq1_fields.sensor_layout); + mq1_fields.sensor_layout = NULL; + free(mq1_fields.chip_select); + mq1_fields.chip_select = NULL; + free(mq1_fields.timestamp); + mq1_fields.timestamp = NULL; + free(mq1_fields.exposure_time_s); + mq1_fields.exposure_time_s = NULL; + free(mq1_fields.counter); + mq1_fields.counter = NULL; + free(mq1_fields.colour_mode); + mq1_fields.colour_mode = NULL; + free(mq1_fields.gain_mode); + mq1_fields.gain_mode = NULL; + free(mq1_fields.threshold0); + mq1_fields.threshold0 = NULL; + free(mq1_fields.threshold1); + mq1_fields.threshold1 = NULL; + free(mq1_fields.threshold2); + mq1_fields.threshold2 = NULL; + free(mq1_fields.threshold3); + mq1_fields.threshold3 = NULL; + free(mq1_fields.threshold4); + mq1_fields.threshold4 = NULL; + free(mq1_fields.threshold5); + mq1_fields.threshold5 = NULL; + free(mq1_fields.threshold6); + mq1_fields.threshold6 = NULL; + free(mq1_fields.threshold7); + mq1_fields.threshold7 = NULL; + free(mq1_fields.header_extension_id); + mq1_fields.header_extension_id = NULL; + free(mq1_fields.extended_timestamp); + mq1_fields.extended_timestamp = NULL; + free(mq1_fields.exposure_time_ns); + mq1_fields.exposure_time_ns = NULL; + free(mq1_fields.bit_depth); + mq1_fields.bit_depth = NULL; +} + +/*for MQ1 single, basically a copy of MQ1 quad*/ +void fill_MQ1_single_fields(MQ1_fields *mq1_field, + unsigned int index, + mq1s mq1_h) +{ + if (index >= mq1_field->max_length) { + const char *current_file = only_file_name(__FILE__); + fprintf(stderr, + "%s:%d: error: the index of the field is larger than " + "the maximum number of the fields\n", + current_file, __LINE__); + exit(1); + } + + mq1_field->sequence_number[index] = mq1_h.sequence_number; + mq1_field->header_bytes[index] = mq1_h.header_bytes; + mq1_field->num_chips[index] = mq1_h.num_chips; + mq1_field->det_x[index] = mq1_h.det_x; + mq1_field->det_y[index] = mq1_h.det_y; + /*header_id is char[4]*/ + snprintf(mq1_field->header_id + index * MQ1_CHAR_LEN_HEADER_ID, + MQ1_CHAR_LEN_HEADER_ID, "%s", mq1_h.header_id); + /*pixel_depth is char[4]*/ + snprintf(mq1_field->pixel_depth + index * MQ1_CHAR_LEN_PIXEL_DEPTH, + MQ1_CHAR_LEN_PIXEL_DEPTH, "%s", mq1_h.pixel_depth); + /*sensor_layout is char[7]*/ + snprintf(mq1_field->sensor_layout + index * MQ1_CHAR_LEN_SENSOR_LAYOUT, + MQ1_CHAR_LEN_SENSOR_LAYOUT, "%s", mq1_h.sensor_layout); + /*chip_select is char[3]*/ + snprintf(mq1_field->chip_select + index * MQ1_CHAR_LEN_CHIP_SELECT, + MQ1_CHAR_LEN_CHIP_SELECT, "%s", mq1_h.chip_select); + /*timestamp is char[27]*/ + snprintf(mq1_field->timestamp + index * MQ1_CHAR_LEN_TIMESTAMP, + MQ1_CHAR_LEN_TIMESTAMP, "%s", mq1_h.timestamp); + mq1_field->exposure_time_s[index] = mq1_h.exposure_time_s; + mq1_field->counter[index] = mq1_h.counter; + mq1_field->colour_mode[index] = mq1_h.colour_mode; + mq1_field->gain_mode[index] = mq1_h.gain_mode; + /*threshold is float[8]*/ + mq1_field->threshold0[index] = mq1_h.threshold[0]; + mq1_field->threshold1[index] = mq1_h.threshold[1]; + mq1_field->threshold2[index] = mq1_h.threshold[2]; + mq1_field->threshold3[index] = mq1_h.threshold[3]; + mq1_field->threshold4[index] = mq1_h.threshold[4]; + mq1_field->threshold5[index] = mq1_h.threshold[5]; + mq1_field->threshold6[index] = mq1_h.threshold[6]; + mq1_field->threshold7[index] = mq1_h.threshold[7]; + /*header_extension_id is char[5]*/ + snprintf(mq1_field->header_extension_id + + index * MQ1_CHAR_LEN_HEADER_EXTENSION_ID, + MQ1_CHAR_LEN_HEADER_EXTENSION_ID, "%s", mq1_h.header_extension_id); + /*extended_timestamp is char[31]*/ + snprintf(mq1_field->extended_timestamp + + index * MQ1_CHAR_LEN_EXTENDED_TIMESTAMP, + MQ1_CHAR_LEN_EXTENDED_TIMESTAMP, "%s", mq1_h.extended_timestamp); + mq1_field->exposure_time_ns[index] = mq1_h.exposure_time_ns; + mq1_field->bit_depth[index] = mq1_h.bit_depth; +} + +/*for MQ1 quad, basically a copy of MQ1 single*/ +void fill_MQ1_quad_fields(MQ1_fields *mq1_field, unsigned int index, mq1q mq1_h) +{ + if (index >= mq1_field->max_length) { + const char *current_file = only_file_name(__FILE__); + fprintf(stderr, + "%s:%d: error: the index of the field is larger than " + "the maximum number of the fields\n", + current_file, __LINE__); + exit(1); + } + + mq1_field->sequence_number[index] = mq1_h.sequence_number; + mq1_field->header_bytes[index] = mq1_h.header_bytes; + mq1_field->num_chips[index] = mq1_h.num_chips; + mq1_field->det_x[index] = mq1_h.det_x; + mq1_field->det_y[index] = mq1_h.det_y; + /*header_id is char[4]*/ + snprintf(mq1_field->header_id + index * MQ1_CHAR_LEN_HEADER_ID, + MQ1_CHAR_LEN_HEADER_ID, "%s", mq1_h.header_id); + /*pixel_depth is char[4]*/ + snprintf(mq1_field->pixel_depth + index * MQ1_CHAR_LEN_PIXEL_DEPTH, + MQ1_CHAR_LEN_PIXEL_DEPTH, "%s", mq1_h.pixel_depth); + /*sensor_layout is char[7]*/ + snprintf(mq1_field->sensor_layout + index * MQ1_CHAR_LEN_SENSOR_LAYOUT, + MQ1_CHAR_LEN_SENSOR_LAYOUT, "%s", mq1_h.sensor_layout); + /*chip_select is char[3]*/ + snprintf(mq1_field->chip_select + index * MQ1_CHAR_LEN_CHIP_SELECT, + MQ1_CHAR_LEN_CHIP_SELECT, "%s", mq1_h.chip_select); + /*timestamp is char[27]*/ + snprintf(mq1_field->timestamp + index * MQ1_CHAR_LEN_TIMESTAMP, + MQ1_CHAR_LEN_TIMESTAMP, "%s", mq1_h.timestamp); + mq1_field->exposure_time_s[index] = mq1_h.exposure_time_s; + mq1_field->counter[index] = mq1_h.counter; + mq1_field->colour_mode[index] = mq1_h.colour_mode; + mq1_field->gain_mode[index] = mq1_h.gain_mode; + /*threshold is float[8]*/ + mq1_field->threshold0[index] = mq1_h.threshold[0]; + mq1_field->threshold1[index] = mq1_h.threshold[1]; + mq1_field->threshold2[index] = mq1_h.threshold[2]; + mq1_field->threshold3[index] = mq1_h.threshold[3]; + mq1_field->threshold4[index] = mq1_h.threshold[4]; + mq1_field->threshold5[index] = mq1_h.threshold[5]; + mq1_field->threshold6[index] = mq1_h.threshold[6]; + mq1_field->threshold7[index] = mq1_h.threshold[7]; + /*header_extension_id is char[5]*/ + snprintf(mq1_field->header_extension_id + + index * MQ1_CHAR_LEN_HEADER_EXTENSION_ID, + MQ1_CHAR_LEN_HEADER_EXTENSION_ID, "%s", mq1_h.header_extension_id); + /*extended_timestamp is char[31]*/ + snprintf(mq1_field->extended_timestamp + + index * MQ1_CHAR_LEN_EXTENDED_TIMESTAMP, + MQ1_CHAR_LEN_EXTENDED_TIMESTAMP, "%s", mq1_h.extended_timestamp); + mq1_field->exposure_time_ns[index] = mq1_h.exposure_time_ns; + mq1_field->bit_depth[index] = mq1_h.bit_depth; +} + +/* Mainly this is just for easy access to mq1_fields and dac + * Please free the returning array after using it + */ + +info *mq1_fields_info(MQ1_fields *fields_struct) +{ + if (!fields_struct) + return NULL; + + info *fields = malloc(MQ1_FIELDS_NUM_FIELDS * sizeof(info)); + if (!fields) + return NULL; + + fields[0] = (info) {"header_id", fields_struct->header_id}; + fields[1] = (info) {"max_length", &fields_struct->max_length}; + fields[2] = (info) {"sequence_number", fields_struct->sequence_number}; + fields[3] = (info) {"header_bytes", fields_struct->header_bytes}; + fields[4] = (info) {"num_chips", fields_struct->num_chips}; + fields[5] = (info) {"det_x", fields_struct->det_x}; + fields[6] = (info) {"det_y", fields_struct->det_y}; + fields[7] = (info) {"pixel_depth", fields_struct->pixel_depth}; + fields[8] = (info) {"sensor_layout", fields_struct->sensor_layout}; + fields[9] = (info) {"chip_select", fields_struct->chip_select}; + fields[10] = (info) {"timestamp", fields_struct->timestamp}; + fields[11] = (info) {"exposure_time_s", fields_struct->exposure_time_s}; + fields[12] = (info) {"counter", fields_struct->counter}; + fields[13] = (info) {"colour_mode", fields_struct->colour_mode}; + fields[14] = (info) {"gain_mode", fields_struct->gain_mode}; + fields[15] = (info) {"threshold0", fields_struct->threshold0}; + fields[16] = (info) {"threshold1", fields_struct->threshold1}; + fields[17] = (info) {"threshold2", fields_struct->threshold2}; + fields[18] = (info) {"threshold3", fields_struct->threshold3}; + fields[19] = (info) {"threshold4", fields_struct->threshold4}; + fields[20] = (info) {"threshold5", fields_struct->threshold5}; + fields[21] = (info) {"threshold6", fields_struct->threshold6}; + fields[22] = (info) {"threshold7", fields_struct->threshold7}; + fields[23] = + (info) {"header_extension_id", fields_struct->header_extension_id}; + fields[24] = (info) {"extended_timestamp", fields_struct->extended_timestamp}; + fields[25] = (info) {"exposure_time_ns", fields_struct->exposure_time_ns}; + fields[26] = (info) {"bit_depth", fields_struct->bit_depth}; + + return fields; +} + +info *dac_info(dac_rx *dac) +{ + if (!dac) + return NULL; + + info *dinfo = malloc(DAC_NUM_FIELDS * sizeof(info)); + if (!dinfo) + return NULL; + + dinfo[0] = (info) {"dac_format", &dac->dac_format}; + dinfo[1] = (info) {"threshold0", &dac->threshold0}; + dinfo[2] = (info) {"threshold1", &dac->threshold1}; + dinfo[3] = (info) {"threshold2", &dac->threshold2}; + dinfo[4] = (info) {"threshold3", &dac->threshold3}; + dinfo[5] = (info) {"threshold4", &dac->threshold4}; + dinfo[6] = (info) {"threshold5", &dac->threshold5}; + dinfo[7] = (info) {"threshold6", &dac->threshold6}; + dinfo[8] = (info) {"threshold7", &dac->threshold7}; + dinfo[9] = (info) {"preamp", &dac->preamp}; + dinfo[10] = (info) {"ikrum", &dac->ikrum}; + dinfo[11] = (info) {"shaper", &dac->shaper}; + dinfo[12] = (info) {"disc", &dac->disc}; + dinfo[13] = (info) {"disc_LS", &dac->disc_LS}; + dinfo[14] = (info) {"shaper_test", &dac->shaper_test}; + dinfo[15] = (info) {"dac_disc_L", &dac->dac_disc_L}; + dinfo[16] = (info) {"dac_test", &dac->dac_test}; + dinfo[17] = (info) {"dac_disc_H", &dac->dac_disc_H}; + dinfo[18] = (info) {"delay", &dac->delay}; + dinfo[19] = (info) {"TP_buff_in", &dac->TP_buff_in}; + dinfo[20] = (info) {"TP_buff_out", &dac->TP_buff_out}; + dinfo[21] = (info) {"RPZ", &dac->RPZ}; + dinfo[22] = (info) {"GND", &dac->GND}; + dinfo[23] = (info) {"TP_ref", &dac->TP_ref}; + dinfo[24] = (info) {"FBK", &dac->FBK}; + dinfo[25] = (info) {"Cas", &dac->Cas}; + dinfo[26] = (info) {"TP_ref_A", &dac->TP_ref_A}; + dinfo[27] = (info) {"TP_ref_B", &dac->TP_ref_B}; + + return dinfo; +} diff --git a/src/io_header.h b/src/io_header.h new file mode 100644 index 0000000..6188456 --- /dev/null +++ b/src/io_header.h @@ -0,0 +1,73 @@ +// clang-format Language: C +#include "mib_header.h" + +#include +#include +#include + +#ifndef IO_HEADER_H +#define IO_HEADER_H + +typedef struct { + char *header_id; + unsigned int max_length; + unsigned int *sequence_number; + unsigned int *header_bytes; + unsigned int *num_chips; + unsigned int *det_x; + unsigned int *det_y; + char *pixel_depth; + char *sensor_layout; + char *chip_select; + char *timestamp; + double *exposure_time_s; + unsigned int *counter; + unsigned int *colour_mode; + unsigned int *gain_mode; + float *threshold0; + float *threshold1; + float *threshold2; + float *threshold3; + float *threshold4; + float *threshold5; + float *threshold6; + float *threshold7; + char *header_extension_id; + char *extended_timestamp; + unsigned int *exposure_time_ns; + unsigned int *bit_depth; +} MQ1_fields; + +MQ1_fields allocate_MQ1_fields(unsigned int nheaders); + +void deallocate_MQ1_fields(MQ1_fields mq1_fields); + +unsigned int mq1_single_from_file(FILE *mib_ptr, + unsigned int nheaders, + unsigned int detector_frame_bytes, + mq1s *mq1s_h, + MQ1_fields *mq1_fields); + +unsigned int mq1_quad_from_file(FILE *mib_ptr, + unsigned int nheaders, + unsigned int detector_frame_bytes, + mq1q *mq1q_h, + MQ1_fields *mq1_fields); + +void fill_MQ1_single_fields(MQ1_fields *mq1_field, + unsigned int index, + mq1s mq1_h); + +void fill_MQ1_quad_fields(MQ1_fields *mq1_field, + unsigned int index, + mq1q mq1_h); + +typedef struct { + const char *name; + void *data; +} info; + +info *mq1_fields_info(MQ1_fields *fields); +info *dac_info(dac_rx *dac); + +#endif diff --git a/src/macros.h b/src/macros.h new file mode 100644 index 0000000..1fe0ea3 --- /dev/null +++ b/src/macros.h @@ -0,0 +1,45 @@ +// clang-format Language: C +#ifndef MACROS_H +#define MACROS_H + +#define MQ1_SINGLE_HEADER_BYTES 384 + +#define MQ1_SINGLE_HEADER_NUM_FIELDS 54 + +#define MQ1_QUAD_HEADER_BYTES 768 + +#define MQ1_QUAD_HEADER_NUM_FIELDS 138 + +#define MQ1_CHAR_LEN_HEADER_ID 4 + +#define MQ1_CHAR_LEN_PIXEL_DEPTH 4 + +#define MQ1_CHAR_LEN_SENSOR_LAYOUT 7 + +#define MQ1_CHAR_LEN_CHIP_SELECT 3 + +#define MQ1_CHAR_LEN_TIMESTAMP 27 + +#define MQ1_FLOAT_LEN_THRESHOLD 8 + +#define MQ1_CHAR_LEN_HEADER_EXTENSION_ID 5 + +#define MQ1_CHAR_LEN_EXTENDED_TIMESTAMP 31 + +#define MQ1_FIELDS_NUM_FIELDS 27 + +#define DAC_NUM_FIELDS 28 + +#define NUM_CHUNKS_IN_CACHE 32 + +#define PRIME_FOR_HASH 521 + +#define ALIGNMENT_THRESHOLD 512 + +#define MIB_HEADER_SIZE_FIELD_OFFSET 10 + +#define MIB_HEADER_SIZE_FIELD_LENGTH 6 + +#define MIB_HEADER_METADATA_BUF_SIZE 16 + +#endif diff --git a/src/main.c b/src/main.c new file mode 100644 index 0000000..dc2a62f --- /dev/null +++ b/src/main.c @@ -0,0 +1,148 @@ +#include "config.h" +#include "mib2h5.h" +#include +#include +#include +#include +#include +#include +#include +#include + +// long options structure for getopt_long +static const struct option long_options[] = { + {"help", no_argument, NULL, 'h'}, + {"version", no_argument, NULL, 'v'}, + {"no-metadata", no_argument, NULL, 'N'}, + {"with-metadata", no_argument, NULL, 'M'}, + {"output-dir", required_argument, NULL, 'o'}, + {"output-directory", required_argument, NULL, 'o'}, // alias for output-dir + {"dataset-key", required_argument, NULL, 'd'}, + {"metadata-key", required_argument, NULL, 'k'}, + {"compress", no_argument, NULL, 'c'}, + {"reshape-to", required_argument, NULL, 'r'}, + {"timeout", required_argument, NULL, 't'}, + {NULL, 0, NULL, 0}}; + +static void print_usage(const char *program_name) +{ + // clang-format off + fprintf(stderr, "Usage: %s [options] file1.mib [file2.mib ...]\n", program_name); + fprintf(stderr, "\nOptions:\n"); + fprintf(stderr, " -o, --output-dir DIR Output directory (default: current)\n"); + fprintf(stderr, " --output-directory DIR Alternative for --output-dir\n"); + fprintf(stderr, " -d, --dataset-key KEY Dataset key in HDF5 (default: /data)\n"); + fprintf(stderr, " -k, --metadata-key KEY Metadata key in HDF5 (default: /metadata)\n"); + fprintf(stderr, " - NOT YET IMPLEMENTED\n"); + fprintf(stderr, " -c, --compress Enable compression\n"); + fprintf(stderr, " -M, --with-metadata Include metadata in output (default)\n"); + fprintf(stderr, " -N, --no-metadata Exclude metadata from output\n"); + fprintf(stderr, " -r, --reshape-to DIMS Reshape dimensions (e.g., '10x10')\n"); + fprintf(stderr, " - NOT YET IMPLEMENTED\n"); + fprintf(stderr, " -t, --timeout SECS Timeout in seconds\n"); + fprintf(stderr, " - NOT YET IMPLEMENTED\n"); + fprintf(stderr, " -v, --version Display the version\n"); + fprintf(stderr, " -h, --help Show this help message\n"); + fprintf(stderr, "\nEnvironment variables:\n"); + fprintf(stderr, " MIB2H5_SHUFFLE Shuffle level for Blosc compression (default: 2)\n"); + fprintf(stderr, " MIB2H5_COMPRESSION_LEVEL Blosc compression level (default: 9)\n"); + // clang-format on +} + +int main(int argc, char *argv[]) +{ + int opt; + char *output_directory = NULL; + char *dataset_key = NULL; + char *metadata_key = NULL; + char *reshape_dims = NULL; + bool use_compression = false; + bool include_metadata = true; + bool report_progress = true; + unsigned int timeout_seconds = 900; + + // parse options + int option_index = 0; + while ((opt = getopt_long(argc, argv, "o:d:k:r:t:cMNvh", long_options, + &option_index)) != -1) { + switch (opt) { + case 'o': + output_directory = optarg; + break; + case 'd': + dataset_key = optarg; + break; + case 'k': + metadata_key = optarg; + break; + case 'r': + reshape_dims = optarg; + break; + case 't': { + char *endptr; + long val = strtol(optarg, &endptr, 10); + if (*endptr != '\0' || val < 0 || val > UINT_MAX) { + fprintf(stderr, "Error: Invalid timeout value: %s\n", optarg); + return 1; + } + timeout_seconds = (unsigned int) val; + } break; + case 'c': + use_compression = true; + break; + case 'M': + include_metadata = true; + break; + case 'N': + include_metadata = false; + break; + case 'v': + printf("%d.%d.%d\n", MIB2H5_VERSION_MAJOR, MIB2H5_VERSION_MINOR, + MIB2H5_VERSION_PATCH); + return 0; + case 'h': + print_usage(argv[0]); + return 0; + default: + print_usage(argv[0]); + return 1; + } + } + + // collect input files from remaining arguments + int num_input_files = argc - optind; + if (num_input_files <= 0) { + fprintf(stderr, "Error: No input files specified\n\n"); + print_usage(argv[0]); + return 1; + } + + const char **input_files = + (const char **) malloc((size_t) num_input_files * sizeof(char *)); + if (input_files == NULL) { + fprintf(stderr, "Error: Cannot allocate memory for input files\n"); + return 1; + } + + for (int i = 0; i < num_input_files; ++i) { + input_files[i] = argv[optind + i]; + } + + int status = + mib_to_h5(input_files, num_input_files, output_directory, include_metadata, + dataset_key, metadata_key, use_compression, reshape_dims, + report_progress, timeout_seconds); + + free(input_files); + + if (status != 0) { + const char *error_msg = mib_to_h5_last_error(); + if (error_msg != NULL) { + fprintf(stderr, "Error: %s\n", error_msg); + } else { + fprintf(stderr, "Error: Unknown conversion error (status: %d)\n", status); + } + } + + return status; +} diff --git a/src/mib2h5.h b/src/mib2h5.h new file mode 100644 index 0000000..111eaa4 --- /dev/null +++ b/src/mib2h5.h @@ -0,0 +1,90 @@ +// clang-format Language: C +/** + * @file mib2h5.h + * @brief Public API for mib2h5 library - MerlinEM MIB to HDF5 converter + * @version 0.0.1 + * + * This header defines the public API for converting MerlinEM detector + * output files (.mib) from Quantum Detector to HDF5 format (.h5). + * + * The library preserves all metadata and supports frame-by-frame processing + * for handling large datasets efficiently. + */ +#ifndef MIB2H5_H +#define MIB2H5_H + +#include + +#define MIB2H5_VERSION_MAJOR 0 +#define MIB2H5_VERSION_MINOR 0 +#define MIB2H5_VERSION_PATCH 1 +#define MIB2H5_VERSION "0.0.1" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @brief Convert MIB files to HDF5 files + * + * Converts MerlinEM .mib files to HDF5 files, saves all the frames and + * optionally keeps their metadata. Each input .mib file produces a separate + * HDF5 output file with the same base name. + * + * @param[in] input_files Array of paths to input .mib files + * @param[in] num_input_files Number of files in input_files (must be >0) + * @param[in] output_dir Output directory (NULL uses current directory) + * @param[in] include_metadata If true, includes metadata of frames in HDF5 + * @param[in] dataset_key HDF5 dataset path for frames (NULL defaults to + * "/data") + * @param[in] metadata_key HDF5 group path for metadata (NULL defaults to + * "/metadata") + * @param[in] use_compression Enable Blosc compression if true + * @param[in] reshape_dims Reshape string like "10x10" (NOT YET IMPLEMENTED) + * @param[in] report_progress Enable progress reporting (NOT YET IMPLEMENTED) + * @param[in] timeout_seconds Conversion timeout in seconds, 0 for no limit + * (NOT YET IMPLEMENTED) + * + * @return 0 on success, -1 on error + * + * @note Output files are named by replacing .mib extension with .h5 + * @note If multiple files are provided, conversion continues even if one fails + * @note Some Blosc compression setting can be controlled by + * environment variables: + * - MIB2H5_SHUFFLE (default: 2, range: 0-2) + * - MIB2H5_COMPRESSION_LEVEL (default: 9, range: 0-9) + * + * @code + * // Example: Convert multiple files with metadata + * const char* files[] = {"data1.mib", "data2.mib"}; + * int result = mib_to_h5(files, 2, "output/", true, + * NULL, NULL, false, NULL, false, 0); + * if (result != 0) { + * fprintf(stderr, "Conversion failed: %s\n", mib_to_h5_last_error()); + * } + * @endcode + */ +int mib_to_h5(const char **input_files, + int num_input_files, + const char *output_dir, + bool include_metadata, + const char *dataset_key, + const char *metadata_key, + bool use_compression, + const char *reshape_dims, + bool report_progress, + unsigned int timeout_seconds); + +/** + * @brief Get the last error message from mib_to_h5 operations + * @warning This function is not yet implemented and always returns a + * placeholder + * @todo Implement proper thread-safe error message handling + */ +const char *mib_to_h5_last_error(void); + +#ifdef __cplusplus +} +#endif + +#endif // MIB2H5_H diff --git a/src/mib_header.h b/src/mib_header.h new file mode 100644 index 0000000..f458347 --- /dev/null +++ b/src/mib_header.h @@ -0,0 +1,115 @@ +// clang-format Language: C +#include "macros.h" +#ifndef MIB_HEADER_H +#define MIB_HEADER_H + +typedef struct { + char dac_format[4]; + unsigned int threshold0; + unsigned int threshold1; + unsigned int threshold2; + unsigned int threshold3; + unsigned int threshold4; + unsigned int threshold5; + unsigned int threshold6; + unsigned int threshold7; + unsigned int preamp; + unsigned int ikrum; + unsigned int shaper; + unsigned int disc; + unsigned int disc_LS; + unsigned int shaper_test; + unsigned int dac_disc_L; + unsigned int dac_test; + unsigned int dac_disc_H; + unsigned int delay; + unsigned int TP_buff_in; + unsigned int TP_buff_out; + unsigned int RPZ; + unsigned int GND; + unsigned int TP_ref; + unsigned int FBK; + unsigned int Cas; + unsigned int TP_ref_A; + unsigned int TP_ref_B; +} dac_rx; + +/* +field 1: header ID (MQ1) +field 2: sequence number of a frame +field 3: the number of bytes of the header +field 4: the number of chips +field 5: the number of columns of the detector (x) +field 6: the number of rows of the detector (y) +field 7: the pixel data type (U01, U08, U16, U32, U64) +field 8: sensor layout (2x2, Nx1, 2x2G, Nx1G, with leading spaces) +field 9: active chip(s) during the capture of the frame, U8 hexadecimal repr +field 10: timestamp +field 11: shutter opening time (seconds) +field 12: counter 0 or 1 +field 13: colour mode, 0=monochrome image, 1=colour image +field 14: gain modem 0 = SLGM, 1 = LGM, 2 = HGM, 3 = SHGM +field 15-22: threshold values (keV) + +DAC of each chips contains 28 entries. + +Header extension + Single / Quad +field 51 / 135 : MQ1A +field 52 / 136 : UTC timestamp (time in ns) +field 53 / 137 : shutter opening time (with suffix "ns") +field 54 / 138 : the pixel data bit depth + +The remainings are null bytes for padding. +*/ + +typedef struct { + char header_id[MQ1_CHAR_LEN_HEADER_ID]; + unsigned int sequence_number; + unsigned int header_bytes; + unsigned int num_chips; + unsigned int det_x; + unsigned int det_y; + char pixel_depth[MQ1_CHAR_LEN_PIXEL_DEPTH]; + char sensor_layout[MQ1_CHAR_LEN_SENSOR_LAYOUT]; + char chip_select[MQ1_CHAR_LEN_CHIP_SELECT]; + char timestamp[MQ1_CHAR_LEN_TIMESTAMP]; + double exposure_time_s; + unsigned int counter; + unsigned int colour_mode; + unsigned int gain_mode; + float threshold[MQ1_FLOAT_LEN_THRESHOLD]; + dac_rx dac0; + char header_extension_id[MQ1_CHAR_LEN_HEADER_EXTENSION_ID]; + char extended_timestamp[MQ1_CHAR_LEN_EXTENDED_TIMESTAMP]; + unsigned int exposure_time_ns; + unsigned int bit_depth; +} mq1s; + +typedef struct { + char header_id[MQ1_CHAR_LEN_HEADER_ID]; + unsigned int sequence_number; + unsigned int header_bytes; + unsigned int num_chips; + unsigned int det_x; + unsigned int det_y; + char pixel_depth[MQ1_CHAR_LEN_PIXEL_DEPTH]; + char sensor_layout[MQ1_CHAR_LEN_SENSOR_LAYOUT]; + char chip_select[MQ1_CHAR_LEN_CHIP_SELECT]; + char timestamp[MQ1_CHAR_LEN_TIMESTAMP]; + double exposure_time_s; + unsigned int counter; + unsigned int colour_mode; + unsigned int gain_mode; + float threshold[MQ1_FLOAT_LEN_THRESHOLD]; + dac_rx dac0; + dac_rx dac1; + dac_rx dac2; + dac_rx dac3; + char header_extension_id[MQ1_CHAR_LEN_HEADER_EXTENSION_ID]; + char extended_timestamp[MQ1_CHAR_LEN_EXTENDED_TIMESTAMP]; + unsigned int exposure_time_ns; + unsigned int bit_depth; +} mq1q; + +#endif diff --git a/src/mib_to_h5.c b/src/mib_to_h5.c new file mode 100644 index 0000000..f884946 --- /dev/null +++ b/src/mib_to_h5.c @@ -0,0 +1,360 @@ +#include "mib_to_h5.h" +#include "append.h" +#include "compress.h" +#include "framebuffer.h" +#include "hdf5_init.h" +#include "hdf5_init_meta.h" +#include "macros.h" +#include "mib2h5.h" +#include "parser.h" +#include "read.h" +#include "utils.h" +#include +#include +#include +#include +#include +#include + +int mib_to_h5_single_file(const char *filename, + const char *output_directory, + const char *dataset_key, + bool include_metadata, + const char *compressor, + unsigned int shuffle, + unsigned int compression_level) +{ + FILE *mib_ptr = NULL; + char *output_file = NULL; + hid_t fapl_id = H5P_DEFAULT; + hid_t fcpl_id = H5P_DEFAULT; + hid_t lcpl_id = H5P_DEFAULT; + hid_t file_id = H5I_INVALID_HID; + hid_t dcpl = H5P_DEFAULT; + hid_t memspace = H5I_INVALID_HID; + hid_t dtype = H5I_INVALID_HID; + hid_t merlin_dataset_id = H5I_INVALID_HID; + hid_t meta_handle[MQ1_FIELDS_NUM_FIELDS] = {0}; + hid_t *dac_handle = NULL; + framebuffer fb = {0}; + int ret = -1; + + // validate inputs + if (!filename || !output_directory || !dataset_key) { + fprintf(stderr, "Error: Missing required parameters\n"); + return -1; + } + + // open MIB file + mib_ptr = fopen(filename, "rb"); + if (!mib_ptr) { + fprintf(stderr, "Error: Cannot open input file %s\n", filename); + return -1; + } + + // get header metadata from first frame + char header_id[MQ1_CHAR_LEN_HEADER_ID] = {0}; + unsigned int header_bytes = 0, num_chips = 0, det_x = 0, det_y = 0; + char pixel_depth[MQ1_CHAR_LEN_PIXEL_DEPTH] = {0}; + + header_meta_from_first(mib_ptr, header_id, &header_bytes, &num_chips, &det_x, + &det_y, pixel_depth); + + // validate pixel_depth string length and format + if (strlen(pixel_depth) != 3 || pixel_depth[0] != 'U' || + pixel_depth[1] < '0' || pixel_depth[1] > '9' || pixel_depth[2] < '0' || + pixel_depth[2] > '9') { + fprintf(stderr, "Error: Invalid pixel depth format: %s\n", pixel_depth); + goto cleanup; + } + + // calculate size of pixel data type + int bufsize = (pixel_depth[1] - '0') * 10 + (pixel_depth[2] - '0'); + bufsize = bufsize / 8; + + // sanity check for detector dimensions + // (arbitrarily limit to max 4096x4096 currently) + if (det_x > 4096 || det_y > 4096) { + fprintf(stderr, + "Error: Detector dimensions exceed maximum (4096x4096): %ux%u\n", + det_x, det_y); + goto cleanup; + } + + // calculate frame size in bytes + size_t frame_size = det_x * det_y * bufsize; + + // calculate stride (header + frame data) + size_t stride_calc = header_bytes + frame_size; + + // check stride fits in unsigned int (just for sure for API compatibility) + if (stride_calc > UINT_MAX) { + fprintf(stderr, "Error: Stride too large for unsigned int\n"); + goto cleanup; + } + unsigned int stride = (unsigned int) stride_calc; + + unsigned int num_frames = num_of_headers(mib_ptr, stride); + + printf("Converting %s:\n", filename); + printf(" Header: %s, %u bytes\n", header_id, header_bytes); + printf(" Detector: %ux%u, %s depth, %u chip(s)\n", det_x, det_y, pixel_depth, + num_chips); + printf(" Frames: %u, stride: %u bytes\n", num_frames, stride); + + // create output filename + output_file = create_output_filename(filename, output_directory); + if (!output_file) { + fprintf(stderr, "Error: Cannot create output filename\n"); + goto cleanup; + } + + // validate output directory exists + if (directory_exists(output_directory) != 0) { + goto cleanup; + } + + // initialise HDF5 property lists + initialize_plist(output_directory, &fapl_id, &fcpl_id, &lcpl_id); + + // create HDF5 file + initialize_file(output_file, &file_id, fapl_id, fcpl_id); + if (file_id < 0) { + fprintf(stderr, "Error: Cannot create HDF5 file\n"); + goto cleanup; + } + + // create dataset creation property list with chunking + dcpl = H5Pcreate(H5P_DATASET_CREATE); + if (dcpl < 0) { + fprintf(stderr, "Error: Cannot create dataset property list\n"); + goto cleanup; + } + + hsize_t chunk_dims[3] = {1, det_y, det_x}; + if (H5Pset_chunk(dcpl, 3, chunk_dims) < 0) { + fprintf(stderr, "Error: Cannot set chunk dimensions\n"); + goto cleanup; + } + + // TODO: Compression functionality to be implemented in a future PR + // The compression parameters (compressor, shuffle, compression_level) are + // currently accepted but not used. The cbytes parameter is set to 0 to + // indicate no compression. + int cbytes = 0; + + // create data memspace + hsize_t frame_dim[2] = {det_y, det_x}; + memspace = H5Screate_simple(2, frame_dim, NULL); + if (memspace < 0) { + fprintf(stderr, "Error: Cannot create memory dataspace\n"); + goto cleanup; + } + + // determine datatype + dtype = bufsize_to_datatype(bufsize); + if (dtype < 0) { + fprintf(stderr, "Error: Invalid buffer size for datatype\n"); + goto cleanup; + } + + // create main dataset + create_merlin_dataset(&merlin_dataset_id, file_id, dataset_key, dtype, dcpl, + lcpl_id, frame_dim); + if (merlin_dataset_id < 0) { + fprintf(stderr, "Error: Cannot create main dataset\n"); + goto cleanup; + } + + // create metadata datasets if requested + if (include_metadata) { + create_meta_mq1_fields_dataset(file_id, lcpl_id, meta_handle); + + // create DAC datasets + size_t dac_size = sizeof(hid_t) * DAC_NUM_FIELDS * num_chips; + dac_handle = malloc(dac_size); + if (!dac_handle) { + fprintf(stderr, "Error: Cannot allocate memory for DAC handles\n"); + goto cleanup; + } + create_dac_dataset(num_chips, file_id, lcpl_id, dac_handle); + } + + // allocate framebuffer + allocate_frame_header(&fb); + + // read first header to get dimensions for frame data allocation + read_header(mib_ptr, 0, &fb); + allocate_frame_data(&fb); + + // check if allocation succeeded + if (!fb.data || !fb.rows) { + fprintf(stderr, "Error: Failed to allocate framebuffer\n"); + goto cleanup; + } + + // process all frames + for (unsigned int i = 0; i < num_frames; ++i) { + unsigned long offset = (unsigned long) i * stride; + + // read frame header and data + read_header(mib_ptr, offset, &fb); + read_frame(mib_ptr, offset, &fb); + + // append frame data to dataset + append_frame_to_dataset(merlin_dataset_id, &fb, cbytes); + + // append metadata if requested + if (include_metadata) { + append_meta_to_dataset(meta_handle, &fb); + + // append DAC data + append_dac_to_dataset(num_chips, dac_handle, &fb); + } + + // progress indicator + if ((i + 1) % 100 == 0 || i == num_frames - 1) { + printf("\rProcessed %u/%u frames", i + 1, num_frames); + fflush(stdout); + } + } + printf("\n"); + + printf("Conversion complete: %s\n", output_file); + ret = 0; // success + +cleanup: + // cleanup framebuffer + if (fb.data || fb.rows) { + deallocate_frame(&fb); + } + + // close HDF5 handles + if (memspace >= 0) + H5Sclose(memspace); + if (merlin_dataset_id >= 0) + H5Dclose(merlin_dataset_id); + + for (int i = 0; i < MQ1_FIELDS_NUM_FIELDS; ++i) { + if (meta_handle[i] > 0 && H5Iis_valid(meta_handle[i])) { + H5Dclose(meta_handle[i]); + } + } + + if (dac_handle) { + for (unsigned int i = 0; i < DAC_NUM_FIELDS * num_chips; ++i) { + if (dac_handle[i] > 0 && H5Iis_valid(dac_handle[i])) { + H5Dclose(dac_handle[i]); + } + } + free(dac_handle); + } + + if (dcpl != H5P_DEFAULT && dcpl >= 0) + H5Pclose(dcpl); + if (fapl_id != H5P_DEFAULT && fapl_id >= 0) + H5Pclose(fapl_id); + if (fcpl_id != H5P_DEFAULT && fcpl_id >= 0) + H5Pclose(fcpl_id); + if (lcpl_id != H5P_DEFAULT && lcpl_id >= 0) + H5Pclose(lcpl_id); + if (file_id >= 0) + H5Fclose(file_id); + + if (mib_ptr) + fclose(mib_ptr); + if (output_file) + free(output_file); + + return ret; +} + +int mib_to_h5(const char **input_files, + int num_input_files, + const char *output_dir, + bool include_metadata, + const char *dataset_key, + const char *metadata_key, + bool use_compression, + const char *reshape_dims, + bool report_progress, + unsigned int timeout_seconds) +{ + // validate inputs + if (!input_files || num_input_files <= 0) { + fprintf(stderr, "Error: No input files specified\n"); + return -1; + } + + // set defaults for optional parameters + const char *actual_output_dir = output_dir ? output_dir : "./"; + const char *actual_dataset_key = dataset_key ? dataset_key : "/data"; + + // handle unimplemented features with user messages + if (metadata_key) { + // TODO: implement custom metadata key support + fprintf(stderr, "Note: metadata_key parameter not yet implemented, using " + "default /metadata\n"); + } + + if (reshape_dims) { + // TODO: implement reshape dimensions support + fprintf(stderr, "Note: reshape_dims not yet implemented\n"); + } + + if (report_progress) { + // TODO: implement progress reporting + fprintf(stderr, "Note: report_progress not yet implemented\n"); + } + + if (timeout_seconds > 0) { + // TODO: implement timeout handling + fprintf(stderr, "Note: timeout_seconds not yet implemented\n"); + } + + // determine compression settings + const char *compressor = NULL; + unsigned int shuffle = 0; + unsigned int compression_level = 0; + + if (use_compression) { + compressor = "blosclz"; + + // check environment variables for compression settings + const char *shuffle_env = getenv("MIB2H5_SHUFFLE"); + const char *level_env = getenv("MIB2H5_COMPRESSION_LEVEL"); + + shuffle = shuffle_env ? (unsigned int) atoi(shuffle_env) : 2; + compression_level = level_env ? (unsigned int) atoi(level_env) : 9; + } + + // process each input file + int total_errors = 0; + for (int i = 0; i < num_input_files; ++i) { + if (!input_files[i]) { + fprintf(stderr, "Error: NULL input file at index %d\n", i); + total_errors++; + continue; + } + + printf("Processing file %d of %d: %s\n", i + 1, num_input_files, + input_files[i]); + + int result = mib_to_h5_single_file(input_files[i], actual_output_dir, + actual_dataset_key, include_metadata, + compressor, shuffle, compression_level); + + if (result != 0) { + fprintf(stderr, "Error: Failed to convert %s\n", input_files[i]); + total_errors++; + // continue processing other files instead of stopping + } + } + + return total_errors > 0 ? -1 : 0; +} + +const char *mib_to_h5_last_error(void) +{ + return "Error handling not yet implemented"; +} diff --git a/src/mib_to_h5.h b/src/mib_to_h5.h new file mode 100644 index 0000000..4c0e66b --- /dev/null +++ b/src/mib_to_h5.h @@ -0,0 +1,16 @@ +// clang-format Language: C +#ifndef MIB_TO_H5_H +#define MIB_TO_H5_H + +#include + +// function for single file conversion +int mib_to_h5_single_file(const char *filename, + const char *output_directory, + const char *dataset_key, + bool include_metadata, + const char *compressor, + unsigned int shuffle, + unsigned int compression_level); + +#endif diff --git a/src/parser.c b/src/parser.c new file mode 100644 index 0000000..acaabed --- /dev/null +++ b/src/parser.c @@ -0,0 +1,362 @@ +#include "parser.h" +#include "io_header.h" +#include "macros.h" +#include "mib_header.h" +#include "utils.h" + +#include +#include +#include + +/* this assumes the provided header is valid + * the header struct will be overwritten + */ +void parse_mq1_single(const char *header, mq1s *mq1_single) +{ + + int n = sscanf( + header, + "%[^,],%u,%u,%u,%u,%u,%[^,],%[^,],%[^,],%[^,]," + "%lf,%u,%u,%u,%f,%f,%f,%f,%f,%f,%f,%f," + "%[^,],%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%" + "u,%u,%u,%u,%u," + "%[^,],%[^,],%uns,%u", + mq1_single->header_id, &mq1_single->sequence_number, + &mq1_single->header_bytes, &mq1_single->num_chips, &mq1_single->det_x, + &mq1_single->det_y, mq1_single->pixel_depth, mq1_single->sensor_layout, + mq1_single->chip_select, mq1_single->timestamp, + &mq1_single->exposure_time_s, &mq1_single->counter, + &mq1_single->colour_mode, &mq1_single->gain_mode, &mq1_single->threshold[0], + &mq1_single->threshold[1], &mq1_single->threshold[2], + &mq1_single->threshold[3], &mq1_single->threshold[4], + &mq1_single->threshold[5], &mq1_single->threshold[6], + &mq1_single->threshold[7], mq1_single->dac0.dac_format, + &mq1_single->dac0.threshold0, &mq1_single->dac0.threshold1, + &mq1_single->dac0.threshold2, &mq1_single->dac0.threshold3, + &mq1_single->dac0.threshold4, &mq1_single->dac0.threshold5, + &mq1_single->dac0.threshold6, &mq1_single->dac0.threshold7, + &mq1_single->dac0.preamp, &mq1_single->dac0.ikrum, &mq1_single->dac0.shaper, + &mq1_single->dac0.disc, &mq1_single->dac0.disc_LS, + &mq1_single->dac0.shaper_test, &mq1_single->dac0.dac_disc_L, + &mq1_single->dac0.dac_test, &mq1_single->dac0.dac_disc_H, + &mq1_single->dac0.delay, &mq1_single->dac0.TP_buff_in, + &mq1_single->dac0.TP_buff_out, &mq1_single->dac0.RPZ, &mq1_single->dac0.GND, + &mq1_single->dac0.TP_ref, &mq1_single->dac0.FBK, &mq1_single->dac0.Cas, + &mq1_single->dac0.TP_ref_A, &mq1_single->dac0.TP_ref_B, + mq1_single->header_extension_id, mq1_single->extended_timestamp, + &mq1_single->exposure_time_ns, &mq1_single->bit_depth); + + if (n == EOF) { + const char *current_file = only_file_name(__FILE__); + fprintf(stderr, "%s:%d: error: the header appears to be empty\n", + current_file, __LINE__); + exit(1); + } else if (n != MQ1_SINGLE_HEADER_NUM_FIELDS) { + const char *current_file = only_file_name(__FILE__); + fprintf(stderr, "%s:%d: error: failed to match the MQ1 Quad header\n", + current_file, __LINE__); + exit(1); + } +} + +void print_single_mib_header(mq1s header) +{ + /* Assume the MQ1 Single header (mq1s) has been initialised, + * otherwise you should treat the printed values as garbages. + */ + printf("header id: %3s\n", header.header_id); + printf("sequence number: %u\n", header.sequence_number); + printf("header bytes: %u\n", header.header_bytes); + printf("number of chip(s): %u\n", header.num_chips); + printf("detector size in x: %u\n", header.det_x); + printf("detector size in y: %u\n", header.det_y); + printf("pixel data type: %3s\n", header.pixel_depth); + printf("sensor layout: %6s\n", header.sensor_layout); + printf("chip selected: %2s\n", header.chip_select); + printf("time stamp: %26s\n", header.timestamp); + printf("exposure time (s): %lf\n", header.exposure_time_s); + printf("counter: %u\n", header.counter); + printf("colour mode: %u\n", header.colour_mode); + printf("gain mode: %u\n", header.gain_mode); + printf("threshold 0: %f keV\n", header.threshold[0]); + printf("threshold 1: %f keV\n", header.threshold[1]); + printf("threshold 2: %f keV\n", header.threshold[2]); + printf("threshold 3: %f keV\n", header.threshold[3]); + printf("threshold 4: %f keV\n", header.threshold[4]); + printf("threshold 5: %f keV\n", header.threshold[5]); + printf("threshold 6: %f keV\n", header.threshold[6]); + printf("threshold 7: %f keV\n", header.threshold[7]); + puts("DAC 0:"); + printf("\t DAC format: %3s\n", header.dac0.dac_format); + printf("\t threshold0: %u\n", header.dac0.threshold0); + printf("\t threshold1: %u\n", header.dac0.threshold1); + printf("\t threshold2: %u\n", header.dac0.threshold2); + printf("\t threshold3: %u\n", header.dac0.threshold3); + printf("\t threshold4: %u\n", header.dac0.threshold4); + printf("\t threshold5: %u\n", header.dac0.threshold5); + printf("\t threshold6: %u\n", header.dac0.threshold6); + printf("\t threshold7: %u\n", header.dac0.threshold7); + printf("\t preamp: %u\n", header.dac0.preamp); + printf("\t ikrum: %u\n", header.dac0.ikrum); + printf("\t shaper: %u\n", header.dac0.shaper); + printf("\t disc: %u\n", header.dac0.disc); + printf("\t disc_LS: %u\n", header.dac0.disc_LS); + printf("\t shaper_test: %u\n", header.dac0.shaper_test); + printf("\t dac_disc_L: %u\n", header.dac0.dac_disc_L); + printf("\t dac_test: %u\n", header.dac0.dac_test); + printf("\t dac_disc_H: %u\n", header.dac0.dac_disc_H); + printf("\t delay: %u\n", header.dac0.delay); + printf("\t TP_buff_in: %u\n", header.dac0.TP_buff_in); + printf("\t TP_buff_out: %u\n", header.dac0.TP_buff_out); + printf("\t RPZ: %u\n", header.dac0.RPZ); + printf("\t GND: %u\n", header.dac0.GND); + printf("\t TP_ref: %u\n", header.dac0.TP_ref); + printf("\t FBK: %u\n", header.dac0.FBK); + printf("\t Cas: %u\n", header.dac0.Cas); + printf("\t TP_ref_A: %u\n", header.dac0.TP_ref_A); + printf("\t TP_ref_B: %u\n", header.dac0.TP_ref_B); + printf("header extension id: %4s\n", header.header_extension_id); + printf("extended time stamp: %28s\n", header.extended_timestamp); + printf("exposure time (ns): %u\n", header.exposure_time_ns); + printf("pixel bit depth: %u\n", header.bit_depth); +} + +/* this assumes the provided header is valid + * the header struct will be overwritten + */ +void parse_mq1_quad(const char *header, mq1q *mq1_quad) +{ + + int n = sscanf( + header, + "%[^,],%u,%u,%u,%u,%u,%[^,],%[^,],%[^,],%[^,]," + "%lf,%u,%u,%u,%f,%f,%f,%f,%f,%f,%f,%f," + "%[^,],%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%" + "u,%u,%u,%u,%u," + "%[^,],%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%" + "u,%u,%u,%u,%u," + "%[^,],%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%" + "u,%u,%u,%u,%u," + "%[^,],%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%" + "u,%u,%u,%u,%u," + "%[^,],%[^,],%uns,%u", + mq1_quad->header_id, &mq1_quad->sequence_number, &mq1_quad->header_bytes, + &mq1_quad->num_chips, &mq1_quad->det_x, &mq1_quad->det_y, + mq1_quad->pixel_depth, mq1_quad->sensor_layout, mq1_quad->chip_select, + mq1_quad->timestamp, &mq1_quad->exposure_time_s, &mq1_quad->counter, + &mq1_quad->colour_mode, &mq1_quad->gain_mode, &mq1_quad->threshold[0], + &mq1_quad->threshold[1], &mq1_quad->threshold[2], &mq1_quad->threshold[3], + &mq1_quad->threshold[4], &mq1_quad->threshold[5], &mq1_quad->threshold[6], + &mq1_quad->threshold[7], mq1_quad->dac0.dac_format, + &mq1_quad->dac0.threshold0, &mq1_quad->dac0.threshold1, + &mq1_quad->dac0.threshold2, &mq1_quad->dac0.threshold3, + &mq1_quad->dac0.threshold4, &mq1_quad->dac0.threshold5, + &mq1_quad->dac0.threshold6, &mq1_quad->dac0.threshold7, + &mq1_quad->dac0.preamp, &mq1_quad->dac0.ikrum, &mq1_quad->dac0.shaper, + &mq1_quad->dac0.disc, &mq1_quad->dac0.disc_LS, &mq1_quad->dac0.shaper_test, + &mq1_quad->dac0.dac_disc_L, &mq1_quad->dac0.dac_test, + &mq1_quad->dac0.dac_disc_H, &mq1_quad->dac0.delay, + &mq1_quad->dac0.TP_buff_in, &mq1_quad->dac0.TP_buff_out, + &mq1_quad->dac0.RPZ, &mq1_quad->dac0.GND, &mq1_quad->dac0.TP_ref, + &mq1_quad->dac0.FBK, &mq1_quad->dac0.Cas, &mq1_quad->dac0.TP_ref_A, + &mq1_quad->dac0.TP_ref_B, mq1_quad->dac1.dac_format, + &mq1_quad->dac1.threshold0, &mq1_quad->dac1.threshold1, + &mq1_quad->dac1.threshold2, &mq1_quad->dac1.threshold3, + &mq1_quad->dac1.threshold4, &mq1_quad->dac1.threshold5, + &mq1_quad->dac1.threshold6, &mq1_quad->dac1.threshold7, + &mq1_quad->dac1.preamp, &mq1_quad->dac1.ikrum, &mq1_quad->dac1.shaper, + &mq1_quad->dac1.disc, &mq1_quad->dac1.disc_LS, &mq1_quad->dac1.shaper_test, + &mq1_quad->dac1.dac_disc_L, &mq1_quad->dac1.dac_test, + &mq1_quad->dac1.dac_disc_H, &mq1_quad->dac1.delay, + &mq1_quad->dac1.TP_buff_in, &mq1_quad->dac1.TP_buff_out, + &mq1_quad->dac1.RPZ, &mq1_quad->dac1.GND, &mq1_quad->dac1.TP_ref, + &mq1_quad->dac1.FBK, &mq1_quad->dac1.Cas, &mq1_quad->dac1.TP_ref_A, + &mq1_quad->dac1.TP_ref_B, mq1_quad->dac2.dac_format, + &mq1_quad->dac2.threshold0, &mq1_quad->dac2.threshold1, + &mq1_quad->dac2.threshold2, &mq1_quad->dac2.threshold3, + &mq1_quad->dac2.threshold4, &mq1_quad->dac2.threshold5, + &mq1_quad->dac2.threshold6, &mq1_quad->dac2.threshold7, + &mq1_quad->dac2.preamp, &mq1_quad->dac2.ikrum, &mq1_quad->dac2.shaper, + &mq1_quad->dac2.disc, &mq1_quad->dac2.disc_LS, &mq1_quad->dac2.shaper_test, + &mq1_quad->dac2.dac_disc_L, &mq1_quad->dac2.dac_test, + &mq1_quad->dac2.dac_disc_H, &mq1_quad->dac2.delay, + &mq1_quad->dac2.TP_buff_in, &mq1_quad->dac2.TP_buff_out, + &mq1_quad->dac2.RPZ, &mq1_quad->dac2.GND, &mq1_quad->dac2.TP_ref, + &mq1_quad->dac2.FBK, &mq1_quad->dac2.Cas, &mq1_quad->dac2.TP_ref_A, + &mq1_quad->dac2.TP_ref_B, mq1_quad->dac3.dac_format, + &mq1_quad->dac3.threshold0, &mq1_quad->dac3.threshold1, + &mq1_quad->dac3.threshold2, &mq1_quad->dac3.threshold3, + &mq1_quad->dac3.threshold4, &mq1_quad->dac3.threshold5, + &mq1_quad->dac3.threshold6, &mq1_quad->dac3.threshold7, + &mq1_quad->dac3.preamp, &mq1_quad->dac3.ikrum, &mq1_quad->dac3.shaper, + &mq1_quad->dac3.disc, &mq1_quad->dac3.disc_LS, &mq1_quad->dac3.shaper_test, + &mq1_quad->dac3.dac_disc_L, &mq1_quad->dac3.dac_test, + &mq1_quad->dac3.dac_disc_H, &mq1_quad->dac3.delay, + &mq1_quad->dac3.TP_buff_in, &mq1_quad->dac3.TP_buff_out, + &mq1_quad->dac3.RPZ, &mq1_quad->dac3.GND, &mq1_quad->dac3.TP_ref, + &mq1_quad->dac3.FBK, &mq1_quad->dac3.Cas, &mq1_quad->dac3.TP_ref_A, + &mq1_quad->dac3.TP_ref_B, mq1_quad->header_extension_id, + mq1_quad->extended_timestamp, &mq1_quad->exposure_time_ns, + &mq1_quad->bit_depth); + + if (n == EOF) { + const char *current_file = only_file_name(__FILE__); + fprintf(stderr, "%s:%d: error: the header appears to be empty\n", + current_file, __LINE__); + exit(1); + } else if (n != MQ1_QUAD_HEADER_NUM_FIELDS) { + const char *current_file = only_file_name(__FILE__); + fprintf(stderr, "%s:%d: error: failed to match the MQ1 Quad header\n", + current_file, __LINE__); + exit(1); + } +} + +void print_quad_mib_header(mq1q header) +{ + /* Assume the MQ1 Quad header (mq1q) has been initialised, + * otherwise you should treat the printed values as garbages. + */ + printf("header id: %3s\n", header.header_id); + printf("sequence number: %u\n", header.sequence_number); + printf("header bytes: %u\n", header.header_bytes); + printf("number of chip(s): %u\n", header.num_chips); + printf("detector size in x: %u\n", header.det_x); + printf("detector size in y: %u\n", header.det_y); + printf("pixel data type: %3s\n", header.pixel_depth); + printf("sensor layout: %6s\n", header.sensor_layout); + printf("chip selected: %2s\n", header.chip_select); + printf("time stamp: %26s\n", header.timestamp); + printf("exposure time (s): %lf\n", header.exposure_time_s); + printf("counter: %u\n", header.counter); + printf("colour mode: %u\n", header.colour_mode); + printf("gain mode: %u\n", header.gain_mode); + printf("threshold 0: %f keV\n", header.threshold[0]); + printf("threshold 1: %f keV\n", header.threshold[1]); + printf("threshold 2: %f keV\n", header.threshold[2]); + printf("threshold 3: %f keV\n", header.threshold[3]); + printf("threshold 4: %f keV\n", header.threshold[4]); + printf("threshold 5: %f keV\n", header.threshold[5]); + printf("threshold 6: %f keV\n", header.threshold[6]); + printf("threshold 7: %f keV\n", header.threshold[7]); + puts("DAC 0:"); + printf("\t DAC format: %3s\n", header.dac0.dac_format); + printf("\t threshold0: %u\n", header.dac0.threshold0); + printf("\t threshold1: %u\n", header.dac0.threshold1); + printf("\t threshold2: %u\n", header.dac0.threshold2); + printf("\t threshold3: %u\n", header.dac0.threshold3); + printf("\t threshold4: %u\n", header.dac0.threshold4); + printf("\t threshold5: %u\n", header.dac0.threshold5); + printf("\t threshold6: %u\n", header.dac0.threshold6); + printf("\t threshold7: %u\n", header.dac0.threshold7); + printf("\t preamp: %u\n", header.dac0.preamp); + printf("\t ikrum: %u\n", header.dac0.ikrum); + printf("\t shaper: %u\n", header.dac0.shaper); + printf("\t disc: %u\n", header.dac0.disc); + printf("\t disc_LS: %u\n", header.dac0.disc_LS); + printf("\t shaper_test: %u\n", header.dac0.shaper_test); + printf("\t dac_disc_L: %u\n", header.dac0.dac_disc_L); + printf("\t dac_test: %u\n", header.dac0.dac_test); + printf("\t dac_disc_H: %u\n", header.dac0.dac_disc_H); + printf("\t delay: %u\n", header.dac0.delay); + printf("\t TP_buff_in: %u\n", header.dac0.TP_buff_in); + printf("\t TP_buff_out: %u\n", header.dac0.TP_buff_out); + printf("\t RPZ: %u\n", header.dac0.RPZ); + printf("\t GND: %u\n", header.dac0.GND); + printf("\t TP_ref: %u\n", header.dac0.TP_ref); + printf("\t FBK: %u\n", header.dac0.FBK); + printf("\t Cas: %u\n", header.dac0.Cas); + printf("\t TP_ref_A: %u\n", header.dac0.TP_ref_A); + printf("\t TP_ref_B: %u\n", header.dac0.TP_ref_B); + puts("DAC 1:"); + printf("\t DAC format: %3s\n", header.dac1.dac_format); + printf("\t threshold0: %u\n", header.dac1.threshold0); + printf("\t threshold1: %u\n", header.dac1.threshold1); + printf("\t threshold2: %u\n", header.dac1.threshold2); + printf("\t threshold3: %u\n", header.dac1.threshold3); + printf("\t threshold4: %u\n", header.dac1.threshold4); + printf("\t threshold5: %u\n", header.dac1.threshold5); + printf("\t threshold6: %u\n", header.dac1.threshold6); + printf("\t threshold7: %u\n", header.dac1.threshold7); + printf("\t preamp: %u\n", header.dac1.preamp); + printf("\t ikrum: %u\n", header.dac1.ikrum); + printf("\t shaper: %u\n", header.dac1.shaper); + printf("\t disc: %u\n", header.dac1.disc); + printf("\t disc_LS: %u\n", header.dac1.disc_LS); + printf("\t shaper_test: %u\n", header.dac1.shaper_test); + printf("\t dac_disc_L: %u\n", header.dac1.dac_disc_L); + printf("\t dac_test: %u\n", header.dac1.dac_test); + printf("\t dac_disc_H: %u\n", header.dac1.dac_disc_H); + printf("\t delay: %u\n", header.dac1.delay); + printf("\t TP_buff_in: %u\n", header.dac1.TP_buff_in); + printf("\t TP_buff_out: %u\n", header.dac1.TP_buff_out); + printf("\t RPZ: %u\n", header.dac1.RPZ); + printf("\t GND: %u\n", header.dac1.GND); + printf("\t TP_ref: %u\n", header.dac1.TP_ref); + printf("\t FBK: %u\n", header.dac1.FBK); + printf("\t Cas: %u\n", header.dac1.Cas); + printf("\t TP_ref_A: %u\n", header.dac1.TP_ref_A); + printf("\t TP_ref_B: %u\n", header.dac1.TP_ref_B); + puts("DAC 2:"); + printf("\t DAC format: %3s\n", header.dac2.dac_format); + printf("\t threshold0: %u\n", header.dac2.threshold0); + printf("\t threshold1: %u\n", header.dac2.threshold1); + printf("\t threshold2: %u\n", header.dac2.threshold2); + printf("\t threshold3: %u\n", header.dac2.threshold3); + printf("\t threshold4: %u\n", header.dac2.threshold4); + printf("\t threshold5: %u\n", header.dac2.threshold5); + printf("\t threshold6: %u\n", header.dac2.threshold6); + printf("\t threshold7: %u\n", header.dac2.threshold7); + printf("\t preamp: %u\n", header.dac2.preamp); + printf("\t ikrum: %u\n", header.dac2.ikrum); + printf("\t shaper: %u\n", header.dac2.shaper); + printf("\t disc: %u\n", header.dac2.disc); + printf("\t disc_LS: %u\n", header.dac2.disc_LS); + printf("\t shaper_test: %u\n", header.dac2.shaper_test); + printf("\t dac_disc_L: %u\n", header.dac2.dac_disc_L); + printf("\t dac_test: %u\n", header.dac2.dac_test); + printf("\t dac_disc_H: %u\n", header.dac2.dac_disc_H); + printf("\t delay: %u\n", header.dac2.delay); + printf("\t TP_buff_in: %u\n", header.dac2.TP_buff_in); + printf("\t TP_buff_out: %u\n", header.dac2.TP_buff_out); + printf("\t RPZ: %u\n", header.dac2.RPZ); + printf("\t GND: %u\n", header.dac2.GND); + printf("\t TP_ref: %u\n", header.dac2.TP_ref); + printf("\t FBK: %u\n", header.dac2.FBK); + printf("\t Cas: %u\n", header.dac2.Cas); + printf("\t TP_ref_A: %u\n", header.dac2.TP_ref_A); + printf("\t TP_ref_B: %u\n", header.dac2.TP_ref_B); + puts("DAC 3:"); + printf("\t DAC format: %3s\n", header.dac3.dac_format); + printf("\t threshold0: %u\n", header.dac3.threshold0); + printf("\t threshold1: %u\n", header.dac3.threshold1); + printf("\t threshold2: %u\n", header.dac3.threshold2); + printf("\t threshold3: %u\n", header.dac3.threshold3); + printf("\t threshold4: %u\n", header.dac3.threshold4); + printf("\t threshold5: %u\n", header.dac3.threshold5); + printf("\t threshold6: %u\n", header.dac3.threshold6); + printf("\t threshold7: %u\n", header.dac3.threshold7); + printf("\t preamp: %u\n", header.dac3.preamp); + printf("\t ikrum: %u\n", header.dac3.ikrum); + printf("\t shaper: %u\n", header.dac3.shaper); + printf("\t disc: %u\n", header.dac3.disc); + printf("\t disc_LS: %u\n", header.dac3.disc_LS); + printf("\t shaper_test: %u\n", header.dac3.shaper_test); + printf("\t dac_disc_L: %u\n", header.dac3.dac_disc_L); + printf("\t dac_test: %u\n", header.dac3.dac_test); + printf("\t dac_disc_H: %u\n", header.dac3.dac_disc_H); + printf("\t delay: %u\n", header.dac3.delay); + printf("\t TP_buff_in: %u\n", header.dac3.TP_buff_in); + printf("\t TP_buff_out: %u\n", header.dac3.TP_buff_out); + printf("\t RPZ: %u\n", header.dac3.RPZ); + printf("\t GND: %u\n", header.dac3.GND); + printf("\t TP_ref: %u\n", header.dac3.TP_ref); + printf("\t FBK: %u\n", header.dac3.FBK); + printf("\t Cas: %u\n", header.dac3.Cas); + printf("\t TP_ref_A: %u\n", header.dac3.TP_ref_A); + printf("\t TP_ref_B: %u\n", header.dac3.TP_ref_B); + printf("header extension id: %4s\n", header.header_extension_id); + printf("extended time stamp: %28s\n", header.extended_timestamp); + printf("exposure time (ns): %u\n", header.exposure_time_ns); + printf("pixel bit depth: %u\n", header.bit_depth); +} diff --git a/src/parser.h b/src/parser.h new file mode 100644 index 0000000..8b59df2 --- /dev/null +++ b/src/parser.h @@ -0,0 +1,13 @@ +// clang-format Language: C +#include "mib_header.h" + +#ifndef PARSER_H +#define PARSER_H +void parse_mq1_single(const char *header, mq1s *mq1_single); + +void print_single_mib_header(mq1s header); + +void parse_mq1_quad(const char *header, mq1q *mq1_quad); + +void print_quad_mib_header(mq1q header); +#endif diff --git a/src/read.c b/src/read.c new file mode 100644 index 0000000..55c3cd8 --- /dev/null +++ b/src/read.c @@ -0,0 +1,199 @@ +#include "read.h" +#include "framebuffer.h" +#include "io_header.h" +#include "macros.h" +#include "parser.h" +#include "utils.h" + +#include +#include +#include +#include + +void read_header(FILE *mib_ptr, unsigned long offset, framebuffer *fb) +{ + char *header; + if (mib_ptr == NULL || fb == NULL) { + fprintf(stderr, "Missing input mib_ptr or fb in read_header\n"); + return; + } + if (fseek(mib_ptr, offset, SEEK_SET) != 0) { + fprintf(stderr, "fseek error in read_header\n"); + return; + } + char buf[MIB_HEADER_METADATA_BUF_SIZE] = {0}; + char headersize_str[MIB_HEADER_SIZE_FIELD_LENGTH] = {0}; + size_t status = + fread(buf, sizeof(char), MIB_HEADER_METADATA_BUF_SIZE, mib_ptr); + if (status != MIB_HEADER_METADATA_BUF_SIZE) { + fprintf(stderr, "fread error in read_header\n"); + return; + } + memcpy(headersize_str, buf + MIB_HEADER_SIZE_FIELD_OFFSET + 1, + MIB_HEADER_SIZE_FIELD_LENGTH - 1); + headersize_str[MIB_HEADER_SIZE_FIELD_LENGTH - 1] = '\0'; + char *end_ptr; + long unsigned int headersize = strtoul(headersize_str, &end_ptr, 10); + if (end_ptr == headersize_str) { + fprintf(stderr, "headersize strtol error in read_header, no digit found\n"); + return; + } else if (*end_ptr != '\0') { + fprintf(stderr, + "headersize strtol error in read_header, invalid character: %c\n", + *end_ptr); + return; + } + + if (headersize != MQ1_SINGLE_HEADER_BYTES && + headersize != MQ1_QUAD_HEADER_BYTES) { + fprintf( + stderr, + "headersize not equal to either single or quad header byte size.\n"); + return; + } + + header = (char *) malloc(sizeof(char) * headersize); + if (!header) { + fprintf(stderr, "malloc fail for header in read_header\n"); + return; + } + + if (fseek(mib_ptr, offset, SEEK_SET) != 0) { + fprintf(stderr, "fseek error in read_header\n"); + goto cleanup; + } + + status = fread(header, sizeof(char), headersize, mib_ptr); + if (status != headersize) { + fprintf(stderr, "fread error in read_header\n"); + goto cleanup; + } + + switch (headersize) { + case MQ1_SINGLE_HEADER_BYTES: { + mq1s mq1_single; + parse_mq1_single(header, &mq1_single); + if (fb->dac0 == NULL) { + fprintf(stderr, "NULL dac pointer in read_header\n"); + goto cleanup; + } + memcpy(fb->dac0, &mq1_single.dac0, sizeof(dac_rx)); + fill_MQ1_single_fields(fb->mq1_header, 0, mq1_single); + break; + } + case MQ1_QUAD_HEADER_BYTES: { + mq1q mq1_quad; + parse_mq1_quad(header, &mq1_quad); + if (fb->dac0 == NULL || fb->dac1 == NULL || fb->dac2 == NULL || + fb->dac3 == NULL) { + fprintf(stderr, "NULL dac pointer in read_header\n"); + goto cleanup; + } + memcpy(fb->dac0, &mq1_quad.dac0, sizeof(dac_rx)); + memcpy(fb->dac1, &mq1_quad.dac1, sizeof(dac_rx)); + memcpy(fb->dac2, &mq1_quad.dac2, sizeof(dac_rx)); + memcpy(fb->dac3, &mq1_quad.dac3, sizeof(dac_rx)); + fill_MQ1_quad_fields(fb->mq1_header, 0, mq1_quad); + break; + } + default: { + fprintf(stderr, "headersize not 384 or 768\n"); + goto cleanup; + } + } + +cleanup: + if (header) { + free(header); + header = NULL; + } +} + +void read_frame(FILE *mib_ptr, unsigned long offset, framebuffer *fb) +{ + if (!mib_ptr) { + fprintf(stderr, "NO MIB_PTR\n"); + return; + } + if (!fb) { + fprintf(stderr, "NO fb\n"); + return; + } + if (!fb->rows) { + fprintf(stderr, "NO fb->rows\n"); + return; + } + + if (fb->mq1_header == NULL) { + fprintf(stderr, "NULL pointer fb->mq1_header in read_frame\n"); + return; + } else { + if (fb->mq1_header->header_bytes == NULL || fb->mq1_header->det_x == NULL || + fb->mq1_header->det_y == NULL || fb->mq1_header->pixel_depth == NULL) { + fprintf(stderr, "NULL pointer inside fb->mq1_header in read_frame\n"); + return; + } + } + + int bufsize = (fb->mq1_header->pixel_depth[1] - '0') * 10 + + (fb->mq1_header->pixel_depth[2] - '0'); + bufsize = bufsize / 8; + if (bufsize != 1 && bufsize != 2 && bufsize != 4 && bufsize != 8) { + fprintf(stderr, "not supported bufsize in read_frame\n"); + return; + } + + int detx = *(fb->mq1_header->det_x); + int dety = *(fb->mq1_header->det_y); + unsigned int header_bytes = *(fb->mq1_header->header_bytes); + + if (fseek(mib_ptr, offset + header_bytes, SEEK_SET) != 0) { + fprintf(stderr, "fseek error in read_frame\n"); + return; + } + + uint8_t *raw_data = malloc(bufsize * detx * dety); + if (!raw_data) { + fprintf(stderr, "malloc failed for raw_data in read_frame\n"); + return; + } + + int status = fread(raw_data, sizeof(char), bufsize * detx * dety, mib_ptr); + if (status != bufsize * detx * dety) { + fprintf(stderr, "fread error in read_frame\n"); + goto cleanup; + } + + for (int i = 0; i < dety; i++) { + for (int j = 0; j < detx; j++) { + size_t index = (i * detx + j) * bufsize; + uint8_t *raw_bytes = &raw_data[index]; + + switch (bufsize) { + case 1: { + ((uint8_t **) fb->rows)[i][j] = raw_bytes[0]; + break; + } + case 2: { + ((uint16_t **) fb->rows)[i][j] = convert_uint16_be(raw_bytes); + break; + } + case 4: { + ((uint32_t **) fb->rows)[i][j] = convert_uint32_be(raw_bytes); + break; + } + case 8: { + ((uint64_t **) fb->rows)[i][j] = convert_uint64_be(raw_bytes); + break; + } + default: { + fprintf(stderr, "not supported bufsize\n"); + goto cleanup; + } + } + } + } +cleanup: + free(raw_data); + raw_data = NULL; +} diff --git a/src/read.h b/src/read.h new file mode 100644 index 0000000..6a46d60 --- /dev/null +++ b/src/read.h @@ -0,0 +1,12 @@ +// clang-format Language: C +#ifndef READ_H +#define READ_H + +#include "framebuffer.h" +#include + +void read_header(FILE *mib_ptr, unsigned long offset, framebuffer *fb); + +void read_frame(FILE *mib_ptr, unsigned long offset, framebuffer *fb); + +#endif diff --git a/src/utils.c b/src/utils.c new file mode 100644 index 0000000..ec5fca9 --- /dev/null +++ b/src/utils.c @@ -0,0 +1,244 @@ +#include "utils.h" + +#include +#include +#include +#include +#include // for strcasecmp +#include +#include +#include +#include + +const char *only_file_name(const char *absolute_file_path) +{ + const char *rslash = strrchr(absolute_file_path, '/'); + return (rslash != NULL) ? rslash + 1 : absolute_file_path; +} + +char *create_output_filename(const char *input_path, const char *output_dir) +{ + // get base filename from path + const char *base_name = only_file_name(input_path); + if (!base_name) { + return NULL; + } + + // check if filename ends with .mib (case-insensitive) + size_t base_len = strlen(base_name); + const char *dot_mib = NULL; + if (base_len > 4) { + // check for .mib or .MIB at the end + if (strcasecmp(base_name + base_len - 4, ".mib") == 0) { + dot_mib = base_name + base_len - 4; + } + } + + // calculate output filename length + size_t name_len = dot_mib ? (size_t) (dot_mib - base_name) : base_len; + // +1 for '/', +4 for '.h5\0' + size_t output_len = strlen(output_dir) + 1 + name_len + 4; + + char *output_file = malloc(output_len); + if (!output_file) { + return NULL; + } + + // build output filename + if (dot_mib) { + // copy basename without .mib, then append .h5 + snprintf(output_file, output_len, "%s/%.*s.h5", output_dir, (int) name_len, + base_name); + } else { + // no .mib extension, just append .h5 + snprintf(output_file, output_len, "%s/%s.h5", output_dir, base_name); + } + + return output_file; +} + +unsigned int num_of_headers(FILE *mib_ptr, const unsigned int stride) +{ + unsigned int num_hdr = 0; + long int original_fpi = 0, file_size = 0; + + /*store the original position*/ + original_fpi = ftell(mib_ptr); + + /*move to the end*/ + if (fseek(mib_ptr, 0L, SEEK_END) != 0) { + const char *current_file = only_file_name(__FILE__); + fprintf(stderr, "%s:%d: error: failed to seek to end of the MIB file\n", + current_file, __LINE__); + goto restore; + } + + /*get the current position (i.e. end position)*/ + file_size = ftell(mib_ptr); + + /*printf("File size: %ld\n", file_size);*/ + /*printf("Stride: %u\n", stride);*/ + + if (stride == 0) { + const char *current_file = only_file_name(__FILE__); + fprintf(stderr, "%s:%d: error: frame stride cannot be zero\n", current_file, + __LINE__); + num_hdr = 0; + } else { + if ((file_size % stride) != 0) { + const char *current_file = only_file_name(__FILE__); + fprintf(stderr, + "%s:%d: error: the stride of the frame is apparently " + "incorrect (it cannot divide the MIB file into an integral " + "number of frames)\n", + current_file, __LINE__); + goto restore; + } + num_hdr = file_size / stride; + } + + /*go back to the original position*/ +restore: + if (fseek(mib_ptr, original_fpi, SEEK_SET) != 0) { + const char *current_file = only_file_name(__FILE__); + fprintf( + stderr, + "%s:%d: error: failed to seek to the original position of the MIB file\n", + current_file, __LINE__); + num_hdr = 0; + } + + return num_hdr; +} + +void header_meta_from_first(FILE *mib_ptr, + char *header_id, + unsigned int *header_bytes, + unsigned int *num_chips, + unsigned int *det_x, + unsigned int *det_y, + char *pixel_depth) +{ + char header_buf[40]; + long int original_fpi = 0; + + /*store the original position*/ + original_fpi = ftell(mib_ptr); + + /*move to the beginning*/ + if (fseek(mib_ptr, 0L, SEEK_SET) != 0) { + const char *current_file = only_file_name(__FILE__); + fprintf(stderr, + "%s:%d: error: failed to seek to beginning of the MIB file\n", + current_file, __LINE__); + goto restore; + } + + /*get enough section of the first header*/ + if (fgets(header_buf, sizeof(header_buf), mib_ptr) == NULL) { + perror("error: failed to read data from header"); + goto restore; + } + + /*assign value (skip sequence number)*/ + int n = sscanf(header_buf, "%3s,%*[^,],%u,%u,%u,%u,%[^,]", header_id, + header_bytes, num_chips, det_x, det_y, pixel_depth); + + /*fail to assign enough value */ + if (n != 6) { + const char *current_file = only_file_name(__FILE__); + fprintf(stderr, "%s:%d: error: failed to assign header meta value\n", + current_file, __LINE__); + goto restore; + } + if (n == EOF) { + const char *current_file = only_file_name(__FILE__); + fprintf(stderr, "%s:%d: error: the first header appears to be empty\n", + current_file, __LINE__); + goto restore; + } + + /*go back to the original position*/ +restore: + if (fseek(mib_ptr, original_fpi, SEEK_SET) != 0) { + const char *current_file = only_file_name(__FILE__); + fprintf( + stderr, + "%s:%d: error: failed to seek to the original position of the MIB file\n", + current_file, __LINE__); + } +} + +hid_t bufsize_to_datatype(int dtype) +{ + hid_t datatype; + switch (dtype) { + case 1: { + datatype = H5T_STD_U8LE; + break; + } + case 2: { + datatype = H5T_STD_U16LE; + break; + } + case 4: { + datatype = H5T_STD_U32LE; + break; + } + case 8: { + datatype = H5T_STD_U64LE; + break; + } + default: { + fprintf(stderr, "Error in datatype, please check input dtype\n"); + return H5I_INVALID_HID; + } + } + return datatype; +} + +unsigned long get_filesystem_block_size(const char *path) +{ + struct statvfs stat; + if (!path) { + fprintf(stderr, "Null pointer in get_filesystem_block_size\n"); + return 1; + } + + if (statvfs(path, &stat) != 0) { + fprintf(stderr, "statvfs failed for '%s': %s (errno=%d)\n", path, + strerror(errno), errno); + return 1; + } + + return stat.f_bsize; +} + +int directory_exists(const char *dir_path) +{ + // check for NULL or empty path + if (!dir_path || *dir_path == '\0') { + fprintf(stderr, "Error: Directory path is empty or NULL\n"); + return -1; + } + + // check if path exists and get its stats + struct stat dir_stat; + if (stat(dir_path, &dir_stat) != 0) { + if (errno == ENOENT) { + fprintf(stderr, "Error: Directory '%s' does not exist\n", dir_path); + } else { + fprintf(stderr, "Error: Cannot access directory '%s': %s (errno=%d)\n", + dir_path, strerror(errno), errno); + } + return -1; + } + + // verify it's actually a directory + if (!S_ISDIR(dir_stat.st_mode)) { + fprintf(stderr, "Error: '%s' exists but is not a directory\n", dir_path); + return -1; + } + + return 0; +} diff --git a/src/utils.h b/src/utils.h new file mode 100644 index 0000000..9751bcf --- /dev/null +++ b/src/utils.h @@ -0,0 +1,66 @@ +// clang-format Language: C +#ifndef UTILS_H +#define UTILS_H + +#include +#include +#include +#include +#include +#include + +static inline uint16_t convert_uint16_be(const uint8_t *bytes) +{ + return ((uint16_t) bytes[0] << 8) | bytes[1]; +} + +static inline uint32_t convert_uint32_be(const uint8_t *bytes) +{ + return ((uint32_t) bytes[0] << 24) | ((uint32_t) bytes[1] << 16) | + ((uint32_t) bytes[2] << 8) | bytes[3]; +} + +static inline uint64_t convert_uint64_be(const uint8_t *bytes) +{ + return ((uint64_t) bytes[0] << 56) | ((uint64_t) bytes[1] << 48) | + ((uint64_t) bytes[2] << 40) | ((uint64_t) bytes[3] << 32) | + ((uint64_t) bytes[4] << 24) | ((uint64_t) bytes[5] << 16) | + ((uint64_t) bytes[6] << 8) | bytes[7]; +} + +static inline void *xmalloc_debug( + size_t size, const char *var, const char *func, const char *file, int line) +{ + void *ptr = malloc(size); + if (!ptr) { + fprintf(stderr, "[ERROR] malloc error for %s: %s (at %s:%d in %s)\n", var, + strerror(errno), file, line, func); + exit(1); + } + return ptr; +} + +#define XMALLOC(size, var) \ + xmalloc_debug((size), (var), __func__, __FILE__, __LINE__) + +const char *only_file_name(const char *absolute_file_path); + +char *create_output_filename(const char *input_path, const char *output_dir); + +unsigned int num_of_headers(FILE *mib_ptr, const unsigned int stride); + +void header_meta_from_first(FILE *mib_ptr, + char *header_id, + unsigned int *header_bytes, + unsigned int *num_chips, + unsigned int *det_x, + unsigned int *det_y, + char *pixel_depth); + +hid_t bufsize_to_datatype(int dtype); + +unsigned long get_filesystem_block_size(const char *path); + +int directory_exists(const char *dir_path); + +#endif diff --git a/tests/test_build/test_build_c.sh b/tests/test_build/test_build_c.sh new file mode 100755 index 0000000..66a4ff4 --- /dev/null +++ b/tests/test_build/test_build_c.sh @@ -0,0 +1,352 @@ +#!/usr/bin/env bash +set -euo pipefail + +# constants +readonly script_dir="$(dirname "$(readlink -f "$0")")" +readonly project_root="$(cd "$script_dir/../.." && pwd)" +readonly log_dir="$script_dir/logs" + +# version requirements +readonly min_gcc_version=7 +readonly min_autoconf_major=2 +readonly min_autoconf_minor=64 + +total_tests=0 +passed_tests=0 +skipped_tests=0 +failed_tests=0 +hdf5_test_path="${HDF5_TEST_PATH:-}" +test_filter="all" + +cleanup() { + cd "$project_root" 2>/dev/null || true + if [[ -f Makefile ]]; then + make distclean >/dev/null 2>&1 || true + fi +} + +# register cleanup on exit +trap cleanup EXIT + +check_prerequisites() { + local errors=0 + + # check gcc + if ! command -v gcc >/dev/null 2>&1; then + echo "ERROR: gcc not found" + errors=$((errors + 1)) + else + local gcc_version + gcc_version=$(gcc -dumpversion | cut -d. -f1) + if [[ "$gcc_version" -lt $min_gcc_version ]]; then + echo "ERROR: gcc version $gcc_version < $min_gcc_version" + errors=$((errors + 1)) + fi + fi + + # check autoconf + if ! command -v autoconf >/dev/null 2>&1; then + echo "ERROR: autoconf not found" + errors=$((errors + 1)) + else + local autoconf_version + autoconf_version=$(autoconf --version | head -n1 | grep -oE '[0-9]+\.[0-9]+' | head -n1) + local major minor + IFS='.' read -r major minor <<< "$autoconf_version" + if [[ "$major" -eq $min_autoconf_major && "$minor" -lt $min_autoconf_minor ]]; then + echo "ERROR: autoconf version $autoconf_version < $min_autoconf_major.$min_autoconf_minor" + errors=$((errors + 1)) + fi + fi + + # check automake + if ! command -v automake >/dev/null 2>&1; then + echo "ERROR: automake not found" + errors=$((errors + 1)) + fi + + # no error should return exit code 0 to indicate success + return "$errors" +} + +# check if test should be run based on filter +should_run_test() { + local test_name="$1" + local filter="$2" + [[ "$filter" == "all" ]] || [[ "$filter" == "$test_name" ]] +} + +# run test in an isolated environment +run_isolated_test() { + local test_name="$1" + local configure_cmd="$2" + local env_setup="${3:-}" + + total_tests=$((total_tests + 1)) + echo -n "Test: $test_name... " + + # run in subshell for complete isolation + if ( + # clear all HDF5 env vars first + unset HDF5_ROOT HDF5_HOME HDF5_DIR + # apply test-specific environment if provided + if [[ -n "$env_setup" ]]; then + # parse space-separated var=value pairs + local -a env_pairs + IFS=' ' read -ra env_pairs <<< "$env_setup" + for pair in "${env_pairs[@]}"; do + # strip 'export ' if present + pair="${pair#export }" + # only export valid var=value pairs + if [[ "$pair" == *=* ]]; then + export "$pair" + fi + done + fi + # run the actual test + run_test_internal "$test_name" "$configure_cmd" + ); then + passed_tests=$((passed_tests + 1)) + else + failed_tests=$((failed_tests + 1)) + fi +} + +# actual test runner +run_test_internal() { + local test_name="$1" + local configure_cmd="$2" + local log_file="$log_dir/${test_name}.log" + + # clean previous build + cleanup + + # parse the command into an array + local -a cmd_array + IFS=' ' read -ra cmd_array <<< "$configure_cmd" + + # run configure (from an array of arguments) + if ! (cd "$project_root" && "${cmd_array[@]}" >>"$log_file" 2>&1); then + # attempt to extract the actual error message + local error_line=$(grep -E "error:|not found at" "$log_file" | tail -1 | sed 's/^configure: //') + if [[ -n "$error_line" ]]; then + echo "FAIL ($error_line)" + else + echo "FAIL (configure)" + fi + return 1 + fi + + # make + if ! (cd "$project_root" && make >>"$log_file" 2>&1); then + # attempt to extract make error + local error_line=$(grep -E "^make.*Error|error:" "$log_file" | tail -1 | head -c 60) + if [[ -n "$error_line" ]]; then + echo "FAIL (make: $error_line...)" + else + echo "FAIL (make)" + fi + return 1 + fi + + # test executable + local mib2h5="$project_root/src/mib2h5" + if [[ ! -f "$mib2h5" ]]; then + echo "FAIL (executable mib2h5 not found)" + return 1 + fi + + # determine hdf5 library path for LD_LIBRARY_PATH + local hdf5_lib="" + if [[ -n "${HDF5_ROOT:-}" ]]; then + hdf5_lib="$HDF5_ROOT" + elif [[ -n "${HDF5_HOME:-}" ]]; then + hdf5_lib="$HDF5_HOME" + elif [[ -n "${HDF5_DIR:-}" ]]; then + hdf5_lib="$HDF5_DIR" + elif [[ -n "$hdf5_test_path" ]] && [[ "$test_name" == "explicit_path" ]]; then + hdf5_lib="$hdf5_test_path" + fi + + local ld_path="" + if [[ -n "$hdf5_lib" ]]; then + ld_path="LD_LIBRARY_PATH=$hdf5_lib/lib:${LD_LIBRARY_PATH:-}" + fi + + # test basic commands + if [[ -n "$ld_path" ]]; then + # use env when ld_path is set + if ! (cd "$project_root" && env "$ld_path" "$mib2h5" --version >>"$log_file" 2>&1); then + echo "FAIL (--version)" + return 1 + fi + + if ! (cd "$project_root" && env "$ld_path" "$mib2h5" --help >>"$log_file" 2>&1); then + echo "FAIL (--help)" + return 1 + fi + else + # run directly when ld_path is empty (system-wide installation) + if ! (cd "$project_root" && "$mib2h5" --version >>"$log_file" 2>&1); then + echo "FAIL (--version)" + return 1 + fi + + if ! (cd "$project_root" && "$mib2h5" --help >>"$log_file" 2>&1); then + echo "FAIL (--help)" + return 1 + fi + fi + + echo "PASS" + return 0 +} + + +main() { + # parse arguments + case "${1:-}" in + --clean) + echo "Cleaning build artefacts..." + cleanup + rm -rf "$log_dir" + echo "Done" + exit 0 + ;; + --list) + echo "Available tests:" + echo " system - Test with system HDF5 installation" + echo " explicit - Test with explicit --with-hdf5 path (requires HDF5_TEST_PATH)" + echo " hdf5_root - Test with HDF5_ROOT environment variable" + echo " hdf5_home - Test with HDF5_HOME environment variable" + echo " hdf5_dir - Test with HDF5_DIR environment variable" + echo "" + echo "Usage:" + echo " $0 # Run all tests" + echo " $0 --only TEST # Run specific test" + echo " $0 --list # Show this list" + echo " $0 --clean # Clean build artefacts" + exit 0 + ;; + --only) + if [[ -z "${2:-}" ]]; then + echo "Error: --only requires a test name" + echo "Use --list to see available tests" + exit 1 + fi + test_filter="$2" + ;; + *) + if [[ -n "${1:-}" ]]; then + echo "Unknown option: $1" + echo "Use --list to see available options" + exit 1 + fi + ;; + esac + + echo "=== C Build Tests ===" + if [[ "$test_filter" != "all" ]]; then + echo "Running only: $test_filter" + fi + echo + + # check prerequisites + echo -n "Prerequisites check... " + if check_prerequisites >/dev/null 2>&1; then + echo "OK" + else + echo "FAIL" + # re-run to show stdout + check_prerequisites + exit 1 + fi + echo + + # create log directory + mkdir -p "$log_dir" + + # run autoreconf once + echo -n "Running autoreconf -i... " + if (cd "$project_root" && autoreconf -i >/dev/null 2>&1); then + echo "OK" + else + echo "FAIL" + exit 1 + fi + echo + + # test 1: system hdf5 + if should_run_test "system" "$test_filter"; then + run_isolated_test "system_hdf5" "./configure" "" + fi + + # test 2: explicit path + if should_run_test "explicit" "$test_filter"; then + if [[ -n "$hdf5_test_path" ]]; then + run_isolated_test "explicit_path" "./configure --with-hdf5=$hdf5_test_path" "" + else + skipped_tests=$((skipped_tests + 1)) + echo "Test: explicit_path... SKIP (set HDF5_TEST_PATH to test)" + fi + fi + + # test 3: hdf5_root environment variable + if should_run_test "hdf5_root" "$test_filter"; then + if [[ -n "${HDF5_ROOT:-}" ]]; then + run_isolated_test "hdf5_root" "./configure" "HDF5_ROOT=$HDF5_ROOT" || true + else + skipped_tests=$((skipped_tests + 1)) + echo "Test: hdf5_root... SKIP (HDF5_ROOT not set)" + fi + fi + + # test 4: hdf5_home environment variable + if should_run_test "hdf5_home" "$test_filter"; then + if [[ -n "${HDF5_HOME:-}" ]]; then + run_isolated_test "hdf5_home" "./configure" "HDF5_HOME=$HDF5_HOME" || true + else + skipped_tests=$((skipped_tests + 1)) + echo "Test: hdf5_home... SKIP (HDF5_HOME not set)" + fi + fi + + # test 5: hdf5_dir environment variable + if should_run_test "hdf5_dir" "$test_filter"; then + if [[ -n "${HDF5_DIR:-}" ]]; then + run_isolated_test "hdf5_dir" "./configure" "HDF5_DIR=$HDF5_DIR" || true + else + skipped_tests=$((skipped_tests + 1)) + echo "Test: hdf5_dir... SKIP (HDF5_DIR not set)" + fi + fi + + # summary + echo + + # handle case where no tests ran + if [[ $total_tests -eq 0 ]] && [[ $skipped_tests -gt 0 ]]; then + echo "Tests run: 0, Skipped: $skipped_tests" + echo "No tests were run. Check --list for available tests." + exit 0 + fi + + # normal summary + echo "Tests: $passed_tests/$total_tests passed" + if [[ $skipped_tests -gt 0 ]]; then + echo "Skipped: $skipped_tests" + fi + + if [[ $failed_tests -eq 0 ]]; then + if [[ $total_tests -gt 0 ]]; then + echo "All tests passed!" + fi + exit 0 + else + echo "Failed: $failed_tests" + echo "Check logs in: $log_dir" + exit 1 + fi +} + +main "$@" diff --git a/tests/test_build/test_build_python.sh b/tests/test_build/test_build_python.sh new file mode 100755 index 0000000..52857ef --- /dev/null +++ b/tests/test_build/test_build_python.sh @@ -0,0 +1,338 @@ +#!/usr/bin/env bash +set -euo pipefail + +# constants +readonly script_dir="$(dirname "$(readlink -f "$0")")" +readonly project_root="$(cd "$script_dir/../.." && pwd)" +readonly python_dir="$project_root/python" +readonly log_dir="$script_dir/logs" + +# version requirements +readonly min_python_major=3 +readonly min_python_minor=10 + +total_tests=0 +passed_tests=0 +failed_tests=0 +skipped_tests=0 +test_filter="all" + +cleanup() { + cd "$project_root" 2>/dev/null || true + if [[ -f Makefile ]]; then + make distclean >/dev/null 2>&1 || true + fi + + # clean python build artefacts + rm -rf "$python_dir/build" 2>/dev/null || true + rm -rf "$python_dir/dist" 2>/dev/null || true + rm -rf "$python_dir/src/mib2h5.egg-info" 2>/dev/null || true + rm -rf "$python_dir/src/mib2h5/_wrapper.c" 2>/dev/null || true + rm -rf "$python_dir/src/mib2h5/_wrapper."*.so 2>/dev/null || true + rm -rf "$python_dir/__pycache__" 2>/dev/null || true + rm -rf "$python_dir/src/mib2h5/__pycache__" 2>/dev/null || true + + # clean pipx installation + if command -v pipx >/dev/null 2>&1; then + pipx uninstall mib2h5 >/dev/null 2>&1 || true + fi +} + +# register cleanup on exit +trap cleanup EXIT + +check_prerequisites() { + local errors=0 + + # check hdf5_root is set + if [[ -z "${HDF5_ROOT:-}" ]]; then + echo "ERROR: HDF5_ROOT environment variable not set" + errors=$((errors + 1)) + elif [[ ! -d "${HDF5_ROOT}" ]]; then + echo "ERROR: HDF5_ROOT directory does not exist: ${HDF5_ROOT}" + errors=$((errors + 1)) + fi + + # check python version + if ! command -v python3 >/dev/null 2>&1; then + echo "ERROR: python3 not found" + errors=$((errors + 1)) + else + local python_version + python_version=$(python3 -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")') + local major minor + IFS='.' read -r major minor <<< "$python_version" + if [[ "$major" -lt $min_python_major ]] || [[ "$major" -eq $min_python_major && "$minor" -lt $min_python_minor ]]; then + echo "ERROR: Python $min_python_major.$min_python_minor+ required, found $python_version" + errors=$((errors + 1)) + fi + fi + + return "$errors" +} + +# check if test should be run based on filter +should_run_test() { + local test_name="$1" + local filter="$2" + [[ "$filter" == "all" ]] || [[ "$filter" == "$test_name" ]] +} + +# run test in an isolated environment +run_isolated_test() { + local test_name="$1" + local install_method="$2" + + total_tests=$((total_tests + 1)) + echo -n "Test: $test_name... " + + # run in subshell for complete isolation + if ( + # clear environment variables that might interfere + unset PYTHONPATH + unset PIP_CONFIG_FILE + unset PIP_CACHE_DIR + + # set required environment + export LD_LIBRARY_PATH="${HDF5_ROOT}/lib:${LD_LIBRARY_PATH:-}" + + # run the actual test + run_test_internal "$test_name" "$install_method" + ); then + passed_tests=$((passed_tests + 1)) + else + failed_tests=$((failed_tests + 1)) + fi +} + +# actual test runner +run_test_internal() { + local test_name="$1" + local install_method="$2" + local log_file="${log_dir}/${test_name}.log" + + # create temp directory for virtual environment + local temp_dir + temp_dir=$(mktemp -d /tmp/mib2h5_test_XXXXXX) + local venv_dir="$temp_dir/venv" + + # ensure temp directory cleanup + trap "rm -rf '$temp_dir'" RETURN + + # clean previous build + cleanup + + # create virtual environment + if ! python3 -m venv "$venv_dir" >>"$log_file" 2>&1; then + echo "FAIL (venv creation)" + return 1 + fi + + # use venv's pip directly (no need to activate in subshell) + local pip_cmd="$venv_dir/bin/pip" + local python_cmd="$venv_dir/bin/python" + + # upgrade pip and install build dependencies + if ! "$pip_cmd" install --upgrade pip 'setuptools>=77.0' wheel 'cython>=3' >>"$log_file" 2>&1; then + echo "FAIL (pip dependencies)" + return 1 + fi + + # change to python directory + cd "$python_dir" + + # install package based on method + case "$install_method" in + "standard") + if ! "$pip_cmd" install . >>"$log_file" 2>&1; then + local error_line + error_line=$(grep -E "error:|ERROR:|FAILED" "$log_file" | tail -1 | head -c 60) + if [[ -n "$error_line" ]]; then + echo "FAIL (install: $error_line...)" + else + echo "FAIL (install)" + fi + return 1 + fi + ;; + "editable") + if ! "$pip_cmd" install -e . >>"$log_file" 2>&1; then + local error_line + error_line=$(grep -E "error:|ERROR:|FAILED" "$log_file" | tail -1 | head -c 60) + if [[ -n "$error_line" ]]; then + echo "FAIL (editable install: $error_line...)" + else + echo "FAIL (editable install)" + fi + return 1 + fi + ;; + "pipx") + # for pipx, we use the global pipx command + export PIPX_DEFAULT_PYTHON=$(which python3) + if ! pipx install . --verbose >>"$log_file" 2>&1; then + local error_line + error_line=$(grep -E "error:|ERROR:|FAILED" "$log_file" | tail -1 | head -c 60) + if [[ -n "$error_line" ]]; then + echo "FAIL (pipx install: $error_line...)" + else + echo "FAIL (pipx install)" + fi + return 1 + fi + ;; + *) + echo "FAIL (unknown install method)" + return 1 + ;; + esac + + # test cli version + local mib2h5_cmd + if [[ "$install_method" == "pipx" ]]; then + mib2h5_cmd="mib2h5" + else + mib2h5_cmd="$venv_dir/bin/mib2h5" + fi + + if ! "$mib2h5_cmd" --version >>"$log_file" 2>&1; then + echo "FAIL (--version)" + return 1 + fi + + # test cli help + if ! "$mib2h5_cmd" --help >>"$log_file" 2>&1; then + echo "FAIL (--help)" + return 1 + fi + + # test python import (skip for pipx as it uses isolated environment) + if [[ "$install_method" != "pipx" ]]; then + if ! "$python_cmd" -c "import mib2h5; print('Import successful')" >>"$log_file" 2>&1; then + echo "FAIL (import)" + return 1 + fi + fi + + # cleanup pipx installation + if [[ "$install_method" == "pipx" ]]; then + pipx uninstall mib2h5 >>"$log_file" 2>&1 || true + fi + + echo "PASS" + return 0 +} + +main() { + # parse arguments + case "${1:-}" in + --clean) + echo "Cleaning build artefacts..." + cleanup + rm -rf "$log_dir" + echo "Done" + exit 0 + ;; + --list) + echo "Require setting HDF5_ROOT to the HDF5 installation" + echo + echo "Available tests:" + echo " standard - Standard pip install" + echo " editable - Editable/development install" + echo " pipx - pipx install" + echo "" + echo "Usage:" + echo " $0 # Run all tests" + echo " $0 --only TEST # Run specific test" + echo " $0 --list # Show this list" + echo " $0 --clean # Clean build artefacts" + exit 0 + ;; + --only) + if [[ -z "${2:-}" ]]; then + echo "Error: --only requires a test name" + echo "Use --list to see available tests" + exit 1 + fi + test_filter="$2" + ;; + *) + if [[ -n "${1:-}" ]]; then + echo "Unknown option: $1" + echo "Use --list to see available options" + exit 1 + fi + ;; + esac + + echo "=== Python Build Tests ===" + if [[ "$test_filter" != "all" ]]; then + echo "Running only: $test_filter" + fi + echo "HDF5_ROOT: ${HDF5_ROOT:-not set}" + echo + + # check prerequisites + echo -n "Prerequisites check... " + if check_prerequisites >/dev/null 2>&1; then + echo "OK" + else + echo "FAIL" + # re-run to show stdout + check_prerequisites + exit 1 + fi + echo + + # create log directory + mkdir -p "$log_dir" + + # test 1: standard install + if should_run_test "standard" "$test_filter"; then + run_isolated_test "standard_install" "standard" + fi + + # test 2: editable install + if should_run_test "editable" "$test_filter"; then + run_isolated_test "editable_install" "editable" + fi + + # test 3: pipx install + if should_run_test "pipx" "$test_filter"; then + if command -v pipx >/dev/null 2>&1; then + run_isolated_test "pipx_install" "pipx" + else + skipped_tests=$((skipped_tests + 1)) + echo "Test: pipx_install... SKIP (pipx not found)" + fi + fi + + # summary + echo + + # handle case where no tests ran + if [[ $total_tests -eq 0 ]] && [[ $skipped_tests -gt 0 ]]; then + echo "Tests run: 0, Skipped: $skipped_tests" + echo "No tests were run. Check --list for available tests." + exit 0 + fi + + # normal summary + echo "Tests: $passed_tests/$total_tests passed" + if [[ $skipped_tests -gt 0 ]]; then + echo "Skipped: $skipped_tests" + fi + + if [[ $failed_tests -eq 0 ]]; then + if [[ $total_tests -gt 0 ]]; then + echo "All tests passed!" + fi + exit 0 + else + echo "Failed: $failed_tests" + echo "Check logs in: $log_dir" + exit 1 + fi +} + +main "$@"