DLPX-97123 Spec 0003 Phase 1: bootstrap merge of sdimitro/sdb + dh-virtualenv overlay

.github/scripts/download-dumps-from-gdrive.sh

@@ -0,0 +1,10 @@
+#!/bin/bash -eux
+
+#
+# Download all the reference core dumps from the public gDrive folder
+# and place them in the root directory.
+#
+python3 -m pip install gdown
+gdown --folder 1fdPVuGXbxNKcMEVwhuda04hZTPCcYJms
+mv SDB-Public/* .
+rmdir SDB-Public

.github/scripts/download-dump-from-s3.sh → .github/scripts/extract-dump.sh

@@ -3,7 +3,7 @@
 #
 # Assumptions for this to work:
 # [1] This script is executed from the root of the SDB repo
-# [2] The archive downloaded from S3 has one of the following profiles:
+# [2] The archive downloaded from gdrive has one of the following profiles:
 #     Profile A - It is lzma-compressed and has the following file structure:
 #           dump-data
 #           ├── dump.201912060006
@@ -33,12 +33,8 @@ if [ ! -d $DATA_DIR ]; then
 	exit 1
 fi
 
-if [ -f "$1" ]; then
-	echo "Found $1 locally, skip download ..."
-else
-	echo "downloading of $1 from S3 ..."
-	wget https://sdb-testing-bucket.s3.us-west-2.amazonaws.com/$1
-	[ $? -eq 0 ] || exit 1
+if [ ! -f "$1" ]; then
+	echo "error: $1: file not found"
 fi
 
 if [[ $1 == *.tar.lzma ]]; then

.github/scripts/install-drgn.sh

@@ -7,8 +7,11 @@
 sudo apt update
 sudo apt install bison flex libelf-dev libdw-dev libomp5 libomp-dev
 
+# Install setuptools for Python 3.12+ compatibility
+python3 -m pip install --upgrade pip setuptools wheel
+
 git clone https://github.com/osandov/drgn.git
 
 cd drgn
-python3 setup.py install
+python3 -m pip install .
 cd -

.github/scripts/install-libkdumpfile.sh

@@ -2,15 +2,13 @@
 
 #
 # These are build requirements of "libkdumpfile"; if we don't install these,
-# the build/install of "libkdumpfile" will fail below. Note that we install
-# all version of python3.X-dev so the Github actions jobs can install
-# libkdumpfile with the right version.
+# the build/install of "libkdumpfile" will fail below. We install python3-dev
+# which will work with whatever Python version is set up by actions/setup-python.
 #
 sudo apt update
-sudo apt install autoconf automake liblzo2-dev libsnappy1v5 libtool pkg-config zlib1g-dev
-sudo apt install python3.8-dev python3.9-dev
+sudo apt install autoconf automake liblzo2-dev libsnappy-dev libtool pkg-config zlib1g-dev binutils-dev python3-dev
 
-git clone https://github.com/ptesarik/libkdumpfile.git
+git clone https://codeberg.org/ptesarik/libkdumpfile.git
 
 cd libkdumpfile
 autoreconf -fi

.github/workflows/cut-release.yml

@@ -0,0 +1,67 @@
+#
+# Cut Release workflow - manually create and push a version tag
+#
+# This workflow:
+# 1. Validates the version format
+# 2. Checks if the tag already exists
+# 3. Creates and pushes the tag
+#
+# After running this, the Release workflow will automatically trigger.
+#
+name: Cut Release
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Version to release (e.g., v0.2.0 or v0.2.0-beta1)'
+        required: true
+        type: string
+
+permissions:
+  contents: write
+
+jobs:
+  cut-release:
+    name: Cut Release
+    runs-on: ubuntu-24.04
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Validate version format
+        run: |
+          VERSION="${{ github.event.inputs.version }}"
+          if [[ ! "$VERSION" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9]+)?$ ]]; then
+            echo "Error: Version must be in format v0.0.0 or v0.0.0-suffix"
+            echo "Got: $VERSION"
+            exit 1
+          fi
+          echo "VERSION=$VERSION" >> $GITHUB_ENV
+
+      - name: Check if tag already exists
+        run: |
+          if git rev-parse "$VERSION" >/dev/null 2>&1; then
+            echo "Error: Tag $VERSION already exists"
+            exit 1
+          fi
+
+      - name: Configure git
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+
+      - name: Create and push tag
+        run: |
+          git tag -a "$VERSION" -m "Release $VERSION"
+          git push origin "$VERSION"
+
+      - name: Summary
+        run: |
+          echo "## Tag $VERSION created!" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "The **Release** workflow should now trigger automatically." >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "You can monitor it at [Actions → Release](/${{ github.repository }}/actions/workflows/release.yml)" >> $GITHUB_STEP_SUMMARY

.github/workflows/main.yml

@@ -1,5 +1,10 @@
 on:
   push:
+    branches:
+      - main
+      - develop
+    tags-ignore:
+      - 'v*'
   pull_request:
   schedule:
     - cron:  '0 0 * * *'
@@ -9,16 +14,17 @@ jobs:
   # Verify the build and installation of SDB.
   #
   install:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     strategy:
       matrix:
-        python-version: [3.8, 3.9]
+        python-version: ['3.10', '3.12']
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v1
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-      - run: python3 setup.py install
+      - run: python3 -m pip install --upgrade pip setuptools wheel
+      - run: python3 -m pip install .
         #
         # The statement below is used for debugging the Github job.
         #
@@ -31,91 +37,122 @@ jobs:
   # the "drgn" from source (there's no package currently available).
   #
   pylint:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v1
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
         with:
-          python-version: '3.8'
+          python-version: '3.10'
       - run: ./.github/scripts/install-drgn.sh
       - run: python3 -m pip install pylint pytest
-      - run: pylint -d duplicate-code -d invalid-name sdb
-      - run: pylint -d duplicate-code -d invalid-name tests
+      - run: python3 -m pip install .
+      - run: pylint -d duplicate-code -d invalid-name -d missing-docstring -d import-outside-toplevel -d too-many-branches -d missing-module-docstring -d missing-function-docstring sdb
+      - run: pylint -d duplicate-code -d invalid-name -d missing-docstring -d import-outside-toplevel -d too-many-branches -d missing-module-docstring -d missing-function-docstring tests
   #
-  # Verify "pytest" runs successfully.
+  # Verify "pytest" runs successfully on unit tests.
   #
-  # Note, we need to have "drgn" installed in order to "pytest". Thus,
+  # Note, we need to have "drgn" installed in order to run "pytest". Thus,
   # prior to running "pytest" we have to clone, build, and install the
-  # "drgn" from source (there's no package currently available). In
-  # addition, we install "libkdumpfile" in the same way beforehand
-  # (there's no package currently for libkdumpfile either) so "drgn"
-  # can open kdump-compressed crash dumps for the integration tests.
+  # "drgn" from source (there's no package currently available).
   #
-  pytest:
-    runs-on: ubuntu-20.04
+  pytest-unit:
+    runs-on: ubuntu-24.04
     strategy:
       matrix:
-        python-version: [3.8, 3.9]
-        dump: [dump.201912060006.tar.lzma, dump.202303131823.tar.gz]
+        python-version: ['3.10', '3.12']
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - run: python3 -m pip install pytest pytest-cov
+      - run: ./.github/scripts/install-drgn.sh
+      - run: python3 -m pip install .
+      - run: pytest -v --cov sdb --cov-report xml tests/unit
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v5
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: ./coverage.xml
+          fail_ci_if_error: false
+          verbose: true
+  #
+  # Verify "pytest" runs successfully on integration tests with crash dumps.
+  #
+  # Note, we need to have "drgn" and "libkdumpfile" installed to run pytest
+  # with crash dumps. We download both reference dumps for full integration testing.
+  #
+  pytest-integration:
+    runs-on: ubuntu-24.04
+    strategy:
+      matrix:
+        python-version: ['3.10', '3.12']
     env:
       AWS_DEFAULT_REGION: 'us-west-2'
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v1
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-      - run: python3 -m pip install aws python-config pytest pytest-cov
+      - run: python3 -m pip install pytest pytest-cov
       - run: ./.github/scripts/install-libkdumpfile.sh
       - run: ./.github/scripts/install-drgn.sh
-      - run: ./.github/scripts/download-dump-from-s3.sh ${{ matrix.dump }}
-      - run: pytest -v --cov sdb --cov-report xml tests
-      - uses: codecov/codecov-action@v1
+      - run: python3 -m pip install .
+      - run: ./.github/scripts/download-dumps-from-gdrive.sh
+      - run: ./.github/scripts/extract-dump.sh dump.201912060006.tar.lzma
+      - run: ./.github/scripts/extract-dump.sh dump.202303131823.tar.gz
+      - run: pytest -v --cov sdb --cov-report xml tests/integration
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
+          files: ./coverage.xml
+          fail_ci_if_error: false
+          verbose: true
   #
   # Verify "yapf" runs successfully.
   #
   yapf:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v1
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
         with:
-          python-version: '3.8'
+          python-version: '3.10'
       - run: python3 -m pip install yapf
       - run: yapf --diff --style google --recursive sdb
       - run: yapf --diff --style google --recursive tests
   #
+  # Verify "ruff" runs successfully.
+  #
+  ruff:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+      - run: python3 -m pip install ruff
+      - run: ruff check sdb tests
+  #
   # Verify "mypy" runs successfully.
   #
   # Note, we need to have "drgn" installed in order to run "mypy".
   # Thus, prior to running "mypy" we have to clone, build, and install
   # the "drgn" from source (there's no package currently available).
   #
-  # Also note the following 2 things specific to mypy:
-  # [1] We expicitly install version 0.730 as the default version that
-  #     comes with Ubuntu is 0.75 which has a couple of bugs triggered
-  #     by our codebase (they've been resolved on trunk though so we
-  #     may want to change this soon).
-  # [2] We supply --ignore-missing-imports to the tests package because
-  #     pytest doesn't provide stubs on typeshed.
+  # Also note that we supply --ignore-missing-imports to the tests package
+  # because pytest doesn't provide stubs on typeshed.
   #
   mypy:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v1
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
         with:
-          python-version: '3.8'
+          python-version: '3.10'
       - run: ./.github/scripts/install-drgn.sh
-      - run: python3 -m pip install mypy==0.730
+      - run: python3 -m pip install mypy pytest
+      - run: python3 -m pip install .
       - run: python3 -m mypy --strict --show-error-codes -p sdb
       - run: python3 -m mypy --strict --ignore-missing-imports --show-error-codes -p tests
-  #
-  # Verify that "shfmt" ran successfully against our shell scripts.
-  #
-  shfmt:
-    runs-on: ubuntu-20.04
-    steps:
-      - uses: actions/checkout@v2
-      - uses: delphix/actions/shfmt@master