Skip to content

Commit 817f546

Browse files
prakashsuryaprakashsurya
authored
Merge pull request #360 from delphix/projects/spec-0003-phase-1
DLPX-97123 Spec 0003 Phase 1: bootstrap merge of sdimitro/sdb + dh-virtualenv overlay
2 parents 437d79e + 3a78e6b commit 817f546

229 files changed

Lines changed: 16235 additions & 4744 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/scripts/download-dumps-from-gdrive.sh‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
#!/bin/bash -eux
2+
3+
#
4+
# Download all the reference core dumps from the public gDrive folder
5+
# and place them in the root directory.
6+
#
7+
python3 -m pip install gdown
8+
gdown --folder 1fdPVuGXbxNKcMEVwhuda04hZTPCcYJms
9+
mv SDB-Public/* .
10+
rmdir SDB-Public

.github/scripts/download-dump-from-s3.sh renamed to .github/scripts/extract-dump.sh

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
#
44
# Assumptions for this to work:
55
# [1] This script is executed from the root of the SDB repo
6-
# [2] The archive downloaded from S3 has one of the following profiles:
6+
# [2] The archive downloaded from gdrive has one of the following profiles:
77
# Profile A - It is lzma-compressed and has the following file structure:
88
# dump-data
99
# ├── dump.201912060006
@@ -33,12 +33,8 @@ if [ ! -d $DATA_DIR ]; then
3333
exit 1
3434
fi
3535

36-
if [ -f "$1" ]; then
37-
echo "Found $1 locally, skip download ..."
38-
else
39-
echo "downloading of $1 from S3 ..."
40-
wget https://sdb-testing-bucket.s3.us-west-2.amazonaws.com/$1
41-
[ $? -eq 0 ] || exit 1
36+
if [ ! -f "$1" ]; then
37+
echo "error: $1: file not found"
4238
fi
4339

4440
if [[ $1 == *.tar.lzma ]]; then

‎.github/scripts/install-drgn.sh‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,11 @@
77
sudo apt update
88
sudo apt install bison flex libelf-dev libdw-dev libomp5 libomp-dev
99

10+
# Install setuptools for Python 3.12+ compatibility
11+
python3 -m pip install --upgrade pip setuptools wheel
12+
1013
git clone https://github.com/osandov/drgn.git
1114

1215
cd drgn
13-
python3 setup.py install
16+
python3 -m pip install .
1417
cd -

‎.github/scripts/install-libkdumpfile.sh‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,15 +2,13 @@
22

33
#
44
# These are build requirements of "libkdumpfile"; if we don't install these,
5-
# the build/install of "libkdumpfile" will fail below. Note that we install
6-
# all version of python3.X-dev so the Github actions jobs can install
7-
# libkdumpfile with the right version.
5+
# the build/install of "libkdumpfile" will fail below. We install python3-dev
6+
# which will work with whatever Python version is set up by actions/setup-python.
87
#
98
sudo apt update
10-
sudo apt install autoconf automake liblzo2-dev libsnappy1v5 libtool pkg-config zlib1g-dev
11-
sudo apt install python3.8-dev python3.9-dev
9+
sudo apt install autoconf automake liblzo2-dev libsnappy-dev libtool pkg-config zlib1g-dev binutils-dev python3-dev
1210

13-
git clone https://github.com/ptesarik/libkdumpfile.git
11+
git clone https://codeberg.org/ptesarik/libkdumpfile.git
1412

1513
cd libkdumpfile
1614
autoreconf -fi

‎.github/workflows/cut-release.yml‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
#
2+
# Cut Release workflow - manually create and push a version tag
3+
#
4+
# This workflow:
5+
# 1. Validates the version format
6+
# 2. Checks if the tag already exists
7+
# 3. Creates and pushes the tag
8+
#
9+
# After running this, the Release workflow will automatically trigger.
10+
#
11+
name: Cut Release
12+
13+
on:
14+
workflow_dispatch:
15+
inputs:
16+
version:
17+
description: 'Version to release (e.g., v0.2.0 or v0.2.0-beta1)'
18+
required: true
19+
type: string
20+
21+
permissions:
22+
contents: write
23+
24+
jobs:
25+
cut-release:
26+
name: Cut Release
27+
runs-on: ubuntu-24.04
28+
29+
steps:
30+
- uses: actions/checkout@v4
31+
with:
32+
fetch-depth: 0
33+
34+
- name: Validate version format
35+
run: |
36+
VERSION="${{ github.event.inputs.version }}"
37+
if [[ ! "$VERSION" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9]+)?$ ]]; then
38+
echo "Error: Version must be in format v0.0.0 or v0.0.0-suffix"
39+
echo "Got: $VERSION"
40+
exit 1
41+
fi
42+
echo "VERSION=$VERSION" >> $GITHUB_ENV
43+
44+
- name: Check if tag already exists
45+
run: |
46+
if git rev-parse "$VERSION" >/dev/null 2>&1; then
47+
echo "Error: Tag $VERSION already exists"
48+
exit 1
49+
fi
50+
51+
- name: Configure git
52+
run: |
53+
git config user.name "github-actions[bot]"
54+
git config user.email "github-actions[bot]@users.noreply.github.com"
55+
56+
- name: Create and push tag
57+
run: |
58+
git tag -a "$VERSION" -m "Release $VERSION"
59+
git push origin "$VERSION"
60+
61+
- name: Summary
62+
run: |
63+
echo "## Tag $VERSION created!" >> $GITHUB_STEP_SUMMARY
64+
echo "" >> $GITHUB_STEP_SUMMARY
65+
echo "The **Release** workflow should now trigger automatically." >> $GITHUB_STEP_SUMMARY
66+
echo "" >> $GITHUB_STEP_SUMMARY
67+
echo "You can monitor it at [Actions → Release](/${{ github.repository }}/actions/workflows/release.yml)" >> $GITHUB_STEP_SUMMARY

‎.github/workflows/main.yml‎

Lines changed: 88 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,10 @@
11
on:
22
push:
3+
branches:
4+
- main
5+
- develop
6+
tags-ignore:
7+
- 'v*'
38
pull_request:
49
schedule:
510
- cron: '0 0 * * *'
@@ -9,16 +14,17 @@ jobs:
914
# Verify the build and installation of SDB.
1015
#
1116
install:
12-
runs-on: ubuntu-20.04
17+
runs-on: ubuntu-24.04
1318
strategy:
1419
matrix:
15-
python-version: [3.8, 3.9]
20+
python-version: ['3.10', '3.12']
1621
steps:
17-
- uses: actions/checkout@v2
18-
- uses: actions/setup-python@v1
22+
- uses: actions/checkout@v4
23+
- uses: actions/setup-python@v5
1924
with:
2025
python-version: ${{ matrix.python-version }}
21-
- run: python3 setup.py install
26+
- run: python3 -m pip install --upgrade pip setuptools wheel
27+
- run: python3 -m pip install .
2228
#
2329
# The statement below is used for debugging the Github job.
2430
#
@@ -31,91 +37,122 @@ jobs:
3137
# the "drgn" from source (there's no package currently available).
3238
#
3339
pylint:
34-
runs-on: ubuntu-20.04
40+
runs-on: ubuntu-24.04
3541
steps:
36-
- uses: actions/checkout@v2
37-
- uses: actions/setup-python@v1
42+
- uses: actions/checkout@v4
43+
- uses: actions/setup-python@v5
3844
with:
39-
python-version: '3.8'
45+
python-version: '3.10'
4046
- run: ./.github/scripts/install-drgn.sh
4147
- run: python3 -m pip install pylint pytest
42-
- run: pylint -d duplicate-code -d invalid-name sdb
43-
- run: pylint -d duplicate-code -d invalid-name tests
48+
- run: python3 -m pip install .
49+
- run: pylint -d duplicate-code -d invalid-name -d missing-docstring -d import-outside-toplevel -d too-many-branches -d missing-module-docstring -d missing-function-docstring sdb
50+
- run: pylint -d duplicate-code -d invalid-name -d missing-docstring -d import-outside-toplevel -d too-many-branches -d missing-module-docstring -d missing-function-docstring tests
4451
#
45-
# Verify "pytest" runs successfully.
52+
# Verify "pytest" runs successfully on unit tests.
4653
#
47-
# Note, we need to have "drgn" installed in order to "pytest". Thus,
54+
# Note, we need to have "drgn" installed in order to run "pytest". Thus,
4855
# prior to running "pytest" we have to clone, build, and install the
49-
# "drgn" from source (there's no package currently available). In
50-
# addition, we install "libkdumpfile" in the same way beforehand
51-
# (there's no package currently for libkdumpfile either) so "drgn"
52-
# can open kdump-compressed crash dumps for the integration tests.
56+
# "drgn" from source (there's no package currently available).
5357
#
54-
pytest:
55-
runs-on: ubuntu-20.04
58+
pytest-unit:
59+
runs-on: ubuntu-24.04
5660
strategy:
5761
matrix:
58-
python-version: [3.8, 3.9]
59-
dump: [dump.201912060006.tar.lzma, dump.202303131823.tar.gz]
62+
python-version: ['3.10', '3.12']
63+
steps:
64+
- uses: actions/checkout@v4
65+
- uses: actions/setup-python@v5
66+
with:
67+
python-version: ${{ matrix.python-version }}
68+
- run: python3 -m pip install pytest pytest-cov
69+
- run: ./.github/scripts/install-drgn.sh
70+
- run: python3 -m pip install .
71+
- run: pytest -v --cov sdb --cov-report xml tests/unit
72+
- name: Upload coverage to Codecov
73+
uses: codecov/codecov-action@v5
74+
with:
75+
token: ${{ secrets.CODECOV_TOKEN }}
76+
files: ./coverage.xml
77+
fail_ci_if_error: false
78+
verbose: true
79+
#
80+
# Verify "pytest" runs successfully on integration tests with crash dumps.
81+
#
82+
# Note, we need to have "drgn" and "libkdumpfile" installed to run pytest
83+
# with crash dumps. We download both reference dumps for full integration testing.
84+
#
85+
pytest-integration:
86+
runs-on: ubuntu-24.04
87+
strategy:
88+
matrix:
89+
python-version: ['3.10', '3.12']
6090
env:
6191
AWS_DEFAULT_REGION: 'us-west-2'
6292
steps:
63-
- uses: actions/checkout@v2
64-
- uses: actions/setup-python@v1
93+
- uses: actions/checkout@v4
94+
- uses: actions/setup-python@v5
6595
with:
6696
python-version: ${{ matrix.python-version }}
67-
- run: python3 -m pip install aws python-config pytest pytest-cov
97+
- run: python3 -m pip install pytest pytest-cov
6898
- run: ./.github/scripts/install-libkdumpfile.sh
6999
- run: ./.github/scripts/install-drgn.sh
70-
- run: ./.github/scripts/download-dump-from-s3.sh ${{ matrix.dump }}
71-
- run: pytest -v --cov sdb --cov-report xml tests
72-
- uses: codecov/codecov-action@v1
100+
- run: python3 -m pip install .
101+
- run: ./.github/scripts/download-dumps-from-gdrive.sh
102+
- run: ./.github/scripts/extract-dump.sh dump.201912060006.tar.lzma
103+
- run: ./.github/scripts/extract-dump.sh dump.202303131823.tar.gz
104+
- run: pytest -v --cov sdb --cov-report xml tests/integration
105+
- name: Upload coverage to Codecov
106+
uses: codecov/codecov-action@v5
73107
with:
74108
token: ${{ secrets.CODECOV_TOKEN }}
109+
files: ./coverage.xml
110+
fail_ci_if_error: false
111+
verbose: true
75112
#
76113
# Verify "yapf" runs successfully.
77114
#
78115
yapf:
79-
runs-on: ubuntu-20.04
116+
runs-on: ubuntu-24.04
80117
steps:
81-
- uses: actions/checkout@v2
82-
- uses: actions/setup-python@v1
118+
- uses: actions/checkout@v4
119+
- uses: actions/setup-python@v5
83120
with:
84-
python-version: '3.8'
121+
python-version: '3.10'
85122
- run: python3 -m pip install yapf
86123
- run: yapf --diff --style google --recursive sdb
87124
- run: yapf --diff --style google --recursive tests
88125
#
126+
# Verify "ruff" runs successfully.
127+
#
128+
ruff:
129+
runs-on: ubuntu-24.04
130+
steps:
131+
- uses: actions/checkout@v4
132+
- uses: actions/setup-python@v5
133+
with:
134+
python-version: '3.10'
135+
- run: python3 -m pip install ruff
136+
- run: ruff check sdb tests
137+
#
89138
# Verify "mypy" runs successfully.
90139
#
91140
# Note, we need to have "drgn" installed in order to run "mypy".
92141
# Thus, prior to running "mypy" we have to clone, build, and install
93142
# the "drgn" from source (there's no package currently available).
94143
#
95-
# Also note the following 2 things specific to mypy:
96-
# [1] We expicitly install version 0.730 as the default version that
97-
# comes with Ubuntu is 0.75 which has a couple of bugs triggered
98-
# by our codebase (they've been resolved on trunk though so we
99-
# may want to change this soon).
100-
# [2] We supply --ignore-missing-imports to the tests package because
101-
# pytest doesn't provide stubs on typeshed.
144+
# Also note that we supply --ignore-missing-imports to the tests package
145+
# because pytest doesn't provide stubs on typeshed.
102146
#
103147
mypy:
104-
runs-on: ubuntu-20.04
148+
runs-on: ubuntu-24.04
105149
steps:
106-
- uses: actions/checkout@v2
107-
- uses: actions/setup-python@v1
150+
- uses: actions/checkout@v4
151+
- uses: actions/setup-python@v5
108152
with:
109-
python-version: '3.8'
153+
python-version: '3.10'
110154
- run: ./.github/scripts/install-drgn.sh
111-
- run: python3 -m pip install mypy==0.730
155+
- run: python3 -m pip install mypy pytest
156+
- run: python3 -m pip install .
112157
- run: python3 -m mypy --strict --show-error-codes -p sdb
113158
- run: python3 -m mypy --strict --ignore-missing-imports --show-error-codes -p tests
114-
#
115-
# Verify that "shfmt" ran successfully against our shell scripts.
116-
#
117-
shfmt:
118-
runs-on: ubuntu-20.04
119-
steps:
120-
- uses: actions/checkout@v2
121-
- uses: delphix/actions/shfmt@master

0 commit comments

Comments
 (0)