diff --git a/.github/workflows/build-binary.yml b/.github/workflows/build-binary.yml new file mode 100644 index 0000000000..40e5076340 --- /dev/null +++ b/.github/workflows/build-binary.yml @@ -0,0 +1,162 @@ +# .github/workflows/build-binary.yml +# Reusable workflow – called by continuous-integration.yml and nightly.yml + +name: Build binary + +on: + workflow_call: + inputs: + build_label: + description: "Label appended to the output folder name, e.g. '20240101-abc1234'" + type: string + required: true + os: + description: "Runner OS: 'ubuntu-22.04' or 'windows-latest'" + type: string + required: true + outputs: + artifact_name: + description: "Name of the uploaded artifact" + value: ${{ jobs.build.outputs.artifact_name }} + +jobs: + build: + runs-on: ${{ inputs.os }} + + outputs: + artifact_name: ${{ steps.set-names.outputs.artifact_name }} + + steps: + - uses: actions/checkout@v6 + + - name: Set up Python 3.11 + uses: actions/setup-python@v6 + with: + python-version: '3.11' + + - name: Compute platform names + id: set-names + shell: bash + run: | + if [ "$RUNNER_OS" = "Windows" ]; then + echo "platform=windows" >> "$GITHUB_OUTPUT" + echo "output_dir=Meshroom-${{ inputs.build_label }}-windows" >> "$GITHUB_OUTPUT" + echo "archive=Meshroom-${{ inputs.build_label }}-windows.zip" >> "$GITHUB_OUTPUT" + echo "artifact_name=binary-windows" >> "$GITHUB_OUTPUT" + else + echo "platform=linux" >> "$GITHUB_OUTPUT" + echo "output_dir=Meshroom-${{ inputs.build_label }}-linux" >> "$GITHUB_OUTPUT" + echo "archive=Meshroom-${{ inputs.build_label }}-linux.tar.gz" >> "$GITHUB_OUTPUT" + echo "artifact_name=binary-linux" >> "$GITHUB_OUTPUT" + fi + + - name: Install system dependencies (Linux only) + if: runner.os == 'Linux' + run: | + sudo apt-get update -qq + sudo apt-get install -y --no-install-recommends \ + libgl1 libglib2.0-0 libdbus-1-3 \ + libxcb-cursor0 libxcb-xinerama0 libxcb-icccm4 \ + libxcb-image0 libxcb-keysyms1 libxcb-randr0 \ + libxcb-render-util0 libxcb-shape0 libxkbcommon-x11-0 + + - name: Install Python dependencies + shell: bash + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt -r dev_requirements.txt + + - name: Build executable + shell: bash + env: + OUTPUT_DIR: ${{ steps.set-names.outputs.output_dir }} + run: | + python setup.py install_exe -d "$OUTPUT_DIR" + + - name: Fix missing PySide6 DLL (Windows only) + if: runner.os == 'Windows' + shell: cmd + env: + OUTPUT_DIR: ${{ steps.set-names.outputs.output_dir }} + run: | + curl -L "https://drive.google.com/uc?export=download&id=1vhPDmDQJJfM_hBD7KVqRfh8tiqTCN7Jv" ^ + -o "%OUTPUT_DIR%\lib\PySide6\Qt63DQuickScene3D.dll" + + - name: Remove unused PySide6 files (Windows) + if: runner.os == 'Windows' + shell: cmd + env: + OUTPUT_DIR: ${{ steps.set-names.outputs.output_dir }} + run: | + cd "%OUTPUT_DIR%\lib\PySide6" + del /s /q Qt6Web*.dll Qt6Designer*.dll *.exe + if exist resources rmdir /s /q resources + if exist translations rmdir /s /q translations + if exist typesystems rmdir /s /q typesystems + if exist examples rmdir /s /q examples + if exist include rmdir /s /q include + + - name: Remove unused PySide6 files (Linux) + if: runner.os == 'Linux' + env: + OUTPUT_DIR: ${{ steps.set-names.outputs.output_dir }} + run: | + cd "${OUTPUT_DIR}/lib/PySide6" + rm -rf resources translations typesystems examples include + cd Qt/lib + rm -f \ + *Qt6WebEngine*.so* \ + *Qt6Designer*.so* \ + *Qt6Multimedia*.so* \ + *Qt6SpatialAudio*.so* \ + *Qt6Charts*.so* \ + *Qt6DataVisualization*.so* \ + *Qt6Graphs*.so* \ + *Qt6Bluetooth*.so* \ + *Qt6Nfc*.so* \ + *Qt6SerialPort*.so* \ + *Qt6SerialBus*.so* \ + *Qt6Positioning*.so* \ + *Qt6Location*.so* \ + *Qt6Sensors*.so* \ + *Qt6TextToSpeech*.so* \ + *Qt6VirtualKeyboard*.so* \ + *Qt6WebSockets*.so* \ + *Qt6WebChannel*.so* \ + *Qt6Pdf*.so* \ + *Qt6Quick3D*.so* \ + *Qt6RemoteObjects*.so* \ + *Qt6Scxml*.so* \ + *Qt6StateMachine*.so* \ + *Qt6NetworkAuth*.so* + + - name: Strip debug symbols from .so files (Linux) + if: runner.os == 'Linux' + env: + OUTPUT_DIR: ${{ steps.set-names.outputs.output_dir }} + run: | + find "$OUTPUT_DIR" -name "*.so*" -not -type l | xargs strip --strip-unneeded + + - name: Create archive (Windows) + if: github.workflow == 'Nightly' && runner.os == 'Windows' + shell: pwsh + env: + OUTPUT_DIR: ${{ steps.set-names.outputs.output_dir }} + ARCHIVE: ${{ steps.set-names.outputs.archive }} + run: Compress-Archive -Path "$env:OUTPUT_DIR" -DestinationPath "$env:ARCHIVE" + + - name: Create archive (Linux) + if: github.workflow == 'Nightly' && runner.os == 'Linux' + env: + OUTPUT_DIR: ${{ steps.set-names.outputs.output_dir }} + ARCHIVE: ${{ steps.set-names.outputs.archive }} + run: tar -czf "$ARCHIVE" "$OUTPUT_DIR" + + - name: Upload artifact + uses: actions/upload-artifact@v7 + if: github.workflow == 'Nightly' + with: + name: ${{ steps.set-names.outputs.artifact_name }} + path: ${{ steps.set-names.outputs.archive }} + retention-days: 1 + diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index e04913250d..248d3e56f9 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -1,3 +1,5 @@ +# .github/workflows/continuous-integration.yml + name: Continuous Integration on: @@ -16,93 +18,19 @@ on: - '**.rst' - 'docs/**' -env: - CI: True - PYTHONPATH: ${{ github.workspace }} - jobs: - build-linux: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: [ 3.11 ] - - steps: - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install flake8 pytest pytest-cov - pip install -r requirements.txt -r dev_requirements.txt --timeout 45 - - name: Lint with flake8 - run: | - # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - name: Test with pytest - run: | - pytest tests/ - pytest --cov --cov-report=xml --junitxml=junit.xml - - name: Upload results to Codecov - uses: codecov/codecov-action@v5 - with: - token: ${{ secrets.CODECOV_TOKEN }} - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} - - name: Set up Python 3.9 - meshroom_compute test - uses: actions/setup-python@v4 - with: - python-version: 3.9 - - name: Install dependencies (Python 3.9) - meshroom_compute test - run: | - python3.9 -m pip install --upgrade pip - python3.9 -m pip install -r requirements.txt --timeout 45 - - name: Run imports - meshroom_compute test - run: | - python3.9 bin/meshroom_compute -h - - build-windows: - runs-on: windows-latest - strategy: - matrix: - python-version: [ 3.11 ] + test-linux: + name: Test (Linux) + uses: ./.github/workflows/run-tests.yml + with: + os: ubuntu-latest + python-version: '3.11' + secrets: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - steps: - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install flake8 pytest - pip install -r requirements.txt -r dev_requirements.txt --timeout 45 - - name: Lint with flake8 - run: | - # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - name: Test with pytest - run: | - pytest tests/ - - name: Set up Python 3.9 - meshroom_compute test - uses: actions/setup-python@v4 - with: - python-version: 3.9 - - name: Install dependencies (Python 3.9) - meshroom_compute test - run: | - python3 -m pip install --upgrade pip - python3 -m pip install -r requirements.txt --timeout 45 - - name: Run imports - meshroom_compute test - run: | - python3 bin/meshroom_compute -h + test-windows: + name: Test (Windows) + uses: ./.github/workflows/run-tests.yml + with: + os: windows-latest + python-version: '3.11' diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml new file mode 100644 index 0000000000..6064e7eabb --- /dev/null +++ b/.github/workflows/nightly.yml @@ -0,0 +1,177 @@ +name: Nightly Build + +on: + schedule: + # Runs every day at 02:00 UTC + - cron: '0 2 * * *' + # Allow manual trigger from the Actions tab + workflow_dispatch: + +permissions: + contents: write # needed to edit releases, upload/delete assets and update "nightly" tag + +jobs: + # ──────────────────────────────────────────────────────────── + # Guard: skip everything if develop hasn't changed since the + # SHA recorded in the single persistent "nightly" release. + # ──────────────────────────────────────────────────────────── + check-changes: + name: Check for new commits on develop + runs-on: ubuntu-latest + outputs: + should_build: ${{ steps.check.outputs.should_build }} + short_sha: ${{ steps.check.outputs.short_sha }} + full_sha: ${{ steps.check.outputs.full_sha }} + build_label: ${{ steps.check.outputs.build_label }} + + steps: + - name: Checkout develop + uses: actions/checkout@v6 + with: + ref: develop + fetch-tags: true # required to find the previous nightly tag + + - name: Determine whether a build is needed + id: check + env: + GH_TOKEN: ${{ github.token }} + run: | + SHORT_SHA=$(git rev-parse --short HEAD) + FULL_SHA=$(git rev-parse HEAD) + BUILD_LABEL="$(date -u '+%Y.%m.%d')-${SHORT_SHA}" + echo "short_sha=$SHORT_SHA" >> "$GITHUB_OUTPUT" + echo "full_sha=$FULL_SHA" >> "$GITHUB_OUTPUT" + echo "build_label=$BUILD_LABEL" >> "$GITHUB_OUTPUT" + + # Read the SHA stored in the body of the persistent "nightly" release. + # The body always contains a line of the form: **Latest commit:** `` + # LAST_SHA=$( + # gh release view nightly \ + # --repo "$GITHUB_REPOSITORY" \ + # --json body \ + # --jq '.body' 2>/dev/null \ + # | grep -oP '(?<=\*\*Latest commit:\*\* `)[\w]+' \ + # || true + # ) + # Read the SHA of the last "nightly" git tag + LAST_SHA=$(git rev-parse --short nightly 2>/dev/null || true) + + if [ "$SHORT_SHA" = "$LAST_SHA" ]; then + echo "develop unchanged since last nightly (SHA: $SHORT_SHA). Skipping." + echo "should_build=false" >> "$GITHUB_OUTPUT" + else + echo "New commits detected (current: $SHORT_SHA, last nightly: ${LAST_SHA:-none}). Building." + echo "should_build=true" >> "$GITHUB_OUTPUT" + fi + + build-windows: + name: Build Windows binary + needs: check-changes + if: needs.check-changes.outputs.should_build == 'true' + uses: ./.github/workflows/build-binary.yml + with: + os: windows-latest + build_label: ${{ needs.check-changes.outputs.build_label }} + + build-linux: + name: Build Linux binary + needs: check-changes + if: needs.check-changes.outputs.should_build == 'true' + uses: ./.github/workflows/build-binary.yml + with: + os: ubuntu-22.04 + build_label: ${{ needs.check-changes.outputs.build_label }} + + # ──────────────────────────────────────────────────────────── + # Update the single persistent "nightly" pre-release: + # - delete old assets + # - upload new assets + # - prepend a build entry to the release body + # - prune body entries older than 3 days + # ──────────────────────────────────────────────────────────── + release: + name: Update nightly release + needs: [check-changes, build-windows, build-linux] + if: needs.check-changes.outputs.should_build == 'true' + runs-on: ubuntu-latest + + permissions: + contents: write # to push git tag nightly + + env: + SHORT_SHA: ${{ needs.check-changes.outputs.short_sha }} + BUILD_LABEL: ${{ needs.check-changes.outputs.build_label }} + GH_TOKEN: ${{ github.token }} + + steps: + - name: Download all build artifacts + uses: actions/download-artifact@v8 + with: + path: dist/ + + # ── Create the release if it doesn't exist yet ────────── + - name: Ensure "nightly" release exists + run: | + if ! gh release view nightly --repo "$GITHUB_REPOSITORY" &>/dev/null; then + echo "Creating initial nightly release…" + gh release create nightly \ + --repo "$GITHUB_REPOSITORY" \ + --title "Nightly build (develop)" \ + --notes "_(first build — history will appear here)_" \ + --prerelease + fi + + # ── Replace assets ─────────────────────────────────────── + # Delete every existing asset on the release, then upload the fresh ones. + # This keeps the release page clean: always exactly two assets. + - name: Delete existing assets + run: | + gh release view nightly \ + --repo "$GITHUB_REPOSITORY" \ + --json assets \ + --jq '.assets[].name' \ + | while read -r ASSET; do + echo " → Deleting asset: $ASSET" + gh release delete-asset nightly "$ASSET" \ + --repo "$GITHUB_REPOSITORY" --yes + done + + - name: Upload new assets + run: | + find dist/ -type f -ls + gh release upload nightly \ + --repo "$GITHUB_REPOSITORY" \ + "dist/binary-windows/Meshroom-${BUILD_LABEL}-windows.zip" \ + "dist/binary-linux/Meshroom-${BUILD_LABEL}-linux.tar.gz" + + # ── Update release body ────────────────────────────────── + # Rewrite the release body with the latest build info. + - name: Update release notes + run: | + NOW=$(date -u '+%Y-%m-%d %H:%M UTC') + + cat > nightly_release_notes.md << EOF + > [!WARNING] + > **Automated nightly build** from the \`develop\` branch. May be unstable. + + | | | + |---|---| + | Date | ${NOW} | + | Commit | [${SHORT_SHA}](https://github.com/${{ github.repository }}/commit/${{ github.sha }}) | + | **Windows** | \`Meshroom-${BUILD_LABEL}-windows.zip\` | + | **Linux** | \`Meshroom-${BUILD_LABEL}-linux.tar.gz\` | + EOF + + gh release edit nightly \ + --repo "$GITHUB_REPOSITORY" \ + --notes-file nightly_release_notes.md + + # ── Move the nightly tag to current HEAD ────── + - name: Re-tag nightly + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + # Force-update the nightly tag to current HEAD via the GitHub API. + # This avoids needing git remote auth or being inside a git directory. + gh api --method PATCH "/repos/${{ github.repository }}/git/refs/tags/nightly" -f sha="${{ github.sha }}" -F force=true || \ + gh api --method POST "/repos/${{ github.repository }}/git/refs" -f ref="refs/tags/nightly" -f sha="${{ github.sha }}" diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml new file mode 100644 index 0000000000..ab4485fecb --- /dev/null +++ b/.github/workflows/run-tests.yml @@ -0,0 +1,97 @@ +# .github/workflows/run-tests.yml +# Reusable workflow – called by continuous-integration.yml +# +# Runs on a single OS/Python combination: +# 1. Lint with flake8 +# 2. Test with pytest (+coverage upload on Linux) +# 3. Smoke-test meshroom_compute with Python 3.9 + +name: Run tests + +on: + workflow_call: + inputs: + os: + description: "Runner OS: 'ubuntu-latest' or 'windows-latest'" + type: string + required: true + python-version: + description: "Python version used for lint + tests" + type: string + default: '3.11' + secrets: + CODECOV_TOKEN: + description: "Codecov upload token (only needed on Linux)" + required: false + +env: + CI: true + PYTHONPATH: ${{ github.workspace }} + +jobs: + test: + runs-on: ${{ inputs.os }} + + steps: + - uses: actions/checkout@v6 + + # ── Primary Python environment ─────────────────────────── + - name: Set up Python ${{ inputs.python-version }} + uses: actions/setup-python@v6 + with: + python-version: ${{ inputs.python-version }} + + - name: Install dependencies + shell: bash + run: | + python -m pip install --upgrade pip + pip install flake8 pytest pytest-cov + pip install -r requirements.txt -r dev_requirements.txt --timeout 45 + + # ── Lint ───────────────────────────────────────────────── + - name: Lint with flake8 + run: | + # Stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # Treat all other errors as warnings (line length 127, GitHub editor width) + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + + # ── Tests ──────────────────────────────────────────────── + - name: Test with pytest + run: pytest tests/ + + # Coverage is generated and uploaded only on Linux to avoid double-counting + - name: Test with pytest (coverage) + if: runner.os == 'Linux' + run: pytest --cov --cov-report=xml --junitxml=junit.xml + + - name: Upload coverage to Codecov + if: runner.os == 'Linux' + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + + - name: Upload test results to Codecov + if: runner.os == 'Linux' && !cancelled() + uses: codecov/test-results-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + + # ── meshroom_compute smoke test (Python 3.9) ───────────── + # Verifies that the CLI entry-point works on the minimum supported version. + - name: Set up Python 3.9 – meshroom_compute smoke test + if: runner.os == 'Linux' + uses: actions/setup-python@v6 + with: + python-version: '3.9' + + - name: Install runtime dependencies (Python 3.9) + if: runner.os == 'Linux' + shell: bash + run: | + python3.9 -m pip install --upgrade pip + python3.9 -m pip install -r requirements.txt --timeout 45 + + - name: Run meshroom_compute –h (Python 3.9) + if: runner.os == 'Linux' + run: python3.9 bin/meshroom_compute -h diff --git a/dev_requirements.txt b/dev_requirements.txt index 22eb613ca0..e12f105ff2 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -1,5 +1,5 @@ # packaging -cx_Freeze==7.2.10 +cx_Freeze==8.6.4 # Python binding packaging numpy==1.* diff --git a/setup.py b/setup.py index 9095d2acc9..4c554cb8eb 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,10 @@ -import platform - import os +import pathlib +import platform +import subprocess import setuptools # for bdist from cx_Freeze import setup, Executable +from cx_Freeze.command.build_exe import build_exe import meshroom currentDir = os.path.dirname(os.path.abspath(__file__)) @@ -40,6 +42,33 @@ def __init__(self, script, initScript=None, base=None, targetName=None, icons=No super(PlatformExecutable, self).__init__(script, initScript, base, targetName, icon, shortcutName, shortcutDir, copyright, trademarks) +# Post-build strip for Linux +cmdclass = {} + +if platform.system() == "Linux": + + class build_exe_and_strip(build_exe): + """Strip debug symbols from all .so files after the normal build.""" + + def run(self): + super().run() + + build_dir = pathlib.Path(self.build_exe) + so_files = [ + f for f in build_dir.rglob("*.so*") + if f.is_file() and not f.is_symlink() + ] + print(f"-- Stripping {len(so_files)} .so files in {build_dir}") + for so in so_files: + result = subprocess.run( + ["strip", "--strip-unneeded", str(so)], + capture_output=True, text=True, + ) + if result.returncode != 0: + print(f" WARNING: strip failed on {so.name}: {result.stderr.strip()}") + print("-- Stripping done.") + + cmdclass = {"build_exe": build_exe_and_strip} build_exe_options = { # include dynamically loaded plugins @@ -58,7 +87,46 @@ def __init__(self, script, initScript=None, base=None, targetName=None, icons=No "cmath", "numpy" ], - "include_files": ["CHANGES.md", "COPYING.md", "LICENSE-MPL2.md", "README.md", "bin"] + "include_files": ["CHANGES.md", "COPYING.md", "LICENSE-MPL2.md", "README.md", "bin"], + "excludes": [ + # Python stdlib bloat + "tkinter", + "unittest", + "email", + "html", + "http.server", + "xmlrpc", + # Unused PySide6/Qt modules + "PySide6.QtWebEngineCore", + "PySide6.QtWebEngineWidgets", + "PySide6.QtWebEngineQuick", + "PySide6.QtMultimedia", + "PySide6.QtMultimediaWidgets", + "PySide6.QtSpatialAudio", + "PySide6.QtCharts", + "PySide6.QtDataVisualization", + "PySide6.QtGraphs", + "PySide6.QtGraphsWidgets", + "PySide6.QtBluetooth", + "PySide6.QtNfc", + "PySide6.QtSerialPort", + "PySide6.QtSerialBus", + "PySide6.QtPositioning", + "PySide6.QtLocation", + "PySide6.QtSensors", + "PySide6.QtTextToSpeech", + "PySide6.QtVirtualKeyboard", + "PySide6.QtWebSockets", + "PySide6.QtWebChannel", + "PySide6.QtPdf", + "PySide6.QtPdfWidgets", + "PySide6.QtQuick3D", + "PySide6.QtRemoteObjects", + "PySide6.QtScxml", + "PySide6.QtStateMachine", + "PySide6.QtNetworkAuth", + "PySide6.QtAxContainer", + ], } if os.path.isdir(os.path.join(currentDir, "tractor")): build_exe_options["packages"].append("tractor") @@ -150,4 +218,5 @@ def __init__(self, script, initScript=None, base=None, targetName=None, icons=No version=meshroom.__version__, options={"build_exe": build_exe_options}, executables=executables, + cmdclass=cmdclass, )