📦 Pack and publish 9b740e80a1434a98b54c0c2b0349a4cfe23a9f86 #2081
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: 📦 Pack and publish | |
| run-name: 📦 Pack and publish ${{ github.sha }} | |
| #on: workflow_call | |
| on: | |
| workflow_run: | |
| workflows: [ 🔬 Test ] | |
| types: [ completed ] | |
| branches: [ master ] | |
| pull_request: | |
| branches: [ master ] | |
| types: [opened, synchronize, reopened, labeled, unlabeled] | |
| push: | |
| branches: | |
| - columnar-* | |
| tags: | |
| - 'release*' | |
| - 'pack_publish' | |
| # cancels the previous workflow run when a new one appears in the same branch (e.g. master or a PR's branch) | |
| concurrency: | |
| group: pack_${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| pack: | |
| name: OK to pack? | |
| runs-on: ubuntu-22.04 | |
| outputs: | |
| version: ${{ steps.semver-tagger.outputs.version }} | |
| version_full: ${{ steps.semver-tagger.outputs.version_full }} | |
| version_rpm: ${{ steps.semver-tagger.outputs.version_rpm }} | |
| target: ${{ steps.semver-tagger.outputs.target }} | |
| should_continue: ${{ steps.check-should-continue.outputs.should_continue }} | |
| if: | | |
| (github.event_name == 'pull_request' && (contains(github.event.pull_request.labels.*.name, 'pack') || contains(github.event.pull_request.labels.*.name, 'publish'))) | |
| || ( github.event_name == 'push' && ( startsWith( github.ref, 'refs/heads/columnar-' ) || contains( github.ref, 'refs/tags/pack_publish' ) ) ) | |
| || (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success' && github.ref == 'refs/heads/master' && github.event.workflow_run.event != 'schedule') | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Update version | |
| id: semver-tagger | |
| uses: manticoresoftware/semver-tagger-action@main | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| conventional_commits_authors: alexey@manticoresearch.com | |
| ignore_patterns: '\.md$|^test/|^manual/|\.clt|\.github|\.patterns|\.yml|\.gitignore' | |
| debug: true | |
| - name: Check if we should continue packing | |
| id: check-should-continue | |
| run: | | |
| # Continue if version was updated, if we have the "pack" label on PR, or if target is "release" | |
| if [[ "${{ steps.semver-tagger.outputs.version_updated }}" == "true" ]]; then | |
| echo "Continuing because version was updated" | |
| echo "should_continue=true" >> $GITHUB_OUTPUT | |
| elif [[ "${{ steps.semver-tagger.outputs.target }}" == "release" ]]; then | |
| echo "Continuing because target is release" | |
| echo "should_continue=true" >> $GITHUB_OUTPUT | |
| elif [[ "${{ github.event_name }}" == "pull_request" && "${{ contains(github.event.pull_request.labels.*.name, 'pack') }}" == "true" ]]; then | |
| echo "Continuing because PR has 'pack' label" | |
| echo "should_continue=true" >> $GITHUB_OUTPUT | |
| elif [[ "${{ contains(github.ref, 'refs/tags/pack_publish') }}" == "true" ]]; then | |
| echo "Continuing because commit has 'pack_publish' tag" | |
| echo "should_continue=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "Skipping packing because version wasn't updated, target is not release, and there's no 'pack' label or tag" | |
| echo "should_continue=false" >> $GITHUB_OUTPUT | |
| fi | |
| - run: | | |
| echo "# Packing and publishing all for commit ${{ github.sha }}" >> $GITHUB_STEP_SUMMARY | |
| echo "* Commit URL: [${{ github.sha }}](/${{ github.repository }}/commit/${{ github.sha }})" >> $GITHUB_STEP_SUMMARY | |
| echo "* Initiated by: [@${{ github.actor }}](https://github.com/${{ github.actor }})" >> $GITHUB_STEP_SUMMARY | |
| echo "* Ref: ${{ github.ref_type }} \"${{ github.ref_name }}\"" >> $GITHUB_STEP_SUMMARY | |
| echo "* Attempt: ${{ github.run_attempt }}" >> $GITHUB_STEP_SUMMARY | |
| echo "* Version updated: ${{ steps.semver-tagger.outputs.version_updated }}" >> $GITHUB_STEP_SUMMARY | |
| echo "* Should continue packing: ${{ steps.check-should-continue.outputs.should_continue }}" >> $GITHUB_STEP_SUMMARY | |
| check_branch: | |
| name: Check branch existence | |
| runs-on: ubuntu-22.04 | |
| outputs: | |
| manticore_locator: ${{ steps.set_locator.outputs.manticore_locator }} | |
| steps: | |
| - name: Check if branch exists in manticoresoftware/manticoresearch | |
| id: check_branch | |
| if: github.ref_name != 'master' | |
| run: | | |
| # Extract the actual branch name for pull requests | |
| if [[ "${{ github.event_name }}" == "pull_request" ]]; then | |
| BRANCH_NAME="${{ github.event.pull_request.head.ref }}" | |
| else | |
| BRANCH_NAME="${{ github.ref_name }}" | |
| fi | |
| HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" https://api.github.com/repos/manticoresoftware/manticoresearch/branches/$BRANCH_NAME) | |
| if [ "$HTTP_STATUS" -eq "200" ]; then | |
| echo "branch_exists=true" >> $GITHUB_OUTPUT | |
| echo "branch_name=$BRANCH_NAME" >> $GITHUB_OUTPUT | |
| else | |
| echo "branch_exists=false" >> $GITHUB_OUTPUT | |
| echo "branch_name=$BRANCH_NAME" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Set Manticore Locator | |
| id: set_locator | |
| run: | | |
| if [[ "${{ github.event.schedule }}" == "00 20 * * *" ]]; then | |
| echo "manticore_locator=GIT_REPOSITORY https://github.com/manticoresoftware/manticoresearch.git GIT_TAG main GIT_SUBMODULES \"\"" >> $GITHUB_OUTPUT | |
| elif [[ "${{ github.ref_name }}" != "master" && "${{ steps.check_branch.outputs.branch_exists }}" == "true" ]]; then | |
| echo "manticore_locator=GIT_REPOSITORY https://github.com/manticoresoftware/manticoresearch.git GIT_TAG ${{ steps.check_branch.outputs.branch_name }} GIT_SUBMODULES \"\"" >> $GITHUB_OUTPUT | |
| else | |
| echo "manticore_locator=" >> $GITHUB_OUTPUT | |
| fi | |
| embedding_builds: | |
| needs: pack | |
| uses: ./.github/workflows/embedding_build_template.yml | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| # Linux builds | |
| - distr: linux | |
| arch: x86_64 | |
| - distr: linux | |
| arch: aarch64 | |
| # macOS builds | |
| - distr: macos | |
| arch: x86_64 | |
| - distr: macos | |
| arch: aarch64 | |
| # Windows builds | |
| - distr: windows | |
| arch: x86_64 | |
| - distr: windows | |
| arch: aarch64 | |
| name: ${{ matrix.distr }} ${{ matrix.arch }} embedding packing | |
| with: | |
| distr: ${{ matrix.distr }} | |
| arch: ${{ matrix.arch }} | |
| pack_debian_ubuntu: | |
| uses: ./.github/workflows/build_template.yml | |
| needs: [embedding_builds, check_branch, pack] | |
| if: needs.pack.outputs.should_continue == 'true' | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| DISTR: [bionic, focal, jammy, buster, bullseye, bookworm] | |
| arch: [x86_64, aarch64] | |
| name: ${{ matrix.DISTR }} ${{ matrix.arch }} packing | |
| with: | |
| MANTICORE_LOCATOR: ${{ needs.check_branch.outputs.manticore_locator }} | |
| DISTR: ${{ matrix.DISTR }} | |
| arch: ${{ matrix.arch }} | |
| cmake_command: | | |
| mkdir build | |
| cd build | |
| cmake -DPACK=1 .. | |
| cmake --build . --target package | |
| cache_key: pack_${{ matrix.DISTR }}_${{ matrix.arch }} | |
| artifact_list: "build/manticore*deb" | |
| embeddings_artifact: embeddings_linux_${{ matrix.arch }} | |
| version: ${{ needs.pack.outputs.version }} | |
| package_version: ${{ needs.pack.outputs.version_full }} | |
| pack_rhel: | |
| uses: ./.github/workflows/build_template.yml | |
| needs: [embedding_builds, check_branch, pack] | |
| if: needs.pack.outputs.should_continue == 'true' | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| DISTR: [rhel7, rhel8, rhel9, rhel10] | |
| arch: [x86_64, aarch64] | |
| name: ${{ matrix.DISTR }} ${{ matrix.arch }} packing | |
| with: | |
| MANTICORE_LOCATOR: ${{ needs.check_branch.outputs.manticore_locator }} | |
| DISTR: ${{ matrix.DISTR }} | |
| arch: ${{ matrix.arch }} | |
| boost: boost_rhel_feb17 | |
| cmake_command: | | |
| ln -s $(pwd) /builds_manticoresearch_dev_usr_src_debug_manticore_component_src_0 | |
| cd /builds_manticoresearch_dev_usr_src_debug_manticore_component_src_0 | |
| mkdir build | |
| cd build | |
| CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX=/builds_manticoresearch_dev_usr_src_debug_manticore_component cmake -DPACK=1 .. | |
| cmake --build . --target package | |
| cache_key: pack_${{ matrix.DISTR }}_${{ matrix.arch }} | |
| artifact_list: "build/manticore*rpm" | |
| embeddings_artifact: embeddings_linux_${{ matrix.arch }} | |
| version: ${{ needs.pack.outputs.version }} | |
| package_version: ${{ needs.pack.outputs.version_full }} | |
| rpm_package_version: ${{ needs.pack.outputs.version_rpm }} | |
| pack_macos: | |
| uses: ./.github/workflows/build_template.yml | |
| needs: [embedding_builds, check_branch, pack] | |
| if: needs.pack.outputs.should_continue == 'true' | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| DISTR: [macos] | |
| arch: [x86_64, arm64] | |
| name: ${{ matrix.DISTR }} ${{ matrix.arch }} packing | |
| with: | |
| MANTICORE_LOCATOR: ${{ needs.check_branch.outputs.manticore_locator }} | |
| DISTR: ${{ matrix.DISTR }} | |
| arch: ${{ matrix.arch }} | |
| HOMEBREW_PREFIX: /opt/homebrew | |
| cmake_command: | | |
| mkdir build | |
| cd build | |
| cmake -DPACK=1 .. | |
| cmake --build . --target package | |
| cache_key: pack_${{ matrix.DISTR }}_${{ matrix.arch }} | |
| artifact_list: "build/manticore*tar.gz" | |
| embeddings_artifact: embeddings_macos_${{ matrix.arch == 'arm64' && 'aarch64' || matrix.arch }} | |
| version: ${{ needs.pack.outputs.version }} | |
| package_version: ${{ needs.pack.outputs.version_full }} | |
| pack_windows: | |
| name: Windows x64 package | |
| uses: ./.github/workflows/build_template.yml | |
| needs: [embedding_builds, check_branch, pack] | |
| if: needs.pack.outputs.should_continue == 'true' | |
| with: | |
| MANTICORE_LOCATOR: ${{ needs.check_branch.outputs.manticore_locator }} | |
| DISTR: windows | |
| arch: x64 | |
| cmake_command: | | |
| mkdir build | |
| cd build | |
| cmake -DPACK=1 .. | |
| cmake --build . --target package | |
| cache_key: pack_${{ matrix.DISTR }}_${{ matrix.arch }} | |
| artifact_list: "build/manticore*exe build/manticore*zip" | |
| embeddings_artifact: embeddings_windows_x86_64 | |
| version: ${{ needs.pack.outputs.version }} | |
| package_version: ${{ needs.pack.outputs.version_full }} | |
| skip_avx2_move: true | |
| # virtual job to simplify the CI | |
| # This job depends on all the package preparation jobs that have to pass before we can start publishing packages | |
| publish: | |
| name: OK to publish? | |
| runs-on: ubuntu-22.04 | |
| needs: [pack_debian_ubuntu, pack_rhel, pack_macos, pack_windows, pack] | |
| if: | | |
| (github.repository == 'manticoresoftware/columnar') | |
| && ( | |
| (github.event_name == 'pull_request' && (contains(github.event.pull_request.labels.*.name, 'publish'))) | |
| || (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success') | |
| || (github.event_name == 'push' && startsWith(github.ref, 'refs/heads/columnar-')) | |
| || (github.event_name == 'push' && contains(github.ref, 'refs/tags/pack_publish')) | |
| || (needs.pack.outputs.target == 'release') | |
| ) | |
| outputs: | |
| target: ${{ steps.set-target.outputs.target }} | |
| steps: | |
| - run: echo "Ready to publish" | |
| publish_debian_ubuntu: | |
| needs: | |
| - pack | |
| - publish | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| fail-fast: true | |
| matrix: | |
| DISTR: [bionic, focal, jammy, buster, bullseye, bookworm] | |
| arch: [x86_64, aarch64] | |
| name: ${{ matrix.DISTR }} ${{ matrix.arch }} publishing | |
| steps: | |
| - uses: manticoresoftware/publish_to_repo@main | |
| with: | |
| ssh_key: ${{ secrets.REPO_SSH_KEY }} | |
| distr: ${{ matrix.DISTR }} | |
| arch: ${{ matrix.arch }} | |
| artifact: build_${{ matrix.DISTR }}_RelWithDebInfo_${{ matrix.arch }} | |
| type: deb | |
| delimiter: "-" | |
| target: ${{ needs.pack.outputs.target }} | |
| publish_rhel: | |
| needs: | |
| - pack | |
| - publish | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| fail-fast: true | |
| matrix: | |
| DISTR: [7, 8, 9, 10] | |
| arch: [x86_64, aarch64] | |
| name: RHEL ${{ matrix.DISTR }} ${{ matrix.arch }} publishing | |
| steps: | |
| - uses: manticoresoftware/publish_to_repo@main | |
| with: | |
| ssh_key: ${{ secrets.REPO_SSH_KEY }} | |
| distr: ${{ matrix.DISTR }} | |
| arch: ${{ matrix.arch }} | |
| artifact: build_rhel${{ matrix.DISTR }}_RelWithDebInfo_${{ matrix.arch }} | |
| type: rpm | |
| delimiter: "_" | |
| target: ${{ needs.pack.outputs.target }} | |
| publish_macos: | |
| needs: | |
| - pack | |
| - publish | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| fail-fast: true | |
| matrix: | |
| arch: [x86_64, arm64] | |
| name: macos ${{ matrix.arch }} publishing | |
| steps: | |
| - uses: manticoresoftware/publish_to_repo@main | |
| with: | |
| ssh_key: ${{ secrets.REPO_SSH_KEY }} | |
| distr: macos | |
| arch: ${{ matrix.arch }} | |
| artifact: build_macos_RelWithDebInfo_${{ matrix.arch }} | |
| type: arc | |
| delimiter: "-" | |
| target: ${{ needs.pack.outputs.target }} | |
| publish_windows: | |
| name: Publishing Windows packages to repo.manticoresearch.com | |
| needs: | |
| - pack | |
| - publish | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: manticoresoftware/publish_to_repo@main | |
| with: | |
| ssh_key: ${{ secrets.REPO_SSH_KEY }} | |
| distr: windows | |
| arch: x64 | |
| artifact: build_windows_RelWithDebInfo_x64 | |
| type: arc | |
| delimiter: "-" | |
| target: ${{ needs.pack.outputs.target }} | |
| update-manticoresearch-deps: | |
| needs: | |
| - pack | |
| - publish | |
| runs-on: ubuntu-22.04 | |
| if: needs.publish.outputs.target != 'release' | |
| steps: | |
| - name: Update deps | |
| uses: manticoresoftware/manticoresearch/actions/update-deps@main | |
| with: | |
| name: mcl | |
| version: ${{ needs.pack.outputs.version_full }} | |
| token: ${{ secrets.PR_TOKEN }} |