Skip to content
huggingface-nightly-build-and-publish

huggingface-nightly-build-and-publish #169

Sign in to view logs

huggingface-nightly-build-and-publish

huggingface-nightly-build-and-publish #169

Workflow file for this run

.github/workflows/huggingface-nightly.yml at 1cf65a1
name: huggingface-nightly-build-and-publish
on:
schedule:
# Nightly at 03:17 UTC
- cron: "17 3 * * *"
workflow_dispatch:
inputs:
upload_to:
description: "Where to upload (none/pypi)"
required: true
default: "pypi"
type: choice
options:
- none
- pypi
skip_existing:
description: "Skip already-uploaded versions"
required: true
default: true
type: boolean
package:
description: "Package to build (stable releases must select a single package)"
required: true
default: "all"
type: choice
options:
- all
- nemotron-ocr-v2
release_type:
description: "Version mode"
required: true
default: "nightly"
type: choice
options:
- nightly
- stable
release_version:
description: "Stable PyPI version when release_type=stable (for example: 2.0.0)"
required: false
default: ""
type: string
permissions:
contents: read
jobs:
# One dev suffix for the whole workflow so matrix legs (e.g. OCR x86 + aarch64) share
# the same version. UTC YYYYMMDDHHmmss (14 digits) is unique per second and stays
# within the uint64 limit that uv/Rust-based installers require for PEP 440 integers.
nightly_coordinate:
runs-on: ubuntu-latest
outputs:
nightly_date_suffix: ${{ steps.suffix.outputs.nightly_date_suffix }}
steps:
- id: suffix
shell: bash
env:
INPUT_PACKAGE: ${{ inputs.package }}
INPUT_RELEASE_TYPE: ${{ inputs.release_type }}
INPUT_RELEASE_VERSION: ${{ inputs.release_version }}
run: |
set -euo pipefail
if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
if [[ "${INPUT_RELEASE_TYPE}" == "stable" ]]; then
if [[ -z "${INPUT_RELEASE_VERSION}" ]]; then
echo "::error::release_version is required when release_type=stable"
exit 1
fi
if [[ "${INPUT_PACKAGE}" == "all" ]]; then
echo "::error::Stable releases must select a single package"
exit 1
fi
elif [[ -n "${INPUT_RELEASE_VERSION}" ]]; then
echo "::error::release_version is only valid when release_type=stable"
exit 1
fi
fi
echo "nightly_date_suffix=$(date -u +%Y%m%d%H%M%S)" >> "$GITHUB_OUTPUT"
build:
needs: nightly_coordinate
if: ${{ github.event_name != 'workflow_dispatch' || inputs.package == 'all' }}
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
repo:
- id: nemotron-page-elements-v3
url: https://huggingface.co/nvidia/nemotron-page-elements-v3
project_subdir: ""
nightly_base_version: "3.0.2"
- id: nemotron-table-structure-v1
url: https://huggingface.co/nvidia/nemotron-table-structure-v1
project_subdir: ""
nightly_base_version: "1.0.1"
- id: nemotron-graphic-elements-v1
url: https://huggingface.co/nvidia/nemotron-graphic-elements-v1
project_subdir: ""
nightly_base_version: "1.0.1"
steps:
- name: Checkout orchestrator repo
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.12"
- name: Install git-lfs
run: |
sudo apt-get update
sudo apt-get install -y git-lfs
git lfs install
- name: Decide upload target
id: target
shell: bash
run: |
set -euo pipefail
# Default for scheduled runs: pypi
upload_to="pypi"
skip_existing="true"
if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
upload_to="${{ inputs.upload_to }}"
skip_existing="${{ inputs.skip_existing }}"
fi
echo "upload_to=${upload_to}" >> "$GITHUB_OUTPUT"
echo "skip_existing=${skip_existing}" >> "$GITHUB_OUTPUT"
- name: Build (and maybe upload)
env:
NIGHTLY_DATE_SUFFIX: ${{ needs.nightly_coordinate.outputs.nightly_date_suffix }}
PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }}
shell: bash
run: |
set -euo pipefail
upload_flag=""
repo_url="https://upload.pypi.org/legacy/"
token_env="PYPI_API_TOKEN"
if [[ "${{ steps.target.outputs.upload_to }}" == "none" ]]; then
upload_flag=""
else
upload_flag="--upload"
fi
skip_existing_flag=""
if [[ "${{ steps.target.outputs.skip_existing }}" == "true" ]]; then
skip_existing_flag="--skip-existing"
fi
python ci/scripts/nightly_build_publish.py \
--repo-id "${{ matrix.repo.id }}" \
--repo-url "${{ matrix.repo.url }}" \
--work-dir ".work" \
--dist-dir "dist-out" \
--project-subdir "${{ matrix.repo.project_subdir }}" \
--nightly-base-version "${{ matrix.repo.nightly_base_version }}" \
${upload_flag} \
--repository-url "${repo_url}" \
--token-env "${token_env}" \
${skip_existing_flag}
- name: Upload build artifacts (GH Actions)
if: always()
uses: actions/upload-artifact@v4
with:
name: dist-${{ matrix.repo.id }}
path: dist-out/${{ matrix.repo.id }}/*
build_ocr_cuda:
needs: nightly_coordinate
if: ${{ github.event_name != 'workflow_dispatch' || inputs.package == 'all' || inputs.package == 'nemotron-ocr-v2' }}
# Nemotron OCR packages need nvcc/CUDA headers to build their extension.
# Build with Python 3.12 to match upstream package constraints and
# avoid producing an extension for the wrong Python ABI.
# Matrix across x86_64 and aarch64 so the wheel works on ARM hosts
# (e.g. DGX Spark) as well as conventional x86 machines.
runs-on: ${{ matrix.platform.runner }}
strategy:
fail-fast: false
matrix:
platform:
- runner: ubuntu-latest
cuda_image: nvidia/cuda:13.0.0-devel-ubuntu24.04
arch: x86_64
- runner: ubuntu-24.04-arm
cuda_image: nvidia/cuda:13.0.0-devel-ubuntu24.04
arch: aarch64
ocr:
- id: nemotron-ocr-v2
url: https://huggingface.co/nvidia/nemotron-ocr-v2
nightly_base_version: "2.0.1"
container:
image: ${{ matrix.platform.cuda_image }}
steps:
- name: Install system deps (git, lfs, build tools)
shell: bash
run: |
set -euo pipefail
apt-get update
apt-get install -y --no-install-recommends \
ca-certificates \
git \
git-lfs \
build-essential \
ninja-build \
patchelf \
python3 \
python3-venv \
python3-pip
git lfs install
- name: Checkout orchestrator repo
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.12"
- name: Decide upload target
id: target
shell: bash
run: |
set -euo pipefail
# Default for scheduled runs: pypi
upload_to="pypi"
skip_existing="true"
if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
upload_to="${{ inputs.upload_to }}"
skip_existing="${{ inputs.skip_existing }}"
fi
echo "upload_to=${upload_to}" >> "$GITHUB_OUTPUT"
echo "skip_existing=${skip_existing}" >> "$GITHUB_OUTPUT"
- name: Build ${{ matrix.ocr.id }} (and maybe upload)
env:
NIGHTLY_DATE_SUFFIX: ${{ needs.nightly_coordinate.outputs.nightly_date_suffix }}
PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }}
CUDA_HOME: /usr/local/cuda
BUILD_CPP_EXTENSION: "1"
BUILD_CPP_FORCE: "1"
OCR_TORCH_VERSION: "2.11.0"
OCR_TORCHVISION_VERSION: "0.26.0"
INPUT_RELEASE_TYPE: ${{ inputs.release_type }}
INPUT_RELEASE_VERSION: ${{ inputs.release_version }}
TORCH_CUDA_ARCH_LIST: "8.0;8.6;8.9;9.0;10.0;12.0+PTX"
PIP_EXTRA_INDEX_URL: "https://download.pytorch.org/whl/cu130"
shell: bash
run: |
set -euo pipefail
upload_flag=""
repo_url="https://upload.pypi.org/legacy/"
token_env="PYPI_API_TOKEN"
if [[ "${{ steps.target.outputs.upload_to }}" == "none" ]]; then
upload_flag=""
else
upload_flag="--upload"
fi
skip_existing_flag=""
if [[ "${{ steps.target.outputs.skip_existing }}" == "true" ]]; then
skip_existing_flag="--skip-existing"
fi
# The upstream build script checks ARCH to skip x86-only SIMD
# flags (e.g. -mavx2) that are invalid on aarch64.
arch_env_arg=""
if [[ "$(uname -m)" == "aarch64" ]]; then
arch_env_arg="--build-env ARCH=arm64"
fi
release_type="nightly"
expected_version="${{ matrix.ocr.nightly_base_version }}.dev"
version_args=(--nightly-base-version "${{ matrix.ocr.nightly_base_version }}")
if [[ "${{ github.event_name }}" == "workflow_dispatch" && "${INPUT_RELEASE_TYPE}" == "stable" ]]; then
release_type="stable"
expected_version="${INPUT_RELEASE_VERSION}"
version_args=(--release-version "${INPUT_RELEASE_VERSION}")
fi
python --version
python -m pip install "packaging>=24"
python ci/scripts/nightly_build_publish.py \
--repo-id "${{ matrix.ocr.id }}" \
--repo-url "${{ matrix.ocr.url }}" \
--work-dir ".work" \
--dist-dir "dist-out" \
--project-subdir "nemotron-ocr" \
"${version_args[@]}" \
--hatch-force-platform-wheel \
--auditwheel-repair \
--auditwheel-exclude libtorch_cpu.so \
--auditwheel-exclude libtorch_cuda.so \
--auditwheel-exclude libtorch.so \
--auditwheel-exclude libc10.so \
--auditwheel-exclude libc10_cuda.so \
--auditwheel-exclude libtorch_python.so \
--build-no-isolation \
--venv-pip-install "hatchling" \
--venv-pip-install "setuptools>=68" \
--venv-pip-install "ninja" \
--venv-pip-install "torch==${OCR_TORCH_VERSION}" \
--venv-pip-install "torchvision==${OCR_TORCHVISION_VERSION}" \
--pin-runtime-dependency "torch" \
--pin-runtime-dependency "torchvision" \
--build-env "BUILD_CPP_EXTENSION=1" \
--build-env "BUILD_CPP_FORCE=1" \
${arch_env_arg} \
${upload_flag} \
--repository-url "${repo_url}" \
--token-env "${token_env}" \
${skip_existing_flag}
export EXPECTED_OCR_VERSION="${expected_version}"
export OCR_RELEASE_TYPE="${release_type}"
python - <<'PY'
import os
import sys
import zipfile
from email.parser import Parser
from pathlib import Path
from packaging.requirements import Requirement
from packaging.specifiers import SpecifierSet
from packaging.utils import canonicalize_name
from packaging.version import Version
dist_dir = Path("dist-out/${{ matrix.ocr.id }}")
wheels = sorted(dist_dir.glob("*.whl"))
if not wheels:
raise SystemExit(f"No wheel found in {dist_dir}")
py_tag = f"cpython-{sys.version_info.major}{sys.version_info.minor}"
expected_project = "nemotron-ocr"
expected_package = "nemotron_ocr"
expected_version = os.environ["EXPECTED_OCR_VERSION"]
release_type = os.environ["OCR_RELEASE_TYPE"]
expected_runtime_dependencies = {
"torch": f"~={os.environ['OCR_TORCH_VERSION']}",
"torchvision": f"~={os.environ['OCR_TORCHVISION_VERSION']}",
}
def expanded_compatible_specifier(specifier):
if not specifier.startswith("~="):
return None
version = Version(specifier[2:])
release = version.release
if len(release) < 3:
return None
upper = f"{release[0]}.{release[1] + 1}.0"
return SpecifierSet(f">={version},<{upper}")
def declares_runtime_dependency(metadata, package, specifier):
expected_name = canonicalize_name(package)
expected_specifier = SpecifierSet(specifier)
expanded_specifier = expanded_compatible_specifier(specifier)
for value in Parser().parsestr(metadata).get_all("Requires-Dist", []):
requirement = Requirement(value)
if canonicalize_name(requirement.name) != expected_name:
continue
if requirement.specifier == expected_specifier:
return True
if expanded_specifier and requirement.specifier == expanded_specifier:
return True
return False
matches = []
for wheel in wheels:
with zipfile.ZipFile(wheel) as zf:
names = zf.namelist()
metadata_names = [n for n in names if n.endswith(".dist-info/METADATA")]
metadata = "\n".join(zf.read(n).decode("utf-8") for n in metadata_names)
metadata_message = Parser().parsestr(metadata)
if f"Name: {expected_project}" not in metadata:
raise SystemExit(f"Built wheel metadata does not declare project {expected_project!r}")
metadata_version = metadata_message.get("Version")
if release_type == "stable" and metadata_version != expected_version:
raise SystemExit(
"Built wheel metadata does not declare expected version "
f"{expected_version!r}; got {metadata_version!r}"
)
if release_type != "stable" and not (
metadata_version and metadata_version.startswith(expected_version)
):
raise SystemExit(
f"Built wheel metadata does not declare a {expected_version} nightly"
)
if not any(name.startswith(f"{expected_package}/") for name in names):
raise SystemExit(f"Built wheel is missing {expected_package} package")
for package, specifier in expected_runtime_dependencies.items():
expected = f"Requires-Dist: {package}{specifier}"
if not declares_runtime_dependency(metadata, package, specifier):
raise SystemExit(
f"Built wheel metadata does not declare runtime dependency {expected!r}"
)
if any(name.startswith("nemotron_ocr_v2/") for name in names):
raise SystemExit("Built wheel still contains the patched nemotron_ocr_v2 package")
for name in names:
if "nemotron_ocr_cpp/_nemotron_ocr_cpp." in name:
matches.append((wheel.name, name))
if not matches:
raise SystemExit("Built wheel is missing nemotron_ocr_cpp extension artifact")
if not any(py_tag in ext for _, ext in matches):
formatted = "\n".join(f"{w}: {ext}" for w, ext in matches)
raise SystemExit(
f"Built extension ABI does not match runner Python ({py_tag}). Found:\n{formatted}"
)
for wheel in wheels:
if "py3-none-any" in wheel.name:
raise SystemExit(
"Wheel is tagged py3-none-any; pip cannot select x86_64 vs aarch64. "
f"Got: {wheel.name}"
)
if wheel.name.endswith("-linux_x86_64.whl") or wheel.name.endswith(
"-linux_aarch64.whl"
):
raise SystemExit(
"Wheel still has a bare linux_* tag; PyPI rejects these. "
"auditwheel repair should emit manylinux_*. Got: "
f"{wheel.name}"
)
print("Verified nemotron_ocr_cpp extension ABI:", py_tag)
PY
- name: Upload build artifacts (GH Actions)
if: always()
uses: actions/upload-artifact@v4
with:
name: dist-${{ matrix.ocr.id }}-${{ matrix.platform.arch }}
path: dist-out/${{ matrix.ocr.id }}/*