Update IANA Data #216
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Update IANA Data | |
| on: | |
| schedule: | |
| # Run daily at 2:00 AM UTC / 18:00 US-Pacific | |
| - cron: "0 2 * * *" | |
| workflow_dispatch: # Allow manual triggering | |
| # Prevent concurrent runs | |
| concurrency: | |
| group: update-iana-data | |
| cancel-in-progress: false | |
| permissions: | |
| contents: write | |
| actions: read # pull the iptoasn artifact from the update-iptoasn workflow | |
| jobs: | |
| update-data: | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v6.0.2 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v8.1.0 | |
| with: | |
| enable-cache: true | |
| version-file: "pyproject.toml" | |
| - uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8 | |
| - name: Install dependencies | |
| run: bin/setup | |
| # Download core IANA files | |
| - name: Download core data (RDAP, TLD list, root zone DB) | |
| id: download-core | |
| continue-on-error: true | |
| run: make download-core | |
| - name: Notification if core download fails | |
| if: steps.download-core.outcome == 'failure' | |
| uses: ./.github/actions/pushover | |
| with: | |
| pushover_api_token: ${{ secrets.PUSHOVER_API_TOKEN }} | |
| pushover_user_key: ${{ secrets.PUSHOVER_USER_KEY }} | |
| title: "IANA core data download failed" | |
| message: "❌ Failed to download IANA core data files" | |
| - name: Download ICANN Registry Agreement Table | |
| id: download-registry-agreement | |
| continue-on-error: true | |
| run: uv run python scripts/registry-agreement-table/download_registry_agreement_table.py | |
| - name: Notification if registry agreement download fails | |
| if: steps.download-registry-agreement.outcome == 'failure' | |
| uses: ./.github/actions/pushover | |
| with: | |
| pushover_api_token: ${{ secrets.PUSHOVER_API_TOKEN }} | |
| pushover_user_key: ${{ secrets.PUSHOVER_USER_KEY }} | |
| title: "ICANN registry agreement download failed" | |
| message: "❌ Failed to download ICANN registry agreement table" | |
| - name: Download ICANN gTLDs JSON report | |
| id: download-gtlds | |
| continue-on-error: true | |
| run: uv run python scripts/gtlds/download_gtlds.py | |
| - name: Notification if gTLDs report download fails | |
| if: steps.download-gtlds.outcome == 'failure' | |
| uses: ./.github/actions/pushover | |
| with: | |
| pushover_api_token: ${{ secrets.PUSHOVER_API_TOKEN }} | |
| pushover_user_key: ${{ secrets.PUSHOVER_USER_KEY }} | |
| title: "ICANN gTLDs report download failed" | |
| message: "❌ Failed to download ICANN gTLDs JSON report" | |
| - name: Generate IDN script mappings | |
| id: generate-idn-mapping | |
| continue-on-error: true | |
| run: make generate-idn-mapping | |
| - name: Notification if IDN mapping fails | |
| if: steps.generate-idn-mapping.outcome == 'failure' | |
| uses: ./.github/actions/pushover | |
| with: | |
| pushover_api_token: ${{ secrets.PUSHOVER_API_TOKEN }} | |
| pushover_user_key: ${{ secrets.PUSHOVER_USER_KEY }} | |
| title: "IANA data - IDN mapping generation failed" | |
| message: "❌ Failed to generate IDN script mappings" | |
| # Download iptoasn artifact (uploaded by update-iptoasn workflow) | |
| - name: Get latest iptoasn workflow run | |
| id: get-iptoasn-run | |
| continue-on-error: true | |
| run: | | |
| RUN_ID=$(gh run list --workflow=update-iptoasn.yaml --status=success --limit=1 --json databaseId -q '.[0].databaseId') | |
| if [ -z "$RUN_ID" ]; then | |
| echo "::error::No successful update-iptoasn run found; cannot fetch required iptoasn data" | |
| exit 1 | |
| fi | |
| echo "run_id=$RUN_ID" >> "$GITHUB_OUTPUT" | |
| echo "Found successful iptoasn workflow run: $RUN_ID" | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| # continue-on-error so a failed iptoasn artifact doesn't block a --preserve-asn | |
| # rebuild (doesn't need it); a bad --all build is caught by the tests below. | |
| - name: Download iptoasn artifact | |
| id: download-iptoasn | |
| continue-on-error: true | |
| uses: actions/download-artifact@v8 | |
| with: | |
| pattern: iptoasn-* | |
| path: data/source/iptoasn | |
| github-token: ${{ github.token }} | |
| run-id: ${{ steps.get-iptoasn-run.outputs.run_id }} | |
| merge-multiple: true | |
| - name: Record iptoasn artifact info | |
| if: steps.download-iptoasn.outcome == 'success' | |
| run: | | |
| # Find the downloaded file and log its info | |
| if [ -f "data/source/iptoasn/ip2asn-combined.tsv.gz" ]; then | |
| SIZE=$(stat -c%s data/source/iptoasn/ip2asn-combined.tsv.gz) | |
| SIZE_MB=$(echo "scale=1; $SIZE / 1048576" | bc) | |
| echo "Downloaded iptoasn data (${SIZE_MB} MB)" | |
| else | |
| echo "Warning: iptoasn file not found after download" | |
| ls -la data/source/iptoasn/ || true | |
| fi | |
| # Download TLD pages in 8 batches for better error isolation | |
| # Total: 1,592 TLDs, ~200 per batch, ~3-4 min each | |
| # Continue on error to allow other batches to complete | |
| - name: "Download TLDs: a-b (198 TLDs)" | |
| id: download-tlds-ab | |
| continue-on-error: true | |
| run: make download-tld-pages GROUPS="a b" | |
| - name: "Download TLDs: c-d (187 TLDs)" | |
| id: download-tlds-cd | |
| continue-on-error: true | |
| run: make download-tld-pages GROUPS="c d" | |
| - name: "Download TLDs: e-h (227 TLDs)" | |
| id: download-tlds-eh | |
| continue-on-error: true | |
| run: make download-tld-pages GROUPS="e f g h" | |
| - name: "Download TLDs: i + IDN (215 TLDs)" | |
| id: download-tlds-i-idn | |
| continue-on-error: true | |
| run: make download-tld-pages GROUPS="i xn--" | |
| - name: "Download TLDs: j-m (221 TLDs)" | |
| id: download-tlds-jm | |
| continue-on-error: true | |
| run: make download-tld-pages GROUPS="j k l m" | |
| - name: "Download TLDs: n-q (158 TLDs)" | |
| id: download-tlds-nq | |
| continue-on-error: true | |
| run: make download-tld-pages GROUPS="n o p q" | |
| - name: "Download TLDs: r-s (190 TLDs)" | |
| id: download-tlds-rs | |
| continue-on-error: true | |
| run: make download-tld-pages GROUPS="r s" | |
| - name: "Download TLDs: t-z (188 TLDs)" | |
| id: download-tlds-tz | |
| continue-on-error: true | |
| run: make download-tld-pages GROUPS="t u v w y z" | |
| - name: Notification if TLD download fails | |
| if: | | |
| steps.download-tlds-ab.outcome == 'failure' || | |
| steps.download-tlds-cd.outcome == 'failure' || | |
| steps.download-tlds-eh.outcome == 'failure' || | |
| steps.download-tlds-i-idn.outcome == 'failure' || | |
| steps.download-tlds-jm.outcome == 'failure' || | |
| steps.download-tlds-nq.outcome == 'failure' || | |
| steps.download-tlds-rs.outcome == 'failure' || | |
| steps.download-tlds-tz.outcome == 'failure' | |
| uses: ./.github/actions/pushover | |
| with: | |
| pushover_api_token: ${{ secrets.PUSHOVER_API_TOKEN }} | |
| pushover_user_key: ${{ secrets.PUSHOVER_USER_KEY }} | |
| title: "IANA TLD data download failed" | |
| message: "❌ Failed to download IANA TLD data files" | |
| - name: Check if source data changed | |
| id: source-check | |
| run: | | |
| # Check if data/source/ has any changes (not just metadata.json) | |
| if git diff --quiet data/source/; then | |
| echo "Source files unchanged (timestamp updates only)" | |
| echo "source_changed=false" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "Source files have content changes" | |
| echo "source_changed=true" >> "$GITHUB_OUTPUT" | |
| fi | |
| # Always rebuild so manual curation (data/manual/) propagates, not just IANA | |
| # source changes. --all on source change (ASN refresh), else --preserve-asn. | |
| - name: Build tlds.json | |
| id: build | |
| env: | |
| SOURCE_CHANGED: ${{ steps.source-check.outputs.source_changed }} | |
| run: | | |
| if [ "$SOURCE_CHANGED" = "true" ]; then | |
| echo "IANA source changed: full build with ASN refresh" | |
| ./bin/build --all | |
| else | |
| echo "No IANA source change: regenerate from manual curation, preserving ASN" | |
| ./bin/build --preserve-asn | |
| fi | |
| - name: Notification if build fails | |
| if: failure() && steps.build.outcome == 'failure' | |
| uses: ./.github/actions/pushover | |
| with: | |
| pushover_api_token: ${{ secrets.PUSHOVER_API_TOKEN }} | |
| pushover_user_key: ${{ secrets.PUSHOVER_USER_KEY }} | |
| title: "IANA data - tlds.json build failed" | |
| message: "❌ Failed to build tlds.json" | |
| # Build always runs now, so always test before committing a regenerated diff. | |
| - name: Run tests | |
| id: run-tests | |
| run: | | |
| set -o pipefail | |
| # Intentional pipe to tee; SC2266 misreads it as a mistaken ||. | |
| # shellcheck disable=SC2266 | |
| bin/test 2>&1 | tee test-output.log | |
| - name: Notification if tests fail | |
| if: failure() && steps.run-tests.outcome == 'failure' | |
| uses: ./.github/actions/pushover | |
| with: | |
| pushover_api_token: ${{ secrets.PUSHOVER_API_TOKEN }} | |
| pushover_user_key: ${{ secrets.PUSHOVER_USER_KEY }} | |
| title: "IANA data - tests failed" | |
| message: "❌ Tests failed for iana-data" | |
| - name: Check for changes | |
| id: git-check | |
| if: always() | |
| run: | | |
| if ! git diff --exit-code data/; then | |
| echo "changed=true" >> "$GITHUB_OUTPUT" | |
| # Get list of changed files in data/ directory | |
| CHANGED_FILES=$(git diff --name-only data/ | sed 's|^data/source/||' | sed 's|^data/generated/||') | |
| { | |
| echo "files<<EOF" | |
| echo "$CHANGED_FILES" | |
| echo "EOF" | |
| } >> "$GITHUB_OUTPUT" | |
| fi | |
| # run-tests.outcome must be success (not just "not failure"): a failed build | |
| # skips tests, and "skipped != failure" would otherwise commit partial data. | |
| - name: Commit and push changes | |
| id: commit-push | |
| if: always() && steps.git-check.outputs.changed == 'true' && steps.run-tests.outcome == 'success' | |
| env: | |
| SOURCE_CHANGED: ${{ steps.source-check.outputs.source_changed }} | |
| CHANGED_FILES: ${{ steps.git-check.outputs.files }} | |
| run: | | |
| git config user.name "github-actions[bot]" | |
| git config user.email "github-actions[bot]@users.noreply.github.com" | |
| git add data/ | |
| # Build commit message: distinguish a source refresh from a manual-data regen. | |
| if [ "$SOURCE_CHANGED" = "true" ]; then | |
| COMMIT_MSG="Update IANA source data files" | |
| else | |
| COMMIT_MSG="Regenerate data from manual curation" | |
| fi | |
| if [ -n "$CHANGED_FILES" ]; then | |
| COMMIT_MSG="${COMMIT_MSG} | |
| Changed files:" | |
| while IFS= read -r file; do | |
| COMMIT_MSG="${COMMIT_MSG} | |
| - ${file}" | |
| done <<< "$CHANGED_FILES" | |
| fi | |
| git commit -m "${COMMIT_MSG}" | |
| git push | |
| - name: Notification if data committed | |
| if: steps.commit-push.outcome == 'success' | |
| uses: ./.github/actions/pushover | |
| with: | |
| pushover_api_token: ${{ secrets.PUSHOVER_API_TOKEN }} | |
| pushover_user_key: ${{ secrets.PUSHOVER_USER_KEY }} | |
| title: "IANA data - successfully updated" | |
| message: "✅ Successfully updated and committed IANA data" | |
| - name: Email notification if data committed | |
| if: steps.commit-push.outcome == 'success' | |
| uses: case/actions-email@v1 | |
| env: | |
| RESEND_API_KEY: ${{ secrets.RESEND_API_KEY }} | |
| EMAIL_FROM: ${{ secrets.EMAIL_FROM }} | |
| EMAIL_TO: ${{ secrets.EMAIL_TO }} | |
| with: | |
| provider: resend | |
| subject: "IANA data - successfully updated" | |
| body_text: "Successfully updated and committed IANA data" |