ci: add package version diff comment on PRs

.github/workflows/nix-build.yml

@@ -35,6 +35,25 @@ jobs:
           nix-signing-key: ${{ secrets.NIX_SIGN_SECRET_KEY }}
           role-to-assume: ${{ secrets.DEV_AWS_ROLE }}
 
+      # shared with version-diff's HEAD side
+      - name: Evaluate drvPaths for version-diff
+        if: github.event_name == 'pull_request'
+        run: |
+          mkdir -p /tmp/drvpaths
+          for system in x86_64-linux aarch64-linux aarch64-darwin; do
+            nix eval --no-pure-eval --json \
+              --expr "(import $PWD/ci/version-diff/drvpaths.nix { dir = toString ./.; system = \"$system\"; })" \
+              > "/tmp/drvpaths/$system.json"
+          done
+
+      - name: Upload drvPaths artifact
+        if: github.event_name == 'pull_request'
+        uses: actions/upload-artifact@v4
+        with:
+          name: head-drvpaths
+          path: /tmp/drvpaths/*.json
+          retention-days: 1
+
       - name: Generate Nix Matrix
         id: gen-matrix
         run: |
@@ -259,3 +278,109 @@ jobs:
         run: |
           jq -rn --argjson needs '${{ toJSON(needs) }}' '$needs|to_entries[]|select(.value.result!="success")|"::error::"+.key+": "+.value.result'
           exit 1
+
+  version-diff:
+    name: Package version diff
+    if: github.event_name == 'pull_request'
+    needs:
+      - eval
+      - builds-x86_64-linux
+      - builds-aarch64-linux
+      - builds-aarch64-darwin
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - system: x86_64-linux
+            runs_on: blacksmith-32vcpu-ubuntu-2404
+            installer: ephemeral
+          - system: aarch64-linux
+            runs_on: blacksmith-4vcpu-ubuntu-2404-arm
+            installer: ephemeral
+          - system: aarch64-darwin
+            runs_on:
+              group: self-hosted-runners-nix
+              labels: [aarch64-darwin]
+            installer: self-hosted
+    runs-on: ${{ matrix.runs_on }}
+    steps:
+      - name: Checkout PR head
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+        with:
+          path: head
+
+      - name: Checkout PR base
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+        with:
+          ref: ${{ github.event.pull_request.base.sha }}
+          path: base
+
+      - name: Install nix (ephemeral)
+        if: matrix.installer == 'ephemeral'
+        uses: ./head/.github/actions/nix-install-ephemeral
+
+      - name: Install nix (self-hosted)
+        if: matrix.installer == 'self-hosted'
+        uses: ./head/.github/actions/nix-install-self-hosted
+
+      # HEAD side already done in eval
+      - name: Download head drvPaths
+        uses: actions/download-artifact@v4
+        with:
+          name: head-drvpaths
+          path: /tmp/head-drvpaths
+
+      # use HEAD's script, base may predate it
+      - name: Evaluate base drvPaths
+        working-directory: base
+        run: |
+          nix eval --no-pure-eval --json \
+            --expr '(import ${{ github.workspace }}/head/ci/version-diff/drvpaths.nix { dir = toString ./.; system = "${{ matrix.system }}"; })' \
+            > /tmp/base-drvpaths.json
+
+      - name: Diff versions with nvd
+        run: ./head/ci/version-diff/diff.sh "${{ matrix.system }}"
+
+      - name: Upload diff artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: diff-${{ matrix.system }}
+          path: /tmp/diff.txt
+          retention-days: 7
+
+  version-diff-comment:
+    name: Post package version diff comment
+    if: always() && github.event_name == 'pull_request'
+    needs: version-diff
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+
+      - name: Download diff artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: diff-*
+          path: diffs
+          merge-multiple: false
+
+      - name: Assemble collapsed comment body
+        run: ./ci/version-diff/comment.sh
+
+      - name: Find existing version diff comment
+        uses: peter-evans/find-comment@v3
+        id: fc
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          comment-author: "github-actions[bot]"
+          body-includes: "<!-- version-diff -->"
+
+      - name: Comment on PR
+        uses: peter-evans/create-or-update-comment@v4
+        with:
+          comment-id: ${{ steps.fc.outputs.comment-id }}
+          issue-number: ${{ github.event.pull_request.number }}
+          body-path: /tmp/comment.md
+          edit-mode: replace

ci/version-diff/comment.sh

@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+set -euo pipefail
+shopt -s nullglob
+
+all_no_diff=true
+for dir in diffs/diff-*; do
+	file="$dir/diff.txt"
+	if [ ! -f "$file" ] || ! grep -q '^No `legacyPackages' "$file"; then
+		all_no_diff=false
+		break
+	fi
+done
+
+if [ "$all_no_diff" = "true" ]; then
+	systems=$(for dir in diffs/diff-*; do echo "\`${dir#diffs/diff-}\`"; done | paste -sd', ')
+	{
+		echo "<!-- version-diff -->"
+		echo "## Package version diff: none"
+		echo 'Compares built package versions in `legacyPackages` between this PR and its base commit, per system.'
+		echo
+		echo "### No Package Differences"
+		echo "All packages are hash-identical on all systems (${systems})."
+	} >/tmp/comment.md
+else
+	{
+		echo "<!-- version-diff -->"
+		echo "## Package version diff"
+		echo 'Compares built package versions in `legacyPackages` between this PR and its base commit, per system.'
+		echo
+		for dir in diffs/diff-*; do
+			system="${dir#diffs/diff-}"
+			file="$dir/diff.txt"
+			if [ ! -f "$file" ]; then
+				echo "<details>"
+				echo "<summary>${system}: diff unavailable (job failed or skipped)</summary>"
+				echo "</details>"
+				echo
+				continue
+			fi
+			if grep -q '^No `legacyPackages' "$file"; then
+				# short, no-op case: summary alone, nothing to expand
+				echo "<details>"
+				echo "<summary>${system}: $(cat "$file")</summary>"
+				echo "</details>"
+				echo
+				continue
+			fi
+			summary=$(grep '^Closure size:' "$file" | tail -1)
+			echo "<details>"
+			echo "<summary>${system}: ${summary:-changed}</summary>"
+			echo
+			echo '```'
+			# cap the visible diff so a single arch can't blow the comment size limit
+			head -c 20000 "$file"
+			if [ "$(wc -c <"$file")" -gt 20000 ]; then
+				echo
+				echo "... truncated, see workflow run for full output ..."
+			fi
+			echo '```'
+			echo "</details>"
+			echo
+		done
+	} >/tmp/comment.md
+fi

ci/version-diff/diff.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+system="$1"
+head_json="/tmp/head-drvpaths/$system.json"
+base_json="/tmp/base-drvpaths.json"
+
+if diff -q <(jq -S . "$head_json") <(jq -S . "$base_json") >/dev/null; then
+	echo "No \`legacyPackages.$system\` version changes in this PR." >/tmp/diff.txt
+	cat /tmp/diff.txt
+	exit 0
+fi
+
+changed=$(jq -r -n --slurpfile h "$head_json" --slurpfile b "$base_json" '
+  ($h[0]) as $H | ($b[0]) as $B |
+  ($H | keys) as $hk | ($B | keys) as $bk |
+  ($hk - ($hk - $bk))[] | select($H[.] != $B[.])
+')
+added=$(jq -r -n --slurpfile h "$head_json" --slurpfile b "$base_json" '
+  (($h[0] | keys) - ($b[0] | keys))[]
+')
+removed=$(jq -r -n --slurpfile h "$head_json" --slurpfile b "$base_json" '
+  (($b[0] | keys) - ($h[0] | keys))[]
+')
+
+: >/tmp/diff.txt
+for attr in $changed; do
+	head_path=$(cd head && nix build ".#legacyPackages.$system.$attr" --no-link --print-out-paths)
+	base_path=$(cd base && nix build ".#legacyPackages.$system.$attr" --no-link --print-out-paths)
+	nix run nixpkgs#nvd -- diff "$base_path" "$head_path" >>/tmp/diff.txt
+done
+for attr in $added; do
+	echo "+ $attr added" >>/tmp/diff.txt
+done
+for attr in $removed; do
+	echo "- $attr removed" >>/tmp/diff.txt
+done
+
+if [ ! -s /tmp/diff.txt ]; then
+	echo "No \`legacyPackages.$system\` version changes in this PR." >/tmp/diff.txt
+fi
+cat /tmp/diff.txt

ci/version-diff/drvpaths.nix

@@ -0,0 +1,33 @@
+# dumps legacyPackages.<system> -> drvPath, recursively; used to diff head vs base
+{ dir, system }:
+let
+  flake = builtins.getFlake dir;
+  isDrv = v: (v.type or "") == "derivation";
+  collect =
+    prefix: set:
+    builtins.concatLists (
+      map (
+        n:
+        let
+          v = set.${n};
+        in
+        if isDrv v then
+          [
+            {
+              name = prefix + n;
+              drvPath = v.drvPath;
+            }
+          ]
+        else if builtins.isAttrs v then
+          collect (prefix + n + ".") v
+        else
+          [ ]
+      ) (builtins.attrNames set)
+    );
+in
+builtins.listToAttrs (
+  map (e: {
+    inherit (e) name;
+    value = e.drvPath;
+  }) (collect "" flake.legacyPackages.${system})
+)