Testing interop retries

.github/workflows/qns.yml

@@ -6,11 +6,8 @@ on:
     branches:
       - main
   schedule:
-    # Run the daily job three times, to account for flakiness
-    # The jobs start running at 8 PM PST / 3 AM UTC
+    # Run the daily job at 8 PM PST / 3 AM UTC
     - cron: '0 3 * * *'
-    - cron: '0 4 * * *'
-    - cron: '0 5 * * *'
 
 name: qns
 
@@ -361,6 +358,10 @@ jobs:
           python3 .github/interop/check.py \
             --required .github/interop/required.json \
             web/logs/latest/result.json
+          if [[ ${{ fromJSON(github.run_attempt) }} == 2 ]]; then
+            exit 0
+          fi
+          exit 1
 
   h3spec:
     runs-on: ubuntu-22.04
@@ -555,40 +556,39 @@ jobs:
           # `timeout` exits with `124` if the time limit was reached
           [[ "$EXIT_CODE" == "124" ]] || exit $EXIT_CODE
 
-  qns-status-report:
+  retry-failures:
     runs-on: ubuntu-latest
-    if: ${{ always() }}
-    needs: [env, s2n-quic-qns, interop, h3spec, perf, perf-report, attack]
+    needs: [interop-report]
+    if: failure() && fromJSON(github.run_attempt) < 3
+    permissions:
+      actions: write
+    env:
+      GH_REPO: ${{ github.repository }}
+      GH_TOKEN: ${{ github.token }}
     steps:
-      - uses: aws-actions/configure-aws-credentials@v4.3.1
-        if: github.event_name != 'pull_request'
-        with:
-          role-to-assume: arn:aws:iam::003495580562:role/GitHubOIDCRole
-          role-session-name: S2nQuicGHASession
-          aws-region: us-west-2
-      - name: Report daily qns run to CloudWatch
-        if: github.event_name != 'pull_request'
+      - name: Run retry.yml
         run: |
-          METRIC_VALUE=${{ contains(needs.*.result, 'failure') && '1' || '0' }}
-          aws cloudwatch put-metric-data --namespace "Github" --metric-name "ActionCIFailure" \
-            --value $METRIC_VALUE --dimensions Initiator=${{github.event_name}} --timestamp $(date +%s)
+          # You cannot rerun an in-progress workflow, so you cannot rerun a workflow
+          # from inside that same workflow. Instead, start a new workflow.
+          # The new workflow will wait for this run to complete, then retry.
+          gh workflow run tshark.yml \
+            -r ${{ github.head_ref || github.ref_name }} \
+            -F version=${{ github.run_id }}
 
-  # Some tests may be flaky and require less strict alarming.
-  # Report those tests separately until the flakiness has been resolved.
-  qns-status-report-flaky:
+  qns-status-report:
     runs-on: ubuntu-latest
-    if: ${{ always() }}
-    needs: [interop-report]
+    if: always() && needs.retry-failures.result == 'skipped'
+    needs: [retry-failures, env, s2n-quic-qns, interop, interop-report, h3spec, perf, perf-report, attack]
     steps:
       - uses: aws-actions/configure-aws-credentials@v4.3.1
         if: github.event_name != 'pull_request'
         with:
           role-to-assume: arn:aws:iam::003495580562:role/GitHubOIDCRole
           role-session-name: S2nQuicGHASession
           aws-region: us-west-2
-      - name: Report potentially flaky qns job runs to CloudWatch
+      - name: Report daily qns run to CloudWatch
         if: github.event_name != 'pull_request'
         run: |
           METRIC_VALUE=${{ contains(needs.*.result, 'failure') && '1' || '0' }}
-          aws cloudwatch put-metric-data --namespace "Github" --metric-name "ActionCIFailureFlaky" \
+          aws cloudwatch put-metric-data --namespace "Github" --metric-name "ActionCIFailure" \
             --value $METRIC_VALUE --dimensions Initiator=${{github.event_name}} --timestamp $(date +%s)

.github/workflows/retry.yml

@@ -0,0 +1,22 @@
+# You cannot rerun an in-progress workflow, so you cannot rerun a workflow
+# from inside that same workflow. Instead, this workflow should be used.
+# It waits for the previous workflow complete, then reruns it.
+
+on:
+  workflow_dispatch:
+    inputs:
+      run_id:
+        required: true
+jobs:
+  retry:
+    runs-on: ubuntu-latest
+    permissions:
+      actions: write
+    steps:
+      - name: rerun ${{ inputs.run_id }}
+        env:
+          GH_REPO: ${{ github.repository }}
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          gh run watch ${{ inputs.run_id }}
+          gh run rerun ${{ inputs.run_id }}

.github/workflows/tshark.yml

@@ -1,64 +1,18 @@
 on:
-  pull_request:
-    branches:
-      - main
-    paths:
-      - ".github/Dockerfile.tshark"
-      - ".github/workflows/tshark.yml"
-
-  push:
-    branches:
-      - main
-    paths:
-      - ".github/Dockerfile.tshark"
-      - ".github/workflows/tshark.yml"
-
-  schedule:
-    # run every morning at 10am Pacific Time
-    # Running this every day makes sure the static dependencies are up to date
-    - cron: '0 17 * * *'
-
   workflow_dispatch:
     inputs:
       version:
-        description: 'wireshark version'
         required: true
-        default: '4.4.2'
-        type: string
-
-# Updating status is relatively safe (doesnt modify source code) and caution
-# should be taken before adding more permissions.
-permissions:
-  statuses: write
-  id-token: write # This is required for requesting the JWT/OIDC
-
 jobs:
-  build:
+  retry:
     runs-on: ubuntu-latest
-    env:
-      WIRESHARK_VERSION: ${{ inputs.version || '4.4.2' }}
+    permissions:
+      actions: write
     steps:
-      - uses: actions/checkout@v5
-
-      - run: mkdir -p target/tshark/v$WIRESHARK_VERSION
-
-      - name: Build
-        working-directory: .github
+      - name: rerun ${{ inputs.version }}
+        env:
+          GH_REPO: ${{ github.repository }}
+          GH_TOKEN: ${{ github.token }}
         run: |
-          docker build -f Dockerfile.tshark -t tshark-static --build-arg version=$WIRESHARK_VERSION .
-          docker run \
-            -v `pwd`/../target/tshark/v$WIRESHARK_VERSION:/host-dir \
-            tshark-static \
-            cp /usr/local/bin/tshark /usr/local/bin/editcap /host-dir/
-
-      - uses: aws-actions/configure-aws-credentials@v4.3.1
-        if: github.event_name == 'schedule' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.repository == github.event.pull_request.head.repo.full_name
-        with:
-          role-to-assume: arn:aws:iam::024603541914:role/GitHubOIDCRole
-          role-session-name: S2nQuicGHAS3Session
-          aws-region: us-west-2
-
-      - name: Upload to S3
-        if: github.event_name == 'schedule' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.repository == github.event.pull_request.head.repo.full_name
-        run: |
-          aws s3 sync target/tshark "s3://s2n-quic-ci-artifacts/tshark" --acl private --follow-symlinks
+          gh run watch ${{ inputs.version }}
+          gh run rerun ${{ inputs.version }}