Bug/node size failures

.github/workflows/node-increase-disk.yml

@@ -18,13 +18,28 @@ on:
         description: "The amount of diskspace to add (GB)"
         default: "50"
         required: true
+      disk:
+        description: "The disk to increase using growpart (e.g., /dev/xvda)"
+        default: "/dev/xvda"
+        required: true
+      partition_number:
+        description: "The partition number to extend using growpart (e.g., 1)"
+        default: "1"
+        required: true
+      partition_device:
+        description: "The partition device to extend using resize2fs (e.g., /dev/xvda1)"
+        default: "/dev/xvda1"
+        required: true
 
 
 jobs:
   increase-disk:
     environment: ${{inputs.network}}
     runs-on: ubuntu-latest
     steps:
+      - name: Install jq
+        run: sudo apt-get install -y jq
+
       - name: Checkout
         uses: actions/checkout@v4
 
@@ -39,14 +54,31 @@ jobs:
           aws_secret_access_key: ${{ secrets.AWS_SECRET_KEY }}
           aws_region: ${{vars.S3_REGION}}
 
-      - name: Run playbook
-        uses: arillso/action.playbook@master
+      - uses: actions/setup-python@v5
         with:
-          # Required, playbook filepath
-          playbook: ./scripts/ansible/aws/ec2-modify-volume-size.yml
-          inventory: ./ansible-hosts
-          galaxy_file: ./scripts/ansible/aws/requirements.yml
-          extra_vars: "aws_region=${{github.event.inputs.aws_region}} instance_id=${{inputs.instance_id}} aws_secret_key=${{secrets.AWS_SECRET_KEY}} aws_access_key_id=${{secrets.AWS_ACCESS_KEY_ID}} volume_size=${{github.event.inputs.disk_size}} ansible_python_interpreter='/usr/bin/python3'"
+          python-version: '3.11'
+
+      - name: Install Ansible + deps
+        run: |
+          python -m pip install --upgrade pip
+          pip install "ansible-core>=2.16" boto3 botocore packaging
+
+      - name: Galaxy install
+        run: ansible-galaxy install -r ./scripts/ansible/aws/requirements.yml
+
+      - name: Run playbook
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_KEY }}
+        run: |
+          ansible-playbook \
+            -i ./ansible-hosts \
+            --extra-vars "aws_region=${{github.event.inputs.aws_region}} instance_id=${{inputs.instance_id}} aws_secret_key=${{secrets.AWS_SECRET_KEY}} aws_access_key_id=${{secrets.AWS_ACCESS_KEY_ID}} volume_size=${{github.event.inputs.disk_size}} ansible_python_interpreter='/usr/bin/python3'" \
+            ./scripts/ansible/aws/ec2-modify-volume-size.yml
+
+      - name: Wait for AWS to finish provisioning the disk
+        run: sleep 30m
+        shell: bash
 
       - name: Extend file system
         id: extend-file-system
@@ -59,14 +91,29 @@ jobs:
             [{"Key":"InstanceIds","Values":["${{ inputs.instance_id }}"]}]
           document-name: AWS-RunShellScript
           parameters: |
-            {"commands":["sudo growpart /dev/nvme0n1 1", "sudo resize2fs /dev/nvme0n1p1"]}
+            {
+              "commands":[
+                "sudo growpart ${{ inputs.disk }} ${{ inputs.partition_number }}",
+                "sudo resize2fs ${{ inputs.partition_device }}"
+              ]
+            }
 
-      - name: Check SSM output
-        if: steps.extend-file-system.outcome == 'success'
+      - name: Get SSM output
+        if: always()
+        id: get-ssm-output
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_KEY }}
           AWS_DEFAULT_REGION: ${{ inputs.aws_region }}
         run: |
-          aws ssm list-command-invocations --command-id "${{ steps.extend-file-system.outputs.command-id }}" --details
+          COMMAND_OUTPUT=$(aws ssm list-command-invocations --command-id ${{ steps.extend-file-system.outputs.command-id }} --details | jq -c .)
+          echo "COMMAND_OUTPUT=$COMMAND_OUTPUT" >> $GITHUB_ENV
+
+      - name: Check SSM output
+        if: steps.get-ssm-output.outcome == 'success' && !contains(env.COMMAND_OUTPUT, 'Status:Success')
+        run: |
+          echo "Error detected trying to extend the partition."
+          echo ${{env.COMMAND_OUTPUT}}
+          exit 1
+
 

scripts/ansible/aws/ec2-modify-volume-size.yml

@@ -6,49 +6,49 @@
     volume_type_default: "{{ volume_type | default('io1') }}"
 
   pre_tasks:
-  - name: Install dependencies
-    pip: name={{ item }}
-    with_items:
-    - boto3
-    - botocore
+    - name: Install dependencies
+      pip: name={{ item }}
+      with_items:
+        - boto3
+        - botocore
 
   tasks:
-
-  - name: Get EC2 instance state
-    amazon.aws.ec2_instance:
-      aws_access_key_id: "{{ aws_access_key_id }}"
-      aws_secret_access_key: "{{ aws_secret_key }}"
-      region: "{{ aws_region }}"
-      instance_ids: [ "{{instance_id}}" ]
-    register: instances
-
-  - name: EC2 instance info
-    ansible.builtin.debug:
-      msg: Instance ID {{ instances['instances'][0].instance_id }}, root volume {{ instances['instances'][0].block_device_mappings[0].ebs }}
-
-  - name: Get current volume info
-    amazon.aws.ec2_vol:
-      aws_access_key_id: "{{ aws_access_key_id }}"
-      aws_secret_access_key: "{{ aws_secret_key }}"
-      region: "{{ aws_region }}"
-      instance: "{{ instance_id }}"
-      id: "{{ instances['instances'][0].block_device_mappings[0].ebs.volume_id }}"
-      modify_volume: false
-    register: volume_info
-
-  - name: EC2 volume info
-    ansible.builtin.debug:
-      msg: Volume ID {{ instances['instances'][0].block_device_mappings[0].ebs.volume_id }}, Old size {{ volume_info.volume.size }}, New size {{ volume_info.volume.size|int + volume_size|int }}
-
-  - name: Modify the volume
-    amazon.aws.ec2_vol:
-      aws_access_key_id: "{{ aws_access_key_id }}"
-      aws_secret_access_key: "{{ aws_secret_key }}"
-      region: "{{ aws_region }}"
-      instance: "{{ instance_id }}"
-      id: "{{ instances['instances'][0].block_device_mappings[0].ebs.volume_id }}"
-      volume_size: "{{ volume_info.volume.size|int + volume_size|int }}"
-      volume_type: "{{ volume_type_default }}"
-      iops: "{{ iops | default(1300) if volume_type_default is regex('^io*') else omit }}"
-      modify_volume: true
+    - name: Get EC2 instance state
+      amazon.aws.ec2_instance:
+        aws_access_key_id: "{{ aws_access_key_id }}"
+        aws_secret_access_key: "{{ aws_secret_key }}"
+        region: "{{ aws_region }}"
+        instance_ids: [ "{{instance_id}}" ]
+      register: instances
+
+    - name: EC2 instance info
+      ansible.builtin.debug:
+        msg: Instance ID {{ instances['instances'][0].instance_id }}, root volume {{ instances['instances'][0].block_device_mappings[0].ebs }}
+
+    - name: Get current volume info
+      amazon.aws.ec2_vol:
+        aws_access_key_id: "{{ aws_access_key_id }}"
+        aws_secret_access_key: "{{ aws_secret_key }}"
+        region: "{{ aws_region }}"
+        instance: "{{ instance_id }}"
+        id: "{{ instances['instances'][0].block_device_mappings[0].ebs.volume_id }}"
+        modify_volume: false
+      register: volume_info
+
+    - name: EC2 volume info
+      ansible.builtin.debug:
+        msg: Volume ID {{ instances['instances'][0].block_device_mappings[0].ebs.volume_id }}, Old size {{ volume_info.volume.size }}, New size {{ volume_info.volume.size|int + volume_size|int }}
+
+    - name: Modify the volume
+      amazon.aws.ec2_vol:
+        aws_access_key_id: "{{ aws_access_key_id }}"
+        aws_secret_access_key: "{{ aws_secret_key }}"
+        region: "{{ aws_region }}"
+        instance: "{{ instance_id }}"
+        id: "{{ instances['instances'][0].block_device_mappings[0].ebs.volume_id }}"
+        volume_size: "{{ volume_info.volume.size|int + volume_size|int }}"
+        volume_type: "{{ volume_type_default }}"
+        iops: "{{ iops | default(1300) if volume_type_default is regex('^io*') else omit }}"
+        modify_volume: true
+