AI analysis of migration failures

.github/workflows/packer.yml

@@ -42,6 +42,7 @@
   V2V_IMG: ${{ vars.REGISTRY || 'quay.io' }}/${{ vars.REPO || 'platform9' }}/vjailbreak-v2v-helper
   CONTROLLER_IMG: ${{ vars.REGISTRY || 'quay.io' }}/${{ vars.REPO || 'platform9' }}/vjailbreak-controller
   VPWNED_IMG: ${{ vars.REGISTRY || 'quay.io' }}/${{ vars.REPO || 'platform9' }}/vjailbreak-vpwned
+  AI_IMG: ${{ vars.REGISTRY || 'quay.io' }}/${{ vars.REPO || 'platform9' }}/vjailbreak-ai
   NIGHTLY_RELEASE: ${{ github.event.inputs.is_nightly }}
   AMPLITUDE_API_KEY: ${{ secrets.AMPLITUDE_API_KEY }}
   BUGSNAG_API_KEY: ${{ secrets.BUGSNAG_API_KEY }}
@@ -63,6 +64,7 @@
       controller_img: ${{ steps.set_env.outputs.controller_img }}
       qcow2_img: ${{ steps.set_env.outputs.qcow2_img }}
       vpwned_img: ${{ steps.set_env.outputs.vpwned_img }}
+      ai_img: ${{ steps.set_env.outputs.ai_img }}
 
     steps:
       - name: Checkout repository
@@ -148,6 +150,7 @@
           echo "v2v_img=${{ env.V2V_IMG }}:${TAG}" >> $GITHUB_OUTPUT
           echo "controller_img=${{ env.CONTROLLER_IMG }}:${TAG}" >> $GITHUB_OUTPUT
           echo "vpwned_img=${{ env.VPWNED_IMG }}:${TAG}" >> $GITHUB_OUTPUT
+          echo "ai_img=${{ env.AI_IMG }}:${TAG}" >> $GITHUB_OUTPUT
           echo "Final tag generated: ${TAG}"
 
   build-ui:
@@ -287,32 +290,65 @@
           path: vpwned-image.tar
           retention-days: 1
 
+  build-vjailbreak-ai:
+    runs-on: ubuntu-latest
+    if: "!startsWith(github.head_ref, 'pre-release/')"
+    needs: determine-release
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.inputs.branch || github.ref }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Set environment variables
+        run: |
+          echo "AI_IMG=${{ needs.determine-release.outputs.ai_img }}" >> $GITHUB_ENV
+
+      - name: Build vjailbreak-ai image
+        run: make vjailbreak-ai
+
+      - name: Save vjailbreak-ai image as tar file
+        run: |
+          docker save ${{ env.AI_IMG }} -o ai-image.tar
+
+      - name: Upload vjailbreak-ai image as artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: ai-docker-image
+          path: ai-image.tar
+          retention-days: 1
+
   push-images:
     runs-on: ubuntu-latest
-    needs: [determine-release, build-ui, build-v2v-helper, build-controller, build-vpwned]
+    needs: [determine-release, build-ui, build-v2v-helper, build-controller, build-vpwned, build-vjailbreak-ai]
     if: |
       !startsWith(github.head_ref, 'pre-release/') &&
       needs.determine-release.result == 'success' &&
-      (needs.build-ui.result == 'success' || needs.build-ui.result == 'skipped') && 
-      (needs.build-v2v-helper.result == 'success' || needs.build-v2v-helper.result == 'skipped') && 
-      (needs.build-controller.result == 'success' || needs.build-controller.result == 'skipped') && 
+      (needs.build-ui.result == 'success' || needs.build-ui.result == 'skipped') &&
+      (needs.build-v2v-helper.result == 'success' || needs.build-v2v-helper.result == 'skipped') &&
+      (needs.build-controller.result == 'success' || needs.build-controller.result == 'skipped') &&
       (needs.build-vpwned.result == 'success' || needs.build-vpwned.result == 'skipped') &&
+      (needs.build-vjailbreak-ai.result == 'success' || needs.build-vjailbreak-ai.result == 'skipped') &&
       (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
-    
+
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
         with:
           ref: ${{ github.event.inputs.branch || github.ref }}
-          
+
       - name: Login to Docker Hub
         if: ${{ !env.ACT }}
         uses: docker/login-action@v3
         with:
           registry: ${{ vars.REGISTRY }}
           username: ${{ secrets.QUAY_ROBOT_USERNAME }}
           password: ${{ secrets.QUAY_ROBOT_PASSWORD }}
-          
+
       - name: Set environment variables
         run: |
           echo "TAG=${{ needs.determine-release.outputs.tag }}" >> $GITHUB_ENV
@@ -321,7 +357,8 @@
           echo "CONTROLLER_IMG=${{ needs.determine-release.outputs.controller_img }}" >> $GITHUB_ENV
           echo "QCOW2_IMG=${{ needs.determine-release.outputs.qcow2_img }}" >> $GITHUB_ENV
           echo "VPWNED_IMG=${{ needs.determine-release.outputs.vpwned_img }}" >> $GITHUB_ENV
-
+          echo "AI_IMG=${{ needs.determine-release.outputs.ai_img }}" >> $GITHUB_ENV
+
       - name: Download UI image artifact
         if: ${{ !env.ACT }}
         uses: actions/download-artifact@v4
@@ -335,55 +372,68 @@
         with:
           name: v2v-docker-image
           path: ./docker-images
-          
+
       - name: Download Controller image artifact
         if: ${{ !env.ACT }}
         uses: actions/download-artifact@v4
         with:
           name: controller-docker-image
           path: ./docker-images
-          
+
       - name: Download VPWNED image artifact
         if: ${{ !env.ACT }}
         uses: actions/download-artifact@v4
         with:
           name: vpwned-docker-image
           path: ./docker-images
-
+
+      - name: Download vjailbreak-ai image artifact
+        if: ${{ !env.ACT }}
+        uses: actions/download-artifact@v4
+        with:
+          name: ai-docker-image
+          path: ./docker-images
+
       - name: Load Docker images
         if: ${{ !env.ACT }}
         run: |
           docker load -i ./docker-images/ui-image.tar
           docker load -i ./docker-images/v2v-image.tar
           docker load -i ./docker-images/controller-image.tar
           docker load -i ./docker-images/vpwned-image.tar
-
+          docker load -i ./docker-images/ai-image.tar
+
       - name: Push UI Image
         if: ${{ !env.ACT }}
         run: docker push ${{ env.UI_IMG }}
-        
+
       - name: Push V2V Helper Image
         if: ${{ !env.ACT }}
         run: docker push ${{ env.V2V_IMG }}
-        
+
       - name: Push Controller Image
         if: ${{ !env.ACT }}
         run: docker push ${{ env.CONTROLLER_IMG }}
-        
+
       - name: Push VPWNED Image
         if: ${{ !env.ACT }}
         run: docker push ${{ env.VPWNED_IMG }}
+
+      - name: Push vjailbreak-ai Image
+        if: ${{ !env.ACT }}
+        run: docker push ${{ env.AI_IMG }}
 
   post-build:
     runs-on: ubuntu-latest
-    needs: [determine-release, build-ui, build-v2v-helper, build-controller, build-vpwned, push-images]
+    needs: [determine-release, build-ui, build-v2v-helper, build-controller, build-vpwned, build-vjailbreak-ai, push-images]
     if: |
       !startsWith(github.head_ref, 'pre-release/') &&
       needs.determine-release.result == 'success' &&
-      (needs.build-ui.result == 'success' || needs.build-ui.result == 'skipped') && 
-      (needs.build-v2v-helper.result == 'success' || needs.build-v2v-helper.result == 'skipped') && 
-      (needs.build-controller.result == 'success' || needs.build-controller.result == 'skipped') && 
-      (needs.build-vpwned.result == 'success' || needs.build-vpwned.result == 'skipped') && 
+      (needs.build-ui.result == 'success' || needs.build-ui.result == 'skipped') &&
+      (needs.build-v2v-helper.result == 'success' || needs.build-v2v-helper.result == 'skipped') &&
+      (needs.build-controller.result == 'success' || needs.build-controller.result == 'skipped') &&
+      (needs.build-vpwned.result == 'success' || needs.build-vpwned.result == 'skipped') &&
+      (needs.build-vjailbreak-ai.result == 'success' || needs.build-vjailbreak-ai.result == 'skipped') &&
       (needs.push-images.result == 'success')
 
     steps:
@@ -411,6 +461,7 @@
           echo "CONTROLLER_IMG=${{ needs.determine-release.outputs.controller_img }}" >> $GITHUB_ENV
           echo "QCOW2_IMG=${{ needs.determine-release.outputs.qcow2_img }}" >> $GITHUB_ENV
           echo "VPWNED_IMG=${{ needs.determine-release.outputs.vpwned_img }}" >> $GITHUB_ENV
+          echo "AI_IMG=${{ needs.determine-release.outputs.ai_img }}" >> $GITHUB_ENV
 
           if [[ "${{ needs.determine-release.outputs.is_release }}" == "true" ]]; then
             echo "release_found=true" >> $GITHUB_ENV
@@ -430,6 +481,12 @@
           input: ./ui/deploy/ui.yaml
           output: ./image_builder/deploy/01ui.yaml
 
+      - name: Substitute image tag in vjailbreak-ai manifest
+        uses: danielr1996/envsubst-action@1.0.0
+        with:
+          input: ./vjailbreak-ai/deploy/vjailbreak-ai.yaml
+          output: ./image_builder/deploy/08vjailbreak-ai.yaml
+
       - name: Substitue image tags in version config
         uses: danielr1996/envsubst-action@1.0.0
         with:

.gitignore

@@ -23,3 +23,4 @@ proto_dependencies
 ui/test-results/*
 ui/playwright-report/*
 ui/coverage/*
+graphify-out/

Makefile

@@ -25,6 +25,7 @@ export UI_IMG ?= ${REGISTRY}/${REPO}/vjailbreak-ui:${TAG}
 export V2V_IMG ?= ${REGISTRY}/${REPO}/v2v-helper:${TAG}
 export CONTROLLER_IMG ?= ${REGISTRY}/${REPO}/vjailbreak-controller:${TAG}
 export VPWNED_IMG ?= ${REGISTRY}/${REPO}/vjailbreak-vpwned:${TAG}
+export AI_IMG ?= ${REGISTRY}/${REPO}/vjailbreak-ai:${TAG}
 export RELEASE_VERSION ?= $(VERSION)
 export KUBECONFIG ?= ~/.kube/config
 export CONTAINER_TOOL ?= docker
@@ -72,6 +73,10 @@ generate-manifests: setup-hooks vjail-controller ui
 build-vpwned: setup-hooks
 	make -C pkg/vpwned docker-build
 
+.PHONY: vjailbreak-ai
+vjailbreak-ai: setup-hooks
+	docker build --platform linux/amd64 -t $(AI_IMG) vjailbreak-ai/
+
 build-installer: setup-hooks
 	make -C k8s/migration/ build-installer 
 

deploy/vjailbreak-ai-context-configmap.yaml

@@ -0,0 +1,75 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: vjailbreak-ai-context
+  namespace: migration-system
+data:
+  additional_context: |
+    ## vJailbreak Architecture
+
+    vJailbreak migrates VMware VMs to OpenStack in this sequence:
+    1. Controller creates a v2v-helper pod per migration
+    2. v2v-helper runs virt-v2v with the nbdkit+VDDK plugin to stream VMDK data
+    3. Converted disk is uploaded to OpenStack Glance as QCOW2 (or written to a Cinder volume via RAW)
+    4. Nova boots the migrated VM using the uploaded image or volume
+
+    Key components:
+    - **migration-controller-manager** — Kubernetes controller in `migration-system` namespace; reconciles Migration CRs
+    - **v2v-helper pod** — named `<migration-name>` in `migration-system`; runs virt-v2v; one pod per Migration
+    - **VDDK** — VMware Virtual Disk Development Kit at `/home/ubuntu/vmware-vix-disklib-distrib/` on the vJailbreak VM
+    - **Conversion scratch space** — `emptyDir` at `/var/tmp/v2v-conversion-disks/` inside v2v-helper pod (~2× VM disk size required)
+    - **storageCopyMethod** — `hot` (live/CBT incremental) or `cold` (offline full-copy); set in MigrationTemplate
+
+    CRD hierarchy: Migration → MigrationPlan → MigrationTemplate → NetworkMapping + StorageMapping
+
+    ---
+
+    ## Common Failure Patterns and Fixes
+
+    ### VDDK / VMware Connectivity
+    | Symptom | Likely Cause | Fix |
+    |---------|-------------|-----|
+    | `VixDiskLib_Open failed` / `Transport error` | VDDK library missing or wrong version | Verify `/home/ubuntu/vmware-vix-disklib-distrib/` exists; VDDK 7.0+ required |
+    | `Failed to connect to host <esxi-fqdn>` | ESXi hostname not resolving in pod | Add ESXi host entries to `/etc/hosts` on vJailbreak VM, then restart controller |
+    | `SSL certificate verify failed` | vCenter TLS cert untrusted | Set `InsecureSkipVerify: true` in VMwareCreds or add cert to trust store |
+    | `NoPermission` / `NotAuthenticated` | Insufficient vCenter permissions | Account needs Datastore.Browse, VirtualMachine.Provisioning.DiskRandomRead, Global.DisableMethods |
+    | `CBT not enabled` / `queryChangedDiskAreas` error | Changed Block Tracking disabled on source VM | Enable CBT in vCenter VM settings, create a snapshot, then retry |
+
+    ### virt-v2v / Conversion
+    | Symptom | Likely Cause | Fix |
+    |---------|-------------|-----|
+    | `No space left on device` | Node disk exhausted during conversion | Ensure node has ≥2× VM disk size free in `/var/tmp`; check `df -h` on vJailbreak VM |
+    | `libguestfs: error: could not create appliance` | libguestfs kernel/QEMU unavailable in pod | Check pod for missing KVM device; vJailbreak VM must have nested virt or bare-metal KVM |
+    | `Failed to locate virtio drivers` / `virtio-win` | virtio-win ISO not found | Confirm virtio-win ISO is present at the expected path in the v2v-helper image |
+    | `qemu-img: Could not open` / `corrupt` | VMDK corrupt or CBT snapshot stale | Remove stale snapshots in vCenter; try `storageCopyMethod: cold` |
+    | `virt-v2v: error: inspection of disk image failed` | Unsupported guest OS | Check virt-v2v support page; Windows requires virtio-win, Linux needs grub accessible |
+    | Conversion stuck / timeout | Very large VM or slow storage | Increase `migrationTimeout` in MigrationPlan; check network throughput between vJailbreak and ESXi |
+
+    ### OpenStack Upload / Deploy
+    | Symptom | Likely Cause | Fix |
+    |---------|-------------|-----|
+    | `No valid host found` | Flavor constraints or resource exhaustion | Verify flavor exists and Nova has capacity; check `openstack flavor list` |
+    | `Quota exceeded` | OpenStack project quota hit | Check `openstack quota show` for instances, cores, RAM, volumes, gigabytes |
+    | `Network <name> could not be found` | NetworkMapping source or target name mismatch | Verify NetworkMapping.Spec matches exact OpenStack network names |
+    | `VolumeType <name> not found` | StorageMapping mismatch | Verify StorageMapping.Spec matches exact Cinder volume type names; case-sensitive |
+    | `Provided image is too large` | Glance upload size limit | Increase `glance-api.conf image_size_cap`; or use `storageCopyMethod: raw-volume` to bypass Glance |
+    | `ImageNotFound` during Nova boot | Glance upload failed or image in error state | Check `openstack image list`; look for image in `killed` or `queued` state |
+
+    ### Kubernetes / Controller
+    | Symptom | Likely Cause | Fix |
+    |---------|-------------|-----|
+    | Pod `OOMKilled` | Insufficient memory for conversion | Large Windows VMs need 4–8 GB; increase v2v-helper memory limit in MigrationPlan |
+    | Pod stuck in `Pending` | Node selector mismatch or node resource shortage | Check `kubectl describe pod <name>` for scheduling events |
+    | `context deadline exceeded` in controller | Migration timeout too short | Increase `migrationTimeout` in MigrationPlan spec |
+    | Migration stuck in `DataCopyStart` | NBD/VDDK stream stalled | Check v2v-helper logs for `nbdkit` errors; restart migration |
+
+    ---
+
+    ## Analysis Guidelines
+
+    - **Always quote specific log lines** as evidence in `root_cause` — vague causes are not actionable
+    - **fix_steps** should be concrete shell commands or UI steps, not generic advice
+    - Check `storageCopyMethod` (`hot` vs `cold`) — hot migrations have extra pre-copy phases that can fail
+    - If `phase` is `Failed` but logs are empty, check controller logs for reconcile errors mentioning the Migration name
+    - Network and storage mapping errors almost always have the exact mismatched name in the log — quote it
+    - If confidence is `low` or `none`, list the specific information that would be needed to diagnose further

deploy/vjailbreak-ai/context-configmap.yaml

@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: vjailbreak-ai-context
+  namespace: migration-system
+data:
+  # Operator-provided context injected into every AI analysis prompt.
+  # Edit the value below to add site-specific information.
+  # Examples:
+  #   - Custom VDDK install path
+  #   - Known ESXi certificate issues in this environment
+  #   - Network restrictions that affect migrations
+  #   - OpenStack volume type mappings in use
+  additional_context: |
+    # Add operator-specific notes here.
+    # Example:
+    # VDDK is installed at /opt/vmware/vddk (non-standard path).
+    # All ESXi hosts use self-signed TLS certificates -- SSL errors are expected.
+    # OpenStack Cinder volume type "ssd" maps to VMware datastore "fast-ssd".