diff --git a/.github/workflows/bs_meta_browser_build_and_test.yml b/.github/workflows/bs_meta_browser_build_and_test.yml new file mode 100644 index 000000000..4bb0bb288 --- /dev/null +++ b/.github/workflows/bs_meta_browser_build_and_test.yml @@ -0,0 +1,451 @@ +name: 'Build and Test Meta-Browser: Build and test workflow' +on: + workflow_call: + inputs: + runner_name: + description: 'Runner name' + required: true + type: string + + github_hosted_runner: + description: 'Whether to use GitHub-hosted runner' + required: false + type: boolean + default: false + + browser: + description: "Electron or Chromium" + required: true + type: string + + build_type: + description: 'Build Type' + required: true + type: string + + chromium_version: + description: 'Display backend/Ozone platform (ozone-wayland or x11)' + required: true + type: string + + arch: + description: 'Architecture' + required: true + type: string + + aws_arn_role: + required: true + type: string + + aws_region: + required: true + type: string + +jobs: + build-and-test-meta-browser: + name: Build and Test Meta-Browser + runs-on: ${{ inputs.runner_name }} + timeout-minutes: 660 # 11 hours - increase for long Yocto builds + defaults: + run: + shell: bash + steps: + # Configure AWS credentials for accessing S3 cache buckets + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: ${{ inputs.aws_region }} + role-to-assume: ${{ inputs.aws_arn_role }} + role-session-name: mb-build-${{ github.run_id }} + role-duration-seconds: 43200 # 12 hours - maximum for most IAM roles + + # Download config file and set environment variables + - name: Download config file and set env vars from it + run: | + aws s3 cp s3://meta-browser-ci-config-bucket/config.json . + aws s3 cp s3://meta-browser-ci-config-bucket/set_github_env_vars.py . + python3 set_github_env_vars.py --file config.json + + # Quick system verification (using pre-configured custom AMI) + - name: Verify AMI configuration + run: | + echo "=== Verifying pre-configured AMI ===" + echo "AMI Info:" + cat /etc/ami-info || echo "No AMI info file found" + + echo "" + echo "Checking KAS installation:" + kas --version || echo "KAS not found" + + echo "" + echo "Checking locale:" + locale + + echo "" + echo "Checking AWS CLI:" + aws --version || echo "AWS CLI not found" + + echo "" + echo "Available disk space:" + df -h + + echo "" + echo "Memory info:" + free -h + + echo "✓ AMI verification complete - all dependencies pre-installed" + + # Setup large swap space on local NVMe SSD for memory-intensive Yocto builds + - name: Setup NVMe storage and swap + run: | + echo "=== Setting up NVMe storage and swap ===" + + # Get current user once for all operations + CURRENT_USER=$(whoami) + echo "Setting up NVMe storage for user: $CURRENT_USER" + + # Check for NVMe storage and set up build directory + if [ -e /dev/nvme1n1 ]; then + echo "NVMe storage detected, setting up build directory..." + + # Create filesystem if not already done + if ! mount | grep -q nvme1n1; then + sudo mkfs.ext4 -F /dev/nvme1n1 + sudo mkdir -p /mnt/nvme + sudo mount /dev/nvme1n1 /mnt/nvme + sudo chown $CURRENT_USER:$CURRENT_USER /mnt/nvme + fi + + # Create build directory on NVMe storage + echo "Setting up build workspace on NVMe storage..." + sudo mkdir -p /mnt/nvme/yocto-build + sudo chown $CURRENT_USER:$CURRENT_USER /mnt/nvme/yocto-build + + # Create symlink from /yocto to NVMe storage + sudo rm -rf /yocto 2>/dev/null || true + sudo ln -sf /mnt/nvme/yocto-build /yocto + sudo chown -h $CURRENT_USER:$CURRENT_USER /yocto + + # Set up swap on NVMe (256GB) + echo "Creating 256GB swap file on NVMe SSD..." + sudo fallocate -l 256G /mnt/nvme/swapfile + sudo chmod 600 /mnt/nvme/swapfile + sudo mkswap /mnt/nvme/swapfile + sudo swapon /mnt/nvme/swapfile + + echo "✓ Build directory and swap configured on NVMe storage" + else + echo "ERROR: No NVMe device found! Instance should have NVMe storage for optimal build performance." + echo "Available block devices:" + lsblk + exit 1 + fi + + # Verify swap is active + echo "=== Memory and swap status ===" + free -h + swapon --show + echo "Available disk space:" + df -h + + # Mount EFS file systems and setup cache directories + - name: Setup EFS mounts and cache directories + run: | + # Get current user for consistent permission management + CURRENT_USER=$(whoami) + echo "Setting up EFS mounts and cache directories for user: $CURRENT_USER" + + # Create mount points and cache directories + sudo mkdir -p /mnt/shared-cache/sstate-cache /mnt/shared-cache/downloads /mnt/git-mirror + + # Debug DNS resolution + echo "=== DNS Troubleshooting ===" + echo "Checking DNS resolution for EFS endpoints..." + nslookup ${{ env.SHARED_CACHE_EFS_ID }}.efs.${{ env.AWS_REGION }}.amazonaws.com || echo "DNS lookup failed" + nslookup ${{ env.GIT_MIRROR_EFS_ID }}.efs.${{ env.AWS_REGION }}.amazonaws.com || echo "DNS lookup failed" + + # Check AWS metadata service and VPC DNS + echo "=== VPC DNS Settings ===" + curl -s http://169.254.169.254/latest/meta-data/network/interfaces/macs/ | head -1 | xargs -I {} curl -s http://169.254.169.254/latest/meta-data/network/interfaces/macs/{}/vpc-id || echo "VPC metadata failed" + + # Check if we can reach AWS services + echo "=== AWS Connectivity Check ===" + if curl -s --connect-timeout 10 https://s3.amazonaws.com; then + echo "✓ S3 reachable" + else + echo "✗ S3 not reachable - this will cause build artifact upload to fail" + echo "Network connectivity issue detected" + exit 1 + fi + + # Alternative: Use mount target IP if DNS fails + echo "=== Alternative EFS Mount Strategy ===" + # Try to resolve mount target IPs through AWS CLI if available + if command -v aws &> /dev/null; then + echo "Attempting to get EFS mount targets..." + aws efs describe-mount-targets --file-system-id ${{ env.SHARED_CACHE_EFS_ID }} --region ${{ env.AWS_REGION }} || echo "AWS CLI describe failed" + fi + + # Try using regional EFS mount target instead + echo "=== Mounting EFS file systems ===" + echo "Shared cache EFS: ${{ env.SHARED_CACHE_EFS_ID }}" + echo "Git mirror EFS: ${{ env.GIT_MIRROR_EFS_ID }}" + echo "AWS Region: ${{ env.AWS_REGION }}" + + # Get EFS mount target IPs for fallback + SHARED_CACHE_IP=$(aws efs describe-mount-targets --file-system-id ${{ env.SHARED_CACHE_EFS_ID }} --region ${{ env.AWS_REGION }} --query 'MountTargets[0].IpAddress' --output text 2>/dev/null || echo "") + GIT_MIRROR_IP=$(aws efs describe-mount-targets --file-system-id ${{ env.GIT_MIRROR_EFS_ID }} --region ${{ env.AWS_REGION }} --query 'MountTargets[0].IpAddress' --output text 2>/dev/null || echo "") + echo "Shared cache EFS IP: $SHARED_CACHE_IP" + echo "Git mirror EFS IP: $GIT_MIRROR_IP" + + # Mount shared cache EFS (sstate, downloads) with DNS and IP fallback + SHARED_MOUNTED=false + for i in {1..2}; do + echo "Attempt $i: Mounting shared cache EFS..." + # Try DNS first + if sudo mount -t nfs4 -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,intr,timeo=600 \ + ${{ env.SHARED_CACHE_EFS_ID }}.efs.${{ env.AWS_REGION }}.amazonaws.com:/ /mnt/shared-cache 2>/dev/null; then + echo "✓ Shared cache EFS mounted successfully via DNS" + SHARED_MOUNTED=true + break + elif [ -n "$SHARED_CACHE_IP" ]; then + echo "DNS failed, trying IP address: $SHARED_CACHE_IP" + if sudo mount -t nfs4 -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,intr,timeo=600 \ + $SHARED_CACHE_IP:/ /mnt/shared-cache; then + echo "✓ Shared cache EFS mounted successfully via IP" + SHARED_MOUNTED=true + break + fi + fi + echo "✗ Mount attempt $i failed, retrying..." + sleep 5 + done + + # Mount git mirror EFS with DNS and IP fallback + GIT_MOUNTED=false + for i in {1..2}; do + echo "Attempt $i: Mounting git mirror EFS..." + # Try DNS first + if sudo mount -t nfs4 -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,intr,timeo=600 \ + ${{ env.GIT_MIRROR_EFS_ID }}.efs.${{ env.AWS_REGION }}.amazonaws.com:/ /mnt/git-mirror 2>/dev/null; then + echo "✓ Git mirror EFS mounted successfully via DNS" + GIT_MOUNTED=true + break + elif [ -n "$GIT_MIRROR_IP" ]; then + echo "DNS failed, trying IP address: $GIT_MIRROR_IP" + if sudo mount -t nfs4 -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,intr,timeo=600 \ + $GIT_MIRROR_IP:/ /mnt/git-mirror; then + echo "✓ Git mirror EFS mounted successfully via IP" + GIT_MOUNTED=true + break + fi + fi + echo "✗ Mount attempt $i failed, retrying..." + sleep 5 + done + + # Report mount results + if [ "$SHARED_MOUNTED" = "false" ]; then + echo "⚠️ WARNING: Failed to mount shared cache EFS - builds will be slower without cache" + fi + if [ "$GIT_MOUNTED" = "false" ]; then + echo "⚠️ WARNING: Failed to mount git mirror EFS - will clone repositories directly" + fi + + if [ "$SHARED_MOUNTED" = "true" ] && [ "$GIT_MOUNTED" = "true" ]; then + echo "🎉 All EFS file systems mounted successfully!" + fi + + # Verify mounts + echo "=== Mount Verification ===" + mount | grep mnt || echo "No /mnt filesystems found" + df -h | grep mnt || echo "No /mnt filesystems in df" + + # Set permissions on mount points only (not recursive to avoid processing massive cache data) + sudo chown $CURRENT_USER:$CURRENT_USER /mnt/shared-cache /mnt/git-mirror + sudo chmod 755 /mnt/shared-cache /mnt/git-mirror + echo "✓ Mount point permissions set (skipped recursive operation to avoid processing 100GB+ of cache data)" + + # Set up build directories and environment + - name: Setup build environment + run: | + # Set environment variables and get current user + export LANG=en_US.UTF-8 + export LC_ALL=en_US.UTF-8 + CURRENT_USER=$(whoami) + echo "Setting up build environment for user: $CURRENT_USER" + + # Setup environment variables for all shells + echo "export PATH=\$PATH:\$HOME/.local/bin" >> ~/.bashrc + echo "export LANG=en_US.UTF-8" >> ~/.bashrc + echo "export LC_ALL=en_US.UTF-8" >> ~/.bashrc + + # Create and setup the build directory structure + sudo mkdir -p /yocto /yocto/yocto_ccache /yocto/test-images + sudo chown -R $CURRENT_USER:$CURRENT_USER /yocto + sudo chmod -R 755 /yocto /yocto/yocto_ccache /yocto/test-images + + # Clean any previous builds in the NVMe-backed directory + cd /yocto + rm -rf meta-browser build/tmp/work/*/*/*/pseudo build/tmp/sysroots-components/*/pseudo 2>/dev/null || true + + # Ensure current directory has proper permissions for symlink creation + sudo chown $CURRENT_USER:$CURRENT_USER . + sudo chmod 755 . + + # Create symlinks to EFS mounts in both build directory and /yocto root + ln -sf /mnt/shared-cache/sstate-cache ./sstate-cache + ln -sf /mnt/shared-cache/downloads ./downloads + ln -sf /mnt/shared-cache/sstate-cache /yocto/yocto_sstate_chromium + ln -sf /mnt/shared-cache/sstate-cache /yocto/yocto_sstate_electron + ln -sf /mnt/shared-cache/downloads /yocto/yocto_dl + + # Clone repositories (check if PR or manual trigger) + - name: Clone repositories + run: | + # Clone directly into the NVMe-backed /yocto directory + cd /yocto + + if [ "${{ github.event_name }}" = "pull_request" ]; then + GH_URL="$GITHUB_SERVER_URL/${{ github.event.pull_request.head.repo.full_name }}" + GH_REV="$GITHUB_HEAD_REF" + else + GH_URL="$GITHUB_SERVER_URL/${{ github.repository }}" + GH_REV="${{ github.ref_name }}" + fi + + echo "Cloning from $GH_URL, branch/ref: $GH_REV" + git clone $GH_URL + git -C meta-browser checkout $GH_REV + + # meta-chromium-test is now integrated into meta-browser + + # Run the build + - name: Build Browser + timeout-minutes: 660 # 11 hours for Yocto build + run: | + # Set environment variables that the Docker container would have + export LANG=en_US.UTF-8 + export LC_ALL=en_US.UTF-8 + export PATH=$PATH:$HOME/.local/bin + + # Debug environment and verify setup + echo "=== Environment Debug ===" + echo "PATH: $PATH" + echo "User: $(whoami)" + echo "Current directory: $(pwd)" + which kas || echo "kas not found in PATH" + + echo "Building ${{ inputs.browser }} with ${{ inputs.chromium_version }}" + + # The build script expects to run from /yocto (repositories already cloned here) + cd /yocto + + # Verify repositories are present + if [ -d "meta-browser" ] && [ -d "meta-browser/meta-chromium-test" ]; then + echo "✓ Repository successfully cloned to /yocto" + ls -la meta-browser/ meta-browser/meta-chromium-test/ + else + echo "✗ ERROR: Repository not found in /yocto" + ls -la + exit 1 + fi + + echo "Building ${{ inputs.browser }} with ${{ inputs.chromium_version }}" + + # Verify the build script exists and is executable + if [ -f "./meta-browser/meta-chromium-test/scripts/build.sh" ]; then + echo "✓ Build script found and ready" + chmod +x ./meta-browser/meta-chromium-test/scripts/build.sh + else + echo "✗ Build script not found!" + find . -name "build.sh" 2>/dev/null || echo "No build.sh found anywhere" + exit 1 + fi + + # Fix test-images directory permissions (build script needs to write here) + CURRENT_USER=$(whoami) + sudo chown -R $CURRENT_USER:$CURRENT_USER /yocto/test-images + sudo chmod 755 /yocto/test-images + + # Run the build script + echo "=== Starting build ===" + df -h + + ./meta-browser/meta-chromium-test/scripts/build.sh ${{ inputs.arch }} ${{ inputs.chromium_version }} ${{ inputs.browser }} + + # Refresh AWS credentials before upload (they may have expired during long build) + - name: Refresh AWS credentials for artifact upload + if: success() + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: ${{ env.AWS_REGION }} + role-to-assume: ${{ inputs.aws_arn_role }} + role-session-name: mb-upload-${{ github.run_id }} + role-duration-seconds: 3600 # 1 hour for upload + + # Upload build artifacts to S3 (if successful) + - name: Upload artifacts + if: success() + run: | + cd /yocto + + # Find and upload build artifacts + # The build directory is created by kas in the meta-browser directory + DEPLOY_DIR=$(find meta-browser/build -name "deploy" -type d | head -1) + if [ -n "$DEPLOY_DIR" ] && [ -d "$DEPLOY_DIR" ]; then + echo "=== Artifact Upload ===" + echo "Deploy directory: $DEPLOY_DIR" + echo "Checking directory contents:" + ls -la "$DEPLOY_DIR" || echo "Failed to list deploy directory" + + # Check available disk space + echo "Available disk space:" + df -h + + # Calculate upload size + UPLOAD_SIZE=$(du -sh "$DEPLOY_DIR" | cut -f1) + echo "Upload size: $UPLOAD_SIZE" + + # S3 destination path + S3_PATH="s3://${ARTIFACT_BUCKET_NAME}/meta-browser/${{ inputs.arch }}/${{ inputs.chromium_version }}/${{ github.run_id }}/" + echo "S3 destination: $S3_PATH" + echo "Bucket name from env: ${ARTIFACT_BUCKET_NAME}" + + # Verify AWS credentials are working + echo "Verifying AWS credentials..." + if aws sts get-caller-identity; then + echo "✓ AWS credentials verified successfully" + else + echo "✗ AWS credentials verification failed" + exit 1 + fi + + echo "Starting S3 upload..." + # Use quiet mode to avoid excessive logging for large uploads + if aws s3 sync "$DEPLOY_DIR/" "$S3_PATH" --cli-read-timeout 0 --cli-connect-timeout 60 --quiet; then + echo "✓ Artifacts uploaded successfully to $S3_PATH" + + # Verify upload + echo "Verifying upload..." + aws s3 ls "$S3_PATH" --recursive --human-readable --summarize + else + echo "✗ S3 upload failed" + exit 1 + fi + else + echo "No deploy directory found" + echo "Available directories:" + find . -name "deploy" -type d 2>/dev/null || echo "No deploy directories found" + echo "Build directory structure:" + find meta-browser/build -type d -maxdepth 3 2>/dev/null || echo "No meta-browser/build directory found" + echo "All meta-browser contents:" + ls -la meta-browser/ 2>/dev/null || echo "No meta-browser directory found" + fi + + # Unmount EFS file systems + - name: Cleanup EFS mounts + if: always() + run: | + sudo umount /mnt/shared-cache || true + sudo umount /mnt/git-mirror || true diff --git a/.github/workflows/bs_meta_browser_ci_ec2.yml b/.github/workflows/bs_meta_browser_ci_ec2.yml new file mode 100644 index 000000000..3b39160f5 --- /dev/null +++ b/.github/workflows/bs_meta_browser_ci_ec2.yml @@ -0,0 +1,143 @@ +name: 'Build and Test Meta-Browser: EC2 controller' +on: + workflow_call: + inputs: + build_type: + description: 'Build Type' + required: true + type: string + + browser: + description: 'Chromium or Electron' + required: true + type: string + + chromium_version: + description: 'Display backend/Ozone platform (ozone-wayland or x11)' + required: true + type: string + + arch: + description: 'Target architecture' + required: true + type: string + + instance_type: + description: 'EC2 instance type' + required: false + type: string + default: c6a.4xlarge + + leave_ec2_instance_running: + description: 'Leave EC2 instance running after use' + type: boolean + default: false + + instance_name_postfix: + description: 'Name to add as postfix to the EC2 machine' + type: string + default: auto-triggered + + aws_arn_role: + required: true + type: string + + aws_region: + required: true + type: string + +jobs: + start-runner: + name: Start self-hosted EC2 runner + runs-on: ubuntu-latest + outputs: + label: ${{ steps.start-ec2-runner.outputs.label }} + ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: ${{ inputs.aws_region }} + role-to-assume: ${{ inputs.aws_arn_role }} + role-session-name: mb-ci-${{ github.run_id }} + role-duration-seconds: 43200 # 12 hours for long builds + + - name: Download config file and set env vars from it + run: | + aws s3 cp s3://meta-browser-ci-config-bucket/config.json . + aws s3 cp s3://meta-browser-ci-config-bucket/set_github_env_vars.py . + python3 set_github_env_vars.py --file config.json + + - name: Clean up any leftover runners + run: | + echo "Checking for any leftover runners from previous runs..." + echo "Current GitHub run context:" + echo "Run ID: ${{ github.run_id }}" + echo "Run attempt: ${{ github.run_attempt }}" + echo "Run number: ${{ github.run_number }}" + echo "This run will use unique label with attempt number to avoid conflicts" + + - name: Start EC2 runner with retry + id: start-ec2-runner + uses: brightsign/ec2-github-runner@0fa8b183dd4124fd191ccdbc48b68f0ea46a9634 + timeout-minutes: 15 # Allow more time for registration + with: + mode: start + github-app-private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} + github-app-id: 287690 + ec2-image-id: ami-08a4255385679596c # Custom AMI with Yocto build dependencies pre-installed + ec2-instance-type: ${{ inputs.instance_type }} + subnet-id: ${{ env.VPC_SUBNET_ID }} + security-group-id: ${{ env.VPC_SG_ID }} + run-as-service-with-user: ubuntu + label: "mb-${{ github.run_id }}-${{ github.run_attempt }}-${{ inputs.browser }}-${{ inputs.arch }}-${{ inputs.chromium_version }}" # Unique per matrix job with attempt + aws-resource-tags: > # optional, requires additional permissions + [ + {"Key": "Name", "Value": "github-runner-meta-browser-${{ inputs.instance_name_postfix }}-${{ github.run_id }}"}, + {"Key": "GitHubRepository", "Value": "${{ github.repository }}"}, + {"Key": "ChromiumVersion", "Value": "${{ inputs.chromium_version }}"}, + {"Key": "Architecture", "Value": "${{ inputs.arch }}"}, + {"Key": "RunId", "Value": "${{ github.run_id }}"}, + {"Key": "RunAttempt", "Value": "${{ github.run_attempt }}"}, + {"Key": "Browser", "Value": "${{ inputs.browser }}"} + ] + + build-and-test-meta-browser: + name: Build and Test Meta-Browser + needs: start-runner # required to start the main job when the runner is ready + uses: ./.github/workflows/bs_meta_browser_build_and_test.yml + secrets: inherit + with: + runner_name: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner + github_hosted_runner: false + browser: ${{ inputs.browser }} + build_type: ${{ inputs.build_type }} + chromium_version: ${{ inputs.chromium_version }} + arch: ${{ inputs.arch }} + aws_arn_role: ${{ inputs.aws_arn_role }} + aws_region: ${{ inputs.aws_region }} + + stop-runner: + name: Stop self-hosted EC2 runner + needs: + - start-runner # required to get output from the start-runner job + - build-and-test-meta-browser # required to wait when the main job is done + runs-on: ubuntu-latest + if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ inputs.aws_arn_role }} + aws-region: ${{ inputs.aws_region }} + role-session-name: mb-cleanup-${{ github.run_id }} + + - name: Stop EC2 runner + uses: brightsign/ec2-github-runner@0fa8b183dd4124fd191ccdbc48b68f0ea46a9634 + with: + mode: stop + github-app-private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} + github-app-id: 287690 + label: ${{ needs.start-runner.outputs.label }} + ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }} + leave-ec2-instance-running: ${{ inputs.leave_ec2_instance_running }} diff --git a/.github/workflows/chromium.yml b/.github/workflows/chromium.yml new file mode 100644 index 000000000..31036a95a --- /dev/null +++ b/.github/workflows/chromium.yml @@ -0,0 +1,95 @@ +name: Chromium build- and smoke-test + +on: + workflow_dispatch: + inputs: + repository: + description: 'Repository to clone for the workflow' + required: true + default: 'OSSystems' + branch: + description: 'Branch to checkout for the workflow' + required: true + default: 'master' + use_aws: + description: 'Use AWS EC2 instances instead of local runner' + required: false + type: boolean + default: true + instance_type: + description: 'EC2 instance type for build (more cores = faster builds)' + required: false + type: choice + options: + - 'c6id.4xlarge' # 16 vCPUs, 32 GB RAM, 1x 950 GB NVMe + - 'c6id.8xlarge' # 32 vCPUs, 64 GB RAM, 1x 1900 GB NVMe + - 'c6id.12xlarge' # 48 vCPUs, 96 GB RAM, 2x 1425 GB NVMe + - 'c6id.16xlarge' # 64 vCPUs, 128 GB RAM, 2x 1900 GB NVMe + default: 'c6id.8xlarge' + pull_request: + branches: + - master + paths: + - 'meta-chromium/recipes-browser/chromium/files/**' + - 'meta-chromium/recipes-browser/chromium/chromium*' + - 'meta-chromium/recipes-browser/chromium/gn*' + - '.github/workflows/chromium.yml' + +permissions: + contents: read + actions: read + checks: write + id-token: write # Required for OIDC authentication + +jobs: + # AWS-based builds (always for PR, default for manual dispatch) + aws-matrix-build: + if: ${{ (github.repository_owner == 'brightsign' || github.repository_owner == 'OSSystems') && (github.event_name == 'pull_request' || inputs.use_aws == true || inputs.use_aws == null) }} + strategy: + fail-fast: false # Continue other matrix jobs even if one fails + matrix: + ozone_platform: [ozone-wayland, x11] + arch: [arm, aarch64, x86-64] + uses: ./.github/workflows/bs_meta_browser_ci_ec2.yml + secrets: inherit + with: + build_type: "release" + browser: "chromium" + chromium_version: ${{ matrix.ozone_platform }} + arch: ${{ matrix.arch }} + aws_arn_role: "arn:aws:iam::195607249165:role/github-actions-meta-browser-repo" + aws_region: "us-east-1" + instance_type: ${{ inputs.instance_type || 'c6id.8xlarge' }} # Default for PR builds, user choice for manual + + # Local runner (manual dispatch only, when explicitly disabled AWS) + local-build: + if: ${{ (github.repository_owner == 'brightsign' || github.repository_owner == 'OSSystems') && github.event_name == 'workflow_dispatch' && inputs.use_aws == false }} + strategy: + fail-fast: false # Continue other matrix jobs even if one fails + matrix: + browser_version: [ozone-wayland, x11] + browser: [chromium] + arch: [arm, aarch64, x86-64] + runs-on: [self-hosted, chromium] + container: + image: skandigraun/yocto:latest + volumes: + - yocto:/yocto + steps: + - run: | + mkdir -p /yocto + cd /yocto + rm -rf meta-browser + # Clean stale pseudo state from any previous interrupted builds + rm -rf build/tmp/work/*/*/*/pseudo build/tmp/sysroots-components/*/pseudo 2>/dev/null || true + if [ "${{ github.event_name }}" = "pull_request" ]; then + GH_URL="$GITHUB_SERVER_URL/${{ github.event.pull_request.head.repo.full_name }}" + GH_REV="$GITHUB_HEAD_REF" + else + GH_URL="$GITHUB_SERVER_URL/${{ github.repository }}" + GH_REV="${{ github.ref_name }}" + fi + git clone $GH_URL + git -C meta-browser checkout $GH_REV + # meta-chromium-test is now integrated into meta-browser + ./meta-browser/meta-chromium-test/scripts/build.sh ${{ matrix.arch }} ${{ matrix.browser_version }} ${{ matrix.browser }} diff --git a/.github/workflows/cleanup-efs-cache.yml b/.github/workflows/cleanup-efs-cache.yml new file mode 100644 index 000000000..95d32a738 --- /dev/null +++ b/.github/workflows/cleanup-efs-cache.yml @@ -0,0 +1,267 @@ +name: 'EFS Cache Cleanup' + +on: + workflow_dispatch: + inputs: + cleanup_downloads: + description: 'Clean downloads directory on EFS' + required: true + type: boolean + default: false + + cleanup_sstate: + description: 'Clean sstate-cache directory on EFS' + required: true + type: choice + options: + - 'none' + - 'all' + - 'older-than-week' + default: 'older-than-week' + + aws_region: + description: 'AWS Region' + required: true + type: string + default: 'us-east-1' + + aws_arn_role: + description: 'AWS IAM Role ARN for authentication' + required: true + type: string + default: 'arn:aws:iam::195607249165:role/github-actions-meta-browser-repo' + + instance_type: + description: 'EC2 instance type for cleanup' + required: false + type: string + default: 't3.medium' + + schedule: + # Run every Monday at 00:00 UTC (weekly cleanup) + - cron: '0 0 * * 1' + +permissions: + contents: read + actions: read + id-token: write + +jobs: + start-runner: + name: Start EC2 Runner + runs-on: ubuntu-latest + outputs: + label: ${{ steps.start-runner.outputs.label }} + ec2-instance-id: ${{ steps.start-runner.outputs.ec2-instance-id }} + steps: + # Configure AWS credentials + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: ${{ inputs.aws_region }} + role-to-assume: ${{ inputs.aws_arn_role }} + + # Download config file and set environment variables + - name: Download config file and set env vars from it + run: | + aws s3 cp s3://meta-browser-ci-config-bucket/config.json . + aws s3 cp s3://meta-browser-ci-config-bucket/set_github_env_vars.py . + python3 set_github_env_vars.py --file config.json + + # Start EC2 runner for cleanup + - name: Start EC2 runner + id: start-runner + uses: brightsign/ec2-github-runner@0fa8b183dd4124fd191ccdbc48b68f0ea46a9634 + with: + mode: start + github-app-private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} + github-app-id: 287690 + ec2-image-id: ami-08a4255385679596c # Custom AMI with Yocto build dependencies pre-installed + ec2-instance-type: ${{ inputs.instance_type }} + subnet-id: ${{ env.VPC_SUBNET_ID }} + security-group-id: ${{ env.VPC_SG_ID }} + run-as-service-with-user: root + label: "mb-cleanup-${{ github.run_id }}-${{ github.run_attempt }}" + aws-resource-tags: > + [ + {"Key": "Name", "Value": "gh-runner-meta-browser-cleanup"}, + {"Key": "GitHubRepository", "Value": "${{ github.repository }}"} + ] + + cleanup-efs: + name: Cleanup EFS Cache + needs: start-runner + runs-on: ${{ needs.start-runner.outputs.label }} + steps: + # Configure AWS credentials on the EC2 runner + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: ${{ inputs.aws_region }} + role-to-assume: ${{ inputs.aws_arn_role }} + + # Download config and get EFS ID + - name: Download config file and set env vars from it + run: | + aws s3 cp s3://meta-browser-ci-config-bucket/config.json . + aws s3 cp s3://meta-browser-ci-config-bucket/set_github_env_vars.py . + python3 set_github_env_vars.py --file config.json + + - name: Mount EFS and perform cleanup + run: | + # Create mount point + sudo mkdir -p /mnt/shared-cache + + # Mount EFS using environment variable from config + echo "Mounting EFS: ${{ env.SHARED_CACHE_EFS_ID }}" + sudo mount -t nfs4 -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,intr,timeo=600 \ + ${{ env.SHARED_CACHE_EFS_ID }}.efs.${{ inputs.aws_region }}.amazonaws.com:/ /mnt/shared-cache + + # Show current disk usage + echo "=== Current EFS disk usage ===" + df -h /mnt/shared-cache + echo "" + echo "=== Directory sizes before cleanup ===" + sudo du -sh /mnt/shared-cache/* 2>/dev/null || echo "No directories found" + echo "" + + # Cleanup downloads based on selection (default false for scheduled runs) + CLEANUP_DOWNLOADS="${{ github.event_name == 'schedule' && 'false' || inputs.cleanup_downloads }}" + echo "=== Processing downloads cleanup: $CLEANUP_DOWNLOADS ===" + if [ -d "/mnt/shared-cache/downloads" ]; then + cd /mnt/shared-cache/downloads + echo "Current downloads directory size:" + du -sh . 2>/dev/null || echo "Cannot calculate size" + + if [ "$CLEANUP_DOWNLOADS" = "true" ]; then + echo "Removing all downloads..." + rm -rf ./* 2>/dev/null || true + echo "Downloads directory size after cleanup:" + du -sh . 2>/dev/null || echo "Cannot calculate size" + else + echo "Skipping downloads cleanup (disabled)" + fi + cd / + else + echo "Downloads directory not found" + fi + + # Cleanup sstate-cache based on selection (use older-than-week for scheduled runs) + CLEANUP_MODE="${{ github.event_name == 'schedule' && 'older-than-week' || inputs.cleanup_sstate }}" + echo "=== Processing sstate-cache cleanup: $CLEANUP_MODE ===" + if [ -d "/mnt/shared-cache/sstate-cache" ]; then + cd /mnt/shared-cache/sstate-cache + echo "Current sstate-cache directory size:" + df -h . 2>/dev/null || echo "Cannot calculate size" + du -sh . 2>/dev/null || echo "Cannot calculate size" + + case "$CLEANUP_MODE" in + "none") + echo "Skipping sstate-cache cleanup as requested" + ;; + "all") + echo "Removing all sstate-cache..." + rm -rf ./* 2>/dev/null || true + ;; + "older-than-week") + echo "Smart pruning sstate-cache files not accessed for 7+ days..." + # Smart sstate pruning function that handles paired tarball and siginfo files + prune_sstate() { + local age="$1" + local pattern="$2" + echo "Pruning ${pattern} accessed ${age} days ago" + + find . -atime "${age}" \( -name "${pattern}" -o -name "${pattern}.siginfo" \) -print | \ + while read file; do + case "${file}" in + *.tgz) + tarball="${file}" + siginfo="${file}.siginfo" + delete=y + ;; + *.siginfo) + tarball="${file%.siginfo}" + siginfo="${file}" + # Only delete siginfo if tarball doesn't exist + if [ -e "${tarball}" ]; then + delete=n + else + delete=y + fi + ;; + esac + + if [ "${delete}" = "y" ]; then + if [ -e "${tarball}" ]; then + echo "Removing tarball: ${tarball}" + rm -f "${tarball}" 2>/dev/null || true + fi + if [ -e "${siginfo}" ]; then + echo "Removing siginfo: ${siginfo}" + rm -f "${siginfo}" 2>/dev/null || true + fi + fi + done + } + + # Prune files not accessed for 7+ days + prune_sstate "+6" "*.tgz" + + # Clean up dangling symlinks + echo "Pruning dangling symlinks..." + find . -type l ! -exec test -e {} \; -print0 | xargs -0 --no-run-if-empty rm -v 2>/dev/null || true + + # Clean up old temporary files (*.tgz.????????) + echo "Pruning old temporary files..." + find . -ctime +1 -name '*.tgz.????????' -print0 | xargs -0 --no-run-if-empty rm -v 2>/dev/null || true + ;; + esac + + if [ "$CLEANUP_MODE" != "none" ]; then + echo "sstate-cache directory size after cleanup:" + df -h . 2>/dev/null || echo "Cannot calculate size" + du -sh . 2>/dev/null || echo "Cannot calculate size" + fi + cd / + else + echo "sstate-cache directory not found" + fi + + # Show final disk usage + echo "" + echo "=== Directory sizes after cleanup ===" + sudo du -sh /mnt/shared-cache/* 2>/dev/null || echo "No directories found" + echo "" + echo "=== Final EFS disk usage (note: EFS may take time to reflect freed space) ===" + # Sync filesystem to help with metadata updates + sync + df -h /mnt/shared-cache + echo "" + echo "✅ Cleanup completed! Directory sizes above show the actual freed space." + echo " EFS filesystem usage (df) may take some time to reflect the changes due to distributed filesystem metadata." + + # Unmount EFS + sudo umount /mnt/shared-cache + + stop-runner: + name: Stop EC2 Runner + needs: + - start-runner + - cleanup-efs + runs-on: ubuntu-latest + if: always() + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: ${{ inputs.aws_region }} + role-to-assume: ${{ inputs.aws_arn_role }} + + - name: Stop EC2 runner + uses: brightsign/ec2-github-runner@0fa8b183dd4124fd191ccdbc48b68f0ea46a9634 + with: + mode: stop + github-app-private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} + github-app-id: 287690 + label: ${{ needs.start-runner.outputs.label }} + ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }} diff --git a/.github/workflows/electron.yml b/.github/workflows/electron.yml new file mode 100644 index 000000000..dccd1ccd3 --- /dev/null +++ b/.github/workflows/electron.yml @@ -0,0 +1,95 @@ +name: Electron build- and smoke-test + +on: + workflow_dispatch: + inputs: + repository: + description: 'Repository to clone for the workflow' + required: true + default: 'OSSystems' + branch: + description: 'Branch to checkout for the workflow' + required: true + default: 'master' + use_aws: + description: 'Use AWS EC2 instances instead of local runner' + required: false + type: boolean + default: true + instance_type: + description: 'EC2 instance type for build (more cores = faster builds)' + required: false + type: choice + options: + - 'c6id.4xlarge' # 16 vCPUs, 32 GB RAM, 1x 950 GB NVMe + - 'c6id.8xlarge' # 32 vCPUs, 64 GB RAM, 1x 1900 GB NVMe + - 'c6id.12xlarge' # 48 vCPUs, 96 GB RAM, 2x 1425 GB NVMe + - 'c6id.16xlarge' # 64 vCPUs, 128 GB RAM, 2x 1900 GB NVMe + default: 'c6id.8xlarge' + pull_request: + branches: + - master + paths: + - 'meta-chromium/recipes-browser/chromium/files/**' + - 'meta-chromium/recipes-browser/chromium/electron*' + - 'meta-chromium/recipes-browser/chromium/gn*' + - '.github/workflows/electron.yml' + +permissions: + contents: read + actions: read + checks: write + id-token: write # Required for OIDC authentication + +jobs: + # AWS-based builds (always for PR, default for manual dispatch) + aws-matrix-build: + if: ${{ (github.repository_owner == 'brightsign' || github.repository_owner == 'OSSystems') && (github.event_name == 'pull_request' || inputs.use_aws == true || inputs.use_aws == null) }} + strategy: + fail-fast: false # Continue other matrix jobs even if one fails + matrix: + ozone_platform: [ozone-wayland, ozone-x11] + arch: [arm, aarch64, x86-64] + uses: ./.github/workflows/bs_meta_browser_ci_ec2.yml + secrets: inherit + with: + build_type: "release" + browser: "electron" + chromium_version: ${{ matrix.ozone_platform }} + arch: ${{ matrix.arch }} + aws_arn_role: "arn:aws:iam::195607249165:role/github-actions-meta-browser-repo" + aws_region: "us-east-1" + instance_type: ${{ github.event.inputs.instance_type || 'c6id.8xlarge' }} + + # Local runner (manual dispatch only, when explicitly disabled AWS) + local-build: + if: ${{ (github.repository_owner == 'brightsign' || github.repository_owner == 'OSSystems') && github.event_name == 'workflow_dispatch' && inputs.use_aws == false }} + strategy: + fail-fast: false # Continue other matrix jobs even if one fails + matrix: + browser_version: [ozone-wayland, ozone-x11] + browser: [electron] + arch: [arm, aarch64, x86-64] + runs-on: [self-hosted, chromium] + container: + image: skandigraun/yocto:latest + volumes: + - yocto:/yocto + steps: + - run: | + mkdir -p /yocto + cd /yocto + rm -rf meta-browser + # Clean stale pseudo state from any previous interrupted builds + rm -rf build/tmp/work/*/*/*/pseudo build/tmp/sysroots-components/*/pseudo 2>/dev/null || true + if [ "${{ github.event_name }}" = "pull_request" ]; then + GH_URL="$GITHUB_SERVER_URL/${{ github.event.pull_request.head.repo.full_name }}" + GH_REV="$GITHUB_HEAD_REF" + else + GH_URL="$GITHUB_SERVER_URL/${{ github.repository }}" + GH_REV="${{ github.ref_name }}" + fi + git clone $GH_URL + git -C meta-browser checkout $GH_REV + # meta-chromium-test is now integrated into meta-browser + ./meta-browser/meta-chromium-test/scripts/build.sh ${{ matrix.arch }} ${{ matrix.browser_version }} ${{ matrix.browser }} diff --git a/meta-chromium-test/COPYING.MIT b/meta-chromium-test/COPYING.MIT new file mode 100644 index 000000000..4028e34f9 --- /dev/null +++ b/meta-chromium-test/COPYING.MIT @@ -0,0 +1,17 @@ +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/meta-chromium-test/README.md b/meta-chromium-test/README.md new file mode 100644 index 000000000..872508e62 --- /dev/null +++ b/meta-chromium-test/README.md @@ -0,0 +1,757 @@ +# Meta-Chromium-Test Layer + +A comprehensive Yocto/OpenEmbedded layer for automated testing of Chromium browser builds across multiple architectures, Yocto versions, and C library implementations. + +> **Note**: This layer is now integrated into the [meta-browser](https://github.com/OSSystems/meta-browser) repository as a subdirectory. When cloning meta-browser, you'll find this layer at `meta-browser/meta-chromium-test/`. + +## Table of Contents + +- [Overview](#overview) +- [Purpose](#purpose) +- [Features](#features) +- [Layer Structure](#layer-structure) +- [Prerequisites](#prerequisites) +- [Quick Start](#quick-start) +- [Configuration Matrix](#configuration-matrix) +- [Scripts Usage](#scripts-usage) +- [KAS Configuration](#kas-configuration) +- [Adding New Combinations](#adding-new-combinations) +- [Testing](#testing) +- [Recipes](#recipes) +- [Docker Infrastructure](#docker-infrastructure) +- [Troubleshooting](#troubleshooting) +- [Contributing](#contributing) + +## Overview + +The `meta-chromium-test` layer provides automated testing infrastructure for the Chromium browser on embedded Linux systems. It builds and tests Chromium for the current branch of meta-browser across multiple combinations of: + +- **Architectures**: ARM, AArch64, RISC-V, x86-64 +- **Chromium Variants**: Ozone Wayland, X11 +- **C Library**: glibc (musl support removed for simplicity) + +This layer is integrated into the [meta-browser](https://github.com/OSSystems/meta-browser) layer and uses [KAS](https://kas.readthedocs.io/) for reproducible builds. The testing infrastructure automatically adapts to the current branch of meta-browser (e.g., scarthgap, master) rather than maintaining separate configurations for each Yocto version. + +## Purpose + +This testing layer serves several key purposes: + +1. **Automated CI/CD Testing**: Provides consistent testing across multiple configurations +2. **Regression Detection**: Catches issues early across different platforms and versions +3. **Quality Assurance**: Ensures Chromium builds work reliably on embedded systems +4. **Development Support**: Facilitates testing new features and configurations +5. **Performance Benchmarking**: Enables systematic performance testing + +## Features + +- **Matrix Testing**: Automated generation of test configurations for all supported combinations +- **KAS Integration**: Uses KAS for reproducible and isolated builds +- **QEMU Testing**: Automated testing using QEMU emulation +- **Smoke Tests**: Built-in smoke tests for basic Chromium functionality +- **Script Automation**: Comprehensive build, test, and validation scripts +- **Flexible Architecture**: Easy to extend with new configurations + +## Layer Structure + +``` +meta-chromium-test/ +├── conf/ +│ └── layer.conf # Layer configuration +├── kas/ # KAS configuration files +│ ├── chromium/ # Chromium variant configs +│ │ ├── ozone-wayland.yml # Wayland/Ozone configuration +│ │ └── x11.yml # X11 configuration +│ ├── repos/ # Repository configurations +│ │ └── common-repos.yml # Common repository definitions +│ ├── common.yml # Common build settings +│ ├── musl.yml # musl C library configuration +│ └── *.yml # Generated test configurations +├── recipes-core/ +│ └── images/ +│ └── chromium-test-image.bb # Test image recipe +├── recipes-test/ +│ └── chromium-tests/ +│ ├── chromium-tests.bb # Test package recipe +│ └── files/ +│ └── chromium-smoke-test.sh # Smoke test script +├── scripts/ +│ ├── generate_kas_files.sh # Generate KAS configurations +│ ├── build.sh # Build automation script +│ ├── test.sh # Test automation script +│ └── validate.sh # Layer validation script +└── README.md # This file +``` + +## Prerequisites + +- **Yocto/OpenEmbedded**: Compatible with kirkstone, scarthgap, walnascar, and master +- **KAS**: Version 3.0 or later +- **Python**: 3.8+ +- **Git**: For repository management +- **Docker**: Required for the recommended testing infrastructure + +### Infrastructure Requirements + +The testing infrastructure expects the following base directory structure: + +- `/yocto/` - Base directory for all builds and test images +- `/yocto/{yocto_version}/` - Version-specific build directories (created by GitHub Actions/CI) + +The following subdirectories are created automatically by the `build.sh` script: +- `/yocto/yocto_dl/` - Download directory for source packages +- `/yocto/yocto_sstate_chromium/` - Shared state cache directory +- `/yocto/yocto_ccache/` - ccache directory for faster rebuilds +- `/yocto/test-images/` - Directory for storing built test images + +**Note**: The recommended approach is to use the provided Docker infrastructure which handles these requirements automatically. Only the root `/yocto` directory needs manual setup. + +### Required Layers + +- `meta-browser/meta-chromium` - Chromium recipes +- `meta-openembedded/meta-oe` - Additional recipes +- `meta-clang` - Clang compiler support (for some configurations) + +## Quick Start + +### Recommended Method: Docker + +The recommended approach is to use the Docker-based testing infrastructure which provides a consistent environment: + +```bash +# Example: Build and test master branch with aarch64 ozone-wayland glibc +docker run --rm -it \ + -v /yocto:/yocto \ + -v /path/to/meta-browser:/src/meta-browser:ro \ + skandigraun/yocto:latest \ + bash -c ' + set -e + mkdir -p /yocto/master && cd /yocto/master + rm -rf meta-browser + cp -r /src/meta-browser ./meta-browser + ./meta-browser/meta-chromium-test/scripts/build.sh master aarch64 ozone-wayland glibc + ./meta-browser/meta-chromium-test/scripts/test.sh master aarch64 ozone-wayland glibc + ' +``` + +**Docker Command Breakdown**: +- `-v /yocto:/yocto` - Mounts host `/yocto` directory for persistent builds +- `-v /path/to/meta-browser:/src/meta-browser:ro` - Mounts meta-browser layer (read-only, includes meta-chromium-test) +- `skandigraun/yocto:latest` - Pre-configured Yocto build environment + +### Alternative Method: Native Setup + +For native builds without Docker: + +1. **Clone the required layers**: + ```bash + git clone https://github.com/OSSystems/meta-browser.git + git clone https://github.com/openembedded/meta-openembedded.git + git clone https://github.com/kraj/meta-clang.git + ``` + + Note: meta-chromium-test is now included as a subdirectory within meta-browser. + +2. **Generate KAS files** (if not already present): + ```bash + cd meta-browser/meta-chromium-test + ./scripts/generate_kas_files.sh + ``` + +3. **Build a test image**: + ```bash + kas build meta-browser/meta-chromium-test/kas/glibc-kirkstone-ozone-wayland-x86-64-test.yml + ``` + +4. **Run automated build and test**: + ```bash + ./meta-browser/meta-chromium-test/scripts/build.sh kirkstone x86-64 ozone-wayland + ./meta-browser/meta-chromium-test/scripts/test.sh kirkstone x86-64 ozone-wayland + ``` + +**Note**: Native setup requires manual creation of the `/yocto/` directory structure and proper environment configuration. + +## Configuration Matrix + +The layer supports the following configuration matrix for the current branch: + +| Architectures | Chromium Variants | C Library | +|---------------|-------------------|-----------| +| arm, aarch64, riscv, x86-64 | ozone-wayland, x11 | glibc | + +This generates **16 unique configurations** for comprehensive testing across all supported combinations. The layer automatically uses the correct Yocto configuration based on the meta-browser branch you're working with (e.g., scarthgap commits for scarthgap branch). + +## Scripts Usage + +**Important**: All scripts are designed to run within the Docker infrastructure. For native execution, ensure the `/yocto/` directory structure exists and is properly configured. + +### Docker Usage Examples + +```bash +# Build and test x86-64 ozone-wayland with glibc +docker run --rm -it \ + -v /yocto:/yocto \ + -v /path/to/meta-browser:/src/meta-browser:ro \ + skandigraun/yocto:latest \ + bash -c ' + set -e + mkdir -p /yocto && cd /yocto + rm -rf meta-browser + cp -r /src/meta-browser ./meta-browser + ./meta-browser/meta-chromium-test/scripts/build.sh x86-64 ozone-wayland chromium + ./meta-browser/meta-chromium-test/scripts/test.sh x86-64 ozone-wayland + ' + +# Test with aarch64 x11 +docker run --rm -it \ + -v /yocto:/yocto \ + -v /path/to/meta-browser:/src/meta-browser:ro \ + skandigraun/yocto:latest \ + bash -c ' + set -e + mkdir -p /yocto && cd /yocto + rm -rf meta-browser + cp -r /src/meta-browser ./meta-browser + ./meta-browser/meta-chromium-test/scripts/build.sh aarch64 x11 chromium + ./meta-browser/meta-chromium-test/scripts/test.sh aarch64 x11 + ' + ./meta-browser/meta-chromium-test/scripts/build.sh aarch64 x11 chromium musl + ./meta-browser/meta-chromium-test/scripts/test.sh aarch64 x11 musl + ' +``` + +### generate_kas_files.sh + +Generates all KAS configuration files for the supported matrix: + +```bash +# Generate all KAS files +./scripts/generate_kas_files.sh +``` + +**Customizing the script**: +- Update `CHROMIUM_VERSIONS` for new Chromium variants +- Add new architectures to `ARCHS` and update `arch_machine_dict` + +### build.sh + +Automated build script for specific configurations: + +```bash +# Usage: build.sh arch chromium_version browser + +# Examples (within Docker container): +./scripts/build.sh x86-64 ozone-wayland chromium +./scripts/build.sh aarch64 x11 chromium +./scripts/build.sh riscv ozone-wayland electron +``` + +**Script behavior**: +- Creates build directory structure in `/yocto/` +- Checks for existing builds to avoid rebuilding +- Uses KAS for reproducible builds +- Copies final images to `/yocto/test-images/` for testing + +### test.sh + +Automated testing using QEMU: + +```bash +# Usage: test.sh arch chromium_version + +# Examples (within Docker container): +./scripts/test.sh x86-64 ozone-wayland +./scripts/test.sh aarch64 x11 +``` + +**Script behavior**: +- Looks for built images in `/yocto/test-images/` +- Launches QEMU with appropriate configuration +- Runs automated smoke tests +- Creates `.done` files to track completed tests + +### validate.sh + +Validates the layer structure and configuration: + +```bash +./scripts/validate.sh +``` + +## KAS Configuration + +The layer uses a modular KAS configuration approach: + +### Base Configuration Files + +- **`common.yml`**: Shared settings for all builds +- **`musl.yml`**: musl C library specific settings +- **`chromium/ozone-wayland.yml`**: Wayland/Ozone variant settings +- **`chromium/x11.yml`**: X11 variant settings + +### Repository Configurations + +- **`repos/common-repos.yml`**: Common repository definitions +- **`repos/{version}.yml`**: Version-specific repository commits and settings + +### Generated Test Configurations + +Each test configuration follows the naming pattern: +``` +{libc}-{yocto_version}-{chromium_version}-{arch}-test.yml +``` + +Examples: +- `glibc-kirkstone-ozone-wayland-x86-64-test.yml` +- `musl-scarthgap-x11-aarch64-test.yml` + +## Adding New Combinations + +### Adding a New Yocto Version + +1. **Create repository configuration**: + ```bash + cp kas/repos/common-repos.yml kas/repos/newfversion.yml + # Edit newfversion.yml with appropriate repository commits + ``` + +2. **Update generate_kas_files.sh**: + ```bash + YOCTO_VERSIONS=("newfversion") + ``` + +3. **Regenerate KAS files**: + ```bash + ./scripts/generate_kas_files.sh + ``` + +### Adding a New Architecture + +1. **Update generate_kas_files.sh**: + ```bash + ARCHS=("arm" "aarch64" "riscv" "x86-64" "newarch") + + # Add machine mapping + arch_machine_dict["newarch"]="qemunewarch" + ``` + +2. **Update build.sh and test.sh** if needed for architecture-specific settings + +3. **Regenerate configurations**: + ```bash + ./scripts/generate_kas_files.sh + ``` + +### Adding a New Chromium Variant + +1. **Create variant configuration**: + ```yaml + # kas/chromium/newvariant.yml + header: + version: 11 + + local_conf_header: + chromium-newvariant: | + PREFERRED_PROVIDER_chromium = "chromium-newvariant" + PACKAGECONFIG:append:pn-chromium-newvariant = " proprietary-codecs" + ``` + +2. **Update generate_kas_files.sh**: + ```bash + CHROMIUM_VERSIONS=("ozone-wayland" "x11" "newvariant") + ``` + +3. **Regenerate configurations**: + ```bash + ./scripts/generate_kas_files.sh + ``` + +## Testing + +### Smoke Tests + +The layer includes basic smoke tests for Chromium functionality: + +```bash +# Run smoke test on target +chromium-smoke-test +``` + +### Manual Testing + +1. **Build and boot image**: + ```bash + # If running from meta-browser root: + kas build meta-browser/meta-chromium-test/kas/glibc-kirkstone-ozone-wayland-x86-64-test.yml + # If running from meta-chromium-test directory: + kas build kas/glibc-kirkstone-ozone-wayland-x86-64-test.yml + runqemu qemux86-64 + ``` + +2. **Test Chromium manually**: + ```bash + # On target system + chromium --no-sandbox + ``` + +### Automated Testing Pipeline + +For CI/CD integration using Docker with ccache: + +```bash +#!/bin/bash +# Example CI script for Docker-based testing with ccache + +DOCKER_IMAGE="skandigraun/yocto:latest" +META_BROWSER_PATH="/path/to/meta-browser" +META_CHROMIUM_TEST_PATH="/path/to/meta-chromium-test" + +# Test configurations +CONFIGS=( + "kirkstone x86-64 ozone-wayland glibc" + "scarthgap aarch64 x11 musl" + "master riscv ozone-wayland glibc" +) + +# Show initial ccache stats +echo "Initial ccache statistics:" +docker run --rm -v yocto:/yocto "$DOCKER_IMAGE" bash -c " + export CCACHE_DIR=/yocto/yocto_ccache + ccache -s 2>/dev/null || echo 'ccache not yet initialized' +" + +for config in "${CONFIGS[@]}"; do + read -r yocto_version arch chromium_version libc <<< "$config" + + echo "Testing: $config" + + docker run --rm -it \ + -v yocto:/yocto \ + -v "$META_BROWSER_PATH":/src/meta-browser:ro \ + -v "$META_CHROMIUM_TEST_PATH":/src/meta-chromium-test:ro \ + "$DOCKER_IMAGE" \ + bash -c " + set -e + # Only create Yocto version directory - build.sh handles the rest + mkdir -p /yocto/$yocto_version && cd /yocto/$yocto_version + rm -rf meta-browser meta-chromium-test + cp -r /src/meta-browser ./meta-browser + cp -r /src/meta-chromium-test ./meta-chromium-test + ./meta-chromium-test/scripts/build.sh $yocto_version $arch $chromium_version $libc + ./meta-chromium-test/scripts/test.sh $yocto_version $arch $chromium_version $libc + " || exit 1 +done + +# Show final ccache stats +echo "Final ccache statistics:" +docker run --rm -v yocto:/yocto "$DOCKER_IMAGE" bash -c " + export CCACHE_DIR=/yocto/yocto_ccache + ccache -s +" + +echo "All tests completed successfully with ccache acceleration!" +``` + +## Recipes + +### chromium-test-image.bb + +A comprehensive test image including: +- Both Chromium variants (Ozone Wayland and X11) +- Development and debugging tools +- Network utilities +- Font packages +- X11 server components + +### chromium-tests.bb + +Provides smoke tests and utility scripts for Chromium testing. + +## Docker Infrastructure + +### Container Setup + +The testing infrastructure uses the `skandigraun/yocto:latest` Docker image which provides: + +- Pre-configured Yocto build environment +- All required tools and dependencies +- Optimized build cache and download directories +- ccache support for faster rebuilds +- Consistent environment across different host systems + +### Directory Structure + +The Docker container uses the following directory structure (created automatically by `build.sh`): + +``` +/yocto/ # Host directory mounted to container +├── yocto_dl/ # Download cache (created by build.sh) +├── yocto_sstate_chromium/ # Shared state cache (created by build.sh) +├── yocto_ccache/ # ccache directory (created by build.sh) +├── test-images/ # Built test images (created by build.sh) +├── kirkstone/ # Kirkstone builds (created as needed) +├── scarthgap/ # Scarthgap builds (created as needed) +├── walnascar/ # Walnascar builds (created as needed) +└── master/ # Master builds (created as needed) +``` + +**Note**: Only the root `/yocto` directory needs to be set up manually. All subdirectories are created automatically by the build script with proper permissions. + +### Volume Mounts + +- `/yocto:/yocto` - Persistent build artifacts and caches +- `/path/to/meta-browser:/src/meta-browser:ro` - Meta-browser layer (read-only) +- `/path/to/meta-chromium-test:/src/meta-chromium-test:ro` - This test layer (read-only) + +### Environment Variables + +The Docker environment includes optimized settings for: +- Download directory: `/yocto/yocto_dl` +- Shared state directory: `/yocto/yocto_sstate_chromium` +- ccache directory: `/yocto/yocto_ccache` (up to 50GB cache) +- Source mirror URL for faster downloads +- Build optimizations for containerized environments + +### ccache Configuration + +The layer is configured with ccache to significantly speed up rebuilds: + +- **Cache Location**: `/yocto/yocto_ccache` +- **Maximum Size**: 50GB (configurable in `kas/common.yml`) +- **Compression**: Enabled with level 6 for space efficiency +- **Shared Across**: All architectures and Yocto versions + +**ccache Benefits**: +- **First Build**: Same speed as without ccache +- **Subsequent Builds**: 2-10x faster depending on changes +- **Cross-Architecture**: Shared cache across ARM, AArch64, x86-64, RISC-V +- **Cross-Version**: Shared cache across different Yocto versions + +### Setting Up the Docker Volume + +Before running any builds, you need to create the Docker volume with the required directory structure and proper permissions. This ensures that the `yoctouser` in the container can write to all necessary directories. + +#### Step 1: Create the Docker Volume + +```bash +# Create the named Docker volume +docker volume create yocto +``` + +#### Step 2: Initialize Basic Structure and Permissions + +```bash +# Run a temporary container as root to set up basic structure and permissions +docker run --rm -v yocto:/yocto --user root skandigraun/yocto:latest bash -c " + # Create the root /yocto directory with proper ownership + mkdir -p /yocto + + # Set proper ownership for yoctouser (UID 1000, GID 1000 typically) + chown -R 1000:1000 /yocto + + # Set proper permissions + chmod -R 755 /yocto + + echo 'Docker volume setup completed successfully!' + echo 'Note: Build-specific directories will be created automatically by build.sh' +" +``` + +**Note**: The build script (`build.sh`) automatically creates all required subdirectories (`yocto_dl`, `yocto_sstate_chromium`, `yocto_ccache`, `test-images`) with proper permissions when you run your first build. This eliminates the need to manually create these directories during setup. + +#### Step 3: Verify the Setup + +```bash +# Verify that yoctouser can write to the volume +docker run --rm -v yocto:/yocto skandigraun/yocto:latest bash -c " + echo 'Testing write permissions...' + touch /yocto/test-file + ls -la /yocto/test-file + rm /yocto/test-file + echo 'Volume setup verified successfully!' + echo 'Ready for builds - subdirectories will be created automatically' +" +``` + +#### Alternative: One-Command Setup + +For convenience, you can set up everything in a single command: + +```bash +# Create volume and set up basic structure in one go +docker volume create yocto && \ +docker run --rm -v yocto:/yocto --user root skandigraun/yocto:latest bash -c " + mkdir -p /yocto + chown -R 1000:1000 /yocto + chmod -R 755 /yocto + echo 'Yocto Docker volume ready for use!' + echo 'Build-specific directories will be created automatically by build.sh' +" && \ +echo "Volume setup complete. You can now run builds with automatic ccache setup." + +#### ccache Management + +Monitor and manage ccache performance: + +```bash +# Check ccache statistics +docker run --rm -v yocto:/yocto skandigraun/yocto:latest bash -c " + export CCACHE_DIR=/yocto/yocto_ccache + ccache -s +" + +# Clear ccache if needed +docker run --rm -v yocto:/yocto skandigraun/yocto:latest bash -c " + export CCACHE_DIR=/yocto/yocto_ccache + ccache -C +" + +# Set different cache size limit (e.g., 100GB) +docker run --rm -v yocto:/yocto skandigraun/yocto:latest bash -c " + export CCACHE_DIR=/yocto/yocto_ccache + ccache -M 100G +" +``` + +#### Troubleshooting Volume Setup + +If you encounter permission issues: + +1. **Check the yoctouser UID/GID in your container**: + ```bash + docker run --rm skandigraun/yocto:latest id yoctouser + ``` + +2. **Adjust ownership if needed** (replace 1000:1000 with actual UID:GID): + ```bash + docker run --rm -v yocto:/yocto --user root skandigraun/yocto:latest bash -c " + chown -R 1000:1000 /yocto + " + ``` + +3. **Remove and recreate volume if corrupted**: + ```bash + docker volume rm yocto + # Then repeat the setup steps above + ``` + +#### Volume Management + +- **View volume information**: + ```bash + docker volume inspect yocto + ``` + +- **Clean up old builds** (preserves download, sstate, and ccache): + ```bash + docker run --rm -v yocto:/yocto skandigraun/yocto:latest bash -c " + rm -rf /yocto/*/build 2>/dev/null || true + rm -rf /yocto/test-images/* 2>/dev/null || true + echo 'Build artifacts cleaned, caches preserved' + echo 'Note: Shared directories will be recreated automatically on next build' + " + ``` + +- **Complete volume cleanup** (removes everything including ccache): + ```bash + docker volume rm yocto + ``` + +#### Performance Optimization + +For optimal ccache performance: + +1. **Monitor cache hit rate**: + ```bash + docker run --rm -v yocto:/yocto skandigraun/yocto:latest bash -c " + export CCACHE_DIR=/yocto/yocto_ccache + ccache -s | grep 'cache hit rate' + " + ``` + +2. **Adjust cache size** based on available disk space: + - **Minimum recommended**: 20GB for basic usage + - **Optimal**: 50GB for full matrix testing + - **Maximum useful**: 100GB+ for extensive development + +3. **Keep cache across container restarts** by using the persistent volume mount + +Once the volume is set up, you can proceed with the Docker usage examples in the Quick Start section. Your builds will automatically benefit from ccache acceleration on subsequent runs. + +## Troubleshooting + +### Common Issues + +1. **Build Failures**: + ```bash + # Clean previous builds (within Docker container) + kas shell kas/your-config.yml -c "bitbake -c clean chromium-ozone-wayland" + ``` + +2. **Missing Dependencies**: + - Ensure all required layers are present in the Docker container + - Check layer compatibility with `bitbake-layers show-layers` + +3. **QEMU Issues**: + - Verify QEMU machine configuration + - Check available system resources + - Ensure `/yocto/test-images/` contains the required images + +4. **Docker Volume Issues**: + - Verify `/yocto` directory exists on host system + - Check volume mount permissions + - Ensure sufficient disk space in `/yocto` directory + +5. **KAS Problems**: + ```bash + # Validate KAS file (within Docker container) + kas dump kas/your-config.yml + ``` + +### Debug Mode + +Enable debug output within Docker container: +```bash +kas --log-level debug build kas/your-config.yml +``` + +### Getting Help + +- Check the [meta-browser documentation](https://github.com/OSSystems/meta-browser) +- Review [KAS documentation](https://kas.readthedocs.io/) +- Examine build logs in `build/tmp/log/` within the container +- For Docker-specific issues, check container logs and volume mounts + +## Contributing + +1. **Fork the repository** +2. **Create a feature branch** +3. **Add your changes**: + - Update configurations as needed + - Add new test cases + - Update documentation +4. **Test your changes**: + ```bash + ./scripts/validate.sh + ./scripts/generate_kas_files.sh + ``` +5. **Submit a pull request** + +### Development Guidelines + +- Follow existing naming conventions +- Update documentation for new features +- Test changes across multiple configurations +- Maintain backward compatibility +- Add appropriate error handling + +## License + +This layer is distributed under the MIT License. See the `COPYING.MIT` file for details. + +## Maintainers + +- See `MAINTAINERS` file for current maintainer information + +--- + +For questions, issues, or contributions, please use the project's issue tracker or contact the maintainers directly. diff --git a/meta-chromium-test/conf/layer.conf b/meta-chromium-test/conf/layer.conf new file mode 100644 index 000000000..defb7ee90 --- /dev/null +++ b/meta-chromium-test/conf/layer.conf @@ -0,0 +1,11 @@ +BBPATH .= ":${LAYERDIR}" +BBFILES += "${LAYERDIR}/recipes-*/*/*.bb ${LAYERDIR}/recipes-*/*/*.bbappend" + +BBFILE_COLLECTIONS += "chromium-test-layer" +BBFILE_PATTERN_chromium-test-layer := "^${LAYERDIR}/" +BBFILE_PRIORITY_chromium-test-layer = "7" + +LAYERVERSION_chromium-test-layer = "1" +LAYERSERIES_COMPAT_chromium-test-layer = "kirkstone scarthgap styhead walnascar whinlatter" + +LAYERDEPENDS_chromium-test-layer = "chromium-browser-layer" diff --git a/meta-chromium-test/kas/chromium-common.yml b/meta-chromium-test/kas/chromium-common.yml new file mode 100644 index 000000000..14f0596d6 --- /dev/null +++ b/meta-chromium-test/kas/chromium-common.yml @@ -0,0 +1,6 @@ +header: + version: 11 + includes: + - meta-chromium-test/kas/common.yml + +target: chromium-test-image diff --git a/meta-chromium-test/kas/chromium/ozone-wayland.yml b/meta-chromium-test/kas/chromium/ozone-wayland.yml new file mode 100644 index 000000000..c722af6bd --- /dev/null +++ b/meta-chromium-test/kas/chromium/ozone-wayland.yml @@ -0,0 +1,10 @@ +header: + version: 11 + +local_conf_header: + chromium-ozone-wayland: | + PREFERRED_PROVIDER_chromium = "chromium-ozone-wayland" + PACKAGECONFIG:append:pn-chromium-ozone-wayland = " proprietary-codecs" + # Enable Wayland as a distribution feature + DISTRO_FEATURES:append = " wayland" + IMAGE_FEATURES:append = " splash package-management ssh-server-dropbear hwcodecs weston" diff --git a/meta-chromium-test/kas/chromium/x11.yml b/meta-chromium-test/kas/chromium/x11.yml new file mode 100644 index 000000000..2542763de --- /dev/null +++ b/meta-chromium-test/kas/chromium/x11.yml @@ -0,0 +1,10 @@ +header: + version: 11 + +local_conf_header: + chromium-x11: | + PREFERRED_PROVIDER_chromium = "chromium-x11" + PACKAGECONFIG:append:pn-chromium-x11 = " proprietary-codecs" + # Ensure X11 is enabled as a distribution feature + DISTRO_FEATURES:append = " x11" + IMAGE_FEATURES:append = " splash package-management x11-base" diff --git a/meta-chromium-test/kas/common-local.yml b/meta-chromium-test/kas/common-local.yml new file mode 100644 index 000000000..478c68f39 --- /dev/null +++ b/meta-chromium-test/kas/common-local.yml @@ -0,0 +1,20 @@ +header: + version: 11 + +target: chromium-test-image + +local_conf_header: + meta-chromium-common: | + LICENSE_FLAGS_ACCEPTED += "commercial" + INHERIT += "rm_work" + PACKAGECONFIG:append:pn-qemu = " sdl " + PACKAGECONFIG:append:pn-qemu:class-nativesdk = " sdl " + PACKAGECONFIG:append:pn-qemu-system-native = " sdl " + DL_DIR = "${TOPDIR}/../downloads" + SSTATE_DIR = "${TOPDIR}/../sstate-cache" + EXTRA_IMAGE_FEATURES:append = " allow-empty-password allow-root-login ssh-server-openssh tools-debug " + IMAGE_FSTYPES:remove = "wic.qcow2" + INHERIT:remove = "create-spdx" + # Chromium specific settings + PACKAGECONFIG:append:pn-wayland = " freeglut" + PACKAGECONFIG:append:pn-mesa = " gbm gallium" diff --git a/meta-chromium-test/kas/common.yml b/meta-chromium-test/kas/common.yml new file mode 100644 index 000000000..6802d60ed --- /dev/null +++ b/meta-chromium-test/kas/common.yml @@ -0,0 +1,19 @@ +header: + version: 11 + +local_conf_header: + meta-common: | + LICENSE_FLAGS_ACCEPTED += "commercial" + INHERIT += "rm_work" + BB_GENERATE_MIRROR_TARBALLS = "1" + PACKAGECONFIG:append:pn-qemu = " sdl " + PACKAGECONFIG:append:pn-qemu:class-nativesdk = " sdl " + PACKAGECONFIG:append:pn-qemu-system-native = " sdl " + DL_DIR = "/yocto/yocto_dl" + SSTATE_DIR = "/yocto/yocto_sstate_chromium" + SOURCE_MIRROR_URL = "" + EXTRA_IMAGE_FEATURES:append = " allow-empty-password allow-root-login ssh-server-openssh tools-debug " + IMAGE_FSTYPES:remove = "wic.qcow2" + INHERIT:remove = "create-spdx" + # ccache disabled for cloud builds - sstate cache provides better acceleration + CCACHE_DISABLE = "1" diff --git a/meta-chromium-test/kas/electron-common.yml b/meta-chromium-test/kas/electron-common.yml new file mode 100644 index 000000000..910d4bc03 --- /dev/null +++ b/meta-chromium-test/kas/electron-common.yml @@ -0,0 +1,6 @@ +header: + version: 11 + includes: + - meta-chromium-test/kas/common.yml + +target: electron-test-image diff --git a/meta-chromium-test/kas/electron/ozone-wayland.yml b/meta-chromium-test/kas/electron/ozone-wayland.yml new file mode 100644 index 000000000..249e58484 --- /dev/null +++ b/meta-chromium-test/kas/electron/ozone-wayland.yml @@ -0,0 +1,10 @@ +header: + version: 11 + +local_conf_header: + electron-ozone-wayland: | + PREFERRED_PROVIDER_electron = "electron-ozone-wayland" + PACKAGECONFIG:append:pn-electron-ozone-wayland = " proprietary-codecs" + # Enable Wayland as a distribution feature + DISTRO_FEATURES:append = " wayland" + IMAGE_FEATURES:append = " splash package-management ssh-server-dropbear hwcodecs weston" diff --git a/meta-chromium-test/kas/electron/ozone-x11.yml b/meta-chromium-test/kas/electron/ozone-x11.yml new file mode 100644 index 000000000..bb20380a1 --- /dev/null +++ b/meta-chromium-test/kas/electron/ozone-x11.yml @@ -0,0 +1,10 @@ +header: + version: 11 + +local_conf_header: + electron-ozone-x11: | + PREFERRED_PROVIDER_electron = "electron-ozone-x11" + PACKAGECONFIG:append:pn-electron-ozone-x11 = " proprietary-codecs" + # Enable X11 distribution feature + DISTRO_FEATURES:append = " x11" + IMAGE_FEATURES:append = " splash package-management x11-base" diff --git a/meta-chromium-test/kas/glibc-ozone-wayland-aarch64-chromium-test.yml b/meta-chromium-test/kas/glibc-ozone-wayland-aarch64-chromium-test.yml new file mode 100644 index 000000000..0d13eadca --- /dev/null +++ b/meta-chromium-test/kas/glibc-ozone-wayland-aarch64-chromium-test.yml @@ -0,0 +1,12 @@ +header: + version: 11 + includes: + - meta-chromium-test/kas/chromium/ozone-wayland.yml + - meta-chromium-test/kas/repos/common-repos.yml + - meta-chromium-test/kas/chromium-common.yml + +machine: qemuarm64 + +local_conf_header: + image_name: | + IMAGE_NAME = "glibc-ozone-wayland-aarch64-chromium" diff --git a/meta-chromium-test/kas/glibc-ozone-wayland-aarch64-electron-test.yml b/meta-chromium-test/kas/glibc-ozone-wayland-aarch64-electron-test.yml new file mode 100644 index 000000000..363143a6f --- /dev/null +++ b/meta-chromium-test/kas/glibc-ozone-wayland-aarch64-electron-test.yml @@ -0,0 +1,12 @@ +header: + version: 11 + includes: + - meta-chromium-test/kas/electron/ozone-wayland.yml + - meta-chromium-test/kas/repos/common-repos.yml + - meta-chromium-test/kas/electron-common.yml + +machine: qemuarm64 + +local_conf_header: + image_name: | + IMAGE_NAME = "glibc-ozone-wayland-aarch64-electron" diff --git a/meta-chromium-test/kas/glibc-ozone-wayland-arm-chromium-test.yml b/meta-chromium-test/kas/glibc-ozone-wayland-arm-chromium-test.yml new file mode 100644 index 000000000..23001d36b --- /dev/null +++ b/meta-chromium-test/kas/glibc-ozone-wayland-arm-chromium-test.yml @@ -0,0 +1,12 @@ +header: + version: 11 + includes: + - meta-chromium-test/kas/chromium/ozone-wayland.yml + - meta-chromium-test/kas/repos/common-repos.yml + - meta-chromium-test/kas/chromium-common.yml + +machine: qemuarm + +local_conf_header: + image_name: | + IMAGE_NAME = "glibc-ozone-wayland-arm-chromium" diff --git a/meta-chromium-test/kas/glibc-ozone-wayland-arm-electron-test.yml b/meta-chromium-test/kas/glibc-ozone-wayland-arm-electron-test.yml new file mode 100644 index 000000000..f22f4244f --- /dev/null +++ b/meta-chromium-test/kas/glibc-ozone-wayland-arm-electron-test.yml @@ -0,0 +1,12 @@ +header: + version: 11 + includes: + - meta-chromium-test/kas/electron/ozone-wayland.yml + - meta-chromium-test/kas/repos/common-repos.yml + - meta-chromium-test/kas/electron-common.yml + +machine: qemuarm + +local_conf_header: + image_name: | + IMAGE_NAME = "glibc-ozone-wayland-arm-electron" diff --git a/meta-chromium-test/kas/glibc-ozone-wayland-riscv-chromium-test.yml b/meta-chromium-test/kas/glibc-ozone-wayland-riscv-chromium-test.yml new file mode 100644 index 000000000..9201cd2bc --- /dev/null +++ b/meta-chromium-test/kas/glibc-ozone-wayland-riscv-chromium-test.yml @@ -0,0 +1,12 @@ +header: + version: 11 + includes: + - meta-chromium-test/kas/chromium/ozone-wayland.yml + - meta-chromium-test/kas/repos/common-repos.yml + - meta-chromium-test/kas/chromium-common.yml + +machine: qemuriscv64 + +local_conf_header: + image_name: | + IMAGE_NAME = "glibc-ozone-wayland-riscv-chromium" diff --git a/meta-chromium-test/kas/glibc-ozone-wayland-riscv-electron-test.yml b/meta-chromium-test/kas/glibc-ozone-wayland-riscv-electron-test.yml new file mode 100644 index 000000000..581b4f21d --- /dev/null +++ b/meta-chromium-test/kas/glibc-ozone-wayland-riscv-electron-test.yml @@ -0,0 +1,12 @@ +header: + version: 11 + includes: + - meta-chromium-test/kas/electron/ozone-wayland.yml + - meta-chromium-test/kas/repos/common-repos.yml + - meta-chromium-test/kas/electron-common.yml + +machine: qemuriscv64 + +local_conf_header: + image_name: | + IMAGE_NAME = "glibc-ozone-wayland-riscv-electron" diff --git a/meta-chromium-test/kas/glibc-ozone-wayland-x86-64-chromium-test.yml b/meta-chromium-test/kas/glibc-ozone-wayland-x86-64-chromium-test.yml new file mode 100644 index 000000000..3f74b158a --- /dev/null +++ b/meta-chromium-test/kas/glibc-ozone-wayland-x86-64-chromium-test.yml @@ -0,0 +1,12 @@ +header: + version: 11 + includes: + - meta-chromium-test/kas/chromium/ozone-wayland.yml + - meta-chromium-test/kas/repos/common-repos.yml + - meta-chromium-test/kas/chromium-common.yml + +machine: qemux86-64 + +local_conf_header: + image_name: | + IMAGE_NAME = "glibc-ozone-wayland-x86_64-chromium" diff --git a/meta-chromium-test/kas/glibc-ozone-wayland-x86-64-electron-test.yml b/meta-chromium-test/kas/glibc-ozone-wayland-x86-64-electron-test.yml new file mode 100644 index 000000000..1a24e0965 --- /dev/null +++ b/meta-chromium-test/kas/glibc-ozone-wayland-x86-64-electron-test.yml @@ -0,0 +1,12 @@ +header: + version: 11 + includes: + - meta-chromium-test/kas/electron/ozone-wayland.yml + - meta-chromium-test/kas/repos/common-repos.yml + - meta-chromium-test/kas/electron-common.yml + +machine: qemux86-64 + +local_conf_header: + image_name: | + IMAGE_NAME = "glibc-ozone-wayland-x86_64-electron" diff --git a/meta-chromium-test/kas/glibc-ozone-x11-aarch64-electron-test.yml b/meta-chromium-test/kas/glibc-ozone-x11-aarch64-electron-test.yml new file mode 100644 index 000000000..6d40e4f71 --- /dev/null +++ b/meta-chromium-test/kas/glibc-ozone-x11-aarch64-electron-test.yml @@ -0,0 +1,12 @@ +header: + version: 11 + includes: + - meta-chromium-test/kas/electron/ozone-x11.yml + - meta-chromium-test/kas/repos/common-repos.yml + - meta-chromium-test/kas/electron-common.yml + +machine: qemuarm64 + +local_conf_header: + image_name: | + IMAGE_NAME = "glibc-ozone-x11-aarch64-electron" diff --git a/meta-chromium-test/kas/glibc-ozone-x11-arm-electron-test.yml b/meta-chromium-test/kas/glibc-ozone-x11-arm-electron-test.yml new file mode 100644 index 000000000..8ca967b48 --- /dev/null +++ b/meta-chromium-test/kas/glibc-ozone-x11-arm-electron-test.yml @@ -0,0 +1,12 @@ +header: + version: 11 + includes: + - meta-chromium-test/kas/electron/ozone-x11.yml + - meta-chromium-test/kas/repos/common-repos.yml + - meta-chromium-test/kas/electron-common.yml + +machine: qemuarm + +local_conf_header: + image_name: | + IMAGE_NAME = "glibc-ozone-x11-arm-electron" diff --git a/meta-chromium-test/kas/glibc-ozone-x11-riscv-electron-test.yml b/meta-chromium-test/kas/glibc-ozone-x11-riscv-electron-test.yml new file mode 100644 index 000000000..f39c878d3 --- /dev/null +++ b/meta-chromium-test/kas/glibc-ozone-x11-riscv-electron-test.yml @@ -0,0 +1,12 @@ +header: + version: 11 + includes: + - meta-chromium-test/kas/electron/ozone-x11.yml + - meta-chromium-test/kas/repos/common-repos.yml + - meta-chromium-test/kas/electron-common.yml + +machine: qemuriscv64 + +local_conf_header: + image_name: | + IMAGE_NAME = "glibc-ozone-x11-riscv-electron" diff --git a/meta-chromium-test/kas/glibc-ozone-x11-x86-64-electron-test.yml b/meta-chromium-test/kas/glibc-ozone-x11-x86-64-electron-test.yml new file mode 100644 index 000000000..8b7db1a78 --- /dev/null +++ b/meta-chromium-test/kas/glibc-ozone-x11-x86-64-electron-test.yml @@ -0,0 +1,12 @@ +header: + version: 11 + includes: + - meta-chromium-test/kas/electron/ozone-x11.yml + - meta-chromium-test/kas/repos/common-repos.yml + - meta-chromium-test/kas/electron-common.yml + +machine: qemux86-64 + +local_conf_header: + image_name: | + IMAGE_NAME = "glibc-ozone-x11-x86_64-electron" diff --git a/meta-chromium-test/kas/glibc-x11-aarch64-chromium-test.yml b/meta-chromium-test/kas/glibc-x11-aarch64-chromium-test.yml new file mode 100644 index 000000000..ac35f8a99 --- /dev/null +++ b/meta-chromium-test/kas/glibc-x11-aarch64-chromium-test.yml @@ -0,0 +1,12 @@ +header: + version: 11 + includes: + - meta-chromium-test/kas/chromium/x11.yml + - meta-chromium-test/kas/repos/common-repos.yml + - meta-chromium-test/kas/chromium-common.yml + +machine: qemuarm64 + +local_conf_header: + image_name: | + IMAGE_NAME = "glibc-x11-aarch64-chromium" diff --git a/meta-chromium-test/kas/glibc-x11-arm-chromium-test.yml b/meta-chromium-test/kas/glibc-x11-arm-chromium-test.yml new file mode 100644 index 000000000..06a1671b9 --- /dev/null +++ b/meta-chromium-test/kas/glibc-x11-arm-chromium-test.yml @@ -0,0 +1,12 @@ +header: + version: 11 + includes: + - meta-chromium-test/kas/chromium/x11.yml + - meta-chromium-test/kas/repos/common-repos.yml + - meta-chromium-test/kas/chromium-common.yml + +machine: qemuarm + +local_conf_header: + image_name: | + IMAGE_NAME = "glibc-x11-arm-chromium" diff --git a/meta-chromium-test/kas/glibc-x11-riscv-chromium-test.yml b/meta-chromium-test/kas/glibc-x11-riscv-chromium-test.yml new file mode 100644 index 000000000..51952792b --- /dev/null +++ b/meta-chromium-test/kas/glibc-x11-riscv-chromium-test.yml @@ -0,0 +1,12 @@ +header: + version: 11 + includes: + - meta-chromium-test/kas/chromium/x11.yml + - meta-chromium-test/kas/repos/common-repos.yml + - meta-chromium-test/kas/chromium-common.yml + +machine: qemuriscv64 + +local_conf_header: + image_name: | + IMAGE_NAME = "glibc-x11-riscv-chromium" diff --git a/meta-chromium-test/kas/glibc-x11-x86-64-chromium-test.yml b/meta-chromium-test/kas/glibc-x11-x86-64-chromium-test.yml new file mode 100644 index 000000000..4536d4765 --- /dev/null +++ b/meta-chromium-test/kas/glibc-x11-x86-64-chromium-test.yml @@ -0,0 +1,12 @@ +header: + version: 11 + includes: + - meta-chromium-test/kas/chromium/x11.yml + - meta-chromium-test/kas/repos/common-repos.yml + - meta-chromium-test/kas/chromium-common.yml + +machine: qemux86-64 + +local_conf_header: + image_name: | + IMAGE_NAME = "glibc-x11-x86_64-chromium" diff --git a/meta-chromium-test/kas/local-dev.yml b/meta-chromium-test/kas/local-dev.yml new file mode 100644 index 000000000..cfdfe9980 --- /dev/null +++ b/meta-chromium-test/kas/local-dev.yml @@ -0,0 +1,12 @@ +header: + version: 11 + +local_conf_header: + local-development: | + # Override CI-specific paths for local development + DL_DIR = "${TOPDIR}/../downloads" + SSTATE_DIR = "${TOPDIR}/../sstate-cache" + # Disable source mirror for local development + SOURCE_MIRROR_URL = "" + # Remove own-mirrors inherit for local development + INHERIT:remove = "own-mirrors" diff --git a/meta-chromium-test/kas/repos/common-repos.yml b/meta-chromium-test/kas/repos/common-repos.yml new file mode 100644 index 000000000..5e6d4479c --- /dev/null +++ b/meta-chromium-test/kas/repos/common-repos.yml @@ -0,0 +1,28 @@ +header: + version: 11 + +repos: + meta-browser: + layers: + meta-chromium: + meta-chromium-test: + meta-oe: + url: https://github.com/openembedded/meta-openembedded + commit: "1d6095b73fa21062fde2b347cd61b6c5104390d9" + layers: + meta-oe: + meta-networking: + meta-multimedia: + meta-python: + poky: + url: https://github.com/yoctoproject/poky.git + commit: "9bc47d6a4d23bff5999b69fa2b99e1b32eaa62fb" + layers: + meta: + meta-poky: + meta-yocto-bsp: + meta-clang: + url: https://github.com/kraj/meta-clang + commit: "02e254ba1fb4937530d94a154b4c713c904c2d06" + layers: + .: diff --git a/meta-chromium-test/recipes-core/images/chromium-test-image.bb b/meta-chromium-test/recipes-core/images/chromium-test-image.bb new file mode 100644 index 000000000..2af144051 --- /dev/null +++ b/meta-chromium-test/recipes-core/images/chromium-test-image.bb @@ -0,0 +1,29 @@ +# Use core-image-minimal as base and build up from there +inherit core-image + +# Install the preferred Chromium provider (set via KAS configuration) +IMAGE_INSTALL += "${PREFERRED_PROVIDER_chromium} chromium-tests" + +SUMMARY = "Test image for Chromium browser builds" +DESCRIPTION = "This image contains the configured Chromium variant for testing purposes." + +# Development and debugging tools +IMAGE_INSTALL += " \ + strace \ + gdb \ + valgrind \ + procps \ + util-linux \ + coreutils \ +" + +# Network tools for testing +IMAGE_INSTALL += " \ + curl \ + wget \ + openssh-sftp-server \ + openssh-scp \ +" + +# Additional features for testing +EXTRA_IMAGE_FEATURES += "allow-empty-password allow-root-login ssh-server-openssh tools-debug" diff --git a/meta-chromium-test/recipes-core/images/electron-test-image.bb b/meta-chromium-test/recipes-core/images/electron-test-image.bb new file mode 100644 index 000000000..4b609ec81 --- /dev/null +++ b/meta-chromium-test/recipes-core/images/electron-test-image.bb @@ -0,0 +1,39 @@ +# Use core-image-minimal as base and build up from there +inherit core-image + +# Install the preferred Electron provider (set via KAS configuration) +IMAGE_INSTALL += "${PREFERRED_PROVIDER_electron} electron-tests" + +SUMMARY = "Test image for Electron application builds" +DESCRIPTION = "This image contains the configured Electron variant for testing purposes." + +# Development and debugging tools +IMAGE_INSTALL += " \ + strace \ + gdb \ + valgrind \ + procps \ + util-linux \ + coreutils \ +" + +# Network tools for testing +IMAGE_INSTALL += " \ + curl \ + wget \ + openssh-sftp-server \ + openssh-scp \ +" + +# Wayland support for electron-ozone-wayland +IMAGE_INSTALL += " \ + weston \ + weston-init \ + wayland-protocols \ +" + +# Additional features for testing +EXTRA_IMAGE_FEATURES += "allow-empty-password allow-root-login ssh-server-openssh tools-debug" + +# Ensure Wayland is enabled for electron testing +REQUIRED_DISTRO_FEATURES = "wayland" diff --git a/meta-chromium-test/recipes-test/chromium-tests/chromium-tests.bb b/meta-chromium-test/recipes-test/chromium-tests/chromium-tests.bb new file mode 100644 index 000000000..7ef5b5ab1 --- /dev/null +++ b/meta-chromium-test/recipes-test/chromium-tests/chromium-tests.bb @@ -0,0 +1,16 @@ +SUMMARY = "Chromium browser smoke tests" +DESCRIPTION = "Simple smoke tests for Chromium browser functionality" +LICENSE = "MIT" +LIC_FILES_CHKSUM = "file://${COMMON_LICENSE_DIR}/MIT;md5=0835ade698e0bcf8506ecda2f7b4f302" + +SRC_URI = "file://chromium-smoke-test.sh" + +S = "${UNPACKDIR}" + +do_install() { + install -d ${D}${bindir} + install -m 0755 chromium-smoke-test.sh ${D}${bindir}/chromium-smoke-test +} + +RDEPENDS:${PN} = "bash" +FILES:${PN} = "${bindir}/chromium-smoke-test" diff --git a/meta-chromium-test/recipes-test/chromium-tests/files/chromium-smoke-test.sh b/meta-chromium-test/recipes-test/chromium-tests/files/chromium-smoke-test.sh new file mode 100644 index 000000000..f71a5d2c2 --- /dev/null +++ b/meta-chromium-test/recipes-test/chromium-tests/files/chromium-smoke-test.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +# Chromium browser smoke test +# Basic functionality test for Chromium browser + +echo "Starting Chromium smoke test..." + +# Check if chromium binaries are available +if command -v chromium-browser &> /dev/null; then + CHROMIUM_CMD="chromium-browser" +elif command -v chromium &> /dev/null; then + CHROMIUM_CMD="chromium" +else + echo "ERROR: No chromium binary found" + exit 1 +fi + +echo "Found chromium binary: $CHROMIUM_CMD" + +# Test 1: Check version +echo "Testing version information..." +$CHROMIUM_CMD --version +if [ $? -ne 0 ]; then + echo "ERROR: Failed to get version information" + exit 1 +fi + +# Test 2: Check help +echo "Testing help output..." +$CHROMIUM_CMD --help > /dev/null +if [ $? -ne 0 ]; then + echo "ERROR: Failed to get help information" + exit 1 +fi + +# Test 3: Check if we can start in headless mode (basic functionality) +echo "Testing headless mode startup..." +timeout 10 $CHROMIUM_CMD --headless --disable-gpu --no-sandbox --dump-dom about:blank > /dev/null 2>&1 +if [ $? -eq 0 ]; then + echo "SUCCESS: Chromium headless mode works" +else + echo "WARNING: Chromium headless mode may have issues (timeout or error)" +fi + +# Test 4: Check for required libraries +echo "Testing library dependencies..." +ldd $(which $CHROMIUM_CMD) | grep -q "not found" +if [ $? -eq 0 ]; then + echo "ERROR: Missing library dependencies" + ldd $(which $CHROMIUM_CMD) | grep "not found" + exit 1 +else + echo "SUCCESS: All library dependencies found" +fi + +echo "Chromium smoke test completed successfully!" +exit 0 diff --git a/meta-chromium-test/recipes-test/electron-tests/electron-tests.bb b/meta-chromium-test/recipes-test/electron-tests/electron-tests.bb new file mode 100644 index 000000000..d74f6a9b6 --- /dev/null +++ b/meta-chromium-test/recipes-test/electron-tests/electron-tests.bb @@ -0,0 +1,16 @@ +SUMMARY = "Electron application smoke tests" +DESCRIPTION = "Simple smoke tests for Electron application functionality" +LICENSE = "MIT" +LIC_FILES_CHKSUM = "file://${COMMON_LICENSE_DIR}/MIT;md5=0835ade698e0bcf8506ecda2f7b4f302" + +SRC_URI = "file://electron-smoke-test.sh" + +S = "${UNPACKDIR}" + +do_install() { + install -d ${D}${bindir} + install -m 0755 electron-smoke-test.sh ${D}${bindir}/electron-smoke-test +} + +RDEPENDS:${PN} = "bash" +FILES:${PN} = "${bindir}/electron-smoke-test" diff --git a/meta-chromium-test/recipes-test/electron-tests/files/electron-smoke-test.sh b/meta-chromium-test/recipes-test/electron-tests/files/electron-smoke-test.sh new file mode 100644 index 000000000..46d1b8f4a --- /dev/null +++ b/meta-chromium-test/recipes-test/electron-tests/files/electron-smoke-test.sh @@ -0,0 +1,104 @@ +#!/bin/bash + +# Electron application smoke test +# Basic functionality test for Electron application + +echo "Starting Electron smoke test..." + +# Check if electron binaries are available +if command -v electron &> /dev/null; then + ELECTRON_CMD="electron" +else + echo "ERROR: No electron binary found" + exit 1 +fi + +echo "Found electron binary: $ELECTRON_CMD" + +# Test 1: Check version +echo "Testing version information..." +$ELECTRON_CMD --version +if [ $? -ne 0 ]; then + echo "ERROR: Failed to get version information" + exit 1 +fi + +# Test 2: Check help +echo "Testing help output..." +$ELECTRON_CMD --help > /dev/null +if [ $? -ne 0 ]; then + echo "ERROR: Failed to get help information" + exit 1 +fi + +# Test 3: Check if we can start in headless mode (basic functionality) +echo "Testing headless mode startup..." +# Create a minimal test app +TEST_APP_DIR="/tmp/electron-test-app" +mkdir -p $TEST_APP_DIR +cat > $TEST_APP_DIR/package.json << EOF +{ + "name": "electron-test", + "version": "1.0.0", + "main": "main.js" +} +EOF + +cat > $TEST_APP_DIR/main.js << EOF +const { app } = require('electron'); + +app.whenReady().then(() => { + console.log('Electron app started successfully'); + app.quit(); +}); + +app.on('window-all-closed', () => { + app.quit(); +}); +EOF + +# Test the minimal app with a timeout +OUTPUT=$(timeout 10 $ELECTRON_CMD $TEST_APP_DIR --no-sandbox --disable-gpu 2>&1) +EXIT_CODE=$? +echo "$OUTPUT" | grep -q "Electron app started successfully" +GREP_CODE=$? +if [ $EXIT_CODE -eq 124 ]; then + echo "WARNING: Electron application startup timed out" +elif [ $EXIT_CODE -ne 0 ]; then + echo "WARNING: Electron application startup failed (exit code $EXIT_CODE)" +elif [ $GREP_CODE -eq 0 ]; then + echo "SUCCESS: Electron application startup works" +else + echo "WARNING: Electron application startup did not produce expected output" +fi + +# Clean up test app +rm -rf $TEST_APP_DIR + +# Test 4: Check for required libraries +echo "Testing library dependencies..." +ldd $(which $ELECTRON_CMD) | grep -q "not found" +if [ $? -eq 0 ]; then + echo "ERROR: Missing library dependencies" + ldd $(which $ELECTRON_CMD) | grep "not found" + exit 1 +else + echo "SUCCESS: All library dependencies found" +fi + +# Test 5: Check Wayland support if WAYLAND_DISPLAY is set +if [ -n "$WAYLAND_DISPLAY" ]; then + echo "Testing Wayland support..." + timeout 5 $ELECTRON_CMD --version --ozone-platform=wayland --no-sandbox > /dev/null 2>&1 + EXIT_CODE=$? + if [ $EXIT_CODE -eq 0 ]; then + echo "SUCCESS: Wayland platform support detected" + elif [ $EXIT_CODE -eq 124 ]; then + echo "WARNING: Wayland platform test timed out (exit code 124)" + else + echo "WARNING: Wayland platform support may have issues (exit code $EXIT_CODE)" + fi +fi + +echo "Electron smoke test completed successfully!" +exit 0 diff --git a/meta-chromium-test/scripts/build.sh b/meta-chromium-test/scripts/build.sh new file mode 100755 index 000000000..6c490c483 --- /dev/null +++ b/meta-chromium-test/scripts/build.sh @@ -0,0 +1,98 @@ +#!/bin/bash + +if [ $# -le 2 ]; then + echo Usage: $0 arch browser_version browser + echo E.g. $0 aarch64 ozone-wayland electron + echo E.g. $0 aarch64 ozone-wayland chromium + exit 1 +fi + +declare -A arch_qemu_dict +arch_qemu_dict["arm"]="qemuarm" +arch_qemu_dict["aarch64"]="qemuarm64" +arch_qemu_dict["riscv"]="qemuriscv64" +arch_qemu_dict["x86-64"]="qemux86-64" + +arch=$1 +browser_version=$2 +browser=$3 + +kas_file_name=glibc-$browser_version-$arch-$browser + +# check if it needs to be built, if the flag has been set +#if [ -d /yocto/test-images/$kas_file_name ]; then +# echo Image has been already built, exiting. +# exit 0 +#fi + +cd /yocto/meta-browser + +# Ensure all required Yocto subdirectories exist with proper permissions +# This creates the shared directories that are used across all builds +mkdir -p /yocto/yocto_dl +mkdir -p /yocto/yocto_sstate_chromium +mkdir -p /yocto/yocto_ccache +mkdir -p /yocto/test-images +chmod 755 /yocto/yocto_dl /yocto/yocto_sstate_chromium /yocto/yocto_ccache /yocto/test-images + +if [ "$arch" = "riscv" ]; then + OPENSBI="opensbi" +else + OPENSBI="" +fi + +qemu_machine=${arch_qemu_dict[$arch]} + +rm -rf ../build/tmp/deploy/images/$qemu_machine + +kas checkout --update meta-chromium-test/kas/$kas_file_name-test.yml || exit 1 + +# Clean previous builds - adjust packages based on browser +if [ "$browser" = "electron" ]; then + BROWSER_PACKAGES="electron-ozone-wayland electron-ozone-x11" +else + BROWSER_PACKAGES="chromium-ozone-wayland chromium-x11" +fi + +kas shell meta-chromium-test/kas/$kas_file_name-test.yml -c "bitbake -c clean $BROWSER_PACKAGES \ + virtual/kernel $OPENSBI" || exit 1 + +# Build the image +kas build meta-chromium-test/kas/$kas_file_name-test.yml || exit 1 + +# Copy the built image to test images directory +echo "Checking for image directories using KAS..." + +# Use KAS to get the actual build directory and DEPLOY_DIR +BUILD_DIR=$(kas shell meta-chromium-test/kas/$kas_file_name-test.yml -c 'echo $BUILDDIR' 2>/dev/null | tail -1) +DEPLOY_DIR=$(kas shell meta-chromium-test/kas/$kas_file_name-test.yml -c 'echo $DEPLOY_DIR' 2>/dev/null | tail -1) + +echo "KAS Build directory: $BUILD_DIR" +echo "KAS Deploy directory: $DEPLOY_DIR" + +# Construct the image path +if [ -n "$DEPLOY_DIR" ]; then + IMAGE_DIR="$DEPLOY_DIR/images/$qemu_machine" +elif [ -n "$BUILD_DIR" ]; then + IMAGE_DIR="$BUILD_DIR/tmp/deploy/images/$qemu_machine" +else + # Fallback to relative path + IMAGE_DIR="build/tmp/deploy/images/$qemu_machine" +fi + +echo "Looking for images in: $IMAGE_DIR" + +if [ -d "$IMAGE_DIR" ]; then + echo "Copying built image to /yocto/test-images/$kas_file_name" + rm -rf /yocto/test-images/$kas_file_name + cp -r "$IMAGE_DIR" /yocto/test-images/$kas_file_name + echo "Successfully created test image: $kas_file_name" + ls -la /yocto/test-images/$kas_file_name +else + echo "ERROR: Built image directory not found: $IMAGE_DIR" + echo "Available directories in deploy area:" + if [ -n "$DEPLOY_DIR" ] && [ -d "$DEPLOY_DIR" ]; then + find "$DEPLOY_DIR" -name images -type d 2>/dev/null || echo "No images directory found in $DEPLOY_DIR" + fi + exit 1 +fi diff --git a/meta-chromium-test/scripts/generate_kas_files.sh b/meta-chromium-test/scripts/generate_kas_files.sh new file mode 100755 index 000000000..dc3d27ddb --- /dev/null +++ b/meta-chromium-test/scripts/generate_kas_files.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +# Generate KAS files for all combinations +# Usage: generate_kas_files.sh [browser] +# browser: chromium, electron, or both (default) + +BROWSER=${1:-both} +ARCHS=("arm" "aarch64" "riscv" "x86-64") + +declare -A arch_machine_dict +arch_machine_dict["arm"]="qemuarm" +arch_machine_dict["aarch64"]="qemuarm64" +arch_machine_dict["riscv"]="qemuriscv64" +arch_machine_dict["x86-64"]="qemux86-64" + +BASE_DIR="${KAS_BASE_DIR:-./kas}" + +# Validate browser argument +if [ "$BROWSER" != "chromium" ] && [ "$BROWSER" != "electron" ] && [ "$BROWSER" != "both" ]; then + echo "Error: Browser must be 'chromium', 'electron', or 'both'" + echo "Usage: generate_kas_files.sh [browser]" + exit 1 +fi + +# Determine which browsers to generate +if [ "$BROWSER" = "both" ]; then + BROWSERS=("chromium" "electron") +else + BROWSERS=("$BROWSER") +fi + +echo "Generating KAS files for: ${BROWSERS[*]}" + +for current_browser in "${BROWSERS[@]}"; do + echo "Processing $current_browser..." + + # Define versions per browser + if [ "$current_browser" = "electron" ]; then + BROWSER_VERSIONS=("ozone-wayland" "ozone-x11") + else + BROWSER_VERSIONS=("ozone-wayland" "x11") + fi + + for browser_version in "${BROWSER_VERSIONS[@]}"; do + for arch in "${ARCHS[@]}"; do + machine=${arch_machine_dict[$arch]} + kas_file_name="glibc-$browser_version-$arch-$current_browser-test.yml" + + echo "Creating $kas_file_name" + + cat > "$BASE_DIR/$kas_file_name" << EOF +header: + version: 11 + includes: + - meta-chromium-test/kas/$current_browser/$browser_version.yml + - meta-chromium-test/kas/repos/common-repos.yml + - meta-chromium-test/kas/$current_browser-common.yml + +machine: $machine + +local_conf_header: + image_name: | + IMAGE_NAME = "glibc-$browser_version-${arch//-/_}-$current_browser" +EOF + done + done +done + +echo "All KAS files generated successfully for: ${BROWSERS[*]}!" diff --git a/meta-chromium-test/scripts/test.sh b/meta-chromium-test/scripts/test.sh new file mode 100755 index 000000000..b4d3448fe --- /dev/null +++ b/meta-chromium-test/scripts/test.sh @@ -0,0 +1,94 @@ +#!/bin/bash + +if [ $# -le 1 ]; then + echo Usage: $0 arch chromium_version + echo E.g. $0 x86-64 ozone-wayland + exit 1 +fi + +declare -A arch_qemu_dict +arch_qemu_dict["arm"]="qemuarm" +arch_qemu_dict["aarch64"]="qemuarm64" +arch_qemu_dict["riscv"]="qemuriscv64" +arch_qemu_dict["x86-64"]="qemux86-64" + +declare -A qemu_params_dict +qemu_params_dict["x86-64"]='qemuparams=" --enable-kvm "' + +arch=$1 +chromium_version=$2 + +image_folder=glibc-$chromium_version-$arch + +qemu_machine=${arch_qemu_dict[$arch]} +qemu_params="${qemu_params_dict[$arch]}" + +if [ ! -d /yocto/test-images/$image_folder ]; then + echo $image_folder image was not created! + exit 1 +fi + +if [ -f /yocto/test-images/$image_folder.done ]; then + echo $image_folder test done already, skipping now. + exit 0 +fi + +cd /yocto/$yocto_version/poky +source oe-init-build-env ../build + +rm -rf tmp/deploy/images/$qemu_machine +cp -r /yocto/test-images/$image_folder tmp/deploy/images/$qemu_machine + +coproc qemu { runqemu "$qemu_machine $qemu_params"; } + +TIMEOUT=100 +i=0 +while [ $i -lt $TIMEOUT ] +do + if [ -f /tmp/qemu_in_use ]; then + echo "qemu socket is in use, waiting..." + sleep 5 + i=$((i+5)) + else + break + fi +done + +if [ $i -ge $TIMEOUT ]; then + echo "qemu socket is busy, exiting..." + exit 1 +fi + +# wait for qemu to start +echo "Waiting for qemu to start..." +sleep 30 + +# Basic test: check if system boots and chromium processes can start +echo "Testing Chromium installation..." + +# Test if chromium-ozone-wayland is available +echo "root" | socat - unix-connect:/tmp/qemu_monitor > /dev/null 2>&1 +if [ $? -ne 0 ]; then + echo "Could not connect to qemu monitor" + kill %qemu 2>/dev/null + exit 1 +fi + +# Simple smoke test - check if chromium binaries are present +echo "info registers" | socat - unix-connect:/tmp/qemu_monitor | grep -q "pc" +if [ $? -eq 0 ]; then + echo "QEMU is running and responsive" +else + echo "QEMU is not responding properly" + kill %qemu 2>/dev/null + exit 1 +fi + +# Clean shutdown +echo "system_powerdown" | socat - unix-connect:/tmp/qemu_monitor +wait %qemu + +# Mark test as done +touch /yocto/test-images/$image_folder.done + +echo "Test completed successfully for $image_folder" diff --git a/meta-chromium-test/scripts/validate.sh b/meta-chromium-test/scripts/validate.sh new file mode 100755 index 000000000..b5b8a6a2d --- /dev/null +++ b/meta-chromium-test/scripts/validate.sh @@ -0,0 +1,99 @@ +#!/bin/bash + +# Validation script for meta-chromium-test layer +# This script checks if all required files are present and properly configured + +echo "=== Meta-Chromium-Test Layer Validation ===" + +LAYER_DIR="/home/cal/work/yocto/poky/sources/meta-chromium-test" +ERRORS=0 + +# Check basic layer structure +echo "1. Checking layer structure..." +if [ ! -f "$LAYER_DIR/conf/layer.conf" ]; then + echo "ERROR: Missing layer.conf" + ERRORS=$((ERRORS + 1)) +fi + +if [ ! -f "$LAYER_DIR/README.md" ]; then + echo "ERROR: Missing README.md" + ERRORS=$((ERRORS + 1)) +fi + +# Check scripts +echo "2. Checking scripts..." +for script in "build.sh" "test.sh" "generate_kas_files.sh"; do + if [ ! -x "$LAYER_DIR/scripts/$script" ]; then + echo "ERROR: Missing or not executable: scripts/$script" + ERRORS=$((ERRORS + 1)) + fi +done + +# Check recipes +echo "3. Checking recipes..." +if [ ! -f "$LAYER_DIR/recipes-core/images/chromium-test-image.bb" ]; then + echo "ERROR: Missing chromium-test-image.bb" + ERRORS=$((ERRORS + 1)) +fi + +if [ ! -f "$LAYER_DIR/recipes-test/chromium-tests/chromium-tests.bb" ]; then + echo "ERROR: Missing chromium-tests.bb" + ERRORS=$((ERRORS + 1)) +fi + +# Check KAS configurations +echo "4. Checking KAS configurations..." +kas_count=$(find "$LAYER_DIR/kas" -name "*.yml" | wc -l) +echo "Found $kas_count KAS configuration files" + +if [ $kas_count -lt 60 ]; then + echo "WARNING: Expected more KAS files (found $kas_count)" +fi + +# Check chromium variants +if [ ! -f "$LAYER_DIR/kas/chromium/ozone-wayland.yml" ]; then + echo "ERROR: Missing ozone-wayland.yml" + ERRORS=$((ERRORS + 1)) +fi + +if [ ! -f "$LAYER_DIR/kas/chromium/x11.yml" ]; then + echo "ERROR: Missing x11.yml" + ERRORS=$((ERRORS + 1)) +fi + +# Check GitHub workflow +echo "5. Checking GitHub workflow..." +if [ ! -f "/home/cal/work/yocto/poky/meta-browser/.github/workflows/chromium.yml" ]; then + echo "ERROR: Missing chromium.yml workflow" + ERRORS=$((ERRORS + 1)) +fi + +# Validate layer.conf syntax +echo "6. Validating layer.conf syntax..." +if ! grep -q "chromium-test-layer" "$LAYER_DIR/conf/layer.conf"; then + echo "ERROR: layer.conf missing chromium-test-layer collection" + ERRORS=$((ERRORS + 1)) +fi + +if ! grep -q "chromium-browser-layer" "$LAYER_DIR/conf/layer.conf"; then + echo "ERROR: layer.conf missing chromium-browser-layer dependency" + ERRORS=$((ERRORS + 1)) +fi + +# Summary +echo "=== Validation Summary ===" +if [ $ERRORS -eq 0 ]; then + echo "✓ All checks passed! Meta-chromium-test layer is properly configured." + echo "" + echo "You can now:" + echo "1. Build with: ./scripts/build.sh kirkstone x86-64 ozone-wayland glibc" + echo "2. Test with: ./scripts/test.sh kirkstone x86-64 ozone-wayland glibc" + echo "3. Use KAS: kas build kas/glibc-kirkstone-ozone-wayland-x86-64-test.yml" + echo "" + echo "GitHub Actions workflow is available at:" + echo " .github/workflows/chromium.yml" +else + echo "✗ $ERRORS errors found. Please fix them before using the layer." +fi + +exit $ERRORS