From 924c880bd398553b1f81ff827f26f5bd064fb606 Mon Sep 17 00:00:00 2001 From: Harmanpreet Kaur Date: Tue, 16 Dec 2025 17:26:26 +0530 Subject: [PATCH 01/20] Enhance CI workflows: update paths for Docker build, deploy, PyLint, and test workflows; upgrade action versions --- .github/workflows/build-docker-images.yml | 12 ++++ .github/workflows/deploy.yml | 72 ++++++++++++++++------- .github/workflows/pylint.yml | 16 ++++- .github/workflows/test.yml | 21 +++++-- 4 files changed, 91 insertions(+), 30 deletions(-) diff --git a/.github/workflows/build-docker-images.yml b/.github/workflows/build-docker-images.yml index 7519d620..47aeeb5f 100644 --- a/.github/workflows/build-docker-images.yml +++ b/.github/workflows/build-docker-images.yml @@ -7,6 +7,12 @@ on: - dev - demo - hotfix + paths: + - 'src/backend/**' + - 'src/frontend/**' + - 'docker/**' + - '.github/workflows/build-docker-images.yml' + - '.github/workflows/build-docker.yml' pull_request: branches: - main @@ -18,6 +24,12 @@ on: - ready_for_review - reopened - synchronize + paths: + - 'src/backend/**' + - 'src/frontend/**' + - 'docker/**' + - '.github/workflows/build-docker-images.yml' + - '.github/workflows/build-docker.yml' merge_group: workflow_dispatch: diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index be98168f..f61a2490 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -1,17 +1,19 @@ name: Deploy-Test-Cleanup Pipeline on: - workflow_run: - workflows: ["Build Docker and Optional Push"] - types: - - completed - branches: - - main - - dev - - demo - schedule: - - cron: '0 5,17 * * *' # Runs at 5:00 AM and 5:00 PM GMT - workflow_dispatch: + push: + branches: + - main + - dev + - demo + paths: + - 'infra/**' + - 'scripts/**' + - 'azure.yaml' + - '.github/workflows/deploy.yml' + schedule: + - cron: '0 5,17 * * *' # Runs at 5:00 AM and 5:00 PM GMT + workflow_dispatch: env: GPT_MIN_CAPACITY: 150 @@ -25,7 +27,7 @@ jobs: WEBAPP_URL: ${{ steps.get_output.outputs.WEBAPP_URL }} steps: - name: Checkout Code - uses: actions/checkout@v3 + uses: actions/checkout@v5 - name: Setup Azure CLI run: | @@ -43,7 +45,6 @@ jobs: export AZURE_TENANT_ID=${{ secrets.AZURE_TENANT_ID }} export AZURE_CLIENT_SECRET=${{ secrets.AZURE_CLIENT_SECRET }} export AZURE_SUBSCRIPTION_ID="${{ secrets.AZURE_SUBSCRIPTION_ID }}" - export GPT_MIN_CAPACITY="${{ env.GPT_MIN_CAPACITY }}" export AZURE_REGIONS="${{ vars.AZURE_REGIONS }}" chmod +x scripts/checkquota.sh if ! scripts/checkquota.sh; then @@ -72,6 +73,11 @@ jobs: - name: Fail Pipeline if Quota Check Fails if: env.QUOTA_FAILED == 'true' run: exit 1 + + - name: Set Deployment Region + run: | + echo "Selected Region: $VALID_REGION" + echo "AZURE_LOCATION=$VALID_REGION" >> $GITHUB_ENV - name: Install Bicep CLI run: az bicep install @@ -94,7 +100,7 @@ jobs: rg_exists=$(az group exists --name ${{ env.RESOURCE_GROUP_NAME }}) if [ "$rg_exists" = "false" ]; then echo "Resource group does not exist. Creating..." - az group create --name ${{ env.RESOURCE_GROUP_NAME }} --location northcentralus || { echo "Error creating resource group"; exit 1; } + az group create --name ${{ env.RESOURCE_GROUP_NAME }} --location australiaeast || { echo "Error creating resource group"; exit 1; } else echo "Resource group already exists." fi @@ -126,17 +132,20 @@ jobs: IMAGE_TAG="latest" fi + # Generate current timestamp in desired format: YYYY-MM-DDTHH:MM:SS.SSSSSSSZ + current_date=$(date -u +"%Y-%m-%dT%H:%M:%S.%7NZ") + az deployment group create \ --name ${{ env.SOLUTION_PREFIX }}-deployment \ --resource-group ${{ env.RESOURCE_GROUP_NAME }} \ --template-file infra/main.bicep \ --parameters \ solutionName="${{ env.SOLUTION_PREFIX }}" \ - aiDeploymentsLocation="eastus" \ - useWafAlignedArchitecture=false \ - capacity=${{ env.GPT_MIN_CAPACITY }} \ + azureAiServiceLocation='${{ env.AZURE_LOCATION }}' \ imageVersion="${IMAGE_TAG}" \ - createdBy="Pipeline" + createdBy="Pipeline" \ + tags="{'SecurityControl':'Ignore','Purpose':'Deploying and Cleaning Up Resources for Validation','CreatedDate':'$current_date'}" + - name: Assign Contributor role to Service Principal if: always() run: | @@ -185,7 +194,26 @@ jobs: - name: Login to Azure run: | - az login --service-principal -u ${{ secrets.AZURE_CLIENT_ID }} -p ${{ secrets.AZURE_CLIENT_SECRET }} --tenant ${{ secrets.AZURE_TENANT_ID }} + az login --service-principal -u ${{ secrets.AZURE_CLIENT_ID }} -p ${{ secrets.AZURE_CLIENT_SECRET }} --tenant ${{ secrets.AZURE_TENANT_ID }} + az account set --subscription "${{ secrets.AZURE_SUBSCRIPTION_ID }}" + + - name: Assign Contributor role to Service Principal + if: always() + run: | + echo "Assigning Contributor role to SPN for RG: ${{ env.RESOURCE_GROUP_NAME }}" + az role assignment create \ + --assignee ${{ secrets.AZURE_CLIENT_ID }} \ + --role "Contributor" \ + --scope /subscriptions/${{ secrets.AZURE_SUBSCRIPTION_ID }}/resourceGroups/${{ env.RESOURCE_GROUP_NAME }} + + echo "Assigning Log Analytics Contributor role for Log Analytics workspace access at RG level..." + az role assignment create \ + --assignee ${{ secrets.AZURE_CLIENT_ID }} \ + --role "Log Analytics Reader" \ + --scope /subscriptions/${{ secrets.AZURE_SUBSCRIPTION_ID }}/resourceGroups/${{ env.RESOURCE_GROUP_NAME }} || echo "Log Analytics Contributor role assignment failed (may already exist)" + + echo "Waiting for role assignment propagation..." + sleep 30 - name: Get Log Analytics Workspace and OpenAI from Resource Group if: always() @@ -356,7 +384,7 @@ jobs: # Purge OpenAI Resource echo "Purging the OpenAI Resource..." - if ! az resource delete --ids /subscriptions/${{ secrets.AZURE_SUBSCRIPTION_ID }}/providers/Microsoft.CognitiveServices/locations/northcentralus/resourceGroups/${{ env.RESOURCE_GROUP_NAME }}/deletedAccounts/${{ env.OPENAI_RESOURCE_NAME }} --verbose; then + if ! az resource delete --ids /subscriptions/${{ secrets.AZURE_SUBSCRIPTION_ID }}/providers/Microsoft.CognitiveServices/locations/australiaeast/resourceGroups/${{ env.RESOURCE_GROUP_NAME }}/deletedAccounts/${{ env.OPENAI_RESOURCE_NAME }} --verbose; then echo "Failed to purge openai resource: ${{ env.OPENAI_RESOURCE_NAME }}" else echo "Purged the openai resource: ${{ env.OPENAI_RESOURCE_NAME }}" @@ -399,7 +427,7 @@ jobs: EMAIL_BODY=$(cat <Dear Team,

We would like to inform you that the DocGen Deployment Automation process has encountered an issue and has failed to complete successfully.

Build URL: ${RUN_URL}

Please investigate the matter at your earliest convenience.

Best regards,
Your Automation Team

" + "body": "

Dear Team,

We would like to inform you that the CodeMod Deployment Automation process has encountered an issue and has failed to complete successfully.

Build URL: ${RUN_URL}

Please investigate the matter at your earliest convenience.

Best regards,
Your Automation Team

" } EOF ) @@ -412,4 +440,4 @@ jobs: if: always() run: | az logout - echo "Logged out from Azure." + echo "Logged out from Azure." \ No newline at end of file diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index d784267d..c486ba8f 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -1,6 +1,16 @@ name: PyLint -on: [push] +on: + push: + paths: + - '**/*.py' + - '**/.flake8' + - '.github/workflows/pylint.yml' + pull_request: + paths: + - '**/*.py' + - '**/.flake8' + - '.github/workflows/pylint.yml' jobs: lint: @@ -12,11 +22,11 @@ jobs: steps: # Step 1: Checkout code - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v5 # Step 2: Set up Python environment - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 34a2f24d..f0c62813 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -6,6 +6,12 @@ on: - main - dev - demo + paths: + - 'src/backend/**/*.py' + - 'src/tests/backend/**' + - '.github/workflows/test.yml' + - 'src/backend/requirements.txt' + - 'src/backend/pyproject.toml' pull_request: types: - opened @@ -16,6 +22,12 @@ on: - main - dev - demo + paths: + - 'src/backend/**/*.py' + - 'src/tests/backend/**' + - '.github/workflows/test.yml' + - 'src/backend/requirements.txt' + - 'src/backend/pyproject.toml' jobs: # frontend_tests: @@ -23,7 +35,7 @@ jobs: # steps: # - name: Checkout code -# uses: actions/checkout@v3 +# uses: actions/checkout@v5 # - name: Set up Node.js # uses: actions/setup-node@v3 @@ -64,10 +76,10 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v5 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v6 with: python-version: '3.11' @@ -75,7 +87,6 @@ jobs: run: | python -m pip install --upgrade pip pip install -r src/backend/requirements.txt - pip install -r src/frontend/requirements.txt pip install pytest-cov pip install pytest-asyncio - name: Set PYTHONPATH @@ -103,4 +114,4 @@ jobs: - name: Skip Backend Tests if: env.skip_backend_tests == 'true' run: | - echo "Skipping backend tests because no test files were found." + echo "Skipping backend tests because no test files were found." \ No newline at end of file From 1c3f5a70f7ddee1a7d898e962aea33cde6f86944 Mon Sep 17 00:00:00 2001 From: Harmanpreet Kaur Date: Tue, 16 Dec 2025 17:43:03 +0530 Subject: [PATCH 02/20] Revert "Enhance CI workflows: update paths for Docker build, deploy, PyLint, and test workflows; upgrade action versions" This reverts commit 924c880bd398553b1f81ff827f26f5bd064fb606. --- .github/workflows/build-docker-images.yml | 12 ---- .github/workflows/deploy.yml | 72 +++++++---------------- .github/workflows/pylint.yml | 16 +---- .github/workflows/test.yml | 21 ++----- 4 files changed, 30 insertions(+), 91 deletions(-) diff --git a/.github/workflows/build-docker-images.yml b/.github/workflows/build-docker-images.yml index 47aeeb5f..7519d620 100644 --- a/.github/workflows/build-docker-images.yml +++ b/.github/workflows/build-docker-images.yml @@ -7,12 +7,6 @@ on: - dev - demo - hotfix - paths: - - 'src/backend/**' - - 'src/frontend/**' - - 'docker/**' - - '.github/workflows/build-docker-images.yml' - - '.github/workflows/build-docker.yml' pull_request: branches: - main @@ -24,12 +18,6 @@ on: - ready_for_review - reopened - synchronize - paths: - - 'src/backend/**' - - 'src/frontend/**' - - 'docker/**' - - '.github/workflows/build-docker-images.yml' - - '.github/workflows/build-docker.yml' merge_group: workflow_dispatch: diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index f61a2490..be98168f 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -1,19 +1,17 @@ name: Deploy-Test-Cleanup Pipeline on: - push: - branches: - - main - - dev - - demo - paths: - - 'infra/**' - - 'scripts/**' - - 'azure.yaml' - - '.github/workflows/deploy.yml' - schedule: - - cron: '0 5,17 * * *' # Runs at 5:00 AM and 5:00 PM GMT - workflow_dispatch: + workflow_run: + workflows: ["Build Docker and Optional Push"] + types: + - completed + branches: + - main + - dev + - demo + schedule: + - cron: '0 5,17 * * *' # Runs at 5:00 AM and 5:00 PM GMT + workflow_dispatch: env: GPT_MIN_CAPACITY: 150 @@ -27,7 +25,7 @@ jobs: WEBAPP_URL: ${{ steps.get_output.outputs.WEBAPP_URL }} steps: - name: Checkout Code - uses: actions/checkout@v5 + uses: actions/checkout@v3 - name: Setup Azure CLI run: | @@ -45,6 +43,7 @@ jobs: export AZURE_TENANT_ID=${{ secrets.AZURE_TENANT_ID }} export AZURE_CLIENT_SECRET=${{ secrets.AZURE_CLIENT_SECRET }} export AZURE_SUBSCRIPTION_ID="${{ secrets.AZURE_SUBSCRIPTION_ID }}" + export GPT_MIN_CAPACITY="${{ env.GPT_MIN_CAPACITY }}" export AZURE_REGIONS="${{ vars.AZURE_REGIONS }}" chmod +x scripts/checkquota.sh if ! scripts/checkquota.sh; then @@ -73,11 +72,6 @@ jobs: - name: Fail Pipeline if Quota Check Fails if: env.QUOTA_FAILED == 'true' run: exit 1 - - - name: Set Deployment Region - run: | - echo "Selected Region: $VALID_REGION" - echo "AZURE_LOCATION=$VALID_REGION" >> $GITHUB_ENV - name: Install Bicep CLI run: az bicep install @@ -100,7 +94,7 @@ jobs: rg_exists=$(az group exists --name ${{ env.RESOURCE_GROUP_NAME }}) if [ "$rg_exists" = "false" ]; then echo "Resource group does not exist. Creating..." - az group create --name ${{ env.RESOURCE_GROUP_NAME }} --location australiaeast || { echo "Error creating resource group"; exit 1; } + az group create --name ${{ env.RESOURCE_GROUP_NAME }} --location northcentralus || { echo "Error creating resource group"; exit 1; } else echo "Resource group already exists." fi @@ -132,20 +126,17 @@ jobs: IMAGE_TAG="latest" fi - # Generate current timestamp in desired format: YYYY-MM-DDTHH:MM:SS.SSSSSSSZ - current_date=$(date -u +"%Y-%m-%dT%H:%M:%S.%7NZ") - az deployment group create \ --name ${{ env.SOLUTION_PREFIX }}-deployment \ --resource-group ${{ env.RESOURCE_GROUP_NAME }} \ --template-file infra/main.bicep \ --parameters \ solutionName="${{ env.SOLUTION_PREFIX }}" \ - azureAiServiceLocation='${{ env.AZURE_LOCATION }}' \ + aiDeploymentsLocation="eastus" \ + useWafAlignedArchitecture=false \ + capacity=${{ env.GPT_MIN_CAPACITY }} \ imageVersion="${IMAGE_TAG}" \ - createdBy="Pipeline" \ - tags="{'SecurityControl':'Ignore','Purpose':'Deploying and Cleaning Up Resources for Validation','CreatedDate':'$current_date'}" - + createdBy="Pipeline" - name: Assign Contributor role to Service Principal if: always() run: | @@ -194,26 +185,7 @@ jobs: - name: Login to Azure run: | - az login --service-principal -u ${{ secrets.AZURE_CLIENT_ID }} -p ${{ secrets.AZURE_CLIENT_SECRET }} --tenant ${{ secrets.AZURE_TENANT_ID }} - az account set --subscription "${{ secrets.AZURE_SUBSCRIPTION_ID }}" - - - name: Assign Contributor role to Service Principal - if: always() - run: | - echo "Assigning Contributor role to SPN for RG: ${{ env.RESOURCE_GROUP_NAME }}" - az role assignment create \ - --assignee ${{ secrets.AZURE_CLIENT_ID }} \ - --role "Contributor" \ - --scope /subscriptions/${{ secrets.AZURE_SUBSCRIPTION_ID }}/resourceGroups/${{ env.RESOURCE_GROUP_NAME }} - - echo "Assigning Log Analytics Contributor role for Log Analytics workspace access at RG level..." - az role assignment create \ - --assignee ${{ secrets.AZURE_CLIENT_ID }} \ - --role "Log Analytics Reader" \ - --scope /subscriptions/${{ secrets.AZURE_SUBSCRIPTION_ID }}/resourceGroups/${{ env.RESOURCE_GROUP_NAME }} || echo "Log Analytics Contributor role assignment failed (may already exist)" - - echo "Waiting for role assignment propagation..." - sleep 30 + az login --service-principal -u ${{ secrets.AZURE_CLIENT_ID }} -p ${{ secrets.AZURE_CLIENT_SECRET }} --tenant ${{ secrets.AZURE_TENANT_ID }} - name: Get Log Analytics Workspace and OpenAI from Resource Group if: always() @@ -384,7 +356,7 @@ jobs: # Purge OpenAI Resource echo "Purging the OpenAI Resource..." - if ! az resource delete --ids /subscriptions/${{ secrets.AZURE_SUBSCRIPTION_ID }}/providers/Microsoft.CognitiveServices/locations/australiaeast/resourceGroups/${{ env.RESOURCE_GROUP_NAME }}/deletedAccounts/${{ env.OPENAI_RESOURCE_NAME }} --verbose; then + if ! az resource delete --ids /subscriptions/${{ secrets.AZURE_SUBSCRIPTION_ID }}/providers/Microsoft.CognitiveServices/locations/northcentralus/resourceGroups/${{ env.RESOURCE_GROUP_NAME }}/deletedAccounts/${{ env.OPENAI_RESOURCE_NAME }} --verbose; then echo "Failed to purge openai resource: ${{ env.OPENAI_RESOURCE_NAME }}" else echo "Purged the openai resource: ${{ env.OPENAI_RESOURCE_NAME }}" @@ -427,7 +399,7 @@ jobs: EMAIL_BODY=$(cat <Dear Team,

We would like to inform you that the CodeMod Deployment Automation process has encountered an issue and has failed to complete successfully.

Build URL: ${RUN_URL}

Please investigate the matter at your earliest convenience.

Best regards,
Your Automation Team

" + "body": "

Dear Team,

We would like to inform you that the DocGen Deployment Automation process has encountered an issue and has failed to complete successfully.

Build URL: ${RUN_URL}

Please investigate the matter at your earliest convenience.

Best regards,
Your Automation Team

" } EOF ) @@ -440,4 +412,4 @@ jobs: if: always() run: | az logout - echo "Logged out from Azure." \ No newline at end of file + echo "Logged out from Azure." diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index c486ba8f..d784267d 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -1,16 +1,6 @@ name: PyLint -on: - push: - paths: - - '**/*.py' - - '**/.flake8' - - '.github/workflows/pylint.yml' - pull_request: - paths: - - '**/*.py' - - '**/.flake8' - - '.github/workflows/pylint.yml' +on: [push] jobs: lint: @@ -22,11 +12,11 @@ jobs: steps: # Step 1: Checkout code - name: Checkout code - uses: actions/checkout@v5 + uses: actions/checkout@v4 # Step 2: Set up Python environment - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 + uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f0c62813..34a2f24d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -6,12 +6,6 @@ on: - main - dev - demo - paths: - - 'src/backend/**/*.py' - - 'src/tests/backend/**' - - '.github/workflows/test.yml' - - 'src/backend/requirements.txt' - - 'src/backend/pyproject.toml' pull_request: types: - opened @@ -22,12 +16,6 @@ on: - main - dev - demo - paths: - - 'src/backend/**/*.py' - - 'src/tests/backend/**' - - '.github/workflows/test.yml' - - 'src/backend/requirements.txt' - - 'src/backend/pyproject.toml' jobs: # frontend_tests: @@ -35,7 +23,7 @@ jobs: # steps: # - name: Checkout code -# uses: actions/checkout@v5 +# uses: actions/checkout@v3 # - name: Set up Node.js # uses: actions/setup-node@v3 @@ -76,10 +64,10 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v5 + uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v6 + uses: actions/setup-python@v4 with: python-version: '3.11' @@ -87,6 +75,7 @@ jobs: run: | python -m pip install --upgrade pip pip install -r src/backend/requirements.txt + pip install -r src/frontend/requirements.txt pip install pytest-cov pip install pytest-asyncio - name: Set PYTHONPATH @@ -114,4 +103,4 @@ jobs: - name: Skip Backend Tests if: env.skip_backend_tests == 'true' run: | - echo "Skipping backend tests because no test files were found." \ No newline at end of file + echo "Skipping backend tests because no test files were found." From 37b9fcea8f8c208cc8d277a0079e67dca3d90489 Mon Sep 17 00:00:00 2001 From: Harmanpreet Kaur Date: Tue, 16 Dec 2025 17:48:36 +0530 Subject: [PATCH 03/20] Update CI workflows: enhance path triggers for Docker build, deploy, PyLint, and test workflows --- .github/workflows/build-docker-images.yml | 12 ++++++++++++ .github/workflows/deploy.yml | 24 ++++++++++++----------- .github/workflows/pylint.yml | 16 ++++++++++++++- .github/workflows/test.yml | 12 ++++++++++++ 4 files changed, 52 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build-docker-images.yml b/.github/workflows/build-docker-images.yml index 7519d620..47aeeb5f 100644 --- a/.github/workflows/build-docker-images.yml +++ b/.github/workflows/build-docker-images.yml @@ -7,6 +7,12 @@ on: - dev - demo - hotfix + paths: + - 'src/backend/**' + - 'src/frontend/**' + - 'docker/**' + - '.github/workflows/build-docker-images.yml' + - '.github/workflows/build-docker.yml' pull_request: branches: - main @@ -18,6 +24,12 @@ on: - ready_for_review - reopened - synchronize + paths: + - 'src/backend/**' + - 'src/frontend/**' + - 'docker/**' + - '.github/workflows/build-docker-images.yml' + - '.github/workflows/build-docker.yml' merge_group: workflow_dispatch: diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index be98168f..b2b8c01b 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -1,17 +1,19 @@ name: Deploy-Test-Cleanup Pipeline on: - workflow_run: - workflows: ["Build Docker and Optional Push"] - types: - - completed - branches: - - main - - dev - - demo - schedule: - - cron: '0 5,17 * * *' # Runs at 5:00 AM and 5:00 PM GMT - workflow_dispatch: + push: + branches: + - main + - dev + - demo + paths: + - 'infra/**' + - 'scripts/**' + - 'azure.yaml' + - '.github/workflows/deploy.yml' + schedule: + - cron: '0 5,17 * * *' # Runs at 5:00 AM and 5:00 PM GMT + workflow_dispatch: env: GPT_MIN_CAPACITY: 150 diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index d784267d..e124825f 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -1,6 +1,20 @@ name: PyLint -on: [push] +on: + push: + paths: + - '**/*.py' + - '**/requirements.txt' + - '**/pyproject.toml' + - '**/.flake8' + - '.github/workflows/pylint.yml' + pull_request: + paths: + - '**/*.py' + - '**/requirements.txt' + - '**/pyproject.toml' + - '**/.flake8' + - '.github/workflows/pylint.yml' jobs: lint: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 34a2f24d..5fcd203e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -6,6 +6,12 @@ on: - main - dev - demo + paths: + - 'src/backend/**/*.py' + - 'src/tests/backend/**' + - '.github/workflows/test.yml' + - 'src/backend/requirements.txt' + - 'src/backend/pyproject.toml' pull_request: types: - opened @@ -16,6 +22,12 @@ on: - main - dev - demo + paths: + - 'src/backend/**/*.py' + - 'src/tests/backend/**' + - '.github/workflows/test.yml' + - 'src/backend/requirements.txt' + - 'src/backend/pyproject.toml' jobs: # frontend_tests: From a6723ab297a40d910dbe4772891008176eaf3016 Mon Sep 17 00:00:00 2001 From: "Prekshith D J (Persistent Systems Inc)" Date: Tue, 16 Dec 2025 17:57:34 +0530 Subject: [PATCH 04/20] Fixed the maintainability code quality issues --- src/backend/common/database/cosmosdb.py | 4 -- src/backend/common/storage/blob_azure.py | 2 +- src/backend/sql_agents/convert_script.py | 9 ++-- .../sql_agents/helpers/comms_manager.py | 14 ------ src/backend/sql_agents/tools/src/Program.cs | 13 +++--- src/frontend/frontend_server.py | 2 +- .../src/components/batchHistoryPanel.tsx | 45 +------------------ src/frontend/src/components/bottomBar.tsx | 3 +- src/frontend/src/components/uploadButton.tsx | 3 -- src/frontend/src/main.jsx | 2 +- src/frontend/src/pages/batchView.tsx | 38 +--------------- src/frontend/src/pages/landingPage.tsx | 1 - src/frontend/src/pages/modernizationPage.tsx | 15 ++----- src/frontend/vite.config.js | 1 - .../backend/common/storage/blob_azure_test.py | 1 - tests/e2e-test/tests/conftest.py | 19 -------- tests/e2e-test/tests/test_codegen_gp_tc.py | 2 +- 17 files changed, 20 insertions(+), 154 deletions(-) diff --git a/src/backend/common/database/cosmosdb.py b/src/backend/common/database/cosmosdb.py index 35071709..7ff043f4 100644 --- a/src/backend/common/database/cosmosdb.py +++ b/src/backend/common/database/cosmosdb.py @@ -352,10 +352,6 @@ async def update_batch_entry( batch["file_count"] = file_count await self.batch_container.replace_item(item=batch_id, body=batch) - # if isinstance(status, ProcessStatus): - # self.logger.info(f"Updated batch {batch_id} to status {status.value}") - # else: - # self.logger.info(f"Updated batch {batch_id} to status {status}") return batch except Exception as e: diff --git a/src/backend/common/storage/blob_azure.py b/src/backend/common/storage/blob_azure.py index da53b2c3..33b126f6 100644 --- a/src/backend/common/storage/blob_azure.py +++ b/src/backend/common/storage/blob_azure.py @@ -52,7 +52,7 @@ async def upload_file( raise try: # Upload the file - upload_results = blob_client.upload_blob( # noqa: F841 + blob_client.upload_blob( file_content, content_type=content_type, metadata=metadata, diff --git a/src/backend/sql_agents/convert_script.py b/src/backend/sql_agents/convert_script.py index 789bf36f..01309af4 100644 --- a/src/backend/sql_agents/convert_script.py +++ b/src/backend/sql_agents/convert_script.py @@ -66,8 +66,7 @@ async def convert_script( # orchestrate the chat current_migration = "No migration" - is_complete: bool = False - while not is_complete: + while True: await comms_manager.group_chat.add_chat_message( ChatMessageContent(role=AuthorRole.USER, content=source_script) ) @@ -274,10 +273,8 @@ async def convert_script( break if comms_manager.group_chat.is_complete: - is_complete = True - - break - + break + migrated_query = current_migration # Handle the case where migration failed and current_migration is None diff --git a/src/backend/sql_agents/helpers/comms_manager.py b/src/backend/sql_agents/helpers/comms_manager.py index 7080e38f..fddc0a7a 100644 --- a/src/backend/sql_agents/helpers/comms_manager.py +++ b/src/backend/sql_agents/helpers/comms_manager.py @@ -270,17 +270,3 @@ async def cleanup(self): except Exception as e: self.logger.error("Error during cleanup: %s", str(e)) - - def __del__(self): - """Destructor to ensure cleanup if not explicitly called.""" - try: - # Only attempt cleanup if there's an active event loop - loop = asyncio.get_running_loop() - if loop and not loop.is_closed(): - # Schedule cleanup as a task - loop.create_task(self.cleanup()) - except RuntimeError: - # No event loop running, can't clean up asynchronously - self.logger.warning("No event loop available for cleanup in destructor") - except Exception as e: - self.logger.error("Error in destructor cleanup: %s", str(e)) diff --git a/src/backend/sql_agents/tools/src/Program.cs b/src/backend/sql_agents/tools/src/Program.cs index b728eefc..06c5f213 100644 --- a/src/backend/sql_agents/tools/src/Program.cs +++ b/src/backend/sql_agents/tools/src/Program.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using Microsoft.SqlServer.TransactSql.ScriptDom; using Newtonsoft.Json; @@ -42,18 +43,14 @@ static void Main(string[] args) IList errors = ParseSqlQuery(sqlQuery); - var errorList = new List>(); - - foreach (var error in errors) - { - var errorDict = new Dictionary + var errorList = errors + .Select(error => new Dictionary { { "Line", error.Line }, { "Column", error.Column }, { "Error", error.Message } - }; - errorList.Add(errorDict); - } + }) + .ToList(); string jsonOutput = JsonConvert.SerializeObject(errorList, Formatting.Indented); Console.WriteLine(jsonOutput); diff --git a/src/frontend/frontend_server.py b/src/frontend/frontend_server.py index 199d1a66..c53af042 100644 --- a/src/frontend/frontend_server.py +++ b/src/frontend/frontend_server.py @@ -4,7 +4,7 @@ from dotenv import load_dotenv from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware -from fastapi.responses import FileResponse, HTMLResponse +from fastapi.responses import FileResponse from fastapi.staticfiles import StaticFiles # Load environment variables from .env file diff --git a/src/frontend/src/components/batchHistoryPanel.tsx b/src/frontend/src/components/batchHistoryPanel.tsx index 0e0259d9..06820737 100644 --- a/src/frontend/src/components/batchHistoryPanel.tsx +++ b/src/frontend/src/components/batchHistoryPanel.tsx @@ -1,6 +1,5 @@ import React, { useState, useEffect, useRef } from "react"; - -import { useDispatch, useSelector } from 'react-redux'; +import { useDispatch } from 'react-redux'; import { Card, Spinner, Tooltip } from "@fluentui/react-components"; import { useNavigate } from "react-router-dom"; import ConfirmationDialog from "../commonComponents/ConfirmationDialog/confirmationDialogue"; @@ -21,7 +20,7 @@ interface BatchHistoryItem { status: string; } const HistoryPanel: React.FC = ({ isOpen, onClose }) => { - const headers = {} + const headers = {}; const [batchHistory, setBatchHistory] = useState([]); const [loading, setLoading] = useState(false); const [error, setError] = useState(null); @@ -81,46 +80,6 @@ const HistoryPanel: React.FC = ({ isOpen, onClose }) => { } }; - // Function to categorize batches - const categorizeBatches = () => { - const now = new Date(); - const userTimeZone = Intl.DateTimeFormat().resolvedOptions().timeZone; - - // Get start of "Today", "Past 7 days", and "Past 30 days" in LOCAL time - const todayStart = new Date(now.getFullYear(), now.getMonth(), now.getDate()); - const past7DaysStart = new Date(todayStart); - const past30DaysStart = new Date(todayStart); - - past7DaysStart.setDate(todayStart.getDate() - 7); - past30DaysStart.setDate(todayStart.getDate() - 30); - - const todayBatches: BatchHistoryItem[] = []; - const past7DaysBatches: BatchHistoryItem[] = []; - const past30DaysBatches: BatchHistoryItem[] = []; - - batchHistory.forEach(batch => { - // Convert UTC timestamp to user's local date - const updatedAtUTC = new Date(batch.created_at); - const updatedAtLocal = new Date(updatedAtUTC.toLocaleString("en-US", { timeZone: userTimeZone })); - - // Extract only the local **date** part for comparison - const updatedDate = new Date(updatedAtLocal.getFullYear(), updatedAtLocal.getMonth(), updatedAtLocal.getDate()); - - // Categorize based on **exact day comparison** - if (updatedDate.getTime() === todayStart.getTime()) { - todayBatches.push(batch); - } else if (updatedDate.getTime() >= past7DaysStart.getTime()) { - past7DaysBatches.push(batch); - } else if (updatedDate.getTime() >= past30DaysStart.getTime()) { - past30DaysBatches.push(batch); - } - }); - - return { todayBatches, past7DaysBatches, past30DaysBatches }; - }; - - // const { todayBatches, past7DaysBatches, past30DaysBatches } = categorizeBatches(); - const deleteBatchFromHistory = (batchId: string) => { // Get the current URL path const currentPath = window.location.pathname; diff --git a/src/frontend/src/components/bottomBar.tsx b/src/frontend/src/components/bottomBar.tsx index 3041a988..c2702d2e 100644 --- a/src/frontend/src/components/bottomBar.tsx +++ b/src/frontend/src/components/bottomBar.tsx @@ -1,6 +1,5 @@ import { Button, Card, Dropdown, DropdownProps, Option } from "@fluentui/react-components" -import React, { useState } from "react" -import { useNavigate } from "react-router-dom" +import React from "react" // Define possible upload states const UploadState = { diff --git a/src/frontend/src/components/uploadButton.tsx b/src/frontend/src/components/uploadButton.tsx index a833f669..0d3e5714 100644 --- a/src/frontend/src/components/uploadButton.tsx +++ b/src/frontend/src/components/uploadButton.tsx @@ -68,7 +68,6 @@ const FileUploadZone: React.FC = ({ const [uploadIntervals, setUploadIntervals] = useState<{ [key: string]: ReturnType }>({}); const [showCancelDialog, setShowCancelDialog] = useState(false); const [showLogoCancelDialog, setShowLogoCancelDialog] = useState(false); - const [uploadState, setUploadState] = useState<'IDLE' | 'UPLOADING' | 'COMPLETED'>('IDLE'); const [batchId, setBatchId] = useState(uuidv4()); const [allUploadsComplete, setAllUploadsComplete] = useState(false); const [fileLimitExceeded, setFileLimitExceeded] = useState(false); @@ -98,7 +97,6 @@ const FileUploadZone: React.FC = ({ } } - setUploadState(newState); onUploadStateChange?.(newState); }, [uploadingFiles, onUploadStateChange]); @@ -280,7 +278,6 @@ const FileUploadZone: React.FC = ({ Object.values(uploadIntervals).forEach(interval => clearInterval(interval)); setUploadIntervals({}); setUploadingFiles([]); - setUploadState('IDLE'); onUploadStateChange?.('IDLE'); setShowCancelDialog(false); setShowLogoCancelDialog(false); diff --git a/src/frontend/src/main.jsx b/src/frontend/src/main.jsx index 7ae29774..da9d477d 100644 --- a/src/frontend/src/main.jsx +++ b/src/frontend/src/main.jsx @@ -50,7 +50,7 @@ const Main = () => { const baseURL = config.API_URL.replace(/\/api$/, ''); // Remove '/api' if it appears at the end console.log('Checking connection to:', baseURL); try { - const response = await fetch(`${baseURL}/health`); + await fetch(`${baseURL}/health`); } catch (error) { console.error('Error connecting to backend:', error); } diff --git a/src/frontend/src/pages/batchView.tsx b/src/frontend/src/pages/batchView.tsx index ba8b9de2..4e9eef95 100644 --- a/src/frontend/src/pages/batchView.tsx +++ b/src/frontend/src/pages/batchView.tsx @@ -33,7 +33,7 @@ import PanelRight from "../components/Panels/PanelRight"; import PanelRightToolbar from "../components/Panels/PanelRightToolbar"; import BatchHistoryPanel from "../components/batchHistoryPanel"; import ConfirmationDialog from "../commonComponents/ConfirmationDialog/confirmationDialogue"; -import { determineFileStatus, filesLogsBuilder, renderErrorSection, useStyles, renderFileError, filesErrorCounter, completedFiles, hasFiles, fileErrorCounter, BatchSummary, fileWarningCounter } from "../api/utils"; +import { determineFileStatus, filesLogsBuilder, renderErrorSection, useStyles, renderFileError, filesErrorCounter, completedFiles, hasFiles, fileErrorCounter, BatchSummary } from "../api/utils"; export const History = bundleIcon(HistoryFilled, HistoryRegular); import { format } from "sql-formatter"; @@ -73,7 +73,6 @@ const BatchStoryPage = () => { const [selectedFileId, setSelectedFileId] = useState(""); const [expandedSections, setExpandedSections] = useState(["errors"]); const [batchSummary, setBatchSummary] = useState(null); - const [selectedFileContent, setSelectedFileContent] = useState(""); const [selectedFileTranslatedContent, setSelectedFileTranslatedContent] = useState(""); @@ -195,7 +194,6 @@ const BatchStoryPage = () => { const data = await response.json(); if (data) { - setSelectedFileContent(data.content || ""); setSelectedFileTranslatedContent(data.translated_content || ""); } @@ -209,40 +207,6 @@ const BatchStoryPage = () => { fetchFileContent(); }, [selectedFileId]); - - const renderWarningContent = () => { - if (!expandedSections.includes("warnings")) return null; - - if (!batchSummary) return null; - - // Group warnings by file - const warningFiles = files.filter(file => file.warningCount && file.warningCount > 0 && file.id !== "summary"); - - if (warningFiles.length === 0) { - return ( -
- No warnings found. -
- ); - } - - return ( -
- {warningFiles.map((file, fileIndex) => ( -
-
- {file.name} ({file.warningCount}) - source -
-
- Warning in file processing. See file for details. -
-
- ))} -
- ); - }; - const renderContent = () => { // Define header content based on selected file const renderHeader = () => { diff --git a/src/frontend/src/pages/landingPage.tsx b/src/frontend/src/pages/landingPage.tsx index b3707393..b86200e6 100644 --- a/src/frontend/src/pages/landingPage.tsx +++ b/src/frontend/src/pages/landingPage.tsx @@ -33,7 +33,6 @@ export const LandingPage = (): JSX.Element => { const dispatch = useDispatch(); // Add dispatch hook const [selectedTargetLanguage, setSelectedTargetLanguage] = useState(["T-SQL"]); const [selectedCurrentLanguage, setSelectedCurrentLanguage] = useState(["Informix"]); - const batchHistoryRef = useRef<{ triggerDeleteAll: () => void } | null>(null); const isPanelOpen = useSelector((state: RootState) => state.historyPanel.isOpen); const navigate = useNavigate(); diff --git a/src/frontend/src/pages/modernizationPage.tsx b/src/frontend/src/pages/modernizationPage.tsx index f5d6d04e..7be5edcf 100644 --- a/src/frontend/src/pages/modernizationPage.tsx +++ b/src/frontend/src/pages/modernizationPage.tsx @@ -477,13 +477,13 @@ const getPrintFileStatus = (status: string): string => { const ModernizationPage = () => { const { batchId } = useParams<{ batchId: string }>(); - const navigate = useNavigate() + const navigate = useNavigate(); // Redux state to listen for start processing completion const batchState = useSelector((state: any) => state.batch); const [batchSummary, setBatchSummary] = useState(null); - const styles = useStyles() + const styles = useStyles(); const [text, setText] = useState(""); const [isPanelOpen, setIsPanelOpen] = React.useState(false); // Add state management @@ -492,16 +492,13 @@ const ModernizationPage = () => { // State for the loading component const [showLoading, setShowLoading] = useState(true); - const [loadingError, setLoadingError] = useState(null); const [selectedFilebg, setSelectedFile] = useState(null); - const [selectedFileId, setSelectedFileId] = React.useState("") + const [selectedFileId, setSelectedFileId] = React.useState(""); const [fileId, setFileId] = React.useState(""); - const [expandedSections, setExpandedSections] = React.useState([]) - const [progressPercentage, setProgressPercentage] = useState(0); + const [expandedSections, setExpandedSections] = React.useState([]); const [allFilesCompleted, setAllFilesCompleted] = useState(false); const [isZipButtonDisabled, setIsZipButtonDisabled] = useState(true); const [fileLoading, setFileLoading] = useState(false); - const [selectedFileTranslatedContent, setSelectedFileTranslatedContent] = useState(""); const [lastActivityTime, setLastActivityTime] = useState(Date.now()); const [pageLoadTime] = useState(Date.now()); @@ -517,11 +514,9 @@ const ModernizationPage = () => { if (!selectedFile || !selectedFile.translatedCode) { setFileLoading(true); const newFileUpdate = await fetchFileFromAPI(selectedFile?.fileId || ""); - setSelectedFileTranslatedContent(newFileUpdate.translatedContent); setFileLoading(false); } else { - setSelectedFileTranslatedContent(selectedFile.translatedCode); } } catch (err) { @@ -799,8 +794,6 @@ const ModernizationPage = () => { } }, [batchId]); - const highestProgressRef = useRef(0); - const currentProcessingFileRef = useRef(null); //new PT FR ends diff --git a/src/frontend/vite.config.js b/src/frontend/vite.config.js index 0d05262f..d239b70e 100644 --- a/src/frontend/vite.config.js +++ b/src/frontend/vite.config.js @@ -1,6 +1,5 @@ import { defineConfig } from 'vite' import react from '@vitejs/plugin-react' -import path from 'path' import tailwindcss from '@tailwindcss/vite' export default defineConfig({ diff --git a/src/tests/backend/common/storage/blob_azure_test.py b/src/tests/backend/common/storage/blob_azure_test.py index 68e5ad0d..afc76e7e 100644 --- a/src/tests/backend/common/storage/blob_azure_test.py +++ b/src/tests/backend/common/storage/blob_azure_test.py @@ -28,7 +28,6 @@ def mock_blob_service(): @pytest.fixture def blob_storage(mock_blob_service): """Fixture to initialize AzureBlobStorage with mocked dependencies""" - service_client, container_client, blob_client = mock_blob_service return AzureBlobStorage(account_name="test_account", container_name="test_container") diff --git a/tests/e2e-test/tests/conftest.py b/tests/e2e-test/tests/conftest.py index b9919e6d..1dd44563 100644 --- a/tests/e2e-test/tests/conftest.py +++ b/tests/e2e-test/tests/conftest.py @@ -33,18 +33,6 @@ def login_logout(): def pytest_html_report_title(report): report.title = "Automation_CodeGen" - -# Add a column for descriptions -# def pytest_html_results_table_header(cells): -# cells.insert(1, html.th("Description")) - - -# def pytest_html_results_table_row(report, cells): -# cells.insert( -# 1, html.td(report.description if hasattr(report, "description") else "") -# ) - - log_streams = {} @@ -124,10 +112,3 @@ def rename_duration_column(): # Add logs and docstring to report # @pytest.hookimpl(hookwrapper=True) -# def pytest_runtest_makereport(item, call): -# outcome = yield -# report = outcome.get_result() -# report.description = str(item.function.__doc__) -# os.makedirs("logs", exist_ok=True) -# extra = getattr(report, "extra", []) -# report.extra = extra diff --git a/tests/e2e-test/tests/test_codegen_gp_tc.py b/tests/e2e-test/tests/test_codegen_gp_tc.py index 6d9e896f..7a239f46 100644 --- a/tests/e2e-test/tests/test_codegen_gp_tc.py +++ b/tests/e2e-test/tests/test_codegen_gp_tc.py @@ -12,7 +12,7 @@ ("01. Validate home page is loaded", lambda home: home.validate_home_page()), ("02. Validate Upload of other than SQL files", lambda home: home.upload_unsupported_files()), ("03. Validate Upload input files for SQL only", lambda home: home.upload_files()), - ("04. Validate translation process for uploaded files", lambda home: _timed_translation(home)), + ("04. Validate translation process for uploaded files", _timed_translation), ("05. Check batch history", lambda home: home.validate_batch_history()), ("06. Download all files and return home", lambda home: home.validate_download_files()), ] From cc7f864add9f74705baaa698d78e35eafd38d2f5 Mon Sep 17 00:00:00 2001 From: Harmanpreet Kaur Date: Wed, 17 Dec 2025 11:10:10 +0530 Subject: [PATCH 05/20] Fix path pattern for .flake8 in PyLint workflow triggers --- .github/workflows/pylint.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index e124825f..f783199b 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -6,14 +6,14 @@ on: - '**/*.py' - '**/requirements.txt' - '**/pyproject.toml' - - '**/.flake8' + - '.flake8' - '.github/workflows/pylint.yml' pull_request: paths: - '**/*.py' - '**/requirements.txt' - '**/pyproject.toml' - - '**/.flake8' + - '.flake8' - '.github/workflows/pylint.yml' jobs: From 1efc3fde28a14a7ecad303ee27957cec314deaef Mon Sep 17 00:00:00 2001 From: Harmanpreet Kaur Date: Wed, 17 Dec 2025 11:20:10 +0530 Subject: [PATCH 06/20] Add 'src/frontend/requirements.txt' to workflow path triggers for push and pull_request events --- .github/workflows/test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5fcd203e..233c0a49 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -11,6 +11,7 @@ on: - 'src/tests/backend/**' - '.github/workflows/test.yml' - 'src/backend/requirements.txt' + - 'src/frontend/requirements.txt' - 'src/backend/pyproject.toml' pull_request: types: @@ -27,6 +28,7 @@ on: - 'src/tests/backend/**' - '.github/workflows/test.yml' - 'src/backend/requirements.txt' + - 'src/frontend/requirements.txt' - 'src/backend/pyproject.toml' jobs: From 1219287a14eab0337e80d1101285226a590c96c6 Mon Sep 17 00:00:00 2001 From: "Prekshith D J (Persistent Systems Inc)" Date: Wed, 17 Dec 2025 13:04:35 +0530 Subject: [PATCH 07/20] Fixed the reliability code quality issues --- src/backend/api/api_routes.py | 1 + src/backend/common/services/batch_service.py | 2 ++ src/backend/sql_agents/helpers/comms_manager.py | 2 ++ src/frontend/src/api/utils.tsx | 2 +- src/frontend/src/pages/batchView.tsx | 2 +- src/frontend/src/pages/modernizationPage.tsx | 2 +- tests/e2e-test/pages/HomePage.py | 1 + tests/e2e-test/pages/loginPage.py | 1 + 8 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/backend/api/api_routes.py b/src/backend/api/api_routes.py index 44718f8a..8a3be72f 100644 --- a/src/backend/api/api_routes.py +++ b/src/backend/api/api_routes.py @@ -279,6 +279,7 @@ async def batch_status_updates( try: await websocket.receive_text() except asyncio.TimeoutError: + # TimeoutError is ignored to keep the WebSocket connection open without receiving data pass except WebSocketDisconnect: diff --git a/src/backend/common/services/batch_service.py b/src/backend/common/services/batch_service.py index 59b1b411..f467820a 100644 --- a/src/backend/common/services/batch_service.py +++ b/src/backend/common/services/batch_service.py @@ -175,6 +175,8 @@ async def delete_batch(self, batch_id: UUID, user_id: str): self.logger.info(f"Successfully deleted batch with ID: {batch_id}") return {"message": "Batch deleted successfully", "batch_id": str(batch_id)} + else: + return {"message": "Batch not found", "batch_id": str(batch_id)} async def delete_file(self, file_id: UUID, user_id: str): """Delete a file and its logs, and update batch file count.""" diff --git a/src/backend/sql_agents/helpers/comms_manager.py b/src/backend/sql_agents/helpers/comms_manager.py index fddc0a7a..da6c9c6c 100644 --- a/src/backend/sql_agents/helpers/comms_manager.py +++ b/src/backend/sql_agents/helpers/comms_manager.py @@ -79,6 +79,8 @@ async def select_agent(self, agents, history): ), None, ) + # No matching case found, so explicitly return None + return None # class for termination strategy class ApprovalTerminationStrategy(TerminationStrategy): diff --git a/src/frontend/src/api/utils.tsx b/src/frontend/src/api/utils.tsx index 95ce5459..50341195 100644 --- a/src/frontend/src/api/utils.tsx +++ b/src/frontend/src/api/utils.tsx @@ -414,7 +414,7 @@ export const renderErrorSection = (batchSummary, expandedSections, setExpandedSe export const renderErrorContent = (batchSummary) => { // Group errors by file - const errorFiles = batchSummary.files.filter(file => file.error_count && file.error_count); + const errorFiles = batchSummary.files.filter(file => file.error_count); if (errorFiles.length === 0) { return (
diff --git a/src/frontend/src/pages/batchView.tsx b/src/frontend/src/pages/batchView.tsx index 4e9eef95..5afcdb03 100644 --- a/src/frontend/src/pages/batchView.tsx +++ b/src/frontend/src/pages/batchView.tsx @@ -349,7 +349,7 @@ const BatchStoryPage = () => { } // Show the summary page when summary is selected - if (selectedFile.id === "summary" && batchSummary) { + if (selectedFile.id === "summary") { // Check if there are no errors and all files are processed successfully const noErrors = (batchSummary.error_count === 0); const allFilesProcessed = (batchSummary.completed_files === batchSummary.total_files); diff --git a/src/frontend/src/pages/modernizationPage.tsx b/src/frontend/src/pages/modernizationPage.tsx index 7be5edcf..dc76f4bd 100644 --- a/src/frontend/src/pages/modernizationPage.tsx +++ b/src/frontend/src/pages/modernizationPage.tsx @@ -1244,7 +1244,7 @@ useEffect(() => { } // Show the full summary page only when all files are completed and summary is selected - if (allFilesCompleted && selectedFile?.id === "summary") { + if (selectedFile?.id === "summary") { const completedCount = files.filter(file => file.status === "completed" && file.file_result !== "error" && file.id !== "summary").length; const totalCount = files.filter(file => file.id !== "summary").length; const errorCount = selectedFile.errorCount || 0; diff --git a/tests/e2e-test/pages/HomePage.py b/tests/e2e-test/pages/HomePage.py index d863ac18..6e184a83 100644 --- a/tests/e2e-test/pages/HomePage.py +++ b/tests/e2e-test/pages/HomePage.py @@ -19,6 +19,7 @@ class HomePage(BasePage): FILE_PROCESSED_MSG = "//span[normalize-space()='3 files processed successfully']" def __init__(self, page): + super().__init__(page) self.page = page def validate_home_page(self): diff --git a/tests/e2e-test/pages/loginPage.py b/tests/e2e-test/pages/loginPage.py index 0b412556..c8f37aa9 100644 --- a/tests/e2e-test/pages/loginPage.py +++ b/tests/e2e-test/pages/loginPage.py @@ -11,6 +11,7 @@ class LoginPage(BasePage): PERMISSION_ACCEPT_BUTTON = "//input[@type='submit']" def __init__(self, page): + super().__init__() self.page = page def authenticate(self, username, password): From 0ad1068c6a36320dbfd85ab141bf515c961743d4 Mon Sep 17 00:00:00 2001 From: "Prekshith D J (Persistent Systems Inc)" Date: Wed, 17 Dec 2025 13:47:21 +0530 Subject: [PATCH 08/20] updated the deployment document --- docs/DeploymentGuide.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/DeploymentGuide.md b/docs/DeploymentGuide.md index 8abf4f58..aa51590c 100644 --- a/docs/DeploymentGuide.md +++ b/docs/DeploymentGuide.md @@ -124,7 +124,7 @@ If you're not using one of the above options for opening the project, then you'l 2. Download the project code: ```shell - azd init -t microsoft/Modernize-your-Code-Solution-Accelerator/ + azd init -t microsoft/Modernize-your-Code-Solution-Accelerator ``` 3. Open the project folder in your terminal or editor. @@ -164,7 +164,7 @@ To adjust quota settings, follow these [steps](../docs/AzureGPTQuotaSettings.md) Reusing an Existing Log Analytics Workspace - Guide to get your [Existing Workspace ID](/docs/re-use-log-analytics.md) + Guide to get your [Existing Workspace ID](./re-use-log-analytics.md) @@ -217,7 +217,7 @@ Rename `azure.yaml` to `azure_original.yaml` and `azure_custom.yaml` to `azure.y Go to `infra` directory -Rename `main.bicep` to `main_original.bicep` and `main_custom.bicep` to `main.bicep`. Continue with the [deploying steps](https://github.com/microsoft/Modernize-your-code-solution-accelerator/blob/main/docs/DeploymentGuide.md#deploying-with-azd). +Rename `main.bicep` to `main_original.bicep` and `main_custom.bicep` to `main.bicep`. Continue with the [deploying steps](#deploying-with-azd). ### 🛠️ Troubleshooting If you encounter any issues during the deployment process, please refer [troubleshooting](../docs/TroubleShootingSteps.md) document for detailed steps and solutions. From 83da9899d5166df6af99d433c70384f5be1b5177 Mon Sep 17 00:00:00 2001 From: "Prekshith D J (Persistent Systems Inc)" Date: Wed, 17 Dec 2025 15:15:28 +0530 Subject: [PATCH 09/20] updated the loginpage --- tests/e2e-test/pages/loginPage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e-test/pages/loginPage.py b/tests/e2e-test/pages/loginPage.py index c8f37aa9..7f013ccb 100644 --- a/tests/e2e-test/pages/loginPage.py +++ b/tests/e2e-test/pages/loginPage.py @@ -11,7 +11,7 @@ class LoginPage(BasePage): PERMISSION_ACCEPT_BUTTON = "//input[@type='submit']" def __init__(self, page): - super().__init__() + super().__init__(page) self.page = page def authenticate(self, username, password): From 9c6b21e66cacfc6edfb1617744169a7ed6273605 Mon Sep 17 00:00:00 2001 From: "Prekshith D J (Persistent Systems Inc)" Date: Wed, 17 Dec 2025 15:23:56 +0530 Subject: [PATCH 10/20] fix the pylint issue --- src/backend/sql_agents/convert_script.py | 2 +- tests/e2e-test/tests/conftest.py | 1 + tests/e2e-test/tests/test_codegen_gp_tc.py | 15 ++++++++------- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/backend/sql_agents/convert_script.py b/src/backend/sql_agents/convert_script.py index 01309af4..eb4a2767 100644 --- a/src/backend/sql_agents/convert_script.py +++ b/src/backend/sql_agents/convert_script.py @@ -274,7 +274,7 @@ async def convert_script( if comms_manager.group_chat.is_complete: break - + migrated_query = current_migration # Handle the case where migration failed and current_migration is None diff --git a/tests/e2e-test/tests/conftest.py b/tests/e2e-test/tests/conftest.py index 1dd44563..b5c5087c 100644 --- a/tests/e2e-test/tests/conftest.py +++ b/tests/e2e-test/tests/conftest.py @@ -33,6 +33,7 @@ def login_logout(): def pytest_html_report_title(report): report.title = "Automation_CodeGen" + log_streams = {} diff --git a/tests/e2e-test/tests/test_codegen_gp_tc.py b/tests/e2e-test/tests/test_codegen_gp_tc.py index 7a239f46..d9ea5c7b 100644 --- a/tests/e2e-test/tests/test_codegen_gp_tc.py +++ b/tests/e2e-test/tests/test_codegen_gp_tc.py @@ -7,6 +7,14 @@ logger = logging.getLogger(__name__) + +def _timed_translation(home): + start = time.time() + home.validate_translate() + end = time.time() + logger.info(f"Translation process for uploaded files took {end - start:.2f} seconds") + + # Define step-wise test actions for Golden Path golden_path_steps = [ ("01. Validate home page is loaded", lambda home: home.validate_home_page()), @@ -18,13 +26,6 @@ ] -def _timed_translation(home): - start = time.time() - home.validate_translate() - end = time.time() - logger.info(f"Translation process for uploaded files took {end - start:.2f} seconds") - - @pytest.mark.parametrize("description, action", golden_path_steps, ids=[desc for desc, _ in golden_path_steps]) def test_codegen_golden_path(login_logout, description, action, request): """ From 21c2c35c6ff7bf5327be79872f45d86711730cce Mon Sep 17 00:00:00 2001 From: Harmanpreet Kaur Date: Thu, 18 Dec 2025 12:01:10 +0530 Subject: [PATCH 11/20] Enhance CI workflows: add path triggers for 'infra/**', 'scripts/**', and 'azure.yaml'; update deploy workflow to trigger on completion of Docker build; remove pull request paths from PyLint workflow --- .github/workflows/build-docker-images.yml | 8 +++ .github/workflows/deploy.yml | 72 +++++++++++++++-------- .github/workflows/pylint.yml | 7 --- 3 files changed, 56 insertions(+), 31 deletions(-) diff --git a/.github/workflows/build-docker-images.yml b/.github/workflows/build-docker-images.yml index 47aeeb5f..ad4d720b 100644 --- a/.github/workflows/build-docker-images.yml +++ b/.github/workflows/build-docker-images.yml @@ -13,6 +13,10 @@ on: - 'docker/**' - '.github/workflows/build-docker-images.yml' - '.github/workflows/build-docker.yml' + - 'infra/**' + - 'scripts/**' + - 'azure.yaml' + - '.github/workflows/deploy.yml' pull_request: branches: - main @@ -30,6 +34,10 @@ on: - 'docker/**' - '.github/workflows/build-docker-images.yml' - '.github/workflows/build-docker.yml' + - 'infra/**' + - 'scripts/**' + - 'azure.yaml' + - '.github/workflows/deploy.yml' merge_group: workflow_dispatch: diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index b2b8c01b..0a131fa0 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -1,19 +1,17 @@ name: Deploy-Test-Cleanup Pipeline on: - push: - branches: - - main - - dev - - demo - paths: - - 'infra/**' - - 'scripts/**' - - 'azure.yaml' - - '.github/workflows/deploy.yml' - schedule: - - cron: '0 5,17 * * *' # Runs at 5:00 AM and 5:00 PM GMT - workflow_dispatch: + workflow_run: + workflows: ["Build Docker and Optional Push"] + types: + - completed + branches: + - main + - dev + - demo + schedule: + - cron: '0 5,17 * * *' # Runs at 5:00 AM and 5:00 PM GMT + workflow_dispatch: env: GPT_MIN_CAPACITY: 150 @@ -27,7 +25,7 @@ jobs: WEBAPP_URL: ${{ steps.get_output.outputs.WEBAPP_URL }} steps: - name: Checkout Code - uses: actions/checkout@v3 + uses: actions/checkout@v5 - name: Setup Azure CLI run: | @@ -45,7 +43,6 @@ jobs: export AZURE_TENANT_ID=${{ secrets.AZURE_TENANT_ID }} export AZURE_CLIENT_SECRET=${{ secrets.AZURE_CLIENT_SECRET }} export AZURE_SUBSCRIPTION_ID="${{ secrets.AZURE_SUBSCRIPTION_ID }}" - export GPT_MIN_CAPACITY="${{ env.GPT_MIN_CAPACITY }}" export AZURE_REGIONS="${{ vars.AZURE_REGIONS }}" chmod +x scripts/checkquota.sh if ! scripts/checkquota.sh; then @@ -74,6 +71,11 @@ jobs: - name: Fail Pipeline if Quota Check Fails if: env.QUOTA_FAILED == 'true' run: exit 1 + + - name: Set Deployment Region + run: | + echo "Selected Region: $VALID_REGION" + echo "AZURE_LOCATION=$VALID_REGION" >> $GITHUB_ENV - name: Install Bicep CLI run: az bicep install @@ -96,7 +98,7 @@ jobs: rg_exists=$(az group exists --name ${{ env.RESOURCE_GROUP_NAME }}) if [ "$rg_exists" = "false" ]; then echo "Resource group does not exist. Creating..." - az group create --name ${{ env.RESOURCE_GROUP_NAME }} --location northcentralus || { echo "Error creating resource group"; exit 1; } + az group create --name ${{ env.RESOURCE_GROUP_NAME }} --location australiaeast || { echo "Error creating resource group"; exit 1; } else echo "Resource group already exists." fi @@ -128,17 +130,20 @@ jobs: IMAGE_TAG="latest" fi + # Generate current timestamp in desired format: YYYY-MM-DDTHH:MM:SS.SSSSSSSZ + current_date=$(date -u +"%Y-%m-%dT%H:%M:%S.%7NZ") + az deployment group create \ --name ${{ env.SOLUTION_PREFIX }}-deployment \ --resource-group ${{ env.RESOURCE_GROUP_NAME }} \ --template-file infra/main.bicep \ --parameters \ solutionName="${{ env.SOLUTION_PREFIX }}" \ - aiDeploymentsLocation="eastus" \ - useWafAlignedArchitecture=false \ - capacity=${{ env.GPT_MIN_CAPACITY }} \ + azureAiServiceLocation='${{ env.AZURE_LOCATION }}' \ imageVersion="${IMAGE_TAG}" \ - createdBy="Pipeline" + createdBy="Pipeline" \ + tags="{'SecurityControl':'Ignore','Purpose':'Deploying and Cleaning Up Resources for Validation','CreatedDate':'$current_date'}" + - name: Assign Contributor role to Service Principal if: always() run: | @@ -187,7 +192,26 @@ jobs: - name: Login to Azure run: | - az login --service-principal -u ${{ secrets.AZURE_CLIENT_ID }} -p ${{ secrets.AZURE_CLIENT_SECRET }} --tenant ${{ secrets.AZURE_TENANT_ID }} + az login --service-principal -u ${{ secrets.AZURE_CLIENT_ID }} -p ${{ secrets.AZURE_CLIENT_SECRET }} --tenant ${{ secrets.AZURE_TENANT_ID }} + az account set --subscription "${{ secrets.AZURE_SUBSCRIPTION_ID }}" + + - name: Assign Contributor role to Service Principal + if: always() + run: | + echo "Assigning Contributor role to SPN for RG: ${{ env.RESOURCE_GROUP_NAME }}" + az role assignment create \ + --assignee ${{ secrets.AZURE_CLIENT_ID }} \ + --role "Contributor" \ + --scope /subscriptions/${{ secrets.AZURE_SUBSCRIPTION_ID }}/resourceGroups/${{ env.RESOURCE_GROUP_NAME }} + + echo "Assigning Log Analytics Contributor role for Log Analytics workspace access at RG level..." + az role assignment create \ + --assignee ${{ secrets.AZURE_CLIENT_ID }} \ + --role "Log Analytics Reader" \ + --scope /subscriptions/${{ secrets.AZURE_SUBSCRIPTION_ID }}/resourceGroups/${{ env.RESOURCE_GROUP_NAME }} || echo "Log Analytics Contributor role assignment failed (may already exist)" + + echo "Waiting for role assignment propagation..." + sleep 30 - name: Get Log Analytics Workspace and OpenAI from Resource Group if: always() @@ -358,7 +382,7 @@ jobs: # Purge OpenAI Resource echo "Purging the OpenAI Resource..." - if ! az resource delete --ids /subscriptions/${{ secrets.AZURE_SUBSCRIPTION_ID }}/providers/Microsoft.CognitiveServices/locations/northcentralus/resourceGroups/${{ env.RESOURCE_GROUP_NAME }}/deletedAccounts/${{ env.OPENAI_RESOURCE_NAME }} --verbose; then + if ! az resource delete --ids /subscriptions/${{ secrets.AZURE_SUBSCRIPTION_ID }}/providers/Microsoft.CognitiveServices/locations/australiaeast/resourceGroups/${{ env.RESOURCE_GROUP_NAME }}/deletedAccounts/${{ env.OPENAI_RESOURCE_NAME }} --verbose; then echo "Failed to purge openai resource: ${{ env.OPENAI_RESOURCE_NAME }}" else echo "Purged the openai resource: ${{ env.OPENAI_RESOURCE_NAME }}" @@ -401,7 +425,7 @@ jobs: EMAIL_BODY=$(cat <Dear Team,

We would like to inform you that the DocGen Deployment Automation process has encountered an issue and has failed to complete successfully.

Build URL: ${RUN_URL}

Please investigate the matter at your earliest convenience.

Best regards,
Your Automation Team

" + "body": "

Dear Team,

We would like to inform you that the CodeMod Deployment Automation process has encountered an issue and has failed to complete successfully.

Build URL: ${RUN_URL}

Please investigate the matter at your earliest convenience.

Best regards,
Your Automation Team

" } EOF ) @@ -414,4 +438,4 @@ jobs: if: always() run: | az logout - echo "Logged out from Azure." + echo "Logged out from Azure." \ No newline at end of file diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index f783199b..46a06b07 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -8,13 +8,6 @@ on: - '**/pyproject.toml' - '.flake8' - '.github/workflows/pylint.yml' - pull_request: - paths: - - '**/*.py' - - '**/requirements.txt' - - '**/pyproject.toml' - - '.flake8' - - '.github/workflows/pylint.yml' jobs: lint: From 6434419945d24021a1d97f37f2ddea2cc842f7d9 Mon Sep 17 00:00:00 2001 From: "Prekshith D J (Persistent Systems Inc)" Date: Fri, 19 Dec 2025 16:37:13 +0530 Subject: [PATCH 12/20] Reverting my changes from dev --- docs/DeploymentGuide.md | 6 +-- src/backend/api/api_routes.py | 1 - src/backend/common/database/cosmosdb.py | 4 ++ src/backend/common/services/batch_service.py | 2 - src/backend/common/storage/blob_azure.py | 2 +- src/backend/sql_agents/convert_script.py | 7 ++- .../sql_agents/helpers/comms_manager.py | 16 ++++++- src/backend/sql_agents/tools/src/Program.cs | 13 +++--- src/frontend/frontend_server.py | 2 +- src/frontend/src/api/utils.tsx | 2 +- .../src/components/batchHistoryPanel.tsx | 45 ++++++++++++++++++- src/frontend/src/components/bottomBar.tsx | 3 +- src/frontend/src/components/uploadButton.tsx | 3 ++ src/frontend/src/main.jsx | 2 +- src/frontend/src/pages/batchView.tsx | 40 ++++++++++++++++- src/frontend/src/pages/landingPage.tsx | 1 + src/frontend/src/pages/modernizationPage.tsx | 17 ++++--- src/frontend/vite.config.js | 1 + .../backend/common/storage/blob_azure_test.py | 1 + tests/e2e-test/pages/HomePage.py | 1 - tests/e2e-test/pages/loginPage.py | 1 - tests/e2e-test/tests/conftest.py | 18 ++++++++ tests/e2e-test/tests/test_codegen_gp_tc.py | 17 ++++--- 23 files changed, 165 insertions(+), 40 deletions(-) diff --git a/docs/DeploymentGuide.md b/docs/DeploymentGuide.md index aa51590c..8abf4f58 100644 --- a/docs/DeploymentGuide.md +++ b/docs/DeploymentGuide.md @@ -124,7 +124,7 @@ If you're not using one of the above options for opening the project, then you'l 2. Download the project code: ```shell - azd init -t microsoft/Modernize-your-Code-Solution-Accelerator + azd init -t microsoft/Modernize-your-Code-Solution-Accelerator/ ``` 3. Open the project folder in your terminal or editor. @@ -164,7 +164,7 @@ To adjust quota settings, follow these [steps](../docs/AzureGPTQuotaSettings.md) Reusing an Existing Log Analytics Workspace - Guide to get your [Existing Workspace ID](./re-use-log-analytics.md) + Guide to get your [Existing Workspace ID](/docs/re-use-log-analytics.md) @@ -217,7 +217,7 @@ Rename `azure.yaml` to `azure_original.yaml` and `azure_custom.yaml` to `azure.y Go to `infra` directory -Rename `main.bicep` to `main_original.bicep` and `main_custom.bicep` to `main.bicep`. Continue with the [deploying steps](#deploying-with-azd). +Rename `main.bicep` to `main_original.bicep` and `main_custom.bicep` to `main.bicep`. Continue with the [deploying steps](https://github.com/microsoft/Modernize-your-code-solution-accelerator/blob/main/docs/DeploymentGuide.md#deploying-with-azd). ### 🛠️ Troubleshooting If you encounter any issues during the deployment process, please refer [troubleshooting](../docs/TroubleShootingSteps.md) document for detailed steps and solutions. diff --git a/src/backend/api/api_routes.py b/src/backend/api/api_routes.py index 8a3be72f..44718f8a 100644 --- a/src/backend/api/api_routes.py +++ b/src/backend/api/api_routes.py @@ -279,7 +279,6 @@ async def batch_status_updates( try: await websocket.receive_text() except asyncio.TimeoutError: - # TimeoutError is ignored to keep the WebSocket connection open without receiving data pass except WebSocketDisconnect: diff --git a/src/backend/common/database/cosmosdb.py b/src/backend/common/database/cosmosdb.py index 7ff043f4..35071709 100644 --- a/src/backend/common/database/cosmosdb.py +++ b/src/backend/common/database/cosmosdb.py @@ -352,6 +352,10 @@ async def update_batch_entry( batch["file_count"] = file_count await self.batch_container.replace_item(item=batch_id, body=batch) + # if isinstance(status, ProcessStatus): + # self.logger.info(f"Updated batch {batch_id} to status {status.value}") + # else: + # self.logger.info(f"Updated batch {batch_id} to status {status}") return batch except Exception as e: diff --git a/src/backend/common/services/batch_service.py b/src/backend/common/services/batch_service.py index f467820a..59b1b411 100644 --- a/src/backend/common/services/batch_service.py +++ b/src/backend/common/services/batch_service.py @@ -175,8 +175,6 @@ async def delete_batch(self, batch_id: UUID, user_id: str): self.logger.info(f"Successfully deleted batch with ID: {batch_id}") return {"message": "Batch deleted successfully", "batch_id": str(batch_id)} - else: - return {"message": "Batch not found", "batch_id": str(batch_id)} async def delete_file(self, file_id: UUID, user_id: str): """Delete a file and its logs, and update batch file count.""" diff --git a/src/backend/common/storage/blob_azure.py b/src/backend/common/storage/blob_azure.py index 33b126f6..da53b2c3 100644 --- a/src/backend/common/storage/blob_azure.py +++ b/src/backend/common/storage/blob_azure.py @@ -52,7 +52,7 @@ async def upload_file( raise try: # Upload the file - blob_client.upload_blob( + upload_results = blob_client.upload_blob( # noqa: F841 file_content, content_type=content_type, metadata=metadata, diff --git a/src/backend/sql_agents/convert_script.py b/src/backend/sql_agents/convert_script.py index eb4a2767..789bf36f 100644 --- a/src/backend/sql_agents/convert_script.py +++ b/src/backend/sql_agents/convert_script.py @@ -66,7 +66,8 @@ async def convert_script( # orchestrate the chat current_migration = "No migration" - while True: + is_complete: bool = False + while not is_complete: await comms_manager.group_chat.add_chat_message( ChatMessageContent(role=AuthorRole.USER, content=source_script) ) @@ -273,7 +274,9 @@ async def convert_script( break if comms_manager.group_chat.is_complete: - break + is_complete = True + + break migrated_query = current_migration diff --git a/src/backend/sql_agents/helpers/comms_manager.py b/src/backend/sql_agents/helpers/comms_manager.py index da6c9c6c..7080e38f 100644 --- a/src/backend/sql_agents/helpers/comms_manager.py +++ b/src/backend/sql_agents/helpers/comms_manager.py @@ -79,8 +79,6 @@ async def select_agent(self, agents, history): ), None, ) - # No matching case found, so explicitly return None - return None # class for termination strategy class ApprovalTerminationStrategy(TerminationStrategy): @@ -272,3 +270,17 @@ async def cleanup(self): except Exception as e: self.logger.error("Error during cleanup: %s", str(e)) + + def __del__(self): + """Destructor to ensure cleanup if not explicitly called.""" + try: + # Only attempt cleanup if there's an active event loop + loop = asyncio.get_running_loop() + if loop and not loop.is_closed(): + # Schedule cleanup as a task + loop.create_task(self.cleanup()) + except RuntimeError: + # No event loop running, can't clean up asynchronously + self.logger.warning("No event loop available for cleanup in destructor") + except Exception as e: + self.logger.error("Error in destructor cleanup: %s", str(e)) diff --git a/src/backend/sql_agents/tools/src/Program.cs b/src/backend/sql_agents/tools/src/Program.cs index 06c5f213..b728eefc 100644 --- a/src/backend/sql_agents/tools/src/Program.cs +++ b/src/backend/sql_agents/tools/src/Program.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Generic; using System.IO; -using System.Linq; using Microsoft.SqlServer.TransactSql.ScriptDom; using Newtonsoft.Json; @@ -43,14 +42,18 @@ static void Main(string[] args) IList errors = ParseSqlQuery(sqlQuery); - var errorList = errors - .Select(error => new Dictionary + var errorList = new List>(); + + foreach (var error in errors) + { + var errorDict = new Dictionary { { "Line", error.Line }, { "Column", error.Column }, { "Error", error.Message } - }) - .ToList(); + }; + errorList.Add(errorDict); + } string jsonOutput = JsonConvert.SerializeObject(errorList, Formatting.Indented); Console.WriteLine(jsonOutput); diff --git a/src/frontend/frontend_server.py b/src/frontend/frontend_server.py index c53af042..199d1a66 100644 --- a/src/frontend/frontend_server.py +++ b/src/frontend/frontend_server.py @@ -4,7 +4,7 @@ from dotenv import load_dotenv from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware -from fastapi.responses import FileResponse +from fastapi.responses import FileResponse, HTMLResponse from fastapi.staticfiles import StaticFiles # Load environment variables from .env file diff --git a/src/frontend/src/api/utils.tsx b/src/frontend/src/api/utils.tsx index 50341195..95ce5459 100644 --- a/src/frontend/src/api/utils.tsx +++ b/src/frontend/src/api/utils.tsx @@ -414,7 +414,7 @@ export const renderErrorSection = (batchSummary, expandedSections, setExpandedSe export const renderErrorContent = (batchSummary) => { // Group errors by file - const errorFiles = batchSummary.files.filter(file => file.error_count); + const errorFiles = batchSummary.files.filter(file => file.error_count && file.error_count); if (errorFiles.length === 0) { return (
diff --git a/src/frontend/src/components/batchHistoryPanel.tsx b/src/frontend/src/components/batchHistoryPanel.tsx index 06820737..0e0259d9 100644 --- a/src/frontend/src/components/batchHistoryPanel.tsx +++ b/src/frontend/src/components/batchHistoryPanel.tsx @@ -1,5 +1,6 @@ import React, { useState, useEffect, useRef } from "react"; -import { useDispatch } from 'react-redux'; + +import { useDispatch, useSelector } from 'react-redux'; import { Card, Spinner, Tooltip } from "@fluentui/react-components"; import { useNavigate } from "react-router-dom"; import ConfirmationDialog from "../commonComponents/ConfirmationDialog/confirmationDialogue"; @@ -20,7 +21,7 @@ interface BatchHistoryItem { status: string; } const HistoryPanel: React.FC = ({ isOpen, onClose }) => { - const headers = {}; + const headers = {} const [batchHistory, setBatchHistory] = useState([]); const [loading, setLoading] = useState(false); const [error, setError] = useState(null); @@ -80,6 +81,46 @@ const HistoryPanel: React.FC = ({ isOpen, onClose }) => { } }; + // Function to categorize batches + const categorizeBatches = () => { + const now = new Date(); + const userTimeZone = Intl.DateTimeFormat().resolvedOptions().timeZone; + + // Get start of "Today", "Past 7 days", and "Past 30 days" in LOCAL time + const todayStart = new Date(now.getFullYear(), now.getMonth(), now.getDate()); + const past7DaysStart = new Date(todayStart); + const past30DaysStart = new Date(todayStart); + + past7DaysStart.setDate(todayStart.getDate() - 7); + past30DaysStart.setDate(todayStart.getDate() - 30); + + const todayBatches: BatchHistoryItem[] = []; + const past7DaysBatches: BatchHistoryItem[] = []; + const past30DaysBatches: BatchHistoryItem[] = []; + + batchHistory.forEach(batch => { + // Convert UTC timestamp to user's local date + const updatedAtUTC = new Date(batch.created_at); + const updatedAtLocal = new Date(updatedAtUTC.toLocaleString("en-US", { timeZone: userTimeZone })); + + // Extract only the local **date** part for comparison + const updatedDate = new Date(updatedAtLocal.getFullYear(), updatedAtLocal.getMonth(), updatedAtLocal.getDate()); + + // Categorize based on **exact day comparison** + if (updatedDate.getTime() === todayStart.getTime()) { + todayBatches.push(batch); + } else if (updatedDate.getTime() >= past7DaysStart.getTime()) { + past7DaysBatches.push(batch); + } else if (updatedDate.getTime() >= past30DaysStart.getTime()) { + past30DaysBatches.push(batch); + } + }); + + return { todayBatches, past7DaysBatches, past30DaysBatches }; + }; + + // const { todayBatches, past7DaysBatches, past30DaysBatches } = categorizeBatches(); + const deleteBatchFromHistory = (batchId: string) => { // Get the current URL path const currentPath = window.location.pathname; diff --git a/src/frontend/src/components/bottomBar.tsx b/src/frontend/src/components/bottomBar.tsx index c2702d2e..3041a988 100644 --- a/src/frontend/src/components/bottomBar.tsx +++ b/src/frontend/src/components/bottomBar.tsx @@ -1,5 +1,6 @@ import { Button, Card, Dropdown, DropdownProps, Option } from "@fluentui/react-components" -import React from "react" +import React, { useState } from "react" +import { useNavigate } from "react-router-dom" // Define possible upload states const UploadState = { diff --git a/src/frontend/src/components/uploadButton.tsx b/src/frontend/src/components/uploadButton.tsx index 0d3e5714..a833f669 100644 --- a/src/frontend/src/components/uploadButton.tsx +++ b/src/frontend/src/components/uploadButton.tsx @@ -68,6 +68,7 @@ const FileUploadZone: React.FC = ({ const [uploadIntervals, setUploadIntervals] = useState<{ [key: string]: ReturnType }>({}); const [showCancelDialog, setShowCancelDialog] = useState(false); const [showLogoCancelDialog, setShowLogoCancelDialog] = useState(false); + const [uploadState, setUploadState] = useState<'IDLE' | 'UPLOADING' | 'COMPLETED'>('IDLE'); const [batchId, setBatchId] = useState(uuidv4()); const [allUploadsComplete, setAllUploadsComplete] = useState(false); const [fileLimitExceeded, setFileLimitExceeded] = useState(false); @@ -97,6 +98,7 @@ const FileUploadZone: React.FC = ({ } } + setUploadState(newState); onUploadStateChange?.(newState); }, [uploadingFiles, onUploadStateChange]); @@ -278,6 +280,7 @@ const FileUploadZone: React.FC = ({ Object.values(uploadIntervals).forEach(interval => clearInterval(interval)); setUploadIntervals({}); setUploadingFiles([]); + setUploadState('IDLE'); onUploadStateChange?.('IDLE'); setShowCancelDialog(false); setShowLogoCancelDialog(false); diff --git a/src/frontend/src/main.jsx b/src/frontend/src/main.jsx index da9d477d..7ae29774 100644 --- a/src/frontend/src/main.jsx +++ b/src/frontend/src/main.jsx @@ -50,7 +50,7 @@ const Main = () => { const baseURL = config.API_URL.replace(/\/api$/, ''); // Remove '/api' if it appears at the end console.log('Checking connection to:', baseURL); try { - await fetch(`${baseURL}/health`); + const response = await fetch(`${baseURL}/health`); } catch (error) { console.error('Error connecting to backend:', error); } diff --git a/src/frontend/src/pages/batchView.tsx b/src/frontend/src/pages/batchView.tsx index 5afcdb03..ba8b9de2 100644 --- a/src/frontend/src/pages/batchView.tsx +++ b/src/frontend/src/pages/batchView.tsx @@ -33,7 +33,7 @@ import PanelRight from "../components/Panels/PanelRight"; import PanelRightToolbar from "../components/Panels/PanelRightToolbar"; import BatchHistoryPanel from "../components/batchHistoryPanel"; import ConfirmationDialog from "../commonComponents/ConfirmationDialog/confirmationDialogue"; -import { determineFileStatus, filesLogsBuilder, renderErrorSection, useStyles, renderFileError, filesErrorCounter, completedFiles, hasFiles, fileErrorCounter, BatchSummary } from "../api/utils"; +import { determineFileStatus, filesLogsBuilder, renderErrorSection, useStyles, renderFileError, filesErrorCounter, completedFiles, hasFiles, fileErrorCounter, BatchSummary, fileWarningCounter } from "../api/utils"; export const History = bundleIcon(HistoryFilled, HistoryRegular); import { format } from "sql-formatter"; @@ -73,6 +73,7 @@ const BatchStoryPage = () => { const [selectedFileId, setSelectedFileId] = useState(""); const [expandedSections, setExpandedSections] = useState(["errors"]); const [batchSummary, setBatchSummary] = useState(null); + const [selectedFileContent, setSelectedFileContent] = useState(""); const [selectedFileTranslatedContent, setSelectedFileTranslatedContent] = useState(""); @@ -194,6 +195,7 @@ const BatchStoryPage = () => { const data = await response.json(); if (data) { + setSelectedFileContent(data.content || ""); setSelectedFileTranslatedContent(data.translated_content || ""); } @@ -207,6 +209,40 @@ const BatchStoryPage = () => { fetchFileContent(); }, [selectedFileId]); + + const renderWarningContent = () => { + if (!expandedSections.includes("warnings")) return null; + + if (!batchSummary) return null; + + // Group warnings by file + const warningFiles = files.filter(file => file.warningCount && file.warningCount > 0 && file.id !== "summary"); + + if (warningFiles.length === 0) { + return ( +
+ No warnings found. +
+ ); + } + + return ( +
+ {warningFiles.map((file, fileIndex) => ( +
+
+ {file.name} ({file.warningCount}) + source +
+
+ Warning in file processing. See file for details. +
+
+ ))} +
+ ); + }; + const renderContent = () => { // Define header content based on selected file const renderHeader = () => { @@ -349,7 +385,7 @@ const BatchStoryPage = () => { } // Show the summary page when summary is selected - if (selectedFile.id === "summary") { + if (selectedFile.id === "summary" && batchSummary) { // Check if there are no errors and all files are processed successfully const noErrors = (batchSummary.error_count === 0); const allFilesProcessed = (batchSummary.completed_files === batchSummary.total_files); diff --git a/src/frontend/src/pages/landingPage.tsx b/src/frontend/src/pages/landingPage.tsx index b86200e6..b3707393 100644 --- a/src/frontend/src/pages/landingPage.tsx +++ b/src/frontend/src/pages/landingPage.tsx @@ -33,6 +33,7 @@ export const LandingPage = (): JSX.Element => { const dispatch = useDispatch(); // Add dispatch hook const [selectedTargetLanguage, setSelectedTargetLanguage] = useState(["T-SQL"]); const [selectedCurrentLanguage, setSelectedCurrentLanguage] = useState(["Informix"]); + const batchHistoryRef = useRef<{ triggerDeleteAll: () => void } | null>(null); const isPanelOpen = useSelector((state: RootState) => state.historyPanel.isOpen); const navigate = useNavigate(); diff --git a/src/frontend/src/pages/modernizationPage.tsx b/src/frontend/src/pages/modernizationPage.tsx index dc76f4bd..f5d6d04e 100644 --- a/src/frontend/src/pages/modernizationPage.tsx +++ b/src/frontend/src/pages/modernizationPage.tsx @@ -477,13 +477,13 @@ const getPrintFileStatus = (status: string): string => { const ModernizationPage = () => { const { batchId } = useParams<{ batchId: string }>(); - const navigate = useNavigate(); + const navigate = useNavigate() // Redux state to listen for start processing completion const batchState = useSelector((state: any) => state.batch); const [batchSummary, setBatchSummary] = useState(null); - const styles = useStyles(); + const styles = useStyles() const [text, setText] = useState(""); const [isPanelOpen, setIsPanelOpen] = React.useState(false); // Add state management @@ -492,13 +492,16 @@ const ModernizationPage = () => { // State for the loading component const [showLoading, setShowLoading] = useState(true); + const [loadingError, setLoadingError] = useState(null); const [selectedFilebg, setSelectedFile] = useState(null); - const [selectedFileId, setSelectedFileId] = React.useState(""); + const [selectedFileId, setSelectedFileId] = React.useState("") const [fileId, setFileId] = React.useState(""); - const [expandedSections, setExpandedSections] = React.useState([]); + const [expandedSections, setExpandedSections] = React.useState([]) + const [progressPercentage, setProgressPercentage] = useState(0); const [allFilesCompleted, setAllFilesCompleted] = useState(false); const [isZipButtonDisabled, setIsZipButtonDisabled] = useState(true); const [fileLoading, setFileLoading] = useState(false); + const [selectedFileTranslatedContent, setSelectedFileTranslatedContent] = useState(""); const [lastActivityTime, setLastActivityTime] = useState(Date.now()); const [pageLoadTime] = useState(Date.now()); @@ -514,9 +517,11 @@ const ModernizationPage = () => { if (!selectedFile || !selectedFile.translatedCode) { setFileLoading(true); const newFileUpdate = await fetchFileFromAPI(selectedFile?.fileId || ""); + setSelectedFileTranslatedContent(newFileUpdate.translatedContent); setFileLoading(false); } else { + setSelectedFileTranslatedContent(selectedFile.translatedCode); } } catch (err) { @@ -794,6 +799,8 @@ const ModernizationPage = () => { } }, [batchId]); + const highestProgressRef = useRef(0); + const currentProcessingFileRef = useRef(null); //new PT FR ends @@ -1244,7 +1251,7 @@ useEffect(() => { } // Show the full summary page only when all files are completed and summary is selected - if (selectedFile?.id === "summary") { + if (allFilesCompleted && selectedFile?.id === "summary") { const completedCount = files.filter(file => file.status === "completed" && file.file_result !== "error" && file.id !== "summary").length; const totalCount = files.filter(file => file.id !== "summary").length; const errorCount = selectedFile.errorCount || 0; diff --git a/src/frontend/vite.config.js b/src/frontend/vite.config.js index d239b70e..0d05262f 100644 --- a/src/frontend/vite.config.js +++ b/src/frontend/vite.config.js @@ -1,5 +1,6 @@ import { defineConfig } from 'vite' import react from '@vitejs/plugin-react' +import path from 'path' import tailwindcss from '@tailwindcss/vite' export default defineConfig({ diff --git a/src/tests/backend/common/storage/blob_azure_test.py b/src/tests/backend/common/storage/blob_azure_test.py index afc76e7e..68e5ad0d 100644 --- a/src/tests/backend/common/storage/blob_azure_test.py +++ b/src/tests/backend/common/storage/blob_azure_test.py @@ -28,6 +28,7 @@ def mock_blob_service(): @pytest.fixture def blob_storage(mock_blob_service): """Fixture to initialize AzureBlobStorage with mocked dependencies""" + service_client, container_client, blob_client = mock_blob_service return AzureBlobStorage(account_name="test_account", container_name="test_container") diff --git a/tests/e2e-test/pages/HomePage.py b/tests/e2e-test/pages/HomePage.py index 6e184a83..d863ac18 100644 --- a/tests/e2e-test/pages/HomePage.py +++ b/tests/e2e-test/pages/HomePage.py @@ -19,7 +19,6 @@ class HomePage(BasePage): FILE_PROCESSED_MSG = "//span[normalize-space()='3 files processed successfully']" def __init__(self, page): - super().__init__(page) self.page = page def validate_home_page(self): diff --git a/tests/e2e-test/pages/loginPage.py b/tests/e2e-test/pages/loginPage.py index 7f013ccb..0b412556 100644 --- a/tests/e2e-test/pages/loginPage.py +++ b/tests/e2e-test/pages/loginPage.py @@ -11,7 +11,6 @@ class LoginPage(BasePage): PERMISSION_ACCEPT_BUTTON = "//input[@type='submit']" def __init__(self, page): - super().__init__(page) self.page = page def authenticate(self, username, password): diff --git a/tests/e2e-test/tests/conftest.py b/tests/e2e-test/tests/conftest.py index b5c5087c..b9919e6d 100644 --- a/tests/e2e-test/tests/conftest.py +++ b/tests/e2e-test/tests/conftest.py @@ -34,6 +34,17 @@ def pytest_html_report_title(report): report.title = "Automation_CodeGen" +# Add a column for descriptions +# def pytest_html_results_table_header(cells): +# cells.insert(1, html.th("Description")) + + +# def pytest_html_results_table_row(report, cells): +# cells.insert( +# 1, html.td(report.description if hasattr(report, "description") else "") +# ) + + log_streams = {} @@ -113,3 +124,10 @@ def rename_duration_column(): # Add logs and docstring to report # @pytest.hookimpl(hookwrapper=True) +# def pytest_runtest_makereport(item, call): +# outcome = yield +# report = outcome.get_result() +# report.description = str(item.function.__doc__) +# os.makedirs("logs", exist_ok=True) +# extra = getattr(report, "extra", []) +# report.extra = extra diff --git a/tests/e2e-test/tests/test_codegen_gp_tc.py b/tests/e2e-test/tests/test_codegen_gp_tc.py index d9ea5c7b..6d9e896f 100644 --- a/tests/e2e-test/tests/test_codegen_gp_tc.py +++ b/tests/e2e-test/tests/test_codegen_gp_tc.py @@ -7,25 +7,24 @@ logger = logging.getLogger(__name__) - -def _timed_translation(home): - start = time.time() - home.validate_translate() - end = time.time() - logger.info(f"Translation process for uploaded files took {end - start:.2f} seconds") - - # Define step-wise test actions for Golden Path golden_path_steps = [ ("01. Validate home page is loaded", lambda home: home.validate_home_page()), ("02. Validate Upload of other than SQL files", lambda home: home.upload_unsupported_files()), ("03. Validate Upload input files for SQL only", lambda home: home.upload_files()), - ("04. Validate translation process for uploaded files", _timed_translation), + ("04. Validate translation process for uploaded files", lambda home: _timed_translation(home)), ("05. Check batch history", lambda home: home.validate_batch_history()), ("06. Download all files and return home", lambda home: home.validate_download_files()), ] +def _timed_translation(home): + start = time.time() + home.validate_translate() + end = time.time() + logger.info(f"Translation process for uploaded files took {end - start:.2f} seconds") + + @pytest.mark.parametrize("description, action", golden_path_steps, ids=[desc for desc, _ in golden_path_steps]) def test_codegen_golden_path(login_logout, description, action, request): """ From e35f862d2a52e89daf6f330d73c98aabcd859e95 Mon Sep 17 00:00:00 2001 From: "Prekshith D J (Persistent Systems Inc)" Date: Mon, 22 Dec 2025 11:05:41 +0530 Subject: [PATCH 13/20] Updated the semicolons, commented lines and identical operands --- src/backend/common/database/cosmosdb.py | 4 ---- src/frontend/src/api/utils.tsx | 2 +- .../src/components/batchHistoryPanel.tsx | 2 +- src/frontend/src/pages/modernizationPage.tsx | 8 +++---- tests/e2e-test/tests/conftest.py | 22 ------------------- 5 files changed, 6 insertions(+), 32 deletions(-) diff --git a/src/backend/common/database/cosmosdb.py b/src/backend/common/database/cosmosdb.py index 35071709..7ff043f4 100644 --- a/src/backend/common/database/cosmosdb.py +++ b/src/backend/common/database/cosmosdb.py @@ -352,10 +352,6 @@ async def update_batch_entry( batch["file_count"] = file_count await self.batch_container.replace_item(item=batch_id, body=batch) - # if isinstance(status, ProcessStatus): - # self.logger.info(f"Updated batch {batch_id} to status {status.value}") - # else: - # self.logger.info(f"Updated batch {batch_id} to status {status}") return batch except Exception as e: diff --git a/src/frontend/src/api/utils.tsx b/src/frontend/src/api/utils.tsx index 95ce5459..50341195 100644 --- a/src/frontend/src/api/utils.tsx +++ b/src/frontend/src/api/utils.tsx @@ -414,7 +414,7 @@ export const renderErrorSection = (batchSummary, expandedSections, setExpandedSe export const renderErrorContent = (batchSummary) => { // Group errors by file - const errorFiles = batchSummary.files.filter(file => file.error_count && file.error_count); + const errorFiles = batchSummary.files.filter(file => file.error_count); if (errorFiles.length === 0) { return (
diff --git a/src/frontend/src/components/batchHistoryPanel.tsx b/src/frontend/src/components/batchHistoryPanel.tsx index 0e0259d9..3440d784 100644 --- a/src/frontend/src/components/batchHistoryPanel.tsx +++ b/src/frontend/src/components/batchHistoryPanel.tsx @@ -21,7 +21,7 @@ interface BatchHistoryItem { status: string; } const HistoryPanel: React.FC = ({ isOpen, onClose }) => { - const headers = {} + const headers = {}; const [batchHistory, setBatchHistory] = useState([]); const [loading, setLoading] = useState(false); const [error, setError] = useState(null); diff --git a/src/frontend/src/pages/modernizationPage.tsx b/src/frontend/src/pages/modernizationPage.tsx index f5d6d04e..489cbca6 100644 --- a/src/frontend/src/pages/modernizationPage.tsx +++ b/src/frontend/src/pages/modernizationPage.tsx @@ -477,13 +477,13 @@ const getPrintFileStatus = (status: string): string => { const ModernizationPage = () => { const { batchId } = useParams<{ batchId: string }>(); - const navigate = useNavigate() + const navigate = useNavigate(); // Redux state to listen for start processing completion const batchState = useSelector((state: any) => state.batch); const [batchSummary, setBatchSummary] = useState(null); - const styles = useStyles() + const styles = useStyles(); const [text, setText] = useState(""); const [isPanelOpen, setIsPanelOpen] = React.useState(false); // Add state management @@ -494,9 +494,9 @@ const ModernizationPage = () => { const [showLoading, setShowLoading] = useState(true); const [loadingError, setLoadingError] = useState(null); const [selectedFilebg, setSelectedFile] = useState(null); - const [selectedFileId, setSelectedFileId] = React.useState("") + const [selectedFileId, setSelectedFileId] = React.useState(""); const [fileId, setFileId] = React.useState(""); - const [expandedSections, setExpandedSections] = React.useState([]) + const [expandedSections, setExpandedSections] = React.useState([]); const [progressPercentage, setProgressPercentage] = useState(0); const [allFilesCompleted, setAllFilesCompleted] = useState(false); const [isZipButtonDisabled, setIsZipButtonDisabled] = useState(true); diff --git a/tests/e2e-test/tests/conftest.py b/tests/e2e-test/tests/conftest.py index b9919e6d..d24d06c6 100644 --- a/tests/e2e-test/tests/conftest.py +++ b/tests/e2e-test/tests/conftest.py @@ -34,17 +34,6 @@ def pytest_html_report_title(report): report.title = "Automation_CodeGen" -# Add a column for descriptions -# def pytest_html_results_table_header(cells): -# cells.insert(1, html.th("Description")) - - -# def pytest_html_results_table_row(report, cells): -# cells.insert( -# 1, html.td(report.description if hasattr(report, "description") else "") -# ) - - log_streams = {} @@ -120,14 +109,3 @@ def rename_duration_column(): # Register this function to run after everything is done atexit.register(rename_duration_column) - - -# Add logs and docstring to report -# @pytest.hookimpl(hookwrapper=True) -# def pytest_runtest_makereport(item, call): -# outcome = yield -# report = outcome.get_result() -# report.description = str(item.function.__doc__) -# os.makedirs("logs", exist_ok=True) -# extra = getattr(report, "extra", []) -# report.extra = extra From bab02b3bce1e781ca5b7dce3e00f40b594523e09 Mon Sep 17 00:00:00 2001 From: "Prekshith D J (Persistent Systems Inc)" Date: Mon, 22 Dec 2025 12:08:46 +0530 Subject: [PATCH 14/20] Updated the others files with minor changes --- src/backend/api/api_routes.py | 1 + src/backend/common/services/batch_service.py | 2 ++ src/backend/sql_agents/helpers/comms_manager.py | 14 -------------- src/backend/sql_agents/tools/src/Program.cs | 13 +++++-------- src/frontend/frontend_server.py | 2 +- tests/e2e-test/pages/HomePage.py | 1 + tests/e2e-test/pages/loginPage.py | 1 + tests/e2e-test/tests/test_codegen_gp_tc.py | 2 +- 8 files changed, 12 insertions(+), 24 deletions(-) diff --git a/src/backend/api/api_routes.py b/src/backend/api/api_routes.py index 44718f8a..8a3be72f 100644 --- a/src/backend/api/api_routes.py +++ b/src/backend/api/api_routes.py @@ -279,6 +279,7 @@ async def batch_status_updates( try: await websocket.receive_text() except asyncio.TimeoutError: + # TimeoutError is ignored to keep the WebSocket connection open without receiving data pass except WebSocketDisconnect: diff --git a/src/backend/common/services/batch_service.py b/src/backend/common/services/batch_service.py index 59b1b411..f467820a 100644 --- a/src/backend/common/services/batch_service.py +++ b/src/backend/common/services/batch_service.py @@ -175,6 +175,8 @@ async def delete_batch(self, batch_id: UUID, user_id: str): self.logger.info(f"Successfully deleted batch with ID: {batch_id}") return {"message": "Batch deleted successfully", "batch_id": str(batch_id)} + else: + return {"message": "Batch not found", "batch_id": str(batch_id)} async def delete_file(self, file_id: UUID, user_id: str): """Delete a file and its logs, and update batch file count.""" diff --git a/src/backend/sql_agents/helpers/comms_manager.py b/src/backend/sql_agents/helpers/comms_manager.py index 7080e38f..fddc0a7a 100644 --- a/src/backend/sql_agents/helpers/comms_manager.py +++ b/src/backend/sql_agents/helpers/comms_manager.py @@ -270,17 +270,3 @@ async def cleanup(self): except Exception as e: self.logger.error("Error during cleanup: %s", str(e)) - - def __del__(self): - """Destructor to ensure cleanup if not explicitly called.""" - try: - # Only attempt cleanup if there's an active event loop - loop = asyncio.get_running_loop() - if loop and not loop.is_closed(): - # Schedule cleanup as a task - loop.create_task(self.cleanup()) - except RuntimeError: - # No event loop running, can't clean up asynchronously - self.logger.warning("No event loop available for cleanup in destructor") - except Exception as e: - self.logger.error("Error in destructor cleanup: %s", str(e)) diff --git a/src/backend/sql_agents/tools/src/Program.cs b/src/backend/sql_agents/tools/src/Program.cs index b728eefc..06c5f213 100644 --- a/src/backend/sql_agents/tools/src/Program.cs +++ b/src/backend/sql_agents/tools/src/Program.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using Microsoft.SqlServer.TransactSql.ScriptDom; using Newtonsoft.Json; @@ -42,18 +43,14 @@ static void Main(string[] args) IList errors = ParseSqlQuery(sqlQuery); - var errorList = new List>(); - - foreach (var error in errors) - { - var errorDict = new Dictionary + var errorList = errors + .Select(error => new Dictionary { { "Line", error.Line }, { "Column", error.Column }, { "Error", error.Message } - }; - errorList.Add(errorDict); - } + }) + .ToList(); string jsonOutput = JsonConvert.SerializeObject(errorList, Formatting.Indented); Console.WriteLine(jsonOutput); diff --git a/src/frontend/frontend_server.py b/src/frontend/frontend_server.py index 199d1a66..c53af042 100644 --- a/src/frontend/frontend_server.py +++ b/src/frontend/frontend_server.py @@ -4,7 +4,7 @@ from dotenv import load_dotenv from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware -from fastapi.responses import FileResponse, HTMLResponse +from fastapi.responses import FileResponse from fastapi.staticfiles import StaticFiles # Load environment variables from .env file diff --git a/tests/e2e-test/pages/HomePage.py b/tests/e2e-test/pages/HomePage.py index d863ac18..6e184a83 100644 --- a/tests/e2e-test/pages/HomePage.py +++ b/tests/e2e-test/pages/HomePage.py @@ -19,6 +19,7 @@ class HomePage(BasePage): FILE_PROCESSED_MSG = "//span[normalize-space()='3 files processed successfully']" def __init__(self, page): + super().__init__(page) self.page = page def validate_home_page(self): diff --git a/tests/e2e-test/pages/loginPage.py b/tests/e2e-test/pages/loginPage.py index 0b412556..c8f37aa9 100644 --- a/tests/e2e-test/pages/loginPage.py +++ b/tests/e2e-test/pages/loginPage.py @@ -11,6 +11,7 @@ class LoginPage(BasePage): PERMISSION_ACCEPT_BUTTON = "//input[@type='submit']" def __init__(self, page): + super().__init__() self.page = page def authenticate(self, username, password): diff --git a/tests/e2e-test/tests/test_codegen_gp_tc.py b/tests/e2e-test/tests/test_codegen_gp_tc.py index 6d9e896f..7a239f46 100644 --- a/tests/e2e-test/tests/test_codegen_gp_tc.py +++ b/tests/e2e-test/tests/test_codegen_gp_tc.py @@ -12,7 +12,7 @@ ("01. Validate home page is loaded", lambda home: home.validate_home_page()), ("02. Validate Upload of other than SQL files", lambda home: home.upload_unsupported_files()), ("03. Validate Upload input files for SQL only", lambda home: home.upload_files()), - ("04. Validate translation process for uploaded files", lambda home: _timed_translation(home)), + ("04. Validate translation process for uploaded files", _timed_translation), ("05. Check batch history", lambda home: home.validate_batch_history()), ("06. Download all files and return home", lambda home: home.validate_download_files()), ] From b8161a33b051775734ff374552ba046a0e1a65e7 Mon Sep 17 00:00:00 2001 From: "Prekshith D J (Persistent Systems Inc)" Date: Mon, 22 Dec 2025 14:40:39 +0530 Subject: [PATCH 15/20] Updated the unused local variable --- src/backend/common/storage/blob_azure.py | 2 +- src/tests/backend/common/storage/blob_azure_test.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/backend/common/storage/blob_azure.py b/src/backend/common/storage/blob_azure.py index da53b2c3..33b126f6 100644 --- a/src/backend/common/storage/blob_azure.py +++ b/src/backend/common/storage/blob_azure.py @@ -52,7 +52,7 @@ async def upload_file( raise try: # Upload the file - upload_results = blob_client.upload_blob( # noqa: F841 + blob_client.upload_blob( file_content, content_type=content_type, metadata=metadata, diff --git a/src/tests/backend/common/storage/blob_azure_test.py b/src/tests/backend/common/storage/blob_azure_test.py index 68e5ad0d..afc76e7e 100644 --- a/src/tests/backend/common/storage/blob_azure_test.py +++ b/src/tests/backend/common/storage/blob_azure_test.py @@ -28,7 +28,6 @@ def mock_blob_service(): @pytest.fixture def blob_storage(mock_blob_service): """Fixture to initialize AzureBlobStorage with mocked dependencies""" - service_client, container_client, blob_client = mock_blob_service return AzureBlobStorage(account_name="test_account", container_name="test_container") From ae0e967abbe814950e611d99e68533faef891943 Mon Sep 17 00:00:00 2001 From: "Prekshith D J (Persistent Systems Inc)" Date: Mon, 22 Dec 2025 14:59:43 +0530 Subject: [PATCH 16/20] Updated the conditional --- src/frontend/src/pages/batchView.tsx | 2 +- src/frontend/src/pages/modernizationPage.tsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/frontend/src/pages/batchView.tsx b/src/frontend/src/pages/batchView.tsx index ba8b9de2..3ce9601a 100644 --- a/src/frontend/src/pages/batchView.tsx +++ b/src/frontend/src/pages/batchView.tsx @@ -385,7 +385,7 @@ const BatchStoryPage = () => { } // Show the summary page when summary is selected - if (selectedFile.id === "summary" && batchSummary) { + if (selectedFile.id === "summary") { // Check if there are no errors and all files are processed successfully const noErrors = (batchSummary.error_count === 0); const allFilesProcessed = (batchSummary.completed_files === batchSummary.total_files); diff --git a/src/frontend/src/pages/modernizationPage.tsx b/src/frontend/src/pages/modernizationPage.tsx index 489cbca6..21c0ce54 100644 --- a/src/frontend/src/pages/modernizationPage.tsx +++ b/src/frontend/src/pages/modernizationPage.tsx @@ -1251,7 +1251,7 @@ useEffect(() => { } // Show the full summary page only when all files are completed and summary is selected - if (allFilesCompleted && selectedFile?.id === "summary") { + if (selectedFile?.id === "summary") { const completedCount = files.filter(file => file.status === "completed" && file.file_result !== "error" && file.id !== "summary").length; const totalCount = files.filter(file => file.id !== "summary").length; const errorCount = selectedFile.errorCount || 0; From 183cd0df49dced66faa35b706498ae68ab9d0fb6 Mon Sep 17 00:00:00 2001 From: "Prekshith D J (Persistent Systems Inc)" Date: Mon, 22 Dec 2025 15:47:05 +0530 Subject: [PATCH 17/20] Updated the all the files --- .../src/components/batchHistoryPanel.tsx | 42 +------------------ src/frontend/src/components/bottomBar.tsx | 3 +- src/frontend/src/components/uploadButton.tsx | 2 - src/frontend/src/main.jsx | 2 +- src/frontend/src/pages/batchView.tsx | 37 +--------------- src/frontend/src/pages/landingPage.tsx | 1 - src/frontend/src/pages/modernizationPage.tsx | 7 ---- src/frontend/vite.config.js | 1 - 8 files changed, 4 insertions(+), 91 deletions(-) diff --git a/src/frontend/src/components/batchHistoryPanel.tsx b/src/frontend/src/components/batchHistoryPanel.tsx index 3440d784..49fd10cb 100644 --- a/src/frontend/src/components/batchHistoryPanel.tsx +++ b/src/frontend/src/components/batchHistoryPanel.tsx @@ -1,6 +1,6 @@ import React, { useState, useEffect, useRef } from "react"; -import { useDispatch, useSelector } from 'react-redux'; +import { useDispatch } from 'react-redux'; import { Card, Spinner, Tooltip } from "@fluentui/react-components"; import { useNavigate } from "react-router-dom"; import ConfirmationDialog from "../commonComponents/ConfirmationDialog/confirmationDialogue"; @@ -81,46 +81,6 @@ const HistoryPanel: React.FC = ({ isOpen, onClose }) => { } }; - // Function to categorize batches - const categorizeBatches = () => { - const now = new Date(); - const userTimeZone = Intl.DateTimeFormat().resolvedOptions().timeZone; - - // Get start of "Today", "Past 7 days", and "Past 30 days" in LOCAL time - const todayStart = new Date(now.getFullYear(), now.getMonth(), now.getDate()); - const past7DaysStart = new Date(todayStart); - const past30DaysStart = new Date(todayStart); - - past7DaysStart.setDate(todayStart.getDate() - 7); - past30DaysStart.setDate(todayStart.getDate() - 30); - - const todayBatches: BatchHistoryItem[] = []; - const past7DaysBatches: BatchHistoryItem[] = []; - const past30DaysBatches: BatchHistoryItem[] = []; - - batchHistory.forEach(batch => { - // Convert UTC timestamp to user's local date - const updatedAtUTC = new Date(batch.created_at); - const updatedAtLocal = new Date(updatedAtUTC.toLocaleString("en-US", { timeZone: userTimeZone })); - - // Extract only the local **date** part for comparison - const updatedDate = new Date(updatedAtLocal.getFullYear(), updatedAtLocal.getMonth(), updatedAtLocal.getDate()); - - // Categorize based on **exact day comparison** - if (updatedDate.getTime() === todayStart.getTime()) { - todayBatches.push(batch); - } else if (updatedDate.getTime() >= past7DaysStart.getTime()) { - past7DaysBatches.push(batch); - } else if (updatedDate.getTime() >= past30DaysStart.getTime()) { - past30DaysBatches.push(batch); - } - }); - - return { todayBatches, past7DaysBatches, past30DaysBatches }; - }; - - // const { todayBatches, past7DaysBatches, past30DaysBatches } = categorizeBatches(); - const deleteBatchFromHistory = (batchId: string) => { // Get the current URL path const currentPath = window.location.pathname; diff --git a/src/frontend/src/components/bottomBar.tsx b/src/frontend/src/components/bottomBar.tsx index 3041a988..c2702d2e 100644 --- a/src/frontend/src/components/bottomBar.tsx +++ b/src/frontend/src/components/bottomBar.tsx @@ -1,6 +1,5 @@ import { Button, Card, Dropdown, DropdownProps, Option } from "@fluentui/react-components" -import React, { useState } from "react" -import { useNavigate } from "react-router-dom" +import React from "react" // Define possible upload states const UploadState = { diff --git a/src/frontend/src/components/uploadButton.tsx b/src/frontend/src/components/uploadButton.tsx index a833f669..b6d90a1e 100644 --- a/src/frontend/src/components/uploadButton.tsx +++ b/src/frontend/src/components/uploadButton.tsx @@ -68,7 +68,6 @@ const FileUploadZone: React.FC = ({ const [uploadIntervals, setUploadIntervals] = useState<{ [key: string]: ReturnType }>({}); const [showCancelDialog, setShowCancelDialog] = useState(false); const [showLogoCancelDialog, setShowLogoCancelDialog] = useState(false); - const [uploadState, setUploadState] = useState<'IDLE' | 'UPLOADING' | 'COMPLETED'>('IDLE'); const [batchId, setBatchId] = useState(uuidv4()); const [allUploadsComplete, setAllUploadsComplete] = useState(false); const [fileLimitExceeded, setFileLimitExceeded] = useState(false); @@ -98,7 +97,6 @@ const FileUploadZone: React.FC = ({ } } - setUploadState(newState); onUploadStateChange?.(newState); }, [uploadingFiles, onUploadStateChange]); diff --git a/src/frontend/src/main.jsx b/src/frontend/src/main.jsx index 7ae29774..da9d477d 100644 --- a/src/frontend/src/main.jsx +++ b/src/frontend/src/main.jsx @@ -50,7 +50,7 @@ const Main = () => { const baseURL = config.API_URL.replace(/\/api$/, ''); // Remove '/api' if it appears at the end console.log('Checking connection to:', baseURL); try { - const response = await fetch(`${baseURL}/health`); + await fetch(`${baseURL}/health`); } catch (error) { console.error('Error connecting to backend:', error); } diff --git a/src/frontend/src/pages/batchView.tsx b/src/frontend/src/pages/batchView.tsx index 3ce9601a..df83a65f 100644 --- a/src/frontend/src/pages/batchView.tsx +++ b/src/frontend/src/pages/batchView.tsx @@ -33,7 +33,7 @@ import PanelRight from "../components/Panels/PanelRight"; import PanelRightToolbar from "../components/Panels/PanelRightToolbar"; import BatchHistoryPanel from "../components/batchHistoryPanel"; import ConfirmationDialog from "../commonComponents/ConfirmationDialog/confirmationDialogue"; -import { determineFileStatus, filesLogsBuilder, renderErrorSection, useStyles, renderFileError, filesErrorCounter, completedFiles, hasFiles, fileErrorCounter, BatchSummary, fileWarningCounter } from "../api/utils"; +import { determineFileStatus, filesLogsBuilder, renderErrorSection, useStyles, renderFileError, filesErrorCounter, completedFiles, hasFiles, fileErrorCounter, BatchSummary } from "../api/utils"; export const History = bundleIcon(HistoryFilled, HistoryRegular); import { format } from "sql-formatter"; @@ -73,7 +73,6 @@ const BatchStoryPage = () => { const [selectedFileId, setSelectedFileId] = useState(""); const [expandedSections, setExpandedSections] = useState(["errors"]); const [batchSummary, setBatchSummary] = useState(null); - const [selectedFileContent, setSelectedFileContent] = useState(""); const [selectedFileTranslatedContent, setSelectedFileTranslatedContent] = useState(""); @@ -209,40 +208,6 @@ const BatchStoryPage = () => { fetchFileContent(); }, [selectedFileId]); - - const renderWarningContent = () => { - if (!expandedSections.includes("warnings")) return null; - - if (!batchSummary) return null; - - // Group warnings by file - const warningFiles = files.filter(file => file.warningCount && file.warningCount > 0 && file.id !== "summary"); - - if (warningFiles.length === 0) { - return ( -
- No warnings found. -
- ); - } - - return ( -
- {warningFiles.map((file, fileIndex) => ( -
-
- {file.name} ({file.warningCount}) - source -
-
- Warning in file processing. See file for details. -
-
- ))} -
- ); - }; - const renderContent = () => { // Define header content based on selected file const renderHeader = () => { diff --git a/src/frontend/src/pages/landingPage.tsx b/src/frontend/src/pages/landingPage.tsx index b3707393..b86200e6 100644 --- a/src/frontend/src/pages/landingPage.tsx +++ b/src/frontend/src/pages/landingPage.tsx @@ -33,7 +33,6 @@ export const LandingPage = (): JSX.Element => { const dispatch = useDispatch(); // Add dispatch hook const [selectedTargetLanguage, setSelectedTargetLanguage] = useState(["T-SQL"]); const [selectedCurrentLanguage, setSelectedCurrentLanguage] = useState(["Informix"]); - const batchHistoryRef = useRef<{ triggerDeleteAll: () => void } | null>(null); const isPanelOpen = useSelector((state: RootState) => state.historyPanel.isOpen); const navigate = useNavigate(); diff --git a/src/frontend/src/pages/modernizationPage.tsx b/src/frontend/src/pages/modernizationPage.tsx index 21c0ce54..dc76f4bd 100644 --- a/src/frontend/src/pages/modernizationPage.tsx +++ b/src/frontend/src/pages/modernizationPage.tsx @@ -492,16 +492,13 @@ const ModernizationPage = () => { // State for the loading component const [showLoading, setShowLoading] = useState(true); - const [loadingError, setLoadingError] = useState(null); const [selectedFilebg, setSelectedFile] = useState(null); const [selectedFileId, setSelectedFileId] = React.useState(""); const [fileId, setFileId] = React.useState(""); const [expandedSections, setExpandedSections] = React.useState([]); - const [progressPercentage, setProgressPercentage] = useState(0); const [allFilesCompleted, setAllFilesCompleted] = useState(false); const [isZipButtonDisabled, setIsZipButtonDisabled] = useState(true); const [fileLoading, setFileLoading] = useState(false); - const [selectedFileTranslatedContent, setSelectedFileTranslatedContent] = useState(""); const [lastActivityTime, setLastActivityTime] = useState(Date.now()); const [pageLoadTime] = useState(Date.now()); @@ -517,11 +514,9 @@ const ModernizationPage = () => { if (!selectedFile || !selectedFile.translatedCode) { setFileLoading(true); const newFileUpdate = await fetchFileFromAPI(selectedFile?.fileId || ""); - setSelectedFileTranslatedContent(newFileUpdate.translatedContent); setFileLoading(false); } else { - setSelectedFileTranslatedContent(selectedFile.translatedCode); } } catch (err) { @@ -799,8 +794,6 @@ const ModernizationPage = () => { } }, [batchId]); - const highestProgressRef = useRef(0); - const currentProcessingFileRef = useRef(null); //new PT FR ends diff --git a/src/frontend/vite.config.js b/src/frontend/vite.config.js index 0d05262f..d239b70e 100644 --- a/src/frontend/vite.config.js +++ b/src/frontend/vite.config.js @@ -1,6 +1,5 @@ import { defineConfig } from 'vite' import react from '@vitejs/plugin-react' -import path from 'path' import tailwindcss from '@tailwindcss/vite' export default defineConfig({ From ead8581b14ceb8cc891a827d06aa5b982d193718 Mon Sep 17 00:00:00 2001 From: "Prekshith D J (Persistent Systems Inc)" Date: Mon, 22 Dec 2025 17:18:16 +0530 Subject: [PATCH 18/20] Fix the pylint issue --- tests/e2e-test/tests/test_codegen_gp_tc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e-test/tests/test_codegen_gp_tc.py b/tests/e2e-test/tests/test_codegen_gp_tc.py index 7a239f46..fb0431d6 100644 --- a/tests/e2e-test/tests/test_codegen_gp_tc.py +++ b/tests/e2e-test/tests/test_codegen_gp_tc.py @@ -12,7 +12,7 @@ ("01. Validate home page is loaded", lambda home: home.validate_home_page()), ("02. Validate Upload of other than SQL files", lambda home: home.upload_unsupported_files()), ("03. Validate Upload input files for SQL only", lambda home: home.upload_files()), - ("04. Validate translation process for uploaded files", _timed_translation), + ("04. Validate translation process for uploaded files",lambda home: _timed_translation(home)), ("05. Check batch history", lambda home: home.validate_batch_history()), ("06. Download all files and return home", lambda home: home.validate_download_files()), ] From 72e0dc5abd31f3f75a3485ee35dfb0eff28b5bc3 Mon Sep 17 00:00:00 2001 From: "Prekshith D J (Persistent Systems Inc)" Date: Mon, 22 Dec 2025 17:24:22 +0530 Subject: [PATCH 19/20] Added white space --- tests/e2e-test/tests/test_codegen_gp_tc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e-test/tests/test_codegen_gp_tc.py b/tests/e2e-test/tests/test_codegen_gp_tc.py index fb0431d6..6d9e896f 100644 --- a/tests/e2e-test/tests/test_codegen_gp_tc.py +++ b/tests/e2e-test/tests/test_codegen_gp_tc.py @@ -12,7 +12,7 @@ ("01. Validate home page is loaded", lambda home: home.validate_home_page()), ("02. Validate Upload of other than SQL files", lambda home: home.upload_unsupported_files()), ("03. Validate Upload input files for SQL only", lambda home: home.upload_files()), - ("04. Validate translation process for uploaded files",lambda home: _timed_translation(home)), + ("04. Validate translation process for uploaded files", lambda home: _timed_translation(home)), ("05. Check batch history", lambda home: home.validate_batch_history()), ("06. Download all files and return home", lambda home: home.validate_download_files()), ] From 0c670a6874273e7e766663794ebadb5e91f44233 Mon Sep 17 00:00:00 2001 From: Shreyas-Microsoft Date: Fri, 26 Dec 2025 12:06:17 +0530 Subject: [PATCH 20/20] add progressPercentage state --- src/frontend/src/pages/modernizationPage.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/src/frontend/src/pages/modernizationPage.tsx b/src/frontend/src/pages/modernizationPage.tsx index dc76f4bd..7f232148 100644 --- a/src/frontend/src/pages/modernizationPage.tsx +++ b/src/frontend/src/pages/modernizationPage.tsx @@ -497,6 +497,7 @@ const ModernizationPage = () => { const [fileId, setFileId] = React.useState(""); const [expandedSections, setExpandedSections] = React.useState([]); const [allFilesCompleted, setAllFilesCompleted] = useState(false); + const [progressPercentage, setProgressPercentage] = useState(0); const [isZipButtonDisabled, setIsZipButtonDisabled] = useState(true); const [fileLoading, setFileLoading] = useState(false); const [lastActivityTime, setLastActivityTime] = useState(Date.now());