Typesense Maintenance Sync #6
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Typesense Maintenance Sync | |
| on: | |
| # Manual trigger only - for maintenance and special operations | |
| workflow_dispatch: | |
| inputs: | |
| operation_type: | |
| description: 'Type of maintenance operation' | |
| required: true | |
| type: choice | |
| options: | |
| - 'incremental-sync' | |
| - 'full-reload' | |
| default: 'incremental-sync' | |
| confirm_deletion: | |
| description: 'For full-reload: Type "DELETE" to confirm collection deletion' | |
| required: false | |
| type: string | |
| default: '' | |
| start_date: | |
| description: 'Start date for data sync (YYYY-MM-DD)' | |
| required: true | |
| default: '2024-01-01' | |
| type: string | |
| end_date: | |
| description: 'End date for data sync (YYYY-MM-DD, default: today)' | |
| required: false | |
| type: string | |
| batch_size: | |
| description: 'Batch size for Typesense upsert operations' | |
| required: false | |
| default: '1000' | |
| type: number | |
| max_records: | |
| description: 'Max records to sync (for testing, 0 = unlimited)' | |
| required: false | |
| default: '0' | |
| type: number | |
| skip_portal_refresh: | |
| description: 'Skip portal cache refresh after sync' | |
| required: false | |
| default: false | |
| type: boolean | |
| env: | |
| PYTHON_VERSION: '3.12' | |
| COLLECTION_NAME: 'news' | |
| jobs: | |
| maintenance-sync: | |
| name: Typesense Maintenance Sync | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| id-token: write | |
| steps: | |
| - name: Validate confirmation input | |
| if: github.event.inputs.operation_type == 'full-reload' | |
| run: | | |
| if [ "${{ github.event.inputs.confirm_deletion }}" != "DELETE" ]; then | |
| echo "❌ Confirmation failed. You must type 'DELETE' for full-reload." | |
| echo " This operation will permanently delete the collection and reload all data." | |
| exit 1 | |
| fi | |
| echo "✅ Confirmation validated for full-reload operation" | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Authenticate to Google Cloud | |
| uses: google-github-actions/auth@v2 | |
| with: | |
| workload_identity_provider: ${{ vars.GCP_WORKLOAD_IDENTITY_PROVIDER }} | |
| service_account: ${{ vars.GCP_SERVICE_ACCOUNT }} | |
| - name: Fetch Typesense Config | |
| id: typesense | |
| uses: destaquesgovbr/reusable-workflows/actions/fetch-typesense-config@v1 | |
| with: | |
| workload_identity_provider: ${{ vars.GCP_WORKLOAD_IDENTITY_PROVIDER }} | |
| service_account: ${{ vars.GCP_SERVICE_ACCOUNT }} | |
| secret_name: typesense-write-conn | |
| - name: Fetch Database URL | |
| id: database | |
| run: | | |
| DATABASE_URL=$(gcloud secrets versions access latest --secret=govbrnews-postgres-connection-string) | |
| echo "::add-mask::$DATABASE_URL" | |
| echo "url=$DATABASE_URL" >> $GITHUB_OUTPUT | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Install Poetry | |
| run: | | |
| curl -sSL https://install.python-poetry.org | python3 - | |
| echo "$HOME/.local/bin" >> $GITHUB_PATH | |
| - name: Cache Poetry dependencies | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/pypoetry | |
| key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-poetry- | |
| - name: Install dependencies | |
| run: | | |
| poetry install --no-interaction --only main | |
| - name: Delete existing collection | |
| if: github.event.inputs.operation_type == 'full-reload' | |
| continue-on-error: true | |
| env: | |
| TYPESENSE_HOST: ${{ steps.typesense.outputs.host }} | |
| TYPESENSE_PORT: ${{ steps.typesense.outputs.port }} | |
| TYPESENSE_API_KEY: ${{ steps.typesense.outputs.api_key }} | |
| run: | | |
| echo "🗑️ Deleting collection '${{ env.COLLECTION_NAME }}'..." | |
| if poetry run data-platform typesense-delete --confirm; then | |
| echo "✅ Collection deleted successfully" | |
| else | |
| echo "⚠️ Collection deletion reported failure" | |
| echo " This may be expected if collection doesn't exist" | |
| echo " Continuing with data load (will create new collection)..." | |
| fi | |
| # Always succeed this step to allow workflow to continue | |
| exit 0 | |
| - name: Calculate date range | |
| id: dates | |
| run: | | |
| START_DATE="${{ github.event.inputs.start_date }}" | |
| END_DATE="${{ github.event.inputs.end_date }}" | |
| if [ -z "$END_DATE" ]; then | |
| END_DATE=$(date +%Y-%m-%d) | |
| fi | |
| echo "start_date=$START_DATE" >> $GITHUB_OUTPUT | |
| echo "end_date=$END_DATE" >> $GITHUB_OUTPUT | |
| echo "📅 Date range: $START_DATE to $END_DATE" | |
| - name: Run data sync | |
| env: | |
| TYPESENSE_HOST: ${{ steps.typesense.outputs.host }} | |
| TYPESENSE_PORT: ${{ steps.typesense.outputs.port }} | |
| TYPESENSE_API_KEY: ${{ steps.typesense.outputs.api_key }} | |
| DATABASE_URL: ${{ steps.database.outputs.url }} | |
| run: | | |
| OPERATION="${{ github.event.inputs.operation_type }}" | |
| if [ "$OPERATION" = "full-reload" ]; then | |
| echo "📥 Starting FULL data reload from PostgreSQL..." | |
| FULL_SYNC_FLAG="--full-sync" | |
| else | |
| echo "📥 Starting INCREMENTAL data sync from PostgreSQL..." | |
| FULL_SYNC_FLAG="" | |
| fi | |
| CMD="poetry run data-platform sync-typesense \ | |
| --start-date \"${{ steps.dates.outputs.start_date }}\" \ | |
| --end-date \"${{ steps.dates.outputs.end_date }}\" \ | |
| --batch-size ${{ github.event.inputs.batch_size }} \ | |
| $FULL_SYNC_FLAG" | |
| if [ "${{ github.event.inputs.max_records }}" != "0" ]; then | |
| CMD="$CMD --max-records ${{ github.event.inputs.max_records }}" | |
| fi | |
| echo "Running: $CMD" | |
| eval $CMD | |
| if [ $? -eq 0 ]; then | |
| echo "✅ Data sync completed successfully" | |
| else | |
| echo "❌ Data sync failed" | |
| exit 1 | |
| fi | |
| - name: Verify data load | |
| env: | |
| TYPESENSE_HOST: ${{ steps.typesense.outputs.host }} | |
| TYPESENSE_PORT: ${{ steps.typesense.outputs.port }} | |
| TYPESENSE_API_KEY: ${{ steps.typesense.outputs.api_key }} | |
| run: | | |
| echo "🔍 Verifying collection health..." | |
| poetry run data-platform typesense-list | |
| - name: Trigger portal cache refresh | |
| if: success() && github.event.inputs.skip_portal_refresh != 'true' | |
| run: | | |
| echo "🔄 Triggering portal deployment to refresh cache..." | |
| gcloud run services update destaquesgovbr-portal \ | |
| --region southamerica-east1 \ | |
| --update-env-vars "CACHE_BUST=$(date +%s)" \ | |
| --quiet | |
| - name: Report final status | |
| if: always() | |
| run: | | |
| echo "================================" | |
| if [ ${{ job.status }} == 'success' ]; then | |
| OPERATION="${{ github.event.inputs.operation_type }}" | |
| echo "✅ Maintenance sync completed successfully" | |
| echo "" | |
| echo "Summary:" | |
| echo " - Operation type: $OPERATION" | |
| if [ "$OPERATION" = "full-reload" ]; then | |
| echo " - Collection '${{ env.COLLECTION_NAME }}' deleted and recreated" | |
| fi | |
| echo " - Date range: ${{ steps.dates.outputs.start_date }} to ${{ steps.dates.outputs.end_date }}" | |
| echo " - Batch size: ${{ github.event.inputs.batch_size }}" | |
| if [ "${{ github.event.inputs.max_records }}" != "0" ]; then | |
| echo " - Max records: ${{ github.event.inputs.max_records }}" | |
| fi | |
| echo " - Collection verified" | |
| if [ "${{ github.event.inputs.skip_portal_refresh }}" != "true" ]; then | |
| echo " - Portal cache refreshed" | |
| fi | |
| else | |
| echo "❌ Maintenance sync failed" | |
| echo "" | |
| echo "Check the logs above for details." | |
| exit 1 | |
| fi | |
| echo "================================" |