run_complete_s3_workflow.py

#!/usr/bin/env python3
"""
Master S3 PDF Manager Script

This script runs the complete workflow:
1. Find all PDF files in folders
2. Create S3 bucket
3. Upload PDFs to S3
4. Extract questions from JSON files
5. Create comprehensive catalog with URLs and questions
"""

import subprocess
import sys
import os

def run_command(command, description):
    """Run a command and handle errors"""
    print(f"\n🚀 {description}")
    print("-" * 50)
    
    try:
        if isinstance(command, str):
            result = subprocess.run(command, shell=True, check=True, text=True)
        else:
            result = subprocess.run(command, check=True, text=True)
        
        print(f"✅ {description} completed successfully")
        return True
        
    except subprocess.CalledProcessError as e:
        print(f"❌ {description} failed with error: {e}")
        return False
    except Exception as e:
        print(f"❌ Unexpected error in {description}: {e}")
        return False

def check_prerequisites():
    """Check if all prerequisites are met"""
    print("🔍 Checking prerequisites...")
    
    # Check Python
    if sys.version_info < (3, 6):
        print("❌ Python 3.6+ required")
        return False
    
    # Check AWS CLI
    try:
        subprocess.run(['aws', '--version'], capture_output=True, check=True)
        print("✅ AWS CLI found")
    except:
        print("❌ AWS CLI not found. Please install AWS CLI")
        return False
    
    # Check AWS configuration
    try:
        subprocess.run(['aws', 'sts', 'get-caller-identity'], capture_output=True, check=True)
        print("✅ AWS credentials configured")
    except:
        print("❌ AWS not configured. Please run 'aws configure'")
        return False
    
    # Check if scripts exist
    scripts = ['s3_pdf_manager.py', 'extract_questions.py']
    for script in scripts:
        if not os.path.exists(script):
            print(f"❌ {script} not found")
            return False
    
    print("✅ All prerequisites met")
    return True

def main():
    """Main workflow"""
    print("="*60)
    print("🚀 MASTER S3 PDF MANAGER")
    print("="*60)
    print("This script will:")
    print("1. 🔍 Search for all PDF files in folders")
    print("2. 🪣 Create S3 bucket (if needed)")
    print("3. 📤 Upload PDFs to S3 with public URLs")
    print("4. 📋 Extract questions from existing JSON files")
    print("5. 🔗 Create comprehensive catalog with everything")
    print("="*60)
    
    # Check prerequisites
    if not check_prerequisites():
        print("\n❌ Prerequisites not met. Please fix the issues above.")
        sys.exit(1)
    
    # Ask for confirmation
    response = input("\n➡️ Continue with the process? (y/N): ").lower().strip()
    if response != 'y' and response != 'yes':
        print("❌ Process cancelled by user")
        sys.exit(0)
    
    print("\n🎬 Starting the complete workflow...")
    
    success_count = 0
    total_steps = 2
    
    # Step 1: Run S3 PDF Manager (includes bucket creation, upload, and basic catalog)
    if run_command(['python3', 's3_pdf_manager.py'], 
                   "Step 1: S3 PDF Upload and Catalog Creation"):
        success_count += 1
    
    # Step 2: Extract questions and merge with catalog
    if run_command(['python3', 'extract_questions.py'], 
                   "Step 2: Question Extraction and Merging"):
        success_count += 1
    
    # Summary
    print("\n" + "="*60)
    print("📊 WORKFLOW SUMMARY")
    print("="*60)
    
    if success_count == total_steps:
        print("🎉 All steps completed successfully!")
        print("\n📄 Generated files:")
        
        files_to_check = [
            ("s3_pdf_catalog.json", "Basic S3 catalog with PDF URLs"),
            ("question_bank.json", "Extracted questions by document and topic"),
            ("s3_pdf_catalog_with_questions.json", "Complete catalog with URLs and questions")
        ]
        
        for filename, description in files_to_check:
            if os.path.exists(filename):
                size_kb = os.path.getsize(filename) / 1024
                print(f"   ✅ {filename} ({size_kb:.1f} KB) - {description}")
            else:
                print(f"   ❌ {filename} - Not found")
        
        print(f"\n💡 Usage tips:")
        print(f"   • Use 's3_pdf_catalog_with_questions.json' for complete data")
        print(f"   • Public URLs are ready to use in your applications")
        print(f"   • Questions are organized by document and topic")
        
    else:
        print(f"⚠️ {success_count}/{total_steps} steps completed successfully")
        print("❌ Some steps failed. Check the output above for details.")
        sys.exit(1)

if __name__ == "__main__":
    main()