ZAMAutoWebsite/processing.py at main · brycepm2/ZAMAutoWebsite · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
import time

from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker

from config import SQLALCHEMY_DATABASE_URI, SQLALCHEMY_ENGINE_OPTIONS
from flask_app import rfpJob

import rfpReaderBackend


engine = create_engine(
    SQLALCHEMY_DATABASE_URI, **SQLALCHEMY_ENGINE_OPTIONS
)
Session = sessionmaker(engine)


def find_pending_job():
    with Session.begin() as session:
        queue = session.query(rfpJob).filter_by(state="Queued")
        if job := queue.first():
            print("Found Job!!!")
            job.state = "Running!!!"
            return job.slug


def getJobData(slug):
    print(f"Processing job: {slug}...", end=" ", flush=True)
    with Session.begin() as session:
        data = session.query(rfpJob).filter_by(slug=slug).all()
        # only work on most recent job
        job = data[-1]
        if job != None:
            prompt = job.prompt
            reportNum = job.reportNum
            rfpStart = job.rfpStart
            rfpEnd = job.rfpEnd
            useGPT4 = job.useGPT4
            return prompt, reportNum, rfpStart, \
                rfpEnd, useGPT4

def runJob(slug, prompt, reportNum, rfpStart, rfpEnd, useGPT4):
    # start clock
    sTime = time.time()
    # scrape desired rfp text
    rfpText, rfpStart, rfpEnd = rfpReaderBackend.getRFPReport(reportNum, rfpStart, rfpEnd)
    if rfpText != False:
        if (useGPT4 == True):
            model = 'gpt-4-1106-preview'
        else:
            model = 'gpt-3.5-turbo-1106'
        print("Probing RFPs", flush=True)
        print("using: ", model)
        outPdfFullPath = '/home/brycepm2/ZAMAutoWebsite/assets/pdfOut/RFPSummary'
        totalCost = rfpReaderBackend.probeRFPs_JSON(rfpText, prompt, model, outPdfFullPath)
        eTime = time.time()
        result = eTime - sTime
        state = "Done!!!"
        print(f"done after {result} seconds!", flush=True)
        print(f"totalCost =  {totalCost}", flush=True)
    else:
        # there are no RFPs in this report
        print("There are no RFPs in this report!!!")
        result = -1
        state = "No RFPs in Report!!!"
    updateJob(slug, rfpStart, rfpEnd, state, totalCost, result)
    return 0

def updateJob(slug, rfpStart, rfpEnd, state, totalCost, result):
    with Session.begin() as session:
        data = session.query(rfpJob).filter_by(slug=slug).all()
        # only work on most recent job
        job = data[-1]
        job.rfpStart = rfpStart
        job.rfpEnd = rfpEnd
        job.state = "Done!!!"
        job.totalCost = totalCost
        job.result = result

if __name__ == "__main__":
    while True:
        if slug := find_pending_job():
            prompt, reportNum, rfpStart, \
                rfpEnd, useGPT4 = getJobData(slug)
            iErr = runJob(slug, prompt, reportNum, rfpStart,
                          rfpEnd, useGPT4)
        else:
            time.sleep(1)