| 1 | import json |
| 2 | import os |
| 3 | import re |
| 4 | |
| 5 | import pandas as pd |
| 6 | from dotenv import load_dotenv |
| 7 | from openai import OpenAI |
| 8 | |
| 9 | load_dotenv() |
| 10 | JSON_REGEX = re.compile(r"```json(.*)```", re.DOTALL) |
| 11 | API_KEY = os.getenv("OPENAI_API_KEY", None) |
| 12 | if API_KEY is None: |
| 13 | print("Please set the OPENAI_API_KEY environment variable.") |
| 14 | exit(1) |
| 15 | |
| 16 | |
| 17 | def parse_lead_file(filename) -> str: |
| 18 | |
| 19 | file = open(filename, "rb") |
| 20 | message_file = client.files.create(file=file, purpose="assistants") |
| 21 | content = """ |
| 22 | Parse the leads from the PDF files into a json list in the following format. |
| 23 | [ |
| 24 | { |
| 25 | "today_date": "", |
| 26 | "agent_name": "ANDREA", |
| 27 | "roofing_company": "CORE FOUR - AUSTIN", |
| 28 | "names": "BORAN ZHAO & TANIA BETANCOURT", |
| 29 | "appointment_date": "4/18/24", |
| 30 | "time": "2PM", |
| 31 | "phone": "979-218-4997", |
| 32 | "email": "TANIA@TXSTATE.EDU", |
| 33 | "address": "524 CARISMATIC LN", |
| 34 | "city": "AUSTIN", |
| 35 | "state": "TX", |
| 36 | "zip_code": "78748", |
| 37 | "additional_address": "", |
| 38 | "insurance_provider": "METROPOLITAN/FARMERS", |
| 39 | "age_of_roof": "3 years", |
| 40 | "animals_in_yard": "Yes", |
| 41 | "last_roof_inspection": "", |
| 42 | "notes": "", |
| 43 | } |
| 44 | ] |
| 45 | """ |
| 46 | thread = client.beta.threads.create( |
| 47 | messages=[ |
| 48 | { |
| 49 | "role": "user", |
| 50 | "content": content, |
| 51 | "attachments": [ |
| 52 | {"file_id": message_file.id, "tools": [{"type": "file_search"}]} |
| 53 | ], |
| 54 | } |
| 55 | ] |
| 56 | ) |
| 57 | run = client.beta.threads.runs.create_and_poll( |
| 58 | thread_id=thread.id, assistant_id=assistant.id |
| 59 | ) |
| 60 | messages = list( |
| 61 | client.beta.threads.messages.list(thread_id=thread.id, run_id=run.id) |
| 62 | ) |
| 63 | return messages[0].content[0].text.value |
| 64 | |
| 65 | |
| 66 | def pdf_to_json(leads: list, filename: str, max_retries=3, attempt=0): |
| 67 | |
| 68 | message = parse_lead_file(filename) |
| 69 | json_match = JSON_REGEX.search(message) |
| 70 | try: |
| 71 | json_content = json_match.group(1) |
| 72 | parsed_json = json.loads(json_content) |
| 73 | leads.extend(parsed_json) |
| 74 | except Exception as e: |
| 75 | print( |
| 76 | f"Error parsing JSON from {filename}:\n{e}\nCurrently on attempt {attempt + 1} of {max_retries}" |
| 77 | ) |
| 78 | pdf_to_json(leads, filename, max_retries, attempt + 1) |
| 79 | |
| 80 | |
| 81 | |
| 82 | pdf_files = [] |
| 83 | for root, dirs, files in os.walk(".data/leads"): |
| 84 | for file in files: |
| 85 | if file.endswith(".pdf"): |
| 86 | pdf_files.append(os.path.join(root, file)) |
| 87 | |
| 88 | |
| 89 | client = OpenAI(api_key=os.getenv("OPENAI_API_KEY")) |
| 90 | assistant = client.beta.assistants.create( |
| 91 | name="Lead Parser", |
| 92 | description="Parse leads from PDF files", |
| 93 | model="gpt-3.5-turbo", |
| 94 | tools=[{"type": "file_search"}], |
| 95 | ) |
| 96 | |
| 97 | |
| 98 | from threading import Thread |
| 99 | |
| 100 | json_leads = [] |
| 101 | threads = [ |
| 102 | Thread(target=pdf_to_json, args=(json_leads, filename), daemon=True) |
| 103 | for filename in pdf_files |
| 104 | ] |
| 105 | for thread in threads: |
| 106 | thread.start() |
| 107 | for thread in threads: |
| 108 | thread.join() |
| 109 | |
| 110 | |
| 111 | df = pd.DataFrame(json_leads) |
| 112 | df.to_csv(".data/leads.csv", index=False) |
huncholane