| 1 | import json |
| 2 | import os |
| 3 | import re |
| 4 | |
| 5 | from dotenv import load_dotenv |
| 6 | from openai import OpenAI |
| 7 | |
| 8 | load_dotenv() |
| 9 | JSON_REGEX = re.compile(r"```json(.*)```", re.DOTALL) |
| 10 | |
| 11 | |
| 12 | def parse_lead_file(file): |
| 13 | |
| 14 | message_file = client.files.create(file=file, purpose="assistants") |
| 15 | content = """ |
| 16 | Parse the leads from the PDF files into a json list in the following format. |
| 17 | [ |
| 18 | { |
| 19 | "todays_date": "", |
| 20 | "agent_name": "ANDREA", |
| 21 | "roofing_company": "CORE FOUR - AUSTIN", |
| 22 | "names": "BORAN ZHAO & TANIA BETANCOURT", |
| 23 | "appointment_date": "4/18/24", |
| 24 | "time": "2PM", |
| 25 | "phone": "979-218-4997", |
| 26 | "email": "TANIA@TXSTATE.EDU", |
| 27 | "address": "524 CARISMATIC LN", |
| 28 | "city": "AUSTIN", |
| 29 | "state": "TX", |
| 30 | "zip_code": "78748", |
| 31 | "additional_address": "", |
| 32 | "insurance_provider": "METROPOLITAN/FARMERS", |
| 33 | "age_of_roof": "3 years", |
| 34 | "animals_in_yard": "Yes", |
| 35 | "last_roof_inspection": "", |
| 36 | "notes": "", |
| 37 | "contact_number": "303-908-3193" |
| 38 | } |
| 39 | ] |
| 40 | """ |
| 41 | thread = client.beta.threads.create( |
| 42 | messages=[ |
| 43 | { |
| 44 | "role": "user", |
| 45 | "content": content, |
| 46 | "attachments": [ |
| 47 | {"file_id": message_file.id, "tools": [{"type": "file_search"}]} |
| 48 | ], |
| 49 | } |
| 50 | ] |
| 51 | ) |
| 52 | run = client.beta.threads.runs.create_and_poll( |
| 53 | thread_id=thread.id, assistant_id=assistant.id |
| 54 | ) |
| 55 | messages = list( |
| 56 | client.beta.threads.messages.list(thread_id=thread.id, run_id=run.id) |
| 57 | ) |
| 58 | return messages[0].content[0].text.value |
| 59 | |
| 60 | |
| 61 | |
| 62 | pdf_files = [] |
| 63 | file_streams = [] |
| 64 | for root, dirs, files in os.walk(".data/leads"): |
| 65 | for file in files: |
| 66 | if file.endswith(".pdf"): |
| 67 | pdf_files.append(os.path.join(root, file)) |
| 68 | file_streams.append(open(os.path.join(root, file), "rb")) |
| 69 | |
| 70 | |
| 71 | client = OpenAI(api_key=os.getenv("OPENAI_API_KEY")) |
| 72 | assistant = client.beta.assistants.create( |
| 73 | name="Lead Parser", |
| 74 | description="Parse leads from PDF files", |
| 75 | model="gpt-3.5-turbo", |
| 76 | tools=[{"type": "file_search"}], |
| 77 | ) |
| 78 | |
| 79 | |
| 80 | file = file_streams[0] |
| 81 | json_leads = [] |
| 82 | for file in file_streams: |
| 83 | message = parse_lead_file(file) |
| 84 | json_match = JSON_REGEX.search(message) |
| 85 | json_content = json_match.group(1) |
| 86 | parsed_json = json.loads(json_content) |
| 87 | json_leads.extend(parsed_json) |
| 88 | |
| 89 | import pandas as pd |
| 90 | |
| 91 | df = pd.DataFrame(json_leads) |
| 92 | df.to_csv("leads.csv", index=False) |
huncholane