| 1 | import json |
| 2 | import os |
| 3 | import re |
| 4 | |
| 5 | from dotenv import load_dotenv |
| 6 | from openai import OpenAI |
| 7 | |
| 8 | load_dotenv() |
| 9 | JSON_REGEX = re.compile(r"```json(.*)```", re.DOTALL) |
| 10 | |
| 11 | |
| 12 | pdf_files = [] |
| 13 | file_streams = [] |
| 14 | for root, dirs, files in os.walk(".data/leads"): |
| 15 | for file in files: |
| 16 | if file.endswith(".pdf"): |
| 17 | pdf_files.append(os.path.join(root, file)) |
| 18 | file_streams.append(open(os.path.join(root, file), "rb")) |
| 19 | |
| 20 | |
| 21 | client = OpenAI(api_key=os.getenv("OPENAI_API_KEY")) |
| 22 | assistant = client.beta.assistants.create( |
| 23 | name="Lead Parser", |
| 24 | description="Parse leads from PDF files", |
| 25 | model="gpt-3.5-turbo", |
| 26 | tools=[{"type": "file_search"}], |
| 27 | ) |
| 28 | |
| 29 | |
| 30 | file = file_streams[0] |
| 31 | message_file = client.files.create(file=file, purpose="assistants") |
| 32 | content = """ |
| 33 | Parse the leads from the PDF files into a json list in the following format. |
| 34 | [ |
| 35 | { |
| 36 | "todays_date": "", |
| 37 | "agent_name": "ANDREA", |
| 38 | "roofing_company": "CORE FOUR - AUSTIN", |
| 39 | "names": "BORAN ZHAO & TANIA BETANCOURT", |
| 40 | "appointment_date": "4/18/24", |
| 41 | "time": "2PM", |
| 42 | "phone": "979-218-4997", |
| 43 | "email": "TANIA@TXSTATE.EDU", |
| 44 | "address": "524 CARISMATIC LN", |
| 45 | "city": "AUSTIN", |
| 46 | "state": "TX", |
| 47 | "zip_code": "78748", |
| 48 | "additional_address": "", |
| 49 | "insurance_provider": "METROPOLITAN/FARMERS", |
| 50 | "age_of_roof": "3 years", |
| 51 | "animals_in_yard": "Yes", |
| 52 | "last_roof_inspection": "", |
| 53 | "notes": "", |
| 54 | "contact_number": "303-908-3193" |
| 55 | } |
| 56 | ] |
| 57 | """ |
| 58 | thread = client.beta.threads.create( |
| 59 | messages=[ |
| 60 | { |
| 61 | "role": "user", |
| 62 | "content": content, |
| 63 | "attachments": [ |
| 64 | {"file_id": message_file.id, "tools": [{"type": "file_search"}]} |
| 65 | ], |
| 66 | } |
| 67 | ] |
| 68 | ) |
| 69 | run = client.beta.threads.runs.create_and_poll( |
| 70 | thread_id=thread.id, assistant_id=assistant.id |
| 71 | ) |
| 72 | messages = list(client.beta.threads.messages.list(thread_id=thread.id, run_id=run.id)) |
| 73 | message_content = messages[0].content[0].text.value |
| 74 | json_match = JSON_REGEX.search(message_content) |
| 75 | if json_match: |
| 76 | json_content = json_match.group(1) |
| 77 | parsed_json = json.loads(json_content) |
| 78 | print(json.dumps(parsed_json, indent=2)) |
| 79 | else: |
| 80 | print("No JSON content found.") |
| 81 | print(message_content) |
huncholane