irongit
Sign inRegister
huncholanehuncholane/Docker-Hackathon-2023Public

An AI project that includes a scraper for NFL data, postgres database, and user interface. It uses a classification algorithm to predict winners.

Docker-Hackathon-2023/nfl_client/url_utils.py
155 lines4.6 KBPython
RawDownloadHistory
1import yaml
2from urllib.parse import urlparse, parse_qs
3import re
4import json
5import inflect
6
7def unpluralize_word(word):
8 p = inflect.engine()
9 singular_form = p.singular_noun(word)
10 return singular_form if singular_form else word
11
12
13def add_brackets(word):
14 return '{'+word+'}'
15
16
17id_re = re.compile(
18 r"([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})")
19
20
21def urlparse_json(url):
22 r = urlparse(url)
23 path = r.path
24 query_params = {}
25 for key, item in parse_qs(r.query).items():
26 item_type = 'string'
27 example = ''
28 if len(item) > 1:
29 item_type = 'array'
30 example = ','.join(item)
31 if len(item) == 1:
32 example = item[0]
33 try:
34 example = float(item[0])
35 item_type = 'number'
36 except:
37 pass
38 try:
39 example = int(item[0])
40 item_type = 'integer'
41 except:
42 pass
43 if item[0] in ['true', 'false']:
44 item_type = 'boolean'
45 query_params.update({
46 key: {
47 'type': item_type,
48 'example': example
49 }
50 })
51
52 # Extract path parameters
53 path_params = {}
54 last_ind = -1
55 path_split = path.split('/')
56 for item in path.split('/'):
57 last = ''
58 if last_ind > 0:
59 last = path_split[last_ind]
60 id_match = id_re.search(item)
61 if id_match:
62 uuid = id_match.group(0)
63 name = unpluralize_word(last)+'_id'
64 path = path.replace(uuid, add_brackets(name))
65 path_params.update({name: {'type': 'string', 'example': uuid}})
66 try:
67 int(item)
68 name = unpluralize_word(last)
69 path = path.replace(item, add_brackets(name))
70 path_params.update({name: {'type': 'integer', 'example': int(item)}})
71 except:
72 pass
73 if item.isupper():
74 name = unpluralize_word(last)
75 path = path.replace(item, add_brackets(name))
76 path_params.update({name: {'type': 'string', 'example': item}})
77 last_ind += 1
78
79 return {
80 path: {
81 'query_parameters': query_params,
82 'path_parameters': path_params
83 }
84 }
85
86
87def har_entry_parse(entry):
88 """Parses an entry from har file into openapi ready json"""
89 url_data = urlparse_json(entry['request']['url'])
90 endpoint = next(iter(url_data))
91 url_data[endpoint]['response_sample'] = json.loads(entry['response']['content']['text'])
92 return url_data
93
94
95def add_api_component(openapi_spec, path, info):
96 """Adds info for a single endpoint"""
97 path_parameters = info['path_parameters']
98 query_parameters = info['query_parameters']
99 openapi_spec['paths'].update({
100 path: {
101 'get': {
102 'parameters': [],
103 'responses': {
104 '200': {
105 'description': 'Successful response',
106 'content': {
107 'application/json': {
108 'example': info['response_sample']
109 }
110 }
111 },
112 },
113 },
114 },
115 })
116
117 if path_parameters:
118 for param_name, param_details in path_parameters.items():
119 openapi_spec['paths'][path]['get']['parameters'].append({
120 'name': param_name,
121 'in': 'path',
122 'required': True,
123 'schema': {
124 'type': param_details['type'],
125 },
126 'description': f'The {param_name} parameter',
127 'example': param_details['example']
128 })
129
130 if query_parameters:
131 for param_name, param_details in query_parameters.items():
132 openapi_spec['paths'][path]['get']['parameters'].append({
133 'name': param_name,
134 'in': 'query',
135 'schema': {
136 'type': param_details['type'],
137 },
138 'description': f'The {param_name} parameter',
139 'example': param_details['example']
140 })
141
142
143def generate_openapi_spec(url_json):
144 """Takes in a dict of url json data to format into openapi"""
145 openapi_spec = {
146 'openapi': '3.0.0',
147 'info': {
148 'title': 'NFL API',
149 'version': '1.0.0',
150 },
151 'paths': {}
152 }
153 for path, info in url_json.items():
154 add_api_component(openapi_spec, path, info)
155 return yaml.dump(openapi_spec, default_flow_style=False)