huncholaneAn AI project that includes a scraper for NFL data, postgres database, and user interface. It uses a classification algorithm to predict winners.
| 1 | { |
| 2 | "cells": [ |
| 3 | { |
| 4 | "cell_type": "markdown", |
| 5 | "metadata": {}, |
| 6 | "source": [ |
| 7 | "## 1. Set up the scraper\n", |
| 8 | "* Download data from nfl.com\n", |
| 9 | "* Use internal api with Bearer Key" |
| 10 | ] |
| 11 | }, |
| 12 | { |
| 13 | "cell_type": "markdown", |
| 14 | "metadata": {}, |
| 15 | "source": [ |
| 16 | "### 1.1 Get a working example for Bearer token" |
| 17 | ] |
| 18 | }, |
| 19 | { |
| 20 | "cell_type": "code", |
| 21 | "execution_count": 2, |
| 22 | "metadata": {}, |
| 23 | "outputs": [], |
| 24 | "source": [ |
| 25 | "import time\n", |
| 26 | "from browsermobproxy import Server\n", |
| 27 | "from selenium import webdriver\n", |
| 28 | "import json\n", |
| 29 | "import os\n", |
| 30 | "BMP_PATH = os.path.abspath('browsermob-proxy-2.1.4/bin/browsermob-proxy.bat')\n", |
| 31 | "\n", |
| 32 | "# Start BrowserMob Proxy\n", |
| 33 | "server = Server(BMP_PATH)\n", |
| 34 | "server.start()\n", |
| 35 | "proxy = server.create_proxy()\n", |
| 36 | "\n", |
| 37 | "# Configure Chrome with the proxy\n", |
| 38 | "chrome_options = webdriver.ChromeOptions()\n", |
| 39 | "chrome_options.add_argument(f'--proxy-server={proxy.proxy}')\n", |
| 40 | "chrome_options.add_argument('--ignore-certificate-errors')\n", |
| 41 | "chrome_options.add_argument('--headless')\n", |
| 42 | "driver = webdriver.Chrome(options=chrome_options)\n", |
| 43 | "# Navigate to the website\n", |
| 44 | "driver.get(\"https://nfl.com/scores\")\n", |
| 45 | "\n", |
| 46 | "# Start capturing network traffic\n", |
| 47 | "proxy.new_har(\"nfl\", options={'captureHeaders': True})\n", |
| 48 | "\n", |
| 49 | "# Wait for the header with the name \"Authorization\"\n", |
| 50 | "timeout = 60 # Adjust the timeout as needed\n", |
| 51 | "def get_auth_token():\n", |
| 52 | " start_time = time.time()\n", |
| 53 | " while time.time() - start_time < timeout:\n", |
| 54 | " har = proxy.har\n", |
| 55 | " for entry in har['log']['entries']:\n", |
| 56 | " request_headers = entry['request']['headers']\n", |
| 57 | " for header in request_headers:\n", |
| 58 | " if header['name'] == 'Authorization':\n", |
| 59 | " return header['value']\n", |
| 60 | " time.sleep(1)\n", |
| 61 | " return None\n", |
| 62 | "auth_token = get_auth_token()\n", |
| 63 | "\n", |
| 64 | "# Store the auth token\n", |
| 65 | "auth_path = 'auth.json'\n", |
| 66 | "auth_json = {\n", |
| 67 | " 'time': time.time(),\n", |
| 68 | " 'token': auth_token\n", |
| 69 | "}\n", |
| 70 | "with open(auth_path, 'w') as f:\n", |
| 71 | " json.dump(auth_json, f)\n", |
| 72 | "\n", |
| 73 | "# Stop capturing network traffic\n", |
| 74 | "har = proxy.har\n", |
| 75 | "\n", |
| 76 | "# Close the browser and BrowserMob Proxy\n", |
| 77 | "driver.quit()\n", |
| 78 | "server.stop()" |
| 79 | ] |
| 80 | }, |
| 81 | { |
| 82 | "cell_type": "markdown", |
| 83 | "metadata": {}, |
| 84 | "source": [ |
| 85 | "### 1.2 Create a class that manages requests to nfl.com" |
| 86 | ] |
| 87 | }, |
| 88 | { |
| 89 | "cell_type": "code", |
| 90 | "execution_count": 5, |
| 91 | "metadata": {}, |
| 92 | "outputs": [], |
| 93 | "source": [ |
| 94 | "from nfl_client import NFLClient\n", |
| 95 | "client = NFLClient(headless=True)" |
| 96 | ] |
| 97 | }, |
| 98 | { |
| 99 | "cell_type": "code", |
| 100 | "execution_count": 7, |
| 101 | "metadata": {}, |
| 102 | "outputs": [ |
| 103 | { |
| 104 | "name": "stdout", |
| 105 | "output_type": "stream", |
| 106 | "text": [ |
| 107 | "Auth Token: Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJjbGllbnRJZCI6ImU1MzVjN2MwLTgxN2YtNDc3Ni04OTkwLTU2NTU2ZjhiMTkyOCIsImNsaWVudEtleSI6IjRjRlVXNkRtd0pwelQ5TDdMckczcVJBY0FCRzVzMDRnIiwiaXNzIjoiTkZMIiwiZGV2aWNlSWQiOiI0YTIzZGM2My00YmQ2LTQ0OTMtYWVmNC0yNDk0OGRkNGMxYTUiLCJwbGFucyI6W3sicGxhbiI6ImZyZWUiLCJleHBpcmF0aW9uRGF0ZSI6IjIwMjQtMTEtMDQiLCJzb3VyY2UiOiJORkwiLCJzdGFydERhdGUiOiIyMDIzLTExLTA1Iiwic3RhdHVzIjoiQUNUSVZFIiwidHJpYWwiOmZhbHNlfV0sIkRpc3BsYXlOYW1lIjoiV0VCX0RFU0tUT1BfREVTS1RPUCIsIk5vdGVzIjoiIiwiZm9ybUZhY3RvciI6IkRFU0tUT1AiLCJsdXJhQXBwS2V5IjoiU1pzNTdkQkdSeGJMNzI4bFZwN0RZUSIsInBsYXRmb3JtIjoiREVTS1RPUCIsInByb2R1Y3ROYW1lIjoiV0VCIiwiY2l0eSI6ImFwcGxlIHZhbGxleSIsImNvdW50cnlDb2RlIjoiVVMiLCJkbWFDb2RlIjoiODAzIiwiaG1hVGVhbXMiOlsiMTA0MDI1MTAtODkzMS0wZDVmLTk4MTUtNzliYjc5NjQ5YTY1IiwiMTA0MDI1MjAtOTZiZi1lOWYyLTRmNjgtODUyMWNhODk2MDYwIiwiMTA0MDQ0MDAtM2IzNS0wNzNmLTE5N2UtMTk0YmI4MjQwNzIzIl0sInJlZ2lvbiI6IkNBIiwiYnJvd3NlciI6IkNocm9tZSBIZWFkbGVzcyIsImNlbGx1bGFyIjpmYWxzZSwiZW52aXJvbm1lbnQiOiJwcm9kdWN0aW9uIiwicm9sZXMiOlsiZnJlZSJdLCJleHAiOjE2OTkxNjQ3MTh9.l60UboGxeZ4ma5dfBircoc9ok_X25aerWyTFFxYTJjo\n" |
| 108 | ] |
| 109 | } |
| 110 | ], |
| 111 | "source": [ |
| 112 | "client.load_auth_token(store_har=True)\n", |
| 113 | "print('Auth Token:', client.auth_token)" |
| 114 | ] |
| 115 | }, |
| 116 | { |
| 117 | "cell_type": "markdown", |
| 118 | "metadata": {}, |
| 119 | "source": [ |
| 120 | "### 1.3 Convert har entry into json\n", |
| 121 | "1. Url to json\n", |
| 122 | "2. Convert har to json " |
| 123 | ] |
| 124 | }, |
| 125 | { |
| 126 | "cell_type": "code", |
| 127 | "execution_count": 1, |
| 128 | "metadata": {}, |
| 129 | "outputs": [ |
| 130 | { |
| 131 | "data": { |
| 132 | "text/plain": [ |
| 133 | "{'/football/v2/games/season/{season}/seasonType/REG/week/{week}': {'query_parameters': {},\n", |
| 134 | " 'path_parameters': {'season': {'type': 'integer', 'example': 2023},\n", |
| 135 | " 'week': {'type': 'integer', 'example': 9}}}}" |
| 136 | ] |
| 137 | }, |
| 138 | "execution_count": 1, |
| 139 | "metadata": {}, |
| 140 | "output_type": "execute_result" |
| 141 | } |
| 142 | ], |
| 143 | "source": [ |
| 144 | "from nfl_client.url_utils import urlparse_json\n", |
| 145 | "url = '/football/v2/games/season/2023/seasonType/REG/week/9'\n", |
| 146 | "urlparse_json(url)" |
| 147 | ] |
| 148 | }, |
| 149 | { |
| 150 | "cell_type": "code", |
| 151 | "execution_count": null, |
| 152 | "metadata": {}, |
| 153 | "outputs": [], |
| 154 | "source": [ |
| 155 | "from nfl_client.url_utils import har_entry_parse\n", |
| 156 | "import json\n", |
| 157 | "with open('nfl_client_data/sample_har_entry.json', 'r') as f:\n", |
| 158 | " data = json.load(f)\n", |
| 159 | "\n", |
| 160 | "url_data = har_entry_parse(data)\n", |
| 161 | "url_data" |
| 162 | ] |
| 163 | }, |
| 164 | { |
| 165 | "cell_type": "markdown", |
| 166 | "metadata": {}, |
| 167 | "source": [ |
| 168 | "#### 1.3.1 Create something to convert this into openapi" |
| 169 | ] |
| 170 | }, |
| 171 | { |
| 172 | "cell_type": "code", |
| 173 | "execution_count": 6, |
| 174 | "metadata": {}, |
| 175 | "outputs": [ |
| 176 | { |
| 177 | "name": "stdout", |
| 178 | "output_type": "stream", |
| 179 | "text": [ |
| 180 | "{'/football/v2/games/season/{season}/blah/{blah_id}': {'query_parameters': {'withExternalIds': {'type': 'boolean', 'example': 'true'}}, 'path_parameters': {'blah_id': {'type': 'string', 'example': 'b07eaa67-f053-11ed-b4a7-bab79e4492fa'}}, 'response_sample': {'id': 'b07eaa67-f053-11ed-b4a7-bab79e4492fa', 'homeTeam': {'id': '10402310-a47e-10ea-7442-16b633633637', 'currentLogo': 'https://static.www.nfl.com/{formatInstructions}/league/api/clubs/logos/KC', 'fullName': 'Kansas City Chiefs'}, 'awayTeam': {'id': '10402700-1662-d8ad-f45c-0b0ea460d045', 'currentLogo': 'https://static.www.nfl.com/{formatInstructions}/league/api/clubs/logos/MIA', 'fullName': 'Miami Dolphins'}, 'category': None, 'date': '2023-11-05', 'time': '2023-11-05T14:30:00Z', 'broadcastInfo': {'homeNetworkChannels': ['NFL NETWORK'], 'awayNetworkChannels': ['NFL NETWORK'], 'internationalWatchOptions': [{'countryCode': 'CA', 'broadcasters': ['DAZN Canada', 'RDS', 'TSN']}, {'countryCode': 'GB', 'broadcasters': ['Sky Sports', 'NFL Game Pass on DAZN']}, {'countryCode': 'MX', 'broadcasters': ['NFL Game Pass on DAZN']}, {'countryCode': 'DE', 'broadcasters': ['NFL Game Pass on DAZN', 'DAZN Germany (OTT)', 'RTL']}, {'countryCode': 'AU', 'broadcasters': ['NFL Game Pass on DAZN', 'ESPN', 'TVNZ+']}, {'countryCode': 'FR', 'broadcasters': ['NFL Game Pass on DAZN', 'beIN 2', '6Play']}, {'countryCode': 'BR', 'broadcasters': ['NFL Game Pass on DAZN', 'ESPN2']}], 'territory': 'NATIONAL'}, 'neutralSite': None, 'venue': {'id': '00082031-5500-35b5-a67a-7f374ebf7168', 'name': 'Frankfurt Stadium'}, 'season': 2023, 'seasonType': 'REG', 'status': 'SCHEDULED', 'week': 9, 'weekType': 'REG', 'externalIds': [{'source': 'elias', 'id': '2023110500'}, {'source': 'gsis', 'id': '59296'}, {'source': 'slug', 'id': 'dolphins-at-chiefs-2023-reg-9'}], 'ticketUrl': None, 'ticketVendors': []}}}\n" |
| 181 | ] |
| 182 | } |
| 183 | ], |
| 184 | "source": [ |
| 185 | "from nfl_client.url_utils import generate_openapi_spec\n", |
| 186 | "\n", |
| 187 | "# Example\n", |
| 188 | "print(url_data)\n", |
| 189 | "openapi_yaml = generate_openapi_spec(url_data)\n", |
| 190 | "with open('openapi.yaml', 'w') as f:\n", |
| 191 | " f.write(openapi_yaml)" |
| 192 | ] |
| 193 | }, |
| 194 | { |
| 195 | "cell_type": "markdown", |
| 196 | "metadata": {}, |
| 197 | "source": [ |
| 198 | "#### 1.3.2 Put it all together" |
| 199 | ] |
| 200 | }, |
| 201 | { |
| 202 | "cell_type": "code", |
| 203 | "execution_count": 1, |
| 204 | "metadata": {}, |
| 205 | "outputs": [ |
| 206 | { |
| 207 | "data": { |
| 208 | "text/plain": [ |
| 209 | "8" |
| 210 | ] |
| 211 | }, |
| 212 | "execution_count": 1, |
| 213 | "metadata": {}, |
| 214 | "output_type": "execute_result" |
| 215 | } |
| 216 | ], |
| 217 | "source": [ |
| 218 | "from nfl_client import NFLClient\n", |
| 219 | "client = NFLClient()\n", |
| 220 | "url_json = client.download_endpoints()\n", |
| 221 | "len(url_json)" |
| 222 | ] |
| 223 | }, |
| 224 | { |
| 225 | "cell_type": "markdown", |
| 226 | "metadata": {}, |
| 227 | "source": [ |
| 228 | "## 2. Collect data into tables from the endpoints\n", |
| 229 | "\n", |
| 230 | "* /football/v2/games/season/{season}/seasonType/{seasonType}/week/{week}\n", |
| 231 | "* /football/v2/standings\n", |
| 232 | "* /football/v2/stats/live/game-summaries/{game-summary_id}" |
| 233 | ] |
| 234 | }, |
| 235 | { |
| 236 | "cell_type": "code", |
| 237 | "execution_count": 4, |
| 238 | "metadata": {}, |
| 239 | "outputs": [ |
| 240 | { |
| 241 | "name": "stdout", |
| 242 | "output_type": "stream", |
| 243 | "text": [ |
| 244 | "https://api.nfl.com/football/v2/teams/history\n", |
| 245 | "AFC Pro Bowl Team\n", |
| 246 | "Arizona Cardinals\n", |
| 247 | "Atlanta Falcons\n", |
| 248 | "Baltimore Ravens\n", |
| 249 | "Buffalo Bills\n", |
| 250 | "Carolina Panthers\n", |
| 251 | "Chicago Bears\n", |
| 252 | "Cincinnati Bengals\n", |
| 253 | "Cleveland Browns\n", |
| 254 | "Dallas Cowboys\n", |
| 255 | "Denver Broncos\n", |
| 256 | "Detroit Lions\n", |
| 257 | "Green Bay Packers\n", |
| 258 | "Houston Texans\n", |
| 259 | "Indianapolis Colts\n", |
| 260 | "Jacksonville Jaguars\n", |
| 261 | "Kansas City Chiefs\n", |
| 262 | "Las Vegas Raiders\n", |
| 263 | "Los Angeles Chargers\n", |
| 264 | "Los Angeles Rams\n", |
| 265 | "Miami Dolphins\n", |
| 266 | "Minnesota Vikings\n", |
| 267 | "New England Patriots\n", |
| 268 | "New Orleans Saints\n", |
| 269 | "New York Giants\n", |
| 270 | "New York Jets\n", |
| 271 | "NFC Pro Bowl Team\n", |
| 272 | "Philadelphia Eagles\n", |
| 273 | "Pittsburgh Steelers\n", |
| 274 | "San Francisco 49ers\n", |
| 275 | "Seattle Seahawks\n", |
| 276 | "Tampa Bay Buccaneers\n", |
| 277 | "Tennessee Titans\n", |
| 278 | "Washington Commanders\n" |
| 279 | ] |
| 280 | } |
| 281 | ], |
| 282 | "source": [ |
| 283 | "from nfl_client import NFLClient\n", |
| 284 | "client = NFLClient()\n", |
| 285 | "r = client.get_teams(2023)\n", |
| 286 | "for team in r.json()['teams']:\n", |
| 287 | " print(team['fullName'])" |
| 288 | ] |
| 289 | }, |
| 290 | { |
| 291 | "cell_type": "code", |
| 292 | "execution_count": null, |
| 293 | "metadata": {}, |
| 294 | "outputs": [], |
| 295 | "source": [] |
| 296 | } |
| 297 | ], |
| 298 | "metadata": { |
| 299 | "kernelspec": { |
| 300 | "display_name": "Python 3", |
| 301 | "language": "python", |
| 302 | "name": "python3" |
| 303 | }, |
| 304 | "language_info": { |
| 305 | "codemirror_mode": { |
| 306 | "name": "ipython", |
| 307 | "version": 3 |
| 308 | }, |
| 309 | "file_extension": ".py", |
| 310 | "mimetype": "text/x-python", |
| 311 | "name": "python", |
| 312 | "nbconvert_exporter": "python", |
| 313 | "pygments_lexer": "ipython3", |
| 314 | "version": "3.10.9" |
| 315 | } |
| 316 | }, |
| 317 | "nbformat": 4, |
| 318 | "nbformat_minor": 2 |
| 319 | } |