huncholaneAn AI project that includes a scraper for NFL data, postgres database, and user interface. It uses a classification algorithm to predict winners.
| 1 | { |
| 2 | "cells": [ |
| 3 | { |
| 4 | "cell_type": "markdown", |
| 5 | "metadata": {}, |
| 6 | "source": [ |
| 7 | "## 1. Set up the scraper\n", |
| 8 | "* Download data from nfl.com\n", |
| 9 | "* Use internal api with Bearer Key" |
| 10 | ] |
| 11 | }, |
| 12 | { |
| 13 | "cell_type": "markdown", |
| 14 | "metadata": {}, |
| 15 | "source": [ |
| 16 | "### 1.1 Get a working example for Bearer token" |
| 17 | ] |
| 18 | }, |
| 19 | { |
| 20 | "cell_type": "code", |
| 21 | "execution_count": 2, |
| 22 | "metadata": {}, |
| 23 | "outputs": [], |
| 24 | "source": [ |
| 25 | "import time\n", |
| 26 | "from browsermobproxy import Server\n", |
| 27 | "from selenium import webdriver\n", |
| 28 | "import json\n", |
| 29 | "import os\n", |
| 30 | "BMP_PATH = os.path.abspath('browsermob-proxy-2.1.4/bin/browsermob-proxy.bat')\n", |
| 31 | "\n", |
| 32 | "# Start BrowserMob Proxy\n", |
| 33 | "server = Server(BMP_PATH)\n", |
| 34 | "server.start()\n", |
| 35 | "proxy = server.create_proxy()\n", |
| 36 | "\n", |
| 37 | "# Configure Chrome with the proxy\n", |
| 38 | "chrome_options = webdriver.ChromeOptions()\n", |
| 39 | "chrome_options.add_argument(f'--proxy-server={proxy.proxy}')\n", |
| 40 | "chrome_options.add_argument('--ignore-certificate-errors')\n", |
| 41 | "chrome_options.add_argument('--headless')\n", |
| 42 | "driver = webdriver.Chrome(options=chrome_options)\n", |
| 43 | "# Navigate to the website\n", |
| 44 | "driver.get(\"https://nfl.com/scores\")\n", |
| 45 | "\n", |
| 46 | "# Start capturing network traffic\n", |
| 47 | "proxy.new_har(\"nfl\", options={'captureHeaders': True})\n", |
| 48 | "\n", |
| 49 | "# Wait for the header with the name \"Authorization\"\n", |
| 50 | "timeout = 60 # Adjust the timeout as needed\n", |
| 51 | "def get_auth_token():\n", |
| 52 | " start_time = time.time()\n", |
| 53 | " while time.time() - start_time < timeout:\n", |
| 54 | " har = proxy.har\n", |
| 55 | " for entry in har['log']['entries']:\n", |
| 56 | " request_headers = entry['request']['headers']\n", |
| 57 | " for header in request_headers:\n", |
| 58 | " if header['name'] == 'Authorization':\n", |
| 59 | " return header['value']\n", |
| 60 | " time.sleep(1)\n", |
| 61 | " return None\n", |
| 62 | "auth_token = get_auth_token()\n", |
| 63 | "\n", |
| 64 | "# Store the auth token\n", |
| 65 | "auth_path = 'auth.json'\n", |
| 66 | "auth_json = {\n", |
| 67 | " 'time': time.time(),\n", |
| 68 | " 'token': auth_token\n", |
| 69 | "}\n", |
| 70 | "with open(auth_path, 'w') as f:\n", |
| 71 | " json.dump(auth_json, f)\n", |
| 72 | "\n", |
| 73 | "# Stop capturing network traffic\n", |
| 74 | "har = proxy.har\n", |
| 75 | "\n", |
| 76 | "# Close the browser and BrowserMob Proxy\n", |
| 77 | "driver.quit()\n", |
| 78 | "server.stop()" |
| 79 | ] |
| 80 | }, |
| 81 | { |
| 82 | "cell_type": "markdown", |
| 83 | "metadata": {}, |
| 84 | "source": [ |
| 85 | "### 1.2 Create a class that manages requests to nfl.com" |
| 86 | ] |
| 87 | }, |
| 88 | { |
| 89 | "cell_type": "code", |
| 90 | "execution_count": 8, |
| 91 | "metadata": {}, |
| 92 | "outputs": [], |
| 93 | "source": [ |
| 94 | "import time\n", |
| 95 | "from browsermobproxy import Server\n", |
| 96 | "from selenium import webdriver\n", |
| 97 | "from pathlib import Path\n", |
| 98 | "import json\n", |
| 99 | "import requests\n", |
| 100 | "import os\n", |
| 101 | "\n", |
| 102 | "class NFLClient():\n", |
| 103 | " \"\"\"Manages access to the nfl.com api\"\"\"\n", |
| 104 | " BMP_PATH = os.path.abspath('browsermob-proxy-2.1.4/bin/browsermob-proxy.bat')\n", |
| 105 | " AUTH_PATH = Path('auth.json')\n", |
| 106 | " AUTH_ENDPOINT = 'https://nfl.com/scores'\n", |
| 107 | " API_ROOT = 'https://api.nfl.com'\n", |
| 108 | " HAR_DIR = Path('nfl_client_data/har_files')\n", |
| 109 | " URL_JSON_PATH = Path('nfl_client_data/urls.json')\n", |
| 110 | " TOKEN_EXPIRE_RATE = 60*60\n", |
| 111 | " TOKEN_AUTH_TIMEOUT = 60\n", |
| 112 | "\n", |
| 113 | " # Har download variables\n", |
| 114 | " har = {}\n", |
| 115 | " har_path = None\n", |
| 116 | " server = None\n", |
| 117 | " proxy = None\n", |
| 118 | " driver = None\n", |
| 119 | "\n", |
| 120 | " # Auth variables\n", |
| 121 | " auth_token = None\n", |
| 122 | " last_auth_download_time = 0\n", |
| 123 | " headers = {}\n", |
| 124 | "\n", |
| 125 | " def __init__(self):\n", |
| 126 | " os.makedirs(self.HAR_DIR, exist_ok=True)\n", |
| 127 | "\n", |
| 128 | " def load_auth_token(self):\n", |
| 129 | " \"\"\"Loads the auth token into the client.\"\"\"\n", |
| 130 | " if os.path.exists(self.AUTH_PATH):\n", |
| 131 | " with open(self.AUTH_PATH, 'r') as f:\n", |
| 132 | " auth_json = json.load(f)\n", |
| 133 | " self.auth_token = auth_json['token']\n", |
| 134 | " self.last_auth_download_time = auth_json['time']\n", |
| 135 | " if self.last_auth_download_time < time.time()-self.TOKEN_EXPIRE_RATE or not self.auth_token:\n", |
| 136 | " self.download_auth_token()\n", |
| 137 | "\n", |
| 138 | " def prep_proxy(self, endpoint=None):\n", |
| 139 | " \"\"\"Prepares the driver and proxy to get the har\"\"\"\n", |
| 140 | " if not endpoint:\n", |
| 141 | " endpoint = self.AUTH_ENDPOINT\n", |
| 142 | " # Start BrowserMob Proxy\n", |
| 143 | " self.server = Server(self.BMP_PATH)\n", |
| 144 | " self.server.start()\n", |
| 145 | " self.proxy = self.server.create_proxy()\n", |
| 146 | "\n", |
| 147 | " # Configure Chrome with the proxy\n", |
| 148 | " chrome_options = webdriver.ChromeOptions()\n", |
| 149 | " chrome_options.add_argument(f'--proxy-server={self.proxy.proxy}')\n", |
| 150 | " chrome_options.add_argument('--ignore-certificate-errors')\n", |
| 151 | " chrome_options.add_argument('--headless')\n", |
| 152 | " self.driver = webdriver.Chrome(options=chrome_options)\n", |
| 153 | " # Navigate to the website\n", |
| 154 | " self.driver.get(endpoint)\n", |
| 155 | "\n", |
| 156 | " # Start capturing network traffic\n", |
| 157 | " self.proxy.new_har(\"nfl\", options={'captureHeaders': True})\n", |
| 158 | "\n", |
| 159 | " # Create the path for storing har data\n", |
| 160 | " try:\n", |
| 161 | " raw_path = endpoint.split('.com')[1]\n", |
| 162 | " raw_path = raw_path.split('?')[0]\n", |
| 163 | " str_path = raw_path.replace('/', '__')\n", |
| 164 | " except:\n", |
| 165 | " str_path = 'nfl'\n", |
| 166 | " self.har_path = Path(str_path+'.har')\n", |
| 167 | "\n", |
| 168 | " def close_proxy(self):\n", |
| 169 | " \"\"\"Closes the driver and proxy\"\"\"\n", |
| 170 | " self.server.stop()\n", |
| 171 | " self.driver.quit()\n", |
| 172 | "\n", |
| 173 | " def wait_for_auth_token(self):\n", |
| 174 | " \"\"\"Downloads the auth token using the scores page of nfl.com\"\"\"\n", |
| 175 | " # Wait for the header with the name \"Authorization\"\n", |
| 176 | " def get_auth_token():\n", |
| 177 | " start_time = time.time()\n", |
| 178 | " while time.time() - start_time < self.TOKEN_AUTH_TIMEOUT:\n", |
| 179 | " self.har = self.proxy.har\n", |
| 180 | " for entry in self.har['log']['entries']:\n", |
| 181 | " request_headers = entry['request']['headers']\n", |
| 182 | " for header in request_headers:\n", |
| 183 | " if header['name'] == 'Authorization':\n", |
| 184 | " return header['value']\n", |
| 185 | " time.sleep(1)\n", |
| 186 | " return None\n", |
| 187 | " self.auth_token = get_auth_token()\n", |
| 188 | "\n", |
| 189 | " def store_auth_token(self):\n", |
| 190 | " \"\"\"Stores the current auth token\"\"\"\n", |
| 191 | " # Store the auth token\n", |
| 192 | " self.time = time.time()\n", |
| 193 | " auth_json = {\n", |
| 194 | " 'time': self.time,\n", |
| 195 | " 'token': self.auth_token\n", |
| 196 | " }\n", |
| 197 | " with open(self.AUTH_PATH, 'w') as f:\n", |
| 198 | " json.dump(auth_json, f)\n", |
| 199 | "\n", |
| 200 | " def store_har(self):\n", |
| 201 | " \"\"\"Stores the current har\"\"\"\n", |
| 202 | " # Store the har as well\n", |
| 203 | " with open(self.har_path, 'w') as f:\n", |
| 204 | " json.dump(self.har, f)\n", |
| 205 | "\n", |
| 206 | " def download_auth_token(self):\n", |
| 207 | " \"\"\"Downloads the auth token\"\"\"\n", |
| 208 | " self.prep_proxy() \n", |
| 209 | " self.wait_for_auth_token()\n", |
| 210 | " self.store_auth_token()\n", |
| 211 | " self.store_har()\n", |
| 212 | "\n", |
| 213 | " def download_endpoints(self, endpoint=None):\n", |
| 214 | " \"\"\"Downloads the endpoints found in har, stores json, updates readme, updates Mixin for the class\"\"\"\n", |
| 215 | " if not self.har:\n", |
| 216 | " self.prep_proxy(endpoint)\n", |
| 217 | " self.wait_for_auth_token()\n", |
| 218 | " self.store_auth_token()\n", |
| 219 | " self.store_har()\n", |
| 220 | " self.close_proxy()\n", |
| 221 | "\n", |
| 222 | " # Gather the list of endpoints\n", |
| 223 | " url_list = []\n", |
| 224 | " for entry in self.har['log']['entries']:\n", |
| 225 | " request = entry['request']\n", |
| 226 | " raw_url = request['url']\n", |
| 227 | " if self.API_ROOT in raw_url:\n", |
| 228 | " url_split = raw_url.split('?')\n", |
| 229 | " full_url = url_split[0]\n", |
| 230 | " new_endpoint = ''\n", |
| 231 | " base_split = full_url.split('.com') \n", |
| 232 | " if len(base_split) > 1:\n", |
| 233 | " new_endpoint = base_split[1]\n", |
| 234 | " param_list = []\n", |
| 235 | " if len(url_split) > 1:\n", |
| 236 | " for param in url_split[1]:\n", |
| 237 | " param, val = param.split('=')\n", |
| 238 | " param_list.append({\n", |
| 239 | " 'param': param,\n", |
| 240 | " 'val': val\n", |
| 241 | " }) \n", |
| 242 | " url_list.append({\n", |
| 243 | " 'raw_url': raw_url,\n", |
| 244 | " 'endpoint': new_endpoint,\n", |
| 245 | " 'params': param_list\n", |
| 246 | " })\n", |
| 247 | "\n", |
| 248 | " # Store the url list json\n", |
| 249 | " with open(self.URL_JSON_PATH, 'w') as f:\n", |
| 250 | " json.dump(url_list, f)\n", |
| 251 | "\n", |
| 252 | " def request(self, endpoint) -> dict:\n", |
| 253 | " \"\"\"Request an endpoint\"\"\"\n", |
| 254 | " self.load_auth_token()\n", |
| 255 | " try:\n", |
| 256 | " return requests.get(endpoint, headers=self.headers)\n", |
| 257 | " except Exception as e:\n", |
| 258 | " return {\n", |
| 259 | " 'error': e\n", |
| 260 | " } " |
| 261 | ] |
| 262 | }, |
| 263 | { |
| 264 | "cell_type": "code", |
| 265 | "execution_count": 9, |
| 266 | "metadata": {}, |
| 267 | "outputs": [ |
| 268 | { |
| 269 | "data": { |
| 270 | "text/plain": [ |
| 271 | "'Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJjbGllbnRJZCI6ImU1MzVjN2MwLTgxN2YtNDc3Ni04OTkwLTU2NTU2ZjhiMTkyOCIsImNsaWVudEtleSI6IjRjRlVXNkRtd0pwelQ5TDdMckczcVJBY0FCRzVzMDRnIiwiaXNzIjoiTkZMIiwiZGV2aWNlSWQiOiJjNDYxYzk0Ny00NjlhLTRmYTQtODI0OS0wYmZhYTA4MTcyMGUiLCJwbGFucyI6W3sicGxhbiI6ImZyZWUiLCJleHBpcmF0aW9uRGF0ZSI6IjIwMjQtMTEtMDQiLCJzb3VyY2UiOiJORkwiLCJzdGFydERhdGUiOiIyMDIzLTExLTA1Iiwic3RhdHVzIjoiQUNUSVZFIiwidHJpYWwiOmZhbHNlfV0sIkRpc3BsYXlOYW1lIjoiV0VCX0RFU0tUT1BfREVTS1RPUCIsIk5vdGVzIjoiIiwiZm9ybUZhY3RvciI6IkRFU0tUT1AiLCJsdXJhQXBwS2V5IjoiU1pzNTdkQkdSeGJMNzI4bFZwN0RZUSIsInBsYXRmb3JtIjoiREVTS1RPUCIsInByb2R1Y3ROYW1lIjoiV0VCIiwiY2l0eSI6ImFwcGxlIHZhbGxleSIsImNvdW50cnlDb2RlIjoiVVMiLCJkbWFDb2RlIjoiODAzIiwiaG1hVGVhbXMiOlsiMTA0MDI1MTAtODkzMS0wZDVmLTk4MTUtNzliYjc5NjQ5YTY1IiwiMTA0MDI1MjAtOTZiZi1lOWYyLTRmNjgtODUyMWNhODk2MDYwIiwiMTA0MDQ0MDAtM2IzNS0wNzNmLTE5N2UtMTk0YmI4MjQwNzIzIl0sInJlZ2lvbiI6IkNBIiwiYnJvd3NlciI6IkNocm9tZSBIZWFkbGVzcyIsImNlbGx1bGFyIjpmYWxzZSwiZW52aXJvbm1lbnQiOiJwcm9kdWN0aW9uIiwicm9sZXMiOlsiZnJlZSJdLCJleHAiOjE2OTkxNTUwNTV9.hFywyQGtR_PyFOTOezj0JxB0UkDQ6dU8yPgakipFagY'" |
| 272 | ] |
| 273 | }, |
| 274 | "execution_count": 9, |
| 275 | "metadata": {}, |
| 276 | "output_type": "execute_result" |
| 277 | } |
| 278 | ], |
| 279 | "source": [ |
| 280 | "client = NFLClient()\n", |
| 281 | "client.load_auth_token()\n", |
| 282 | "client.auth_token" |
| 283 | ] |
| 284 | }, |
| 285 | { |
| 286 | "cell_type": "code", |
| 287 | "execution_count": null, |
| 288 | "metadata": {}, |
| 289 | "outputs": [], |
| 290 | "source": [] |
| 291 | }, |
| 292 | { |
| 293 | "cell_type": "code", |
| 294 | "execution_count": 103, |
| 295 | "metadata": {}, |
| 296 | "outputs": [ |
| 297 | { |
| 298 | "data": { |
| 299 | "text/plain": [ |
| 300 | "'Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJjbGllbnRJZCI6ImU1MzVjN2MwLTgxN2YtNDc3Ni04OTkwLTU2NTU2ZjhiMTkyOCIsImNsaWVudEtleSI6IjRjRlVXNkRtd0pwelQ5TDdMckczcVJBY0FCRzVzMDRnIiwiaXNzIjoiTkZMIiwiZGV2aWNlSWQiOiI3ZWRmZDk4Yy03MDY0LTQyMTUtYjFiMi0yOTY4MGFhODgyNzgiLCJwbGFucyI6W3sicGxhbiI6ImZyZWUiLCJleHBpcmF0aW9uRGF0ZSI6IjIwMjQtMTEtMDQiLCJzb3VyY2UiOiJORkwiLCJzdGFydERhdGUiOiIyMDIzLTExLTA0Iiwic3RhdHVzIjoiQUNUSVZFIiwidHJpYWwiOmZhbHNlfV0sIkRpc3BsYXlOYW1lIjoiV0VCX0RFU0tUT1BfREVTS1RPUCIsIk5vdGVzIjoiIiwiZm9ybUZhY3RvciI6IkRFU0tUT1AiLCJsdXJhQXBwS2V5IjoiU1pzNTdkQkdSeGJMNzI4bFZwN0RZUSIsInBsYXRmb3JtIjoiREVTS1RPUCIsInByb2R1Y3ROYW1lIjoiV0VCIiwiY2l0eSI6ImFwcGxlIHZhbGxleSIsImNvdW50cnlDb2RlIjoiVVMiLCJkbWFDb2RlIjoiODAzIiwiaG1hVGVhbXMiOlsiMTA0MDI1MTAtODkzMS0wZDVmLTk4MTUtNzliYjc5NjQ5YTY1IiwiMTA0MDI1MjAtOTZiZi1lOWYyLTRmNjgtODUyMWNhODk2MDYwIiwiMTA0MDQ0MDAtM2IzNS0wNzNmLTE5N2UtMTk0YmI4MjQwNzIzIl0sInJlZ2lvbiI6IkNBIiwiYnJvd3NlciI6IkNocm9tZSIsImNlbGx1bGFyIjpmYWxzZSwiZW52aXJvbm1lbnQiOiJwcm9kdWN0aW9uIiwicm9sZXMiOlsiZnJlZSJdLCJleHAiOjE2OTkxNDA3OTB9.L9qs6t9J4yU89RO4Jq1u3YUEIxsmxQ4ffZeXzmHlGn0'" |
| 301 | ] |
| 302 | }, |
| 303 | "execution_count": 103, |
| 304 | "metadata": {}, |
| 305 | "output_type": "execute_result" |
| 306 | } |
| 307 | ], |
| 308 | "source": [ |
| 309 | "import json\n", |
| 310 | "\n", |
| 311 | "\n", |
| 312 | "def get_token():\n", |
| 313 | " with open('nfl.har', 'r') as f:\n", |
| 314 | " har_json = json.load(f)\n", |
| 315 | " for entry in har_json['log']['entries']:\n", |
| 316 | " for header in entry['request']['headers']:\n", |
| 317 | " if header['name'].lower() == 'authorization':\n", |
| 318 | " return header['value']\n", |
| 319 | "get_token()\n" |
| 320 | ] |
| 321 | }, |
| 322 | { |
| 323 | "cell_type": "code", |
| 324 | "execution_count": 93, |
| 325 | "metadata": {}, |
| 326 | "outputs": [ |
| 327 | { |
| 328 | "name": "stdout", |
| 329 | "output_type": "stream", |
| 330 | "text": [ |
| 331 | "Authorization header found in the HAR\n" |
| 332 | ] |
| 333 | } |
| 334 | ], |
| 335 | "source": [ |
| 336 | "from browsermobproxy import Server\n", |
| 337 | "from selenium import webdriver\n", |
| 338 | "from selenium.webdriver.common.by import By\n", |
| 339 | "from selenium.webdriver.support.ui import WebDriverWait\n", |
| 340 | "from selenium.webdriver.support import expected_conditions as EC\n", |
| 341 | "\n", |
| 342 | "# Start BrowserMob Proxy\n", |
| 343 | "server = Server(BMP_PATH)\n", |
| 344 | "server.start()\n", |
| 345 | "proxy = server.create_proxy()\n", |
| 346 | "\n", |
| 347 | "# Configure Chrome with the proxy\n", |
| 348 | "chrome_options = webdriver.ChromeOptions()\n", |
| 349 | "chrome_options.add_argument(f'--proxy-server={proxy.proxy}')\n", |
| 350 | "chrome_options.add_argument('--ignore-certificate-errors')\n", |
| 351 | "driver = webdriver.Chrome(options=chrome_options)\n", |
| 352 | "\n", |
| 353 | "# Navigate to the website\n", |
| 354 | "driver.get(\"https://nfl.com\")\n", |
| 355 | "\n", |
| 356 | "# Start capturing network traffic\n", |
| 357 | "proxy.new_har(\"nfl\", options={'captureHeaders': True})\n", |
| 358 | "\n", |
| 359 | "# Perform interactions on the website as needed\n", |
| 360 | "\n", |
| 361 | "# Wait for the \"Authorization\" header to appear in the HAR\n", |
| 362 | "def is_authorization_in_har(har):\n", |
| 363 | " for entry in har['log']['entries']:\n", |
| 364 | " for header in entry['request']['headers']:\n", |
| 365 | " if header['name'] == 'Authorization':\n", |
| 366 | " return True\n", |
| 367 | " return False\n", |
| 368 | "\n", |
| 369 | "wait = WebDriverWait(driver, 60) # Adjust the timeout as needed\n", |
| 370 | "try:\n", |
| 371 | " wait.until(lambda driver: is_authorization_in_har(proxy.har))\n", |
| 372 | " print(\"Authorization header found in the HAR\")\n", |
| 373 | "except Exception:\n", |
| 374 | " print(\"Authorization header not found in the HAR\")\n", |
| 375 | "\n", |
| 376 | "# Close the browser and BrowserMob Proxy\n", |
| 377 | "driver.quit()\n", |
| 378 | "server.stop()\n" |
| 379 | ] |
| 380 | }, |
| 381 | { |
| 382 | "cell_type": "code", |
| 383 | "execution_count": 38, |
| 384 | "metadata": {}, |
| 385 | "outputs": [], |
| 386 | "source": [ |
| 387 | "from selenium import webdriver\n", |
| 388 | "\n", |
| 389 | "\n", |
| 390 | "options = webdriver.ChromeOptions()\n", |
| 391 | "options.add_argument('--headless')\n", |
| 392 | "\n", |
| 393 | "driver = webdriver.Chrome(options)\n", |
| 394 | "\n", |
| 395 | "root_url = 'https://google.com'\n", |
| 396 | "root_url = 'https://nfl.com'\n", |
| 397 | "driver.get(root_url)" |
| 398 | ] |
| 399 | }, |
| 400 | { |
| 401 | "cell_type": "code", |
| 402 | "execution_count": 48, |
| 403 | "metadata": {}, |
| 404 | "outputs": [ |
| 405 | { |
| 406 | "ename": "WebDriverException", |
| 407 | "evalue": "Message: unknown error: network conditions must be set before it can be retrieved\n (Session info: headless chrome=118.0.5993.120)\nStacktrace:\n\tGetHandleVerifier [0x00007FF7D1028EF2+54786]\n\t(No symbol) [0x00007FF7D0F95612]\n\t(No symbol) [0x00007FF7D0E4A64B]\n\t(No symbol) [0x00007FF7D0EB8E10]\n\t(No symbol) [0x00007FF7D0EABE30]\n\t(No symbol) [0x00007FF7D0E80941]\n\t(No symbol) [0x00007FF7D0E81B84]\n\tGetHandleVerifier [0x00007FF7D1377F52+3524194]\n\tGetHandleVerifier [0x00007FF7D13CD800+3874576]\n\tGetHandleVerifier [0x00007FF7D13C5D7F+3843215]\n\tGetHandleVerifier [0x00007FF7D10C5086+694166]\n\t(No symbol) [0x00007FF7D0FA0A88]\n\t(No symbol) [0x00007FF7D0F9CA94]\n\t(No symbol) [0x00007FF7D0F9CBC2]\n\t(No symbol) [0x00007FF7D0F8CC83]\n\tBaseThreadInitThunk [0x00007FFD49F27344+20]\n\tRtlUserThreadStart [0x00007FFD4A5E26B1+33]\n", |
| 408 | "output_type": "error", |
| 409 | "traceback": [ |
| 410 | "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", |
| 411 | "\u001b[1;31mWebDriverException\u001b[0m Traceback (most recent call last)", |
| 412 | "\u001b[1;32mc:\\Users\\huncho\\code\\hackathons\\DockerAiHackathon\\main.ipynb Cell 4\u001b[0m line \u001b[0;36m1\n\u001b[1;32m----> <a href='vscode-notebook-cell:/c%3A/Users/huncho/code/hackathons/DockerAiHackathon/main.ipynb#X11sZmlsZQ%3D%3D?line=0'>1</a>\u001b[0m driver\u001b[39m.\u001b[39;49mget_network_conditions()\n", |
| 413 | "File \u001b[1;32mc:\\Users\\huncho\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\selenium\\webdriver\\chromium\\webdriver.py:82\u001b[0m, in \u001b[0;36mChromiumDriver.get_network_conditions\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m 75\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mget_network_conditions\u001b[39m(\u001b[39mself\u001b[39m):\n\u001b[0;32m 76\u001b[0m \u001b[39m\"\"\"Gets Chromium network emulation settings.\u001b[39;00m\n\u001b[0;32m 77\u001b[0m \n\u001b[0;32m 78\u001b[0m \u001b[39m :Returns: A dict. For example: {'latency': 4,\u001b[39;00m\n\u001b[0;32m 79\u001b[0m \u001b[39m 'download_throughput': 2, 'upload_throughput': 2, 'offline':\u001b[39;00m\n\u001b[0;32m 80\u001b[0m \u001b[39m False}\u001b[39;00m\n\u001b[0;32m 81\u001b[0m \u001b[39m \"\"\"\u001b[39;00m\n\u001b[1;32m---> 82\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mexecute(\u001b[39m\"\u001b[39;49m\u001b[39mgetNetworkConditions\u001b[39;49m\u001b[39m\"\u001b[39;49m)[\u001b[39m\"\u001b[39m\u001b[39mvalue\u001b[39m\u001b[39m\"\u001b[39m]\n", |
| 414 | "File \u001b[1;32mc:\\Users\\huncho\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\selenium\\webdriver\\remote\\webdriver.py:347\u001b[0m, in \u001b[0;36mWebDriver.execute\u001b[1;34m(self, driver_command, params)\u001b[0m\n\u001b[0;32m 345\u001b[0m response \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcommand_executor\u001b[39m.\u001b[39mexecute(driver_command, params)\n\u001b[0;32m 346\u001b[0m \u001b[39mif\u001b[39;00m response:\n\u001b[1;32m--> 347\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49merror_handler\u001b[39m.\u001b[39;49mcheck_response(response)\n\u001b[0;32m 348\u001b[0m response[\u001b[39m\"\u001b[39m\u001b[39mvalue\u001b[39m\u001b[39m\"\u001b[39m] \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_unwrap_value(response\u001b[39m.\u001b[39mget(\u001b[39m\"\u001b[39m\u001b[39mvalue\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39mNone\u001b[39;00m))\n\u001b[0;32m 349\u001b[0m \u001b[39mreturn\u001b[39;00m response\n", |
| 415 | "File \u001b[1;32mc:\\Users\\huncho\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\selenium\\webdriver\\remote\\errorhandler.py:229\u001b[0m, in \u001b[0;36mErrorHandler.check_response\u001b[1;34m(self, response)\u001b[0m\n\u001b[0;32m 227\u001b[0m alert_text \u001b[39m=\u001b[39m value[\u001b[39m\"\u001b[39m\u001b[39malert\u001b[39m\u001b[39m\"\u001b[39m]\u001b[39m.\u001b[39mget(\u001b[39m\"\u001b[39m\u001b[39mtext\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[0;32m 228\u001b[0m \u001b[39mraise\u001b[39;00m exception_class(message, screen, stacktrace, alert_text) \u001b[39m# type: ignore[call-arg] # mypy is not smart enough here\u001b[39;00m\n\u001b[1;32m--> 229\u001b[0m \u001b[39mraise\u001b[39;00m exception_class(message, screen, stacktrace)\n", |
| 416 | "\u001b[1;31mWebDriverException\u001b[0m: Message: unknown error: network conditions must be set before it can be retrieved\n (Session info: headless chrome=118.0.5993.120)\nStacktrace:\n\tGetHandleVerifier [0x00007FF7D1028EF2+54786]\n\t(No symbol) [0x00007FF7D0F95612]\n\t(No symbol) [0x00007FF7D0E4A64B]\n\t(No symbol) [0x00007FF7D0EB8E10]\n\t(No symbol) [0x00007FF7D0EABE30]\n\t(No symbol) [0x00007FF7D0E80941]\n\t(No symbol) [0x00007FF7D0E81B84]\n\tGetHandleVerifier [0x00007FF7D1377F52+3524194]\n\tGetHandleVerifier [0x00007FF7D13CD800+3874576]\n\tGetHandleVerifier [0x00007FF7D13C5D7F+3843215]\n\tGetHandleVerifier [0x00007FF7D10C5086+694166]\n\t(No symbol) [0x00007FF7D0FA0A88]\n\t(No symbol) [0x00007FF7D0F9CA94]\n\t(No symbol) [0x00007FF7D0F9CBC2]\n\t(No symbol) [0x00007FF7D0F8CC83]\n\tBaseThreadInitThunk [0x00007FFD49F27344+20]\n\tRtlUserThreadStart [0x00007FFD4A5E26B1+33]\n" |
| 417 | ] |
| 418 | } |
| 419 | ], |
| 420 | "source": [ |
| 421 | "driver.get_network_conditions()" |
| 422 | ] |
| 423 | }, |
| 424 | { |
| 425 | "cell_type": "code", |
| 426 | "execution_count": 50, |
| 427 | "metadata": {}, |
| 428 | "outputs": [], |
| 429 | "source": [ |
| 430 | "driver.quit()" |
| 431 | ] |
| 432 | }, |
| 433 | { |
| 434 | "cell_type": "code", |
| 435 | "execution_count": 34, |
| 436 | "metadata": {}, |
| 437 | "outputs": [], |
| 438 | "source": [ |
| 439 | "import requests\n", |
| 440 | "import json\n", |
| 441 | "\n", |
| 442 | "# Define the access token\n", |
| 443 | "access_token = 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJjbGllbnRJZCI6ImU1MzVjN2MwLTgxN2YtNDc3Ni04OTkwLTU2NTU2ZjhiMTkyOCIsImNsaWVudEtleSI6IjRjRlVXNkRtd0pwelQ5TDdMckczcVJBY0FCRzVzMDRnIiwiaXNzIjoiTkZMIiwiZGV2aWNlSWQiOiI5NTIzOGE1YS05MTIxLTQzNTEtOTY4MC0wNzZlN2Y2ZTQ1NmYiLCJwbGFucyI6W3sicGxhbiI6ImZyZWUiLCJleHBpcmF0aW9uRGF0ZSI6IjIwMjQtMTEtMDMiLCJzb3VyY2UiOiJORkwiLCJzdGFydERhdGUiOiIyMDIzLTExLTA0Iiwic3RhdHVzIjoiQUNUSVZFIiwidHJpYWwiOmZhbHNlfV0sIkRpc3BsYXlOYW1lIjoiV0VCX0RFU0tUT1BfREVTS1RPUCIsIk5vdGVzIjoiIiwiZm9ybUZhY3RvciI6IkRFU0tUT1AiLCJsdXJhQXBwS2V5IjoiU1pzNTdkQkdSeGJMNzI4bFZwN0RZUSIsInBsYXRmb3JtIjoiREVTS1RPUCIsInByb2R1Y3ROYW1lIjoiV0VCIiwiY2l0eSI6ImxvcyBhbmdlbGVzIiwiY291bnRyeUNvZGUiOiJVUyIsImRtYUNvZGUiOiI4MDMiLCJobWFUZWFtcyI6WyIxMDQwMjUxMC04OTMxLTBkNWYtOTgxNS03OWJiNzk2NDlhNjUiLCIxMDQwMjUyMC05NmJmLWU5ZjItNGY2OC04NTIxY2E4OTYwNjAiLCIxMDQwNDQwMC0zYjM1LTA3M2YtMTk3ZS0xOTRiYjgyNDA3MjMiXSwicmVnaW9uIjoiQ0EiLCJicm93c2VyIjoiQ2hyb21lIiwiY2VsbHVsYXIiOmZhbHNlLCJlbnZpcm9ubWVudCI6InByb2R1Y3Rpb24iLCJyb2xlcyI6WyJmcmVlIl0sImV4cCI6MTY5OTA2NTUwMH0.3a0m-SwXS5ElV9NHNFy2EXwTUDtDCzZkpdPpYs-eW_g'\n", |
| 444 | "headers = {\n", |
| 445 | " 'Authorization': access_token\n", |
| 446 | "}\n", |
| 447 | "\n", |
| 448 | "def request(api_url):\n", |
| 449 | " return requests.get(api_url, headers={\n", |
| 450 | " 'Authorization': access_token\n", |
| 451 | " })" |
| 452 | ] |
| 453 | }, |
| 454 | { |
| 455 | "cell_type": "markdown", |
| 456 | "metadata": {}, |
| 457 | "source": [ |
| 458 | "### 1.2 Get team data through the teams endpoint" |
| 459 | ] |
| 460 | }, |
| 461 | { |
| 462 | "cell_type": "code", |
| 463 | "execution_count": 35, |
| 464 | "metadata": {}, |
| 465 | "outputs": [ |
| 466 | { |
| 467 | "data": { |
| 468 | "text/plain": [ |
| 469 | "<Response [200]>" |
| 470 | ] |
| 471 | }, |
| 472 | "execution_count": 35, |
| 473 | "metadata": {}, |
| 474 | "output_type": "execute_result" |
| 475 | } |
| 476 | ], |
| 477 | "source": [ |
| 478 | "api_url = 'https://api.nfl.com/football/v2/teams?limit=100&season=2023'\n", |
| 479 | "request(api_url)" |
| 480 | ] |
| 481 | }, |
| 482 | { |
| 483 | "cell_type": "code", |
| 484 | "execution_count": null, |
| 485 | "metadata": {}, |
| 486 | "outputs": [], |
| 487 | "source": [] |
| 488 | } |
| 489 | ], |
| 490 | "metadata": { |
| 491 | "kernelspec": { |
| 492 | "display_name": "Python 3", |
| 493 | "language": "python", |
| 494 | "name": "python3" |
| 495 | }, |
| 496 | "language_info": { |
| 497 | "codemirror_mode": { |
| 498 | "name": "ipython", |
| 499 | "version": 3 |
| 500 | }, |
| 501 | "file_extension": ".py", |
| 502 | "mimetype": "text/x-python", |
| 503 | "name": "python", |
| 504 | "nbconvert_exporter": "python", |
| 505 | "pygments_lexer": "ipython3", |
| 506 | "version": "3.10.9" |
| 507 | } |
| 508 | }, |
| 509 | "nbformat": 4, |
| 510 | "nbformat_minor": 2 |
| 511 | } |