Merge pull request #245 from dpguthrie/fix-session-setup

Fix session setup

Author: dpguthrie

CHANGELOG.rst

@@ -1,6 +1,11 @@
 Change Log
 ==========
 
+2.3.6
+-----
+## Fix
+- Use the previously instantiated session within the `Ticker` class (that may also contain proxies, other important session info) to retrieve both cookies and a crumb
+
 2.3.5
 -----
 ## Fix

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "yahooquery"
-version = "2.3.5"
+version = "2.3.6"
 description = "Python wrapper for an unofficial Yahoo Finance API"
 authors = ["Doug Guthrie <douglas.p.guthrie@gmail.com>"]
 documentation = "https://yahooquery.dpguthrie.com"

yahooquery/__init__.py

@@ -1,7 +1,7 @@
 """Python interface to unofficial Yahoo Finance API endpoints"""
 
 name = "yahooquery"
-__version__ = "2.3.5"
+__version__ = "2.3.6"
 
 from .research import Research  # noqa
 from .ticker import Ticker  # noqa

yahooquery/base.py

@@ -933,19 +933,17 @@ class _YahooFinance(object):
     MODULES = _CONFIG["quoteSummary"]["query"]["modules"]["options"]
 
     def __init__(self, **kwargs):
-        self.country = kwargs.get("country", "united states").lower()
+        self.country = kwargs.pop("country", "united states").lower()
         self.formatted = kwargs.pop("formatted", False)
-        self.session = initialize_session(kwargs.pop("session", None), **kwargs)
         self.progress = kwargs.pop("progress", False)
-        self.username = kwargs.get("username", os.getenv("YF_USERNAME", None))
-        self.password = kwargs.get("password", os.getenv("YF_PASSWORD", None))
+        self.username = kwargs.pop("username", os.getenv("YF_USERNAME", None))
+        self.password = kwargs.pop("password", os.getenv("YF_PASSWORD", None))
+        self.session = initialize_session(kwargs.pop("session", None), **kwargs)
         if self.username and self.password:
             self.login()
-            self.crumb = get_crumb(self.session)
         else:
-            host = self._country_params["corsDomain"]
-            cookies, self.crumb = setup_session(host)
-            self.session.cookies = cookies
+            self.session = setup_session(self.session)
+        self.crumb = get_crumb(self.session)
 
     @property
     def symbols(self):

yahooquery/headless.py

@@ -27,6 +27,7 @@ class YahooFinanceHeadless:
     def __init__(self, username: str, password: str):
         self.username = username
         self.password = password
+        self.cookies = RequestsCookieJar()
         chrome_options = Options()
         chrome_options.add_argument("--headless")
         chrome_options.add_argument("--no-sandbox")
@@ -51,16 +52,15 @@ def login(self):
             self.driver.find_element(By.XPATH, "//button[@id='login-signin']").click()
             cookies = self.driver.get_cookies()
             self.driver.quit()
-            self.cookies = self._create_cookie_jar(cookies)
+            self._add_cookies_to_jar(cookies)
 
         except TimeoutException:
             return (
                 "A timeout exception has occured.  Most likely it's due "
                 "to invalid login credentials.  Please try again."
             )
 
-    def _create_cookie_jar(cookies: List[Dict]):
-        cookie_jar = RequestsCookieJar()
+    def _add_cookies_to_jar(self, cookies: List[Dict]):
         for cookie in cookies:
             cookie_dict = {
                 "name": cookie["name"],
@@ -69,5 +69,4 @@ def _create_cookie_jar(cookies: List[Dict]):
                 "path": cookie["path"],
                 "expires": None,  # You can set the expiration if available
             }
-            cookie_jar.set(**cookie_dict)
-        return cookie_jar
+            self.cookies.set(**cookie_dict)

yahooquery/misc.py

@@ -1,7 +1,7 @@
 # third party
 import pandas as pd
 
-from .utils import get_crumb, initialize_session
+from .utils import get_crumb, initialize_session, setup_session
 from .utils.countries import COUNTRIES
 
 BASE_URL = "https://query2.finance.yahoo.com"
@@ -21,6 +21,7 @@ def _make_request(
                 )
             )
     session = initialize_session(**kwargs)
+    session = setup_session(session)
     crumb = get_crumb(session)
     if crumb is not None:
         params["crumb"] = crumb

yahooquery/utils/__init__.py

@@ -18,6 +18,10 @@
 
 DEFAULT_TIMEOUT = 5
 
+CRUMB_FAILURE = (
+    "Failed to obtain crumb.  Ability to retrieve data will be significantly limited."
+)
+
 HEADERS = [
     {
         "upgrade-insecure-requests": "1",
@@ -1362,42 +1366,47 @@ def initialize_session(session=None, **kwargs):
     return session
 
 
-def setup_session(host: str):
-    url = f"https://{host}/quote/tsla"
-    session = requests.Session()
-    session.headers = random.choice(HEADERS)
+def setup_session(session: requests.Session):
+    url = "https://finance.yahoo.com"
     try:
-        _ = session.get(url, allow_redirects=True)
+        response = session.get(url, allow_redirects=True)
     except SSLError:
         counter = 0
         while counter < 5:
             try:
                 session.headers = random.choice(HEADERS)
-                _ = session.get(url, verify=False)
+                response = session.get(url, verify=False)
                 break
             except SSLError:
                 counter += 1
 
-    if not session.cookies:
-        return None, None
+    if not isinstance(session, FuturesSession):
+        return session
 
-    crumb = get_crumb(session)
-    return session.cookies, crumb
+    _ = response.result()
+    return session
 
 
 def get_crumb(session):
     try:
         response = session.get("https://query2.finance.yahoo.com/v1/test/getcrumb")
-        return response.text
 
     except (ConnectionError, RetryError):
-        logger.critical(
-            "Failed to obtain crumb.  Ability to retrieve data will be significantly "
-            "limited."
-        )
+        logger.critical(CRUMB_FAILURE)
         # Cookies most likely not set in previous request
         return None
 
+    if isinstance(session, FuturesSession):
+        crumb = response.result().text
+    else:
+        crumb = response.text
+
+    if crumb is None or crumb == "" or "<html>" in crumb:
+        logger.critical(CRUMB_FAILURE)
+        return None
+
+    return crumb
+
 
 def flatten_list(ls):
     return [item for sublist in ls for item in sublist]