Improve Yahoo crumb handling

compile.sh

@@ -64,11 +64,11 @@ cp -f "${PWD}"/src/financials_coinbase.py "${PWD}"/build/
 
 TMPFILE=`mktemp`
 
-wget "https://files.pythonhosted.org/packages/36/7a/87837f39d0296e723bb9b62bbb257d0355c7f6128853c78955f57342a56d/python_dateutil-2.8.2-py2.py3-none-any.whl" -O $TMPFILE
+wget "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl" -O $TMPFILE
 unzip $TMPFILE dateutil/\* -d "${PWD}"/build/
 rm $TMPFILE
 
-wget "https://files.pythonhosted.org/packages/7f/99/ad6bd37e748257dd70d6f85d916cafe79c0b0f5e2e95b11f7fbc82bf3110/pytz-2023.3-py2.py3-none-any.whl" -O $TMPFILE
+wget "https://files.pythonhosted.org/packages/9c/3d/a121f284241f08268b21359bd425f7d4825cffc5ac5cd0e1b3d82ffd2b10/pytz-2024.1-py2.py3-none-any.whl" -O $TMPFILE
 unzip $TMPFILE pytz/\* -d "${PWD}"/build/
 rm $TMPFILE
 

src/baseclient.py

@@ -33,9 +33,20 @@ class RedirectException(HTTPException):
 
 
 class HttpException(HTTPException):
-    def __init__(self, url, status):
+    def __init__(self, url, response):
         self.url = url
-        self.status = status
+        self.response = response
+
+    def __str__(self):
+        if self.response is None:
+            return f"url='{self.url}'"
+        if type(self.response) is str:
+            return f"url='{self.url}' status='{self.response}'"
+        if self.response.headers:
+            h = '\n'.join(sorted(self.response.headers.__str__().splitlines(), key=lambda l: l.lower()))
+            return f"url='{self.url}' status={self.response.status} reason='{self.response.reason}'{h}\n"
+        else:
+            return f"url='{self.url}' status={self.response.status} reason='{self.response.reason}'"
 
 
 class BaseClient:
@@ -49,18 +60,16 @@ class BaseClient:
         os.makedirs(self.basedir, exist_ok=True)
 
         user_agents = [
-            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/110.0',
-            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/111.0',
-            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/112.0',
-            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/113.0',
-            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/114.0',
-            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/115.0',
-            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/116.0',
-            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/117.0',
-            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/118.0',
-            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/119.0',
-            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/120.0',
-            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/121.0'
+            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:120.0) Gecko/20100101 Firefox/120.0',
+            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:121.0) Gecko/20100101 Firefox/121.0',
+            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:122.0) Gecko/20100101 Firefox/122.0',
+            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:123.0) Gecko/20100101 Firefox/123.0',
+            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:124.0) Gecko/20100101 Firefox/124.0',
+            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:125.0) Gecko/20100101 Firefox/125.0',
+            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:126.0) Gecko/20100101 Firefox/126.0',
+            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:127.0) Gecko/20100101 Firefox/127.0',
+            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0',
+            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:129.0) Gecko/20100101 Firefox/129.0',
         ]
 
         self.default_headers = {
@@ -168,9 +177,10 @@ class BaseClient:
                 raise RedirectException(location)
 
         if self.response.status >= 400:
-            logger.warning("last_url='%s' status=%s headers=%s", self.last_url, self.response.status,
+            logger.warning("last_url='%s' status=%s reason='%s' headers=%s", self.last_url, self.response.status,
+                           self.response.reason,
                            '\n'.join(sorted(self.response.headers.__str__().splitlines(), key=lambda l: l.lower())))
-            raise HttpException(url, self.response.status)
+            raise HttpException(url, self.response)
 
         if self.response.getheader('Content-Encoding') == 'gzip':
             text = gzip.decompress(text)

src/financials_yahoo.py

@@ -183,12 +183,21 @@ class Yahoo(BaseClient):
                 del self.realtime[ticker]
                 return 'Yahoo.getRealtime({}, {}) - handleCookiesAndConsent'.format(ticker, datacode)
 
+            # crumbs like 'TKkC\u002FZBwoUA' may contain unicode _text_ (not encoded code points)
             try:
-                r = '"crumb":"([^"]{11})"'
+                r = r'\bcrumb=([^"]{11,})"'
                 pattern = re.compile(r)
                 match = pattern.search(text)
                 if match:
-                    self.crumb = match.group(1)
+                    self.crumb = urllib.parse.unquote(match.group(1).encode('unicode-escape').decode('ascii'))
+                    logger.debug(f"crumb='{match.group(1)}' self.crumb='{self.crumb}'")
+                else:
+                    r = r'"crumb"\s*:\s*"([^"]{11,})"'
+                    pattern = re.compile(r)
+                    match = pattern.search(text)
+                    if match:
+                        self.crumb = urllib.parse.unquote(match.group(1).encode('unicode-escape').decode('ascii'))
+                        logger.debug(f"crumb='{match.group(1)}' self.crumb='{self.crumb}'")
             except BaseException as e:
                 logger.exception("BaseException ticker=%s datacode=%s", ticker, datacode)
                 del self.realtime[ticker]

src/generate_metainfo.py

@@ -14,7 +14,7 @@ import os
 cur_dir = os.getcwd()
 
 addin_id = "com.financials.getinfo"
-addin_version = "3.6.0"
+addin_version = "3.6.2"
 addin_displayname = "Financial Market Extension"
 addin_publisher_link = "https://github.com/cmallwitz/Financials-Extension"
 addin_publisher_name = "The Publisher"

src/test_ft.py

@@ -275,7 +275,7 @@ class Test(unittest.TestCase):
         self.assertTrue(testutils.is_date(s), 'test_DE_equity EX_DIV_DATE {}'.format(s))
 
         s = financials.getRealtime('ISHAX:GER', 'NAME', 'FT')
-        self.assertEqual('INTERSHOP Communications AG', s, 'test_DE_equity NAME {}'.format(s))
+        self.assertEqual('Intershop Communications AG', s, 'test_DE_equity NAME {}'.format(s))
 
         s = financials.getRealtime('ISHAX:GER', 'BETA', 'FT')
         self.assertTrue(testutils.is_positive_float(s), 'test_DE_equity BETA {}'.format(s))

src/test_yahoo.py

@@ -220,16 +220,16 @@ class Test(unittest.TestCase):
 
         s = financials.getRealtime('ES=F', Datacode.NAME.value, 'YAHOO')
         self.assertEqual(str, type(s), 'test_realtime_US_futures NAME {}'.format(s))
-        self.assertEqual('E-Mini S&P 500 Jun 24', s, 'test_realtime_US_futures NAME {}'.format(s))
+        self.assertEqual('E-Mini S&P 500 Sep 24', s, 'test_realtime_US_futures NAME {}'.format(s))
 
         s = financials.getRealtime('ES=F', Datacode.TICKER.value, 'YAHOO')
         self.assertEqual(str, type(s), 'test_realtime_US_futures TICKER {}'.format(s))
-        self.assertEqual('ESM24.CME', s, 'test_realtime_US_futures TICKER {}'.format(s))
+        self.assertEqual('ESU24.CME', s, 'test_realtime_US_futures TICKER {}'.format(s))
 
         s = financials.getRealtime('ES=F', Datacode.SETTLEMENT_DATE.value, 'YAHOO')
         self.assertEqual(str, type(s), 'test_realtime_US_futures SETTLEMENT_DATE {}'.format(s))
         self.assertTrue(testutils.is_date(s), 'test_realtime_US_futures SETTLEMENT_DATE {}'.format(s))
-        self.assertEqual("2024-06-21", s, 'test_realtime_US_futures SETTLEMENT_DATE {}'.format(s))
+        self.assertEqual("2024-09-20", s, 'test_realtime_US_futures SETTLEMENT_DATE {}'.format(s))
 
         s = financials.getRealtime('ES=F', Datacode.LAST_PRICE.value, 'YAHOO')
         self.assertEqual(float, type(s), 'test_realtime_US_futures LAST_PRICE {}'.format(s))