Reworked Yahoo cookie and general network error handling

.gitignore

@@ -1,4 +1,5 @@
 .idea
 src/__pycache__
 build/*
-Financials-Extension.oxt
+Financials-Extension.oxt
+_trial_temp

README.md

@@ -1,5 +1,9 @@
 # Financials-Extension
 
+Version 3.3.0 includes improved cookie handling and somewhat improved logic to deal with network issues.   
+
+## Overview
+
 This is a Python based extension for LibreOffice Calc to make stock market, index and FX data available in Calc 
 spreadsheets - currently supporting Yahoo's and Financial Times' finance websites using old-fashioned web scraping. 
 
@@ -62,8 +66,8 @@ You can either specify numbers or names (lower or upper case) - not all bits are
 | PREV_CLOSE           | 5    |       Yes        | Yes |||
 | OPEN                 | 6    |       Yes        | Yes |       Yes        | Yes      ||
 | CHANGE               | 7    |       Yes        | Yes |||
-| LAST_PRICE_DATE      | 8    |       Yes        | Yes |                  |          | yyyy-mm-dd |
-| LAST_PRICE_TIME      | 10   |       Yes        | Yes |||
+| LAST_PRICE_DATE      | 8    |                  | Yes |                  |          | yyyy-mm-dd |
+| LAST_PRICE_TIME      | 10   |                  | Yes |||
 | CHANGE_IN_PERCENT    | 11   |       Yes        | Yes |||
 | LOW                  | 14   |       Yes        | Yes |       Yes        | Yes      ||
 | HIGH                 | 16   |       Yes        | Yes |       Yes        | Yes      ||
@@ -87,6 +91,7 @@ You can either specify numbers or names (lower or upper case) - not all bits are
 | EXPIRY_DATE          | 74   | Yes (on options) | No  |                  |          | yyyy-mm-dd |
 | SHARES_OUT           | 75   |       Yes        | Yes |                  |          ||
 | FREE_FLOAT           | 76   |       Yes        | Yes |                  |          ||
+| SETTLEMENT_DATE      | 77   | Yes (on futures) |     |                  |          | yyyy-mm-dd |
 | CLOSE                | 90   |        No        | No  |       Yes        |          ||
 | ADJ_CLOSE            | 91   |        No        | No  |       Yes        |          ||
 | SECTOR               | 98   |       Yes        | Yes |||

src/baseclient.py

@@ -43,30 +43,21 @@ class BaseClient:
         self.connections = {}
         self.cookies = cookiejar.CookieJar()
         self.last_url = None
+        self.redirect_count = 0  # will be set later
 
         self.basedir = os.path.join(str(pathlib.Path.home()), '.financials-extension')
         os.makedirs(self.basedir, exist_ok=True)
 
         user_agents = [
-            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:96.0) Gecko/20100101 Firefox/96.0',
-            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:97.0) Gecko/20100101 Firefox/97.0',
-            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:98.0) Gecko/20100101 Firefox/96.0',
-            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:99.0) Gecko/20100101 Firefox/97.0',
-            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:100.0) Gecko/20100101 Firefox/97.0',
-            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:101.0) Gecko/20100101 Firefox/97.0',
-            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:102.0) Gecko/20100101 Firefox/97.0',
-            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:103.0) Gecko/20100101 Firefox/97.0',
-            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:104.0) Gecko/20100101 Firefox/97.0',
-            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:105.0) Gecko/20100101 Firefox/97.0',
-            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:106.0) Gecko/20100101 Firefox/97.0',
-
-            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36',
-            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36',
-            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36',
-            'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.106 Safari/537.36',
-            'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4104.0 Safari/537.36',
-            'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4149.0 Safari/537.36',
-            'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36'
+            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:100.0) Gecko/20100101 Firefox/100.0',
+            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:101.0) Gecko/20100101 Firefox/101.0',
+            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:102.0) Gecko/20100101 Firefox/102.0',
+            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:103.0) Gecko/20100101 Firefox/103.0',
+            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:104.0) Gecko/20100101 Firefox/104.0',
+            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:105.0) Gecko/20100101 Firefox/105.0',
+            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:106.0) Gecko/20100101 Firefox/106.0',
+            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:107.0) Gecko/20100101 Firefox/107.0',
+            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:108.0) Gecko/20100101 Firefox/108.0'
         ]
 
         self.default_headers = {
@@ -107,7 +98,7 @@ class BaseClient:
             connection = HTTPConnection(host, **kwargs) if scheme == 'http:' else HTTPSConnection(host, **kwargs)
 
         logger.debug('Creating request -----------------------------------------------------')
-        logger.info("url='%s'", url)
+        logger.debug("url='%s'", url)
 
         self.last_url = url
 
@@ -127,7 +118,7 @@ class BaseClient:
 
         logger.debug('Processing response --------------------------------------------------')
 
-        # logger.debug('response.status={}'.format(response.status))
+        logger.debug('response.status=%s', response.status)
         for key, value in response.getheaders():
             logger.debug('Header: %s=%s', key, value)
 
@@ -138,15 +129,24 @@ class BaseClient:
 
     def urlopen(self, url, redirect=True, data=None, headers={}, cookies=[], **kwargs):
 
+        self.last_url = None
+
         self.response = self.request('POST' if data else 'GET', url, data, headers, cookies, **kwargs)
         text = self.response.read()
 
         # Allow redirects - used by Yahoo for some cookie based consent
-        redirect_count = 3
+        self.redirect_count = 5
 
-        while 300 <= self.response.status < 400 and redirect_count >= 0:
+        # (for Yahoo) AWS CloudFront occasionally returns an incorrect, cached error responses
+        # try mitigating by re-requesting straight away
+        if 400 <= self.response.status < 500:
+            if self.response.getheader('X-Cache') == 'Error from cloudfront':
+                self.response = self.request('POST' if data else 'GET', url, data, headers, cookies, **kwargs)
+                text = self.response.read()
 
-            redirect_count -= 1
+        while 300 <= self.response.status < 400 and self.redirect_count >= 0:
+
+            self.redirect_count -= 1
             location = self.response.getheader('Location')
 
             if location and redirect:
@@ -162,6 +162,8 @@ class BaseClient:
                 raise RedirectException(location)
 
         if self.response.status >= 400:
+            logger.warning("last_url='%s' status=%s headers=%s", self.last_url, self.response.status,
+                           '\n'.join(sorted(self.response.headers.__str__().splitlines(), key=lambda l: l.lower())))
             raise HttpException(url, self.response.status)
 
         if self.response.getheader('Content-Encoding') == 'gzip':
@@ -195,6 +197,7 @@ class BaseClient:
         tick[Datacode.EXPIRY_DATE] = None
         tick[Datacode.EX_DIV_DATE] = None
         tick[Datacode.FREE_FLOAT] = None
+        tick[Datacode.SETTLEMENT_DATE] = None
         tick[Datacode.HIGH] = None
         tick[Datacode.HIGH_52_WEEK] = None
         tick[Datacode.INDUSTRY] = None
@@ -215,6 +218,11 @@ class BaseClient:
         tick[Datacode.TIMEZONE] = None
         tick[Datacode.VOLUME] = None
 
+        tick[Datacode.YAHOO_SUMMARY_RECEIVED] = False
+        tick[Datacode.YAHOO_STATISTIC_RECEIVED] = False
+        tick[Datacode.YAHOO_PROFILE_RECEIVED] = False
+        tick[Datacode.TIMESTAMP] = None
+
         return tick
 
     def _return_value(self, data: dict, datacode: int):
@@ -324,6 +332,12 @@ class BaseClient:
             elif datacode == Datacode.FREE_FLOAT.value and Datacode.FREE_FLOAT in data:
                 return data[Datacode.FREE_FLOAT]
 
+            elif datacode == Datacode.SETTLEMENT_DATE.value and Datacode.SETTLEMENT_DATE in data:
+                if data[Datacode.SETTLEMENT_DATE]:
+                    return data[Datacode.SETTLEMENT_DATE].isoformat()
+                else:
+                    return data[Datacode.SETTLEMENT_DATE]
+
             elif datacode == Datacode.SHARES_OUT.value and Datacode.SHARES_OUT in data:
                 return data[Datacode.SHARES_OUT]
 

src/datacode.py

@@ -46,6 +46,7 @@ class Datacode(Enum):
     EXPIRY_DATE = 74
     SHARES_OUT = 75
     FREE_FLOAT = 76
+    SETTLEMENT_DATE = 77
 
     CLOSE = 90
     ADJ_CLOSE = 91
@@ -59,6 +60,9 @@ class Datacode(Enum):
     NAME = 104
     TIMEZONE = 105
 
+    YAHOO_SUMMARY_RECEIVED = 996
+    YAHOO_STATISTIC_RECEIVED = 997
+    YAHOO_PROFILE_RECEIVED = 998
     TIMESTAMP = 999
 
     @classmethod

src/financials.py

@@ -30,7 +30,7 @@ os.makedirs(basedir, exist_ok=True)
 logging.basicConfig(
     handlers=[logging.FileHandler(filename=os.path.join(basedir, 'extension.log'), encoding='utf-8', mode='a+')],
     format="%(asctime)s %(name)s %(levelname)s %(message)s",
-    level=logging.WARN)
+    level=logging.INFO)
 
 # Add current directory to import path
 current_dir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))

src/financials_coinbase.py

@@ -53,7 +53,7 @@ class Coinbase(BaseClient):
         # use cached value for up to 60 seconds
         if ticker in self.realtime:
             tick = self.realtime[ticker]
-            if time.time() - 60 < tick[Datacode.TIMESTAMP]:
+            if Datacode.TIMESTAMP in tick and type(tick[Datacode.TIMESTAMP]) == float and time.time() - 60 < tick[Datacode.TIMESTAMP]:
                 return self._return_value(tick, datacode)
             else:
                 del self.realtime[ticker]
@@ -63,20 +63,22 @@ class Coinbase(BaseClient):
         try:
             text = self.urlopen(url, redirect=True, data=None, headers=None)
         except BaseException as e:
-            logger.exception("BaseException ticker=%s datacode=%s", ticker, datacode)
+            logger.exception("BaseException ticker=%s datacode=%s last_url=%s redirect_count=%s", ticker, datacode, self.last_url, self.redirect_count)
+            del self.realtime[ticker]
             return 'Coinbase.getRealtime({}, {}) - urlopen: {}'.format(ticker, datacode, e)
 
         try:
             with open(os.path.join(self.basedir, 'coinbase-{}.json'.format(ticker)), "w", encoding="utf-8") as text_file:
-                print(f"<!-- '{url}' -->\r\n\r\n{text}", file=text_file)
+                print(f"<!-- '{self.last_url}' -->\r\n\r\n{text}", file=text_file)
         except BaseException:
-            logger.exception("BaseException ticker=%s datacode=%s", ticker, datacode)
+            logger.exception("BaseException open/write ticker=%s datacode=%s", ticker, datacode)
 
         try:
             results = json.loads(text)
 
         except BaseException as e:
             logger.exception("BaseException ticker=%s datacode=%s", ticker, datacode)
+            del self.realtime[ticker]
             return 'Coinbase.getRealtime({}, {}) - crumb: {}'.format(ticker, datacode, e)
 
         try:
@@ -101,6 +103,7 @@ class Coinbase(BaseClient):
 
         except BaseException as e:
             logger.exception("BaseException ticker=%s datacode=%s", ticker, datacode)
+            del self.realtime[ticker]
             return 'Coinbase.getRealtime({}, {}) - process: {}'.format(ticker, datacode, e)
 
         return self._return_value(self.realtime[ticker], datacode)

src/financials_ft.py

@@ -63,7 +63,7 @@ class FT(BaseClient):
         # use cached value for up to 60 seconds
         if ticker in self.realtime:
             tick = self.realtime[ticker]
-            if time.time() - 60 < tick[Datacode.TIMESTAMP]:
+            if Datacode.TIMESTAMP in tick and type(tick[Datacode.TIMESTAMP]) == float and time.time() - 60 < tick[Datacode.TIMESTAMP]:
                 return self._return_value(tick, datacode)
             else:
                 del self.realtime[ticker]
@@ -80,7 +80,8 @@ class FT(BaseClient):
         try:
             text = self.urlopen(url, redirect=True, data=None, headers=None)
         except BaseException as e:
-            logger.exception("BaseException ticker=%s datacode=%s", ticker, datacode)
+            logger.exception("BaseException ticker=%s datacode=%s last_url=%s redirect_count=%s", ticker, datacode, self.last_url, self.redirect_count)
+            del self.realtime[ticker]
             return f'FT.getRealtime({ticker}, {datacode}) - urlopen endpoint: {str(e)}'
 
         try:
@@ -88,7 +89,7 @@ class FT(BaseClient):
             with open(os.path.join(self.basedir, f'ft-{temp}.html'), "w", encoding="utf-8") as text_file:
                 print(f"<!-- '{self.last_url}' -->\r\n\r\n{text}", file=text_file)
         except BaseException:
-            logger.exception("BaseException ticker=%s datacode=%s %s", ticker, datacode)
+            logger.exception("BaseException open/write ticker=%s datacode=%s %s", ticker, datacode)
 
         tick[Datacode.TIMESTAMP] = time.time()
 
@@ -299,9 +300,10 @@ class FT(BaseClient):
 
         except BaseException as e:
             logger.exception("BaseException ticker=%s datacode=%s", ticker, datacode)
+            del self.realtime[ticker]
             return f'FT.getRealtime({ticker}, {datacode}) - process: {str(e)}'
 
-        logger.info(tick)
+        logger.debug(tick)
 
         return self._return_value(self.realtime[ticker], datacode)
 

src/financials_google.py

@@ -66,7 +66,7 @@ class Google(BaseClient):
         # use cached value for up to 60 seconds
         if ticker in self.realtime:
             tick = self.realtime[ticker]
-            if time.time() - 60 < tick[Datacode.TIMESTAMP]:
+            if Datacode.TIMESTAMP in tick and type(tick[Datacode.TIMESTAMP]) == float and time.time() - 60 < tick[Datacode.TIMESTAMP]:
                 return self._return_value(tick, datacode)
             else:
                 del self.realtime[ticker]

src/financials_yahoo.py

@@ -9,22 +9,20 @@
 
 
 import csv
-import datetime
 import html
 import logging
 import os
-import pprint
 import re
 import time
 import urllib.parse
 from http import cookiejar
 
 import dateutil.parser
-import pytz
 
 import jsonParser
 from baseclient import BaseClient, HttpException
 from datacode import Datacode
+from naivehtmlparser import NaiveHTMLParser
 
 logger = logging.getLogger(__name__)
 
@@ -32,22 +30,31 @@ logger = logging.getLogger(__name__)
 # logger.setLevel(logging.DEBUG)
 
 
-def raw(m, key, default=0.0):
+def default(obj, prop, fallback=''):
     try:
-        return m[key]['raw']
+        if obj is None or property is None:
+            return fallback
+
+        v = None
+        if hasattr(obj, prop):
+            v = getattr(obj, prop)
+        elif prop in obj:
+            v = obj[prop]
+        return v if v is not None else fallback
     except:
         pass
-
-    return default
+    return fallback
 
 
-def fmt(m, key, default=0.0):
-    try:
-        return m[key]['fmt']
-    except:
-        pass
-
-    return default
+def handle_abbreviations(s):
+    s = str(s).strip()
+    if s.endswith('M'):
+        return float(s[:-1]) * 1000000
+    elif s.endswith('B'):
+        return float(s[:-1]) * 1000000000
+    elif s.endswith('T'):
+        return float(s[:-1]) * 1000000000000
+    return float(s)
 
 
 def cookie(name, value):
@@ -57,6 +64,17 @@ def cookie(name, value):
                             discard=False, comment=None, comment_url=None, rest=dict())
 
 
+def get_cookies():
+    return [
+        cookie("A1", "d=AQABBNAVmWICEEFBM1xh-RmAmPpJJIsAz3YFEgABBwFcmmJlY_bPb2UB9iMAAAcIzhWZYm7SAIg&S=AQAAAucqV1HMdCsRf6key1gdaFs"),
+        cookie("A1S", "d=AQABBNAVmWICEEFBM1xh-RmAmPpJJIsAz3YFEgABBwFcmmJlY_bPb2UB9iMAAAcIzhWZYm7SAIg&S=AQAAAucqV1HMdCsRf6key1gdaFs&j=GDPR"),
+        cookie("A3", "d=AQABBNAVmWICEEFBM1xh-RmAmPpJJIsAz3YFEgABBwFcmmJlY_bPb2UB9iMAAAcIzhWZYm7SAIg&S=AQAAAucqV1HMdCsRf6key1gdaFs"),
+        cookie("GUC", "AQABBwFimlxjZUIcxQRM"),
+        cookie("maex", "{\"v2\":{}}"),
+        cookie("thamba", "1")
+    ]
+
+
 class Yahoo(BaseClient):
     def __init__(self, ctx):
         super().__init__()
@@ -66,6 +84,61 @@ class Yahoo(BaseClient):
         self.historicdata = {}
         self.js = jsonParser.jsonObject
 
+        self.create_cookies()
+
+    def create_cookies(self):
+
+        cookiejar_path = os.path.join(self.basedir, 'yahoo.cookiejar')
+        cookiejar_exists = os.path.isfile(cookiejar_path) and os.stat(cookiejar_path).st_size >= 0
+
+        if cookiejar_exists:
+            try:
+                self.cookies = cookiejar.LWPCookieJar()
+                self.cookies.load(cookiejar_path, ignore_discard=True)
+
+                required_cookie_names = ["A1", "A1S", "A3", "GUC", "maex", "thamba"]
+                for c in self.cookies:
+                    if c.name in required_cookie_names:
+                        required_cookie_names.remove(c.name)
+
+                if len(required_cookie_names) > 0:
+                    cookiejar_exists = False
+                    logger.info("Overriding cookiejar '%s'", cookiejar_path)
+
+            except BaseException:
+                cookiejar_exists = False
+                logger.exception("BaseException initial loading cookiejar_path=%s", cookiejar_path)
+
+        if not cookiejar_exists:
+            try:
+                lwp_cookiejar = cookiejar.LWPCookieJar()
+                for c in get_cookies():
+                    lwp_cookiejar.set_cookie(c)
+                lwp_cookiejar.save(cookiejar_path, ignore_discard=True)
+                logger.info("Created cookiejar '%s'", cookiejar_path)
+            except BaseException:
+                logger.exception("BaseException creating cookiejar_path=%s", cookiejar_path)
+
+        try:
+            self.cookies = cookiejar.LWPCookieJar()
+            self.cookies.load(cookiejar_path, ignore_discard=True)
+
+            logger.info("Loaded cookiejar '%s'", cookiejar_path)
+
+            for c in self.cookies:
+                logger.info("Cookie name'%s' value='%s' path='%s'", c.name, c.value, c.path)
+
+        except BaseException:
+            logger.exception("BaseException loading cookiejar_path=%s", cookiejar_path)
+
+    def save_cookies(self):
+        cookiejar_path = os.path.join(self.basedir, 'yahoo.cookiejar')
+        try:
+            self.cookies.save(cookiejar_path, ignore_discard=True)
+            logger.debug("Saved cookiejar '%s'", cookiejar_path)
+        except BaseException:
+            logger.exception("BaseException saving cookiejar_path=%s", cookiejar_path)
+
     def _read_ticker_csv_file(self, ticker):
 
         fn = os.path.join(self.basedir, 'yahoo-{}.csv'.format(ticker))
@@ -79,7 +152,7 @@ class Yahoo(BaseClient):
             ticks = {}
 
             for row in reader:
-                tick  = self.get_ticker()
+                tick = self.get_ticker()
                 try:
                     tick[Datacode.OPEN] = float(row['Open'])
                     tick[Datacode.LOW] = float(row['Low'])
@@ -108,159 +181,334 @@ class Yahoo(BaseClient):
         # remove white space
         ticker = "".join(ticker.split())
 
+        needStatistics = datacode in [Datacode.SHARES_OUT.value, Datacode.FREE_FLOAT.value, Datacode.PAYOUT_RATIO.value]
+        needProfile = datacode in [Datacode.SECTOR.value, Datacode.INDUSTRY.value]
+
         # use cached value for up to 60 seconds
         if ticker in self.realtime:
             tick = self.realtime[ticker]
-            if time.time() - 60 < tick[Datacode.TIMESTAMP]:
-                return self._return_value(tick, datacode)
+            if Datacode.TIMESTAMP in tick and type(tick[Datacode.TIMESTAMP]) == float and time.time() - 60 < tick[Datacode.TIMESTAMP]:
+                if (tick[Datacode.YAHOO_STATISTIC_RECEIVED] or not needStatistics) and (
+                        tick[Datacode.YAHOO_PROFILE_RECEIVED] or not needProfile) and (
+                        tick[Datacode.YAHOO_SUMMARY_RECEIVED]):
+                    return self._return_value(tick, datacode)
             else:
                 del self.realtime[ticker]
 
+        if ticker not in self.realtime:
+            self.realtime[ticker] = self.get_ticker()
+
+        if needStatistics:
+            return self.getRealtimeStatistics(ticker, datacode)
+
+        if needProfile:
+            return self.getRealtimeProfile(ticker, datacode)
+
+        return self.getRealtimeSummary(ticker, datacode)
+
+    def getRealtimeSummary(self, ticker, datacode):
+
+        """
+        Retrieve realtime data from Yahoo Finance - Summary tab
+        """
+
+        tick = self.realtime[ticker]
+
         url = 'https://finance.yahoo.com/quote/{}?p={}'.format(ticker, ticker)
 
-        cookies = [
-            cookie("A1", "d=AQABBDcIZWMCEHYhFYqQ7qyTvvD2eAT87mcFEgABCAGDlGPBY_bPb2UB9qMAAAcILwhlY6iIogg&S=AQAAAjZvTuAn1nH4h71eKJtCEHk"),
-            cookie("A1S", "d=AQABBDcIZWMCEHYhFYqQ7qyTvvD2eAT87mcFEgABCAGDlGPBY_bPb2UB9qMAAAcILwhlY6iIogg&S=AQAAAjZvTuAn1nH4h71eKJtCEHk&j=GDPR"),
-            cookie("A3", "d=AQABBDcIZWMCEHYhFYqQ7qyTvvD2eAT87mcFEgABCAGDlGPBY_bPb2UB9qMAAAcILwhlY6iIogg&S=AQAAAjZvTuAn1nH4h71eKJtCEHk"),
-            cookie("GUC", "AQABCAFjlINjwUIcFQQQ&s=AQAAAFOQKXn7&g=Y5M5Jg"),
-            cookie("GUCS", "ASHFadZS"),
-            cookie("maex", "{\"v2\":{}}"),
-            cookie("PRF", "t=TQQQ%2BASTO.L%2BCHMI%2BVFIAX%2BIBM%2BXMR-USD%2BMVV%2BSECU-B.ST%2BMSFT"),
-            cookie("thamba", "1")
-        ]
-
         try:
-            text = self.urlopen(url, redirect=True, data=None, headers=None, cookies=cookies)
+            text = self.urlopen(url, redirect=True)
+            self.save_cookies()
         except BaseException as e:
-            logger.exception("BaseException ticker=%s datacode=%s", ticker, datacode)
-            return 'Yahoo.getRealtime({}, {}) - urlopen: {}'.format(ticker, datacode, e)
+            logger.exception("BaseException ticker=%s datacode=%s last_url=%s redirect_count=%s", ticker, datacode, self.last_url, self.redirect_count)
+            del self.realtime[ticker]
+            return 'Yahoo.getRealtimeSummary({}, {}) - urlopen: {}'.format(ticker, datacode, e)
 
         try:
             with open(os.path.join(self.basedir, 'yahoo-{}.html'.format(ticker)), "w", encoding="utf-8") as text_file:
-                print(f"<!-- '{url}' -->\r\n\r\n{text}", file=text_file)
+                print(f"<!-- '{self.last_url}' -->\r\n\r\n{text}", file=text_file)
         except BaseException:
             logger.exception("BaseException ticker=%s datacode=%s", ticker, datacode)
 
         try:
-            text = urllib.parse.unquote(text)
-            text = text.replace('\\u002F', '/')
-
-            r = '"CrumbStore":{"crumb":"([^"]{11})"'
+            r = '"crumb":"([^"]{11})"'
             pattern = re.compile(r)
             match = pattern.search(text)
-
-            if  match:
+            if match:
                 self.crumb = match.group(1)
-
         except BaseException as e:
             logger.exception("BaseException ticker=%s datacode=%s", ticker, datacode)
-            return 'Yahoo.getRealtime({}, {}) - crumb: {}'.format(ticker, datacode, e)
+            del self.realtime[ticker]
+            return 'Yahoo.getRealtimeSummary({}, {}) - crumb: {}'.format(ticker, datacode, e)
 
         try:
-            start = text.find('"QuoteSummaryStore":{')
-
-            if start < 0:
-                return None
-
-            start = start + len('"QuoteSummaryStore":')
-            results = self.js.parseString(text[start:])
-
-            if not results:
-                return None
-
+            parser = NaiveHTMLParser()
+            root = parser.feed(text)
+            parser.close()
         except BaseException as e:
             logger.exception("BaseException ticker=%s datacode=%s", ticker, datacode)
-            return 'Yahoo.getRealtime({}, {}) - parsing: {}'.format(ticker, datacode, e)
-
-        with open(os.path.join(self.basedir, 'yahoo-{}.js'.format(ticker)), "w", encoding="utf-8") as text_file:
-            print(f"// '{url}' QuoteSummaryStore:\n", file=text_file)
-            pprint.pprint(results.asList(), stream=text_file)
+            return 'Yahoo.getRealtimeSummary({}, {}) - HTML parsing: {}'.format(ticker, datacode, e)
 
         try:
-            price = results['price']
-            quoteType = results['quoteType']
-            summaryDetail = results['summaryDetail']
-            defaultKeyStatistics = results['defaultKeyStatistics'] if 'defaultKeyStatistics' in results else dict()
-
-            if not price:
-                return 'Could not find price for \'{}\''.format(ticker)
-
-            if ticker not in self.realtime:
-                self.realtime[ticker] =  self.get_ticker()
-
-            tick = self.realtime[ticker]
-
+            tick[Datacode.TICKER] = ticker
             tick[Datacode.TIMESTAMP] = time.time()
+            tick[Datacode.YAHOO_SUMMARY_RECEIVED] = True
+
+            parsed = {}
+
+            found = root.findall(f".//fin-streamer[@data-symbol='{ticker}']")
+            for d in found:
+                if hasattr(d, 'attrib') and 'data-field' in d.attrib:
+                    parsed[d.attrib['data-field']] = default(d.attrib, 'value').replace('−', '-').replace(',', '').strip()
+
+            # for futures "regularMarketVolume" is from actual future ticker (potentially different to requested one)
+            if 'regularMarketVolume' not in parsed:
+                found = root.findall(f".//fin-streamer[@data-field='regularMarketVolume']")
+                for d in found:
+                    if hasattr(d, 'attrib') and 'data-field' in d.attrib and 'data-symbol' in d.attrib:
+                        parsed[d.attrib['data-field']] = default(d.attrib, 'value').replace('−', '-').replace(',', '').strip()
+                        tick[Datacode.TICKER] = default(d.attrib, 'data-symbol').strip()
+
+            found = root.findall(f".//td[@data-test]")
+            for d in found:
+                if d:
+                    span = d.find('./span')
+                    if hasattr(d, 'attrib') and hasattr(span, 'text'):
+                        parsed[d.attrib['data-test']] = default(span, 'text').replace('−', '-').replace(',', '').strip()
+                else:
+                    if hasattr(d, 'attrib') and hasattr(d, 'text'):
+                        parsed[d.attrib['data-test']] = default(d, 'text').replace('−', '-').replace(',', '').strip()
+
+            if 'regularMarketPrice' not in parsed:
+                return None
+
+            tick[Datacode.PREV_CLOSE] = self.save_wrapper(lambda: float(parsed['PREV_CLOSE-value']))
+            tick[Datacode.OPEN] = self.save_wrapper(lambda: float(parsed['OPEN-value']))
+            tick[Datacode.CHANGE] = self.save_wrapper(lambda: float(parsed['regularMarketChange']))
+            tick[Datacode.CHANGE_IN_PERCENT] = self.save_wrapper(lambda: float(parsed['regularMarketChangePercent']))
+
+            t = default(parsed, 'DAYS_RANGE-value').split(' - ')
+            tick[Datacode.LOW] = self.save_wrapper(lambda: float(t[0]))
+            tick[Datacode.HIGH] = self.save_wrapper(lambda: float(t[1]))
+
+            tick[Datacode.LAST_PRICE] = self.save_wrapper(lambda: float(parsed['regularMarketPrice']))
+            tick[Datacode.VOLUME] = self.save_wrapper(lambda: float(parsed['regularMarketVolume']))
+            tick[Datacode.AVG_DAILY_VOL_3MONTH] = self.save_wrapper(lambda: float(parsed['AVERAGE_VOLUME_3MONTH-value']))
+            tick[Datacode.BETA] = self.save_wrapper(lambda: float(parsed['BETA_5Y-value']))
+            tick[Datacode.EPS] = self.save_wrapper(lambda: float(parsed['EPS_RATIO-value']))
+            tick[Datacode.PE_RATIO] = self.save_wrapper(lambda: float(parsed['PE_RATIO-value']))
+
+            t = default(parsed, 'DIVIDEND_AND_YIELD-value').replace('(', '').replace(')', '').replace('%', '').strip().split(' ')
+            tick[Datacode.DIV] = self.save_wrapper(lambda: float(t[0]))
+            tick[Datacode.DIV_YIELD] = self.save_wrapper(lambda: float(t[1])/100.0)
 
-            tick[Datacode.PREV_CLOSE] = float(raw(price, 'regularMarketPreviousClose'))
-            tick[Datacode.OPEN] = float(raw(price, 'regularMarketOpen'))
-            tick[Datacode.CHANGE] = float(raw(price, 'regularMarketChange'))
-            tick[Datacode.CHANGE_IN_PERCENT] = 100 * float(raw(price, 'regularMarketChangePercent'))
-            tick[Datacode.LOW] = float(raw(price, 'regularMarketDayLow'))
-            tick[Datacode.HIGH] = float(raw(price, 'regularMarketDayHigh'))
-            tick[Datacode.LAST_PRICE] = float(raw(price, 'regularMarketPrice'))
-            tick[Datacode.VOLUME] = float(raw(price, 'regularMarketVolume'))
-            tick[Datacode.AVG_DAILY_VOL_3MONTH] = float(raw(price, 'averageDailyVolume3Month'))
-            tick[Datacode.BETA] = float(raw(summaryDetail, 'beta'))
-            tick[Datacode.EPS] = self.save_wrapper(lambda: float(raw(results['defaultKeyStatistics'], 'trailingEps')))
-            tick[Datacode.PE_RATIO] = float(raw(summaryDetail, 'trailingPE'))
-            tick[Datacode.DIV] = float(raw(summaryDetail, 'dividendRate'))
-            tick[Datacode.DIV_YIELD] = float(raw(summaryDetail, 'dividendYield'))
             tick[Datacode.EX_DIV_DATE] = self.save_wrapper(
-                lambda: dateutil.parser.parse(str(fmt(summaryDetail, 'exDividendDate')), yearfirst=True, dayfirst=False).date())
-            tick[Datacode.SHARES_OUT] = float(raw(defaultKeyStatistics, 'sharesOutstanding'))
-            tick[Datacode.FREE_FLOAT] = float(raw(defaultKeyStatistics, 'floatShares'))
+                lambda: dateutil.parser.parse(parsed['EX_DIVIDEND_DATE-value'], yearfirst=True, dayfirst=False).date())
 
-            tick[Datacode.PAYOUT_RATIO] = float(raw(summaryDetail, 'payoutRatio'))
-            tick[Datacode.LOW_52_WEEK] = float(raw(summaryDetail, 'fiftyTwoWeekLow'))
-            tick[Datacode.HIGH_52_WEEK] = float(raw(summaryDetail, 'fiftyTwoWeekHigh'))
-            tick[Datacode.MARKET_CAP] = float(raw(summaryDetail, 'marketCap'))
+            t = default(parsed, 'FIFTY_TWO_WK_RANGE-value').split(' - ')
+            tick[Datacode.LOW_52_WEEK] = self.save_wrapper(lambda: float(t[0]))
+            tick[Datacode.HIGH_52_WEEK] = self.save_wrapper(lambda: float(t[1]))
 
-            tick[Datacode.BID] = float(raw(summaryDetail, 'bid'))
-            tick[Datacode.ASK] = float(raw(summaryDetail, 'ask'))
-            tick[Datacode.BIDSIZE] = float(raw(summaryDetail, 'bidSize'))
-            tick[Datacode.ASKSIZE] = float(raw(summaryDetail, 'askSize'))
+            tick[Datacode.MARKET_CAP] = self.save_wrapper(lambda: float(handle_abbreviations(parsed['MARKET_CAP-value'])))
+
+            t = default(parsed, 'BID-value').split(' x ')
+            tick[Datacode.BID] = self.save_wrapper(lambda: float(t[0]))
+            tick[Datacode.BIDSIZE] = self.save_wrapper(lambda: float(t[1]))
+
+            t = default(parsed, 'ASK-value').split(' x ')
+            tick[Datacode.ASK] = self.save_wrapper(lambda: float(t[0]))
+            tick[Datacode.ASKSIZE] = self.save_wrapper(lambda: float(t[1]))
 
             tick[Datacode.EXPIRY_DATE] = self.save_wrapper(
-                lambda: dateutil.parser.parse(str(fmt(summaryDetail, 'expireDate')), yearfirst=True, dayfirst=False).date())
+                lambda: dateutil.parser.parse(parsed['EXPIRE_DATE-value'], yearfirst=True, dayfirst=False).date())
 
-            if quoteType:
-                t = int(price['regularMarketTime'])
-                tz = pytz.timezone(quoteType['exchangeTimezoneName'])
+            tick[Datacode.SETTLEMENT_DATE] = self.save_wrapper(
+                lambda: dateutil.parser.parse(parsed['SETTLEMENT_DATE-value'], yearfirst=True, dayfirst=False).date())
 
-                tick[Datacode.TIMEZONE] = tz
-                dt = datetime.datetime.fromtimestamp(t, tz)
+            r = '<div id="quote-market-notice"[^>]*><span>([^>]*?)(. Market open.)?</span></div>'
+            match = re.compile(r, flags=re.DOTALL).search(text)
+            if match:
+                t = html.unescape(match.group(1)).strip().split(' ')
+                tick[Datacode.TIMEZONE] = self.save_wrapper(lambda: t[-1])
 
-                tick[Datacode.LAST_PRICE_DATE] = dt.date()
-                tick[Datacode.LAST_PRICE_TIME] = dt.time()
+            # if quoteType:
+            #     t = int(price['regularMarketTime'])
+            #     tz = pytz.timezone(quoteType['exchangeTimezoneName'])
+            #
+            #     tick[Datacode.TIMEZONE] = tz
+            #     dt = datetime.datetime.fromtimestamp(t, tz)
+            #
+            #     tick[Datacode.LAST_PRICE_DATE] = dt.date()
+            #     tick[Datacode.LAST_PRICE_TIME] = dt.time()
 
-            tick[Datacode.TICKER] = self.save_wrapper(lambda: str(price['symbol']))
-            tick[Datacode.EXCHANGE] = self.save_wrapper(lambda: str(price['exchange']))
-            tick[Datacode.CURRENCY] = self.save_wrapper(lambda: str(price['currency']))
+            r = '<span>([ \\w]+?) - [^>]*Currency in ([\\w]+)[^>]*</span>'
+            match = re.compile(r, flags=re.DOTALL).search(text)
+            if match:
+                tick[Datacode.EXCHANGE] = self.save_wrapper(lambda: html.unescape(match.group(1)).strip())
+                tick[Datacode.CURRENCY] = self.save_wrapper(lambda: html.unescape(match.group(2)).strip())
 
-            # some Moscow symbols miss currency in data block but show it in text e.g. VTBBA.ME, TBIOA.ME
-            if not tick[Datacode.CURRENCY]:
-                r = r'Currency in ([A-Z]{3})\b'
-                match = re.compile(r, flags=re.DOTALL).search(text)
-                if match:
-                    tick[Datacode.CURRENCY] = match.group(1)
-
-            # fallback for yield on US mutual funds and ETFs, which is in different field
+            # fallback for dividend/yield on mutual funds and ETFs
+            if not tick[Datacode.DIV]:
+                tick[Datacode.DIV] = self.save_wrapper(lambda: float(parsed['LAST_DIVIDEND-value']))
             if not tick[Datacode.DIV_YIELD]:
-                tick[Datacode.DIV_YIELD] = float(raw(summaryDetail, 'yield'))
+                tick[Datacode.DIV_YIELD] = self.save_wrapper(lambda: float(parsed['TD_YIELD-value'].replace('%', '').strip())/100.0)
 
-            name = price['longName'] or price['shortName']
-            if name:
-                tick[Datacode.NAME] = html.unescape(str(name))
-            else:
+            tick[Datacode.NAME] = self.save_wrapper(
+                lambda: html.unescape(root.find('.//h1').text).strip())
+
+            if not tick[Datacode.NAME]:
                 tick[Datacode.NAME] = tick[Datacode.TICKER]
 
-            tick[Datacode.SECTOR] = self.save_wrapper(lambda: str(results['summaryProfile']['sector']))
-            tick[Datacode.INDUSTRY] = self.save_wrapper(lambda: str(results['summaryProfile']['industry']))
-
         except BaseException as e:
             logger.exception("BaseException ticker=%s datacode=%s", ticker, datacode)
-            return 'Yahoo.getRealtime({}, {}) - process: {}'.format(ticker, datacode, e)
+            del self.realtime[ticker]
+            return 'Yahoo.getRealtimeSummary({}, {}) - process: {}'.format(ticker, datacode, e)
+
+        return self._return_value(self.realtime[ticker], datacode)
+
+    def getRealtimeStatistics(self, ticker, datacode):
+
+        """
+        Retrieve realtime data from Yahoo Finance - Statistics tab
+        """
+
+        tick = self.realtime[ticker]
+
+        url = 'https://finance.yahoo.com/quote/{}/key-statistics?p={}'.format(ticker, ticker)
+
+        try:
+            text = self.urlopen(url, redirect=True)
+            self.save_cookies()
+        except BaseException as e:
+            logger.exception("BaseException ticker=%s datacode=%s last_url=%s redirect_count=%s", ticker, datacode, self.last_url, self.redirect_count)
+            del self.realtime[ticker]
+            return 'Yahoo.getRealtimeStatistics({}, {}) - urlopen: {}'.format(ticker, datacode, e)
+
+        try:
+            with open(os.path.join(self.basedir, 'yahoo-{}-statistics.html'.format(ticker)), "w", encoding="utf-8") as text_file:
+                print(f"<!-- '{url}' -->\r\n\r\n{text}", file=text_file)
+        except BaseException:
+            logger.exception("BaseException open/write ticker=%s datacode=%s", ticker, datacode)
+
+        try:
+            parser = NaiveHTMLParser()
+            root = parser.feed(text)
+            parser.close()
+        except BaseException as e:
+            logger.exception("BaseException ticker=%s datacode=%s", ticker, datacode)
+            del self.realtime[ticker]
+            return 'Yahoo.getRealtimeStatistics({}, {}) - HTML parsing: {}'.format(ticker, datacode, e)
+
+        try:
+
+            parsed = {}
+
+            statistics = root.find(".//section[@data-test='qsp-statistics']")
+
+            if statistics is None:
+                return None
+
+            tick[Datacode.TICKER] = ticker
+            tick[Datacode.TIMESTAMP] = time.time()
+            tick[Datacode.YAHOO_STATISTIC_RECEIVED] = True
+
+            # Valuation Measures
+            found = statistics.find('./div[2]/div[1]//table')
+            if found:
+                for d in found.findall('.//tr'):
+                    key = d.find('./td[1]/span').text
+                    if key is not None:
+                        parsed[key] = d.find('./td[2]').text
+
+            # Stock Price History
+            found = statistics.find('./div[2]/div[2]/div[1]/div[1]//table')
+            if found:
+                for d in found.findall('.//tr'):
+                    key = d.find('./td[1]/span').text
+                    if key is not None:
+                        parsed[key] = d.find('./td[2]').text
+
+            # Share Statistics
+            found = statistics.find('./div[2]/div[2]/div[1]/div[2]//table')
+            if found:
+                for d in found.findall('.//tr'):
+                    key = d.find('./td[1]/span').text
+                    if key is not None:
+                        parsed[key] = d.find('./td[2]').text
+
+            # Dividends & Splits
+            found = statistics.find('./div[2]/div[2]/div[1]/div[3]//table')
+            if found:
+                for d in found.findall('.//tr'):
+                    key = d.find('./td[1]/span').text
+                    if key is not None:
+                        parsed[key] = d.find('./td[2]').text
+
+            tick[Datacode.SHARES_OUT] = self.save_wrapper(
+                lambda: float(handle_abbreviations(parsed['Shares Outstanding'])))
+            tick[Datacode.FREE_FLOAT] = self.save_wrapper(
+                lambda: float(handle_abbreviations(parsed['Float'])))
+            tick[Datacode.PAYOUT_RATIO] = self.save_wrapper(
+                lambda: float(handle_abbreviations(parsed['Payout Ratio'].replace('%', '').strip()))/100.0)
+
+        except BaseException as e:
+            logger.exception("BaseException ticker=%s datacode=%s", ticker, datacode)
+            del self.realtime[ticker]
+            return 'Yahoo.getRealtimeStatistics({}, {}) - process: {}'.format(ticker, datacode, e)
+
+        return self._return_value(self.realtime[ticker], datacode)
+
+    def getRealtimeProfile(self, ticker, datacode):
+
+        """
+        Retrieve realtime data from Yahoo Finance - Profile tab
+        """
+
+        tick = self.realtime[ticker]
+
+        url = 'https://finance.yahoo.com/quote/{}/profile?p={}'.format(ticker, ticker)
+
+        try:
+            text = self.urlopen(url, redirect=True)
+            self.save_cookies()
+        except BaseException as e:
+            logger.exception("BaseException ticker=%s datacode=%s last_url=%s redirect_count=%s", ticker, datacode, self.last_url, self.redirect_count)
+            del self.realtime[ticker]
+            return 'Yahoo.getRealtimeProfile({}, {}) - urlopen: {}'.format(ticker, datacode, e)
+
+        try:
+            with open(os.path.join(self.basedir, 'yahoo-{}-profile.html'.format(ticker)), "w", encoding="utf-8") as text_file:
+                print(f"<!-- '{url}' -->\r\n\r\n{text}", file=text_file)
+        except BaseException:
+            logger.exception("BaseException open/write ticker=%s datacode=%s", ticker, datacode)
+
+        try:
+            parser = NaiveHTMLParser()
+            root = parser.feed(text)
+            parser.close()
+        except BaseException as e:
+            logger.exception("BaseException ticker=%s datacode=%s", ticker, datacode)
+            del self.realtime[ticker]
+            return 'Yahoo.getRealtimeProfile({}, {}) - HTML parsing: {}'.format(ticker, datacode, e)
+
+        try:
+
+            p = root.find(".//*[span='Sector(s)']")
+
+            if p is None:
+                return None
+
+            tick[Datacode.TICKER] = ticker
+            tick[Datacode.TIMESTAMP] = time.time()
+            tick[Datacode.YAHOO_PROFILE_RECEIVED] = True
+
+            tick[Datacode.SECTOR] = self.save_wrapper(lambda: p.find("./span[2]").text)
+            tick[Datacode.INDUSTRY] = self.save_wrapper(lambda: p.find("./span[4]").text)
+
+        except BaseException as e:
+            logger.exception("BaseException ticker=%s datacode=%s", ticker, datacode)
+            del self.realtime[ticker]
+            return 'Yahoo.getRealtimeProfile({}, {}) - process: {}'.format(ticker, datacode, e)
 
         return self._return_value(self.realtime[ticker], datacode)
 
@@ -288,7 +536,7 @@ class Yahoo(BaseClient):
         try:
             date_as_dt = dateutil.parser.parse(date, yearfirst=True, dayfirst=False)
         except BaseException as e:
-            logger.exception("BaseException ticker=%s datacode=%s", ticker, datacode)
+            logger.exception("BaseException ticker=%s datacode=%s last_url=%s redirect_count=%s", ticker, datacode, self.last_url, self.redirect_count)
             return 'Yahoo.getHistoric({}, {}, {}) - date_as_dt: {}'.format(ticker, datacode, date, e)
 
         if ticker in self.historicdata:
@@ -311,7 +559,7 @@ class Yahoo(BaseClient):
                 min_tick_date = int(dateutil.parser.parse(min(ticks), yearfirst=True, dayfirst=False).timestamp())  # remember current earliest date
 
         if not self.crumb:
-            self.getRealtime(ticker, datacode)
+            self.getRealtime(ticker, Datacode.LAST_PRICE)
 
         if not self.crumb:
             return 'Yahoo.getHistoric({}, {}, {}) - crumb missing'.format(ticker, datacode, date)

src/generate_metainfo.py

@@ -14,7 +14,7 @@ import os
 cur_dir = os.getcwd()
 
 addin_id = "com.financials.getinfo"
-addin_version = "3.1.0"
+addin_version = "3.3.0"
 addin_displayname = "Financial Market Extension"
 addin_publisher_link = "https://github.com/cmallwitz/Financials-Extension"
 addin_publisher_name = "The Publisher"

src/test_ft.py

@@ -15,6 +15,7 @@ import unittest
 logging.basicConfig(level=logging.ERROR, format="%(asctime)s %(name)s %(levelname)s %(message)s")
 
 import financials
+from datacode import Datacode
 import testutils
 
 financials = financials.createInstance(None)
@@ -134,6 +135,33 @@ class Test(unittest.TestCase):
         s = financials.getRealtime('VFIAX', 'LAST_PRICE_TIME', 'FT')
         self.assertEqual(str, type(s), 'test_US_mutuals LAST_PRICE_TIME {}'.format(s))
 
+    def test_US_futures(self):
+
+        s = financials.getRealtime('ESH3:IOM', Datacode.NAME.value, 'FT')
+        self.assertEqual(str, type(s), 'test_realtime_US_futures NAME {}'.format(s))
+        self.assertEqual('EMINI S&P MAR3', s, 'test_US_futures NAME {}'.format(s))
+
+        s = financials.getRealtime('ESH3:IOM', Datacode.LAST_PRICE.value, 'FT')
+        self.assertEqual(float, type(s), 'test_US_futures LAST_PRICE {}'.format(s))
+
+        # s = financials.getRealtime('ESH3:IOM', Datacode.OPEN.value, 'FT')
+        # self.assertEqual(float, type(s), 'test_US_futures OPEN {}'.format(s))
+
+        s = financials.getRealtime('ESH3:IOM', Datacode.VOLUME.value, 'FT')
+        self.assertEqual(float, type(s), 'test_US_futures VOLUME {}'.format(s))
+
+        s = financials.getRealtime('ESH3:IOM', Datacode.LOW_52_WEEK.value, 'FT')
+        self.assertEqual(float, type(s), 'test_US_futures LOW_52_WEEK {}'.format(s))
+
+        s = financials.getRealtime('ESH3:IOM', Datacode.HIGH_52_WEEK.value, 'FT')
+        self.assertEqual(float, type(s), 'test_US_futures HIGH_52_WEEK {}'.format(s))
+
+        s = financials.getRealtime('ESH3:IOM', Datacode.CHANGE.value, 'FT')
+        self.assertEqual(float, type(s), 'test_US_futures CHANGE {}'.format(s))
+
+        s = financials.getRealtime('ESH3:IOM', Datacode.CHANGE_IN_PERCENT.value, 'FT')
+        self.assertEqual(float, type(s), 'test_US_futures CHANGE_IN_PERCENT {}'.format(s))
+
     def test_UK_ETF(self):
         s = financials.getRealtime('CSP1:LSE:GBX', 'NAME', 'FT')
         self.assertEqual(str, type(s), 'test_UK_ETF NAME {}'.format(s))

src/test_yahoo.py

@@ -16,6 +16,7 @@ import unittest
 
 logging.basicConfig(level=logging.ERROR, format="%(asctime)s %(name)s %(levelname)s %(message)s")
 
+import baseclient
 import financials
 from datacode import Datacode
 import testutils
@@ -23,8 +24,26 @@ import testutils
 financials = financials.createInstance(None)
 
 
+def urlopen_fail(self, url, redirect=True, data=None, headers={}, cookies=[], **kwargs):
+    raise baseclient.HttpException(url, 'simulated urlopen() failed')
+
+
 class Test(unittest.TestCase):
 
+    def test_recovery_from_urlopen_error_issue(self):
+
+        financials.yahoo.last_url = 'blank'
+
+        urlopen_saved = baseclient.BaseClient.urlopen
+        baseclient.BaseClient.urlopen = urlopen_fail
+
+        s = financials.getRealtime('U1IH.F', Datacode.LAST_PRICE.value, 'YAHOO')
+
+        baseclient.BaseClient.urlopen = urlopen_saved
+
+        s = financials.getRealtime('U1IH.F', Datacode.PREV_CLOSE.value, 'YAHOO')
+        self.assertEqual(float, type(s), 'test_recovery_from_urlopen_error_issue PREV_CLOSE {}'.format(s))
+
     def test_currency(self):
         s = financials.getRealtime('EURGBP=X', Datacode.CURRENCY.value, 'YAHOO')
         self.assertEqual(str, type(s), 'test_currency CURRENCY')
@@ -32,6 +51,17 @@ class Test(unittest.TestCase):
         s = financials.getRealtime('EURGBP=X', Datacode.LAST_PRICE.value, 'YAHOO')
         self.assertEqual(float, type(s), 'test_currency LAST_PRICE')
 
+    def test_realtime_US_ZVZZT(self):
+
+        s = financials.getRealtime('ZVZZT', Datacode.PAYOUT_RATIO.value, 'YAHOO')
+        self.assertIsNone(s, 'test_realtime_US_ZVZZT PAYOUT_RATIO {}'.format(s))
+
+        s = financials.getRealtime('ZVZZT', Datacode.SECTOR.value, 'YAHOO')
+        self.assertIsNone(s, 'test_realtime_US_ZVZZT SECTOR {}'.format(s))
+
+        s = financials.getRealtime('ZVZZT', Datacode.LAST_PRICE.value, 'YAHOO')
+        self.assertEqual(float, type(s), 'test_realtime_US_ZVZZT LAST_PRICE {}'.format(s))
+
     def test_realtime_US_equity(self):
 
         s = financials.getRealtime('^GSPC', Datacode.NAME.value, 'YAHOO')
@@ -70,7 +100,7 @@ class Test(unittest.TestCase):
 
         s = financials.getRealtime('IBM', Datacode.NAME.value, 'YAHOO')
         self.assertEqual(str, type(s), 'test_realtime_US_equity NAME {}'.format(s))
-        self.assertEqual(s, 'International Business Machines Corporation',
+        self.assertEqual(s, 'International Business Machines Corporation (IBM)',
                          'test_realtime_US_equity NAME {}'.format(s))
 
         s = financials.getRealtime('IBM', Datacode.SECTOR.value, 'YAHOO')
@@ -82,7 +112,7 @@ class Test(unittest.TestCase):
         self.assertEqual(s, 'Information Technology Services', 'test_realtime_US_equity INDUSTRY {}'.format(s))
 
         s = financials.getRealtime('IBM', Datacode.TIMEZONE.value, 'YAHOO')
-        self.assertEqual(s, 'America/New_York', 'test_realtime_US_equity TIMEZONE {}'.format(s))
+        self.assertEqual(s, 'EST', 'test_realtime_US_equity TIMEZONE {}'.format(s))
 
         s = financials.getRealtime('IBM', Datacode.BETA.value, 'YAHOO')
         self.assertTrue(testutils.is_positive_float(s), 'test_realtime_US_equity BETA {}'.format(s))
@@ -113,7 +143,7 @@ class Test(unittest.TestCase):
         self.assertTrue(testutils.is_positive_float(s), 'test_realtime_US_equity FREE_FLOAT {}'.format(s))
 
         s = financials.getRealtime('IBM', Datacode.EXCHANGE.value, 'YAHOO')
-        self.assertEqual(s, 'NYQ', 'test_realtime_US_equity EXCHANGE')
+        self.assertEqual(s, 'NYSE', 'test_realtime_US_equity EXCHANGE')
 
     def test_realtime_US_mutuals(self):
 
@@ -123,51 +153,104 @@ class Test(unittest.TestCase):
         s = financials.getRealtime('VFIAX', Datacode.LAST_PRICE.value, 'YAHOO')
         self.assertEqual(float, type(s), 'test_realtime_US_mutuals LAST_PRICE {}'.format(s))
 
-        s = financials.getRealtime('VFIAX', Datacode.LAST_PRICE_DATE.value, 'YAHOO')
-        self.assertEqual(str, type(s), 'test_realtime_US_mutuals LAST_PRICE_DATE {}'.format(s))
-
-        s = financials.getRealtime('VFIAX', Datacode.LAST_PRICE_TIME.value, 'YAHOO')
-        self.assertEqual(str, type(s), 'test_realtime_US_mutuals LAST_PRICE_TIME {}'.format(s))
+        s = financials.getRealtime('VFIAX', Datacode.DIV.value, 'YAHOO')
+        self.assertEqual(float, type(s), 'test_realtime_US_mutuals DIV {}'.format(s))
 
         s = financials.getRealtime('VFIAX', Datacode.DIV_YIELD.value, 'YAHOO')
+        # self.assertIsNone(s, 'test_realtime_US_mutuals DIV_YIELD {}'.format(s)) # no yield
         self.assertTrue(testutils.is_positive_float(s), 'test_realtime_US_mutuals DIV_YIELD {}'.format(s))
 
+        s = financials.getRealtime('SHRAX', Datacode.DIV.value, 'YAHOO')
+        self.assertEqual(float, type(s), 'test_realtime_US_mutuals DIV {}'.format(s))
+
+        s = financials.getRealtime('SHRAX', Datacode.DIV_YIELD.value, 'YAHOO')
+        # self.assertIsNone(s, 'test_realtime_US_mutuals DIV_YIELD {}'.format(s)) # no yield
+        self.assertEqual(float, type(s), 'test_realtime_US_mutuals DIV_YIELD {}'.format(s))
+
+        # s = financials.getRealtime('VERX.L', Datacode.DIV.value, 'YAHOO')
+        # self.assertIsNone(s, 'test_realtime_US_mutuals DIV {}'.format(s)) # no dividend
+
+        s = financials.getRealtime('VERX.L', Datacode.DIV_YIELD.value, 'YAHOO')
+        self.assertEqual(float, type(s), 'test_realtime_US_mutuals DIV_YIELD {}'.format(s))
+
     def test_realtime_US_options(self):
 
         # symbol from https://finance.yahoo.com/quote/IBM/options?p=IBM
         
-        s = financials.getRealtime('IBM230120C00130000', Datacode.PREV_CLOSE.value, 'YAHOO')
+        s = financials.getRealtime('IBM240119C00120000', Datacode.PREV_CLOSE.value, 'YAHOO')
         self.assertEqual(float, type(s), 'test_realtime_US_options PREV_CLOSE {}'.format(s))
 
-        s = financials.getRealtime('IBM230120C00130000', Datacode.NAME.value, 'YAHOO')
+        s = financials.getRealtime('IBM240119C00120000', Datacode.NAME.value, 'YAHOO')
         self.assertEqual(str, type(s), 'test_realtime_US_options NAME {}'.format(s))
-        self.assertEqual('IBM Jan 2023 130.000 call', s, 'test_realtime_US_options NAME {}'.format(s))
+        self.assertEqual('IBM Jan 2024 120.000 call', s, 'test_realtime_US_options NAME {}'.format(s))
 
-        s = financials.getRealtime('IBM230120C00130000', Datacode.EXPIRY_DATE.value, 'YAHOO')
+        s = financials.getRealtime('IBM240119C00120000', Datacode.EXPIRY_DATE.value, 'YAHOO')
         self.assertEqual(str, type(s), 'test_realtime_US_options EXPIRY_DATE {}'.format(s))
         self.assertTrue(testutils.is_date(s), 'test_realtime_US_options EXPIRY_DATE {}'.format(s))
-        self.assertEqual("2023-01-20", s, 'test_realtime_US_options EXPIRY_DATE {}'.format(s))
+        self.assertEqual("2024-01-19", s, 'test_realtime_US_options EXPIRY_DATE {}'.format(s))
 
-        s = financials.getRealtime('IBM230120C00130000', Datacode.LAST_PRICE.value, 'YAHOO')
+        s = financials.getRealtime('IBM240119C00120000', Datacode.LAST_PRICE.value, 'YAHOO')
         self.assertEqual(float, type(s), 'test_realtime_US_options LAST_PRICE {}'.format(s))
 
-        s = financials.getRealtime('IBM230120C00130000', Datacode.OPEN.value, 'YAHOO')
+        s = financials.getRealtime('IBM240119C00120000', Datacode.OPEN.value, 'YAHOO')
         self.assertEqual(float, type(s), 'test_realtime_US_options OPEN {}'.format(s))
 
-        s = financials.getRealtime('IBM230120C00130000', Datacode.VOLUME.value, 'YAHOO')
+        s = financials.getRealtime('IBM240119C00120000', Datacode.VOLUME.value, 'YAHOO')
         self.assertEqual(float, type(s), 'test_realtime_US_options VOLUME {}'.format(s))
 
-        s = financials.getRealtime('IBM230120C00130000', Datacode.BID.value, 'YAHOO')
+        s = financials.getRealtime('IBM240119C00120000', Datacode.BID.value, 'YAHOO')
         self.assertEqual(float, type(s), 'test_realtime_US_options BID {}'.format(s))
 
-        s = financials.getRealtime('IBM230120C00130000', Datacode.ASK.value, 'YAHOO')
+        s = financials.getRealtime('IBM240119C00120000', Datacode.ASK.value, 'YAHOO')
         self.assertEqual(float, type(s), 'test_realtime_US_options ASK {}'.format(s))
 
-        s = financials.getRealtime('IBM230120C00130000', Datacode.BIDSIZE.value, 'YAHOO')
-        self.assertEqual(float, type(s), 'test_realtime_US_options BIDSIZE {}'.format(s))
+        s = financials.getRealtime('IBM240119C00120000', Datacode.PAYOUT_RATIO.value, 'YAHOO')
+        self.assertIsNone(s, 'test_realtime_US_options PAYOUT_RATIO {}'.format(s))
 
-        s = financials.getRealtime('IBM230120C00130000', Datacode.ASKSIZE.value, 'YAHOO')
-        self.assertEqual(float, type(s), 'test_realtime_US_options ASKSIZE {}'.format(s))
+        s = financials.getRealtime('IBM230120C00130000', Datacode.SECTOR.value, 'YAHOO')
+        self.assertIsNone(s, 'test_realtime_US_options SECTOR {}'.format(s))
+
+    def test_realtime_US_futures(self):
+
+        s = financials.getRealtime('ES=F', Datacode.NAME.value, 'YAHOO')
+        self.assertEqual(str, type(s), 'test_realtime_US_futures NAME {}'.format(s))
+        self.assertEqual('E-Mini S&P 500 Mar 23 (ES=F)', s, 'test_realtime_US_futures NAME {}'.format(s))
+
+        s = financials.getRealtime('ES=F', Datacode.TICKER.value, 'YAHOO')
+        self.assertEqual(str, type(s), 'test_realtime_US_futures TICKER {}'.format(s))
+        self.assertEqual('ESH23.CME', s, 'test_realtime_US_futures TICKER {}'.format(s))
+
+        s = financials.getRealtime('ES=F', Datacode.SETTLEMENT_DATE.value, 'YAHOO')
+        self.assertEqual(str, type(s), 'test_realtime_US_futures SETTLEMENT_DATE {}'.format(s))
+        self.assertTrue(testutils.is_date(s), 'test_realtime_US_futures SETTLEMENT_DATE {}'.format(s))
+        self.assertEqual("2023-03-17", s, 'test_realtime_US_futures SETTLEMENT_DATE {}'.format(s))
+
+        s = financials.getRealtime('ES=F', Datacode.LAST_PRICE.value, 'YAHOO')
+        self.assertEqual(float, type(s), 'test_realtime_US_futures LAST_PRICE {}'.format(s))
+
+        s = financials.getRealtime('ES=F', Datacode.OPEN.value, 'YAHOO')
+        self.assertEqual(float, type(s), 'test_realtime_US_futures OPEN {}'.format(s))
+
+        s = financials.getRealtime('ES=F', Datacode.VOLUME.value, 'YAHOO')
+        self.assertEqual(float, type(s), 'test_realtime_US_futures VOLUME {}'.format(s))
+
+        s = financials.getRealtime('ES=F', Datacode.BID.value, 'YAHOO')
+        self.assertEqual(float, type(s), 'test_realtime_US_futures BID {}'.format(s))
+
+        s = financials.getRealtime('ES=F', Datacode.ASK.value, 'YAHOO')
+        self.assertEqual(float, type(s), 'test_realtime_US_futures ASK {}'.format(s))
+
+        s = financials.getRealtime('ES=F', Datacode.CHANGE.value, 'YAHOO')
+        self.assertEqual(float, type(s), 'test_realtime_US_futures CHANGE {}'.format(s))
+
+        s = financials.getRealtime('ES=F', Datacode.CHANGE_IN_PERCENT.value, 'YAHOO')
+        self.assertEqual(float, type(s), 'test_realtime_US_futures CHANGE_IN_PERCENT {}'.format(s))
+
+        s = financials.getRealtime('ES=F', Datacode.LOW.value, 'YAHOO')
+        self.assertEqual(float, type(s), 'test_realtime_US_futures LOW {}'.format(s))
+
+        s = financials.getRealtime('ES=F', Datacode.HIGH.value, 'YAHOO')
+        self.assertEqual(float, type(s), 'test_realtime_US_futures HIGH {}'.format(s))
 
     def test_realtime_UK_ETF(self):
 
@@ -175,11 +258,11 @@ class Test(unittest.TestCase):
         self.assertEqual(float, type(s), 'test_realtime_UK_ETF LAST_PRICE {}'.format(s))
 
         s = financials.getRealtime('VERX.L', Datacode.TIMEZONE.value, 'YAHOO')
-        self.assertEqual('Europe/London', s, 'test_realtime_UK_ETF TIMEZONE {}'.format(s))
+        self.assertEqual('GMT', s, 'test_realtime_UK_ETF TIMEZONE {}'.format(s))
 
         s = financials.getRealtime('CSP1.L', Datacode.NAME.value, 'YAHOO')
         self.assertEqual(str, type(s), 'test_realtime_UK_ETF NAME {}'.format(s))
-        self.assertEqual('iShares Core S&P 500 UCITS ETF USD (Acc)', s, 'test_realtime_UK_ETF NAME {}'.format(s))
+        self.assertEqual('iShares Core S&P 500 UCITS ETF USD (Acc) (CSP1.L)', s, 'test_realtime_UK_ETF NAME {}'.format(s))
 
     def test_realtime_DE_equity(self):
 
@@ -187,7 +270,7 @@ class Test(unittest.TestCase):
         self.assertEqual(float, type(s), 'test_realtime_DE_equity LAST_PRICE {}'.format(s))
 
         s = financials.getRealtime('SAP.DE', Datacode.TIMEZONE.value, 'YAHOO')
-        self.assertEqual('Europe/Berlin', s, 'test_realtime_DE_equity TIMEZONE {}'.format(s))
+        self.assertEqual('CET', s, 'test_realtime_DE_equity TIMEZONE {}'.format(s))
 
         s = financials.getRealtime('SAP.DE', Datacode.SECTOR.value, 'YAHOO')
         self.assertEqual(str, type(s), 'test_realtime_DE_equity SECTOR {}'.format(s))
@@ -201,14 +284,51 @@ class Test(unittest.TestCase):
         self.assertEqual(str, type(s), 'test_realtime_DE_equity NAME {}'.format(s))
 
         s = financials.getRealtime('C060.DE', Datacode.EXCHANGE.value, 'YAHOO')
-        self.assertEqual(s, 'GER', 'test_realtime_DE_equity EXCHANGE')
+        self.assertEqual(s, 'XETRA', 'test_realtime_DE_equity EXCHANGE')
+
+    def test_TA_equity(self):
+
+        s = financials.getRealtime('LUMI.TA', 'LAST_PRICE', 'YAHOO')
+        self.assertEqual(float, type(s), 'test_TA_equity LAST_PRICE {}'.format(s))
+
+        s = financials.getRealtime('LUMI.TA', 'OPEN', 'YAHOO')
+        self.assertEqual(float, type(s), 'test_TA_equity OPEN {}'.format(s))
+
+        s = financials.getRealtime('LUMI.TA', 'LOW', 'YAHOO')
+        self.assertEqual(float, type(s), 'test_TA_equity LOW {}'.format(s))
+
+        s = financials.getRealtime('LUMI.TA', 'HIGH', 'YAHOO')
+        self.assertEqual(float, type(s), 'test_TA_equity HIGH {}'.format(s))
+
+        s = financials.getRealtime('LUMI.TA', 'LOW_52_WEEK', 'YAHOO')
+        self.assertEqual(float, type(s), 'test_TA_equity LOW_52_WEEK {}'.format(s))
+
+        s = financials.getRealtime('LUMI.TA', 'HIGH_52_WEEK', 'YAHOO')
+        self.assertEqual(float, type(s), 'test_TA_equity HIGH_52_WEEK {}'.format(s))
+
+        s = financials.getRealtime('LUMI.TA', 'MARKET_CAP', 'YAHOO')
+        self.assertEqual(float, type(s), 'test_TA_equity MARKET_CAP {}'.format(s))
+
+        s = financials.getRealtime('LUMI.TA', 'VOLUME', 'YAHOO')
+        self.assertEqual(float, type(s), 'test_TA_equity VOLUME {}'.format(s))
+
+        s = financials.getRealtime('LUMI.TA', 'CURRENCY', 'YAHOO')
+        self.assertEqual('ILA', s, 'test_TA_equity CURRENCY')
+
+        s = financials.getRealtime('LUMI.TA', 'SECTOR', 'YAHOO')
+        self.assertEqual(str, type(s), 'test_TLV_equity SECTOR {}'.format(s))
+        self.assertEqual('Financial Services', s, 'test_TA_equity SECTOR {}'.format(s))
+
+        s = financials.getRealtime('LUMI.TA', 'INDUSTRY', 'YAHOO')
+        self.assertEqual(str, type(s), 'test_TLV_equity INDUSTRY {}'.format(s))
+        self.assertEqual('Banks—Regional', s, 'test_TA_equity INDUSTRY {}'.format(s))
 
     def test_DK_equity(self):
         s = financials.getRealtime('NOVO-B.CO', 'last_price', 'YAHOO')
         self.assertEqual(float, type(s), 'test_DK_equity LAST_PRICE {}'.format(s))
 
         s = financials.getRealtime('NOVO-B.CO', 'name', 'YAHOO')
-        self.assertEqual('Novo Nordisk A/S', s, 'test_DK_equity NAME {}'.format(s))
+        self.assertEqual('Novo Nordisk A/S (NOVO-B.CO)', s, 'test_DK_equity NAME {}'.format(s))
 
         s = financials.getRealtime('NOVO-B.CO', 'currency', 'YAHOO')
         self.assertEqual('DKK', s, 'test_DK_equity CURRENCY {}'.format(s))
@@ -217,7 +337,18 @@ class Test(unittest.TestCase):
         self.assertEqual(str, type(s), 'test_DK_equity INDUSTRY {}'.format(s))
         self.assertEqual('Biotechnology', s, 'test_DK_equity INDUSTRY {}'.format(s))
 
+        s = financials.getRealtime('MAERSK-B.CO', 'currency', 'YAHOO')
+        self.assertEqual('DKK', s, 'test_DK_equity CURRENCY {}'.format(s))
+
     def test_realtime_TY_equity(self):
+        s = financials.getRealtime('6503.T', Datacode.SECTOR.value, 'YAHOO')
+        self.assertEqual(str, type(s), 'test_TY_equity SECTOR {}'.format(s))
+        self.assertEqual(s, 'Industrials', 'test_TY_equity SECTOR {}'.format(s))
+
+        s = financials.getRealtime('6503.T', Datacode.INDUSTRY.value, 'YAHOO')
+        self.assertEqual(str, type(s), 'test_TY_equity INDUSTRY {}'.format(s))
+        self.assertEqual(s, 'Electrical Equipment & Parts', 'test_TY_equity INDUSTRY {}'.format(s))
+
         s = financials.getRealtime('6503.T', Datacode.OPEN.value, 'YAHOO')
         self.assertEqual(float, type(s), 'test_TY_equity OPEN {}'.format(s))
 
@@ -242,13 +373,8 @@ class Test(unittest.TestCase):
         s = financials.getRealtime('6503.T', Datacode.CURRENCY.value, 'YAHOO')
         self.assertEqual(s, 'JPY', 'test_TY_equity CURRENCY')
 
-        s = financials.getRealtime('6503.T', Datacode.SECTOR.value, 'YAHOO')
-        self.assertEqual(str, type(s), 'test_TY_equity SECTOR {}'.format(s))
-        self.assertEqual(s, 'Industrials', 'test_TY_equity SECTOR {}'.format(s))
-
-        s = financials.getRealtime('6503.T', Datacode.INDUSTRY.value, 'YAHOO')
-        self.assertEqual(str, type(s), 'test_TY_equity INDUSTRY {}'.format(s))
-        self.assertEqual(s, 'Electrical Equipment & Parts', 'test_TY_equity INDUSTRY {}'.format(s))
+        s = financials.getRealtime('6503.T', Datacode.TIMEZONE.value, 'YAHOO')
+        self.assertEqual(s, 'JST', 'test_TY_equity TIMEZONE')
 
     def test_historic_US_equity(self):
 
@@ -337,6 +463,12 @@ class Test(unittest.TestCase):
         s = financials.getRealtime('NO_NAME', Datacode.LAST_PRICE.value, 'YAHOO')
         self.assertIsNone(s, 'test_realtime_errors LAST_PRICE {}'.format(s))
 
+        s = financials.getRealtime('NO_NAME', Datacode.PAYOUT_RATIO.value, 'YAHOO')
+        self.assertIsNone(s, 'test_realtime_errors PAYOUT_RATIO {}'.format(s))
+
+        s = financials.getRealtime('NO_NAME', Datacode.SECTOR.value, 'YAHOO')
+        self.assertIsNone(s, 'test_realtime_errors SECTOR {}'.format(s))
+
         s = financials.getRealtime('C060.DE', -1, 'YAHOO')
         self.assertEqual('Datacode -1 not supported', s, 'test_realtime_errors -1 {}'.format(s))