Fix for FT redirect to unencoded URLs

Handle the case where there is a data-value but no value

compile.sh

@@ -29,13 +29,23 @@ mkdir "${PWD}"/build/META-INF/
 
 # Compile the binaries
 
-echo "Calling idlc..."
-idlc -w -verbose "${PWD}"/idl/XFinancials.idl
+if command -v idlc &> /dev/null
+then
+  echo "Calling idlc..."
+  idlc -w -verbose "${PWD}"/idl/XFinancials.idl
 
-echo "Calling regmerge..."
-regmerge -v "${PWD}"/build/XFinancials.rdb UCR "${PWD}"/idl/XFinancials.urd
+  echo "Calling regmerge..."
+  regmerge -v "${PWD}"/build/XFinancials.rdb UCR "${PWD}"/idl/XFinancials.urd
 
-rm "${PWD}"/idl/XFinancials.urd
+  rm "${PWD}"/idl/XFinancials.urd
+else
+  # LibreOffice 7.4+
+  export unoTypes=/usr/lib/libreoffice/program/types.rdb
+  export offTypes=/usr/lib/libreoffice/program/types/offapi.rdb
+
+  echo "Calling unoidl-write..."
+  unoidl-write $unoTypes $offTypes "${PWD}"/idl/XFinancials.idl "${PWD}"/build/XFinancials.rdb
+fi
 
 echo "Generating meta files..."
 python3 "${PWD}"/src/generate_metainfo.py
@@ -47,7 +57,6 @@ cp -f "${PWD}"/src/jsonParser.py "${PWD}"/build/
 cp -f "${PWD}"/src/naivehtmlparser.py "${PWD}"/build/
 cp -f "${PWD}"/src/tz.py "${PWD}"/build/
 cp -f "${PWD}"/src/financials_ft.py "${PWD}"/build/
-cp -f "${PWD}"/src/financials_google.py "${PWD}"/build/
 cp -f "${PWD}"/src/financials_yahoo.py "${PWD}"/build/
 cp -f "${PWD}"/src/financials_coinbase.py "${PWD}"/build/
 
@@ -55,11 +64,11 @@ cp -f "${PWD}"/src/financials_coinbase.py "${PWD}"/build/
 
 TMPFILE=`mktemp`
 
-wget "https://files.pythonhosted.org/packages/36/7a/87837f39d0296e723bb9b62bbb257d0355c7f6128853c78955f57342a56d/python_dateutil-2.8.2-py2.py3-none-any.whl" -O $TMPFILE
+wget "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl" -O $TMPFILE
 unzip $TMPFILE dateutil/\* -d "${PWD}"/build/
 rm $TMPFILE
 
-wget "https://files.pythonhosted.org/packages/7f/99/ad6bd37e748257dd70d6f85d916cafe79c0b0f5e2e95b11f7fbc82bf3110/pytz-2023.3-py2.py3-none-any.whl" -O $TMPFILE
+wget "https://files.pythonhosted.org/packages/9c/3d/a121f284241f08268b21359bd425f7d4825cffc5ac5cd0e1b3d82ffd2b10/pytz-2024.1-py2.py3-none-any.whl" -O $TMPFILE
 unzip $TMPFILE pytz/\* -d "${PWD}"/build/
 rm $TMPFILE
 

src/baseclient.py

@@ -16,6 +16,7 @@ import pathlib
 import random
 import select
 import urllib.request
+import urllib.parse
 from http import cookiejar
 from http.client import HTTPConnection, HTTPSConnection, HTTPException
 
@@ -33,9 +34,20 @@ class RedirectException(HTTPException):
 
 
 class HttpException(HTTPException):
-    def __init__(self, url, status):
+    def __init__(self, url, response):
         self.url = url
-        self.status = status
+        self.response = response
+
+    def __str__(self):
+        if self.response is None:
+            return f"url='{self.url}'"
+        if type(self.response) is str:
+            return f"url='{self.url}' status='{self.response}'"
+        if self.response.headers:
+            h = '\n'.join(sorted(self.response.headers.__str__().splitlines(), key=lambda l: l.lower()))
+            return f"url='{self.url}' status={self.response.status} reason='{self.response.reason}'{h}\n"
+        else:
+            return f"url='{self.url}' status={self.response.status} reason='{self.response.reason}'"
 
 
 class BaseClient:
@@ -49,15 +61,16 @@ class BaseClient:
         os.makedirs(self.basedir, exist_ok=True)
 
         user_agents = [
-            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:100.0) Gecko/20100101 Firefox/100.0',
-            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:101.0) Gecko/20100101 Firefox/101.0',
-            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:102.0) Gecko/20100101 Firefox/102.0',
-            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:103.0) Gecko/20100101 Firefox/103.0',
-            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:104.0) Gecko/20100101 Firefox/104.0',
-            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:105.0) Gecko/20100101 Firefox/105.0',
-            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:106.0) Gecko/20100101 Firefox/106.0',
-            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:107.0) Gecko/20100101 Firefox/107.0',
-            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:108.0) Gecko/20100101 Firefox/108.0'
+            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:120.0) Gecko/20100101 Firefox/120.0',
+            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:121.0) Gecko/20100101 Firefox/121.0',
+            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:122.0) Gecko/20100101 Firefox/122.0',
+            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:123.0) Gecko/20100101 Firefox/123.0',
+            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:124.0) Gecko/20100101 Firefox/124.0',
+            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:125.0) Gecko/20100101 Firefox/125.0',
+            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:126.0) Gecko/20100101 Firefox/126.0',
+            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:127.0) Gecko/20100101 Firefox/127.0',
+            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0',
+            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:129.0) Gecko/20100101 Firefox/129.0',
         ]
 
         self.default_headers = {
@@ -71,16 +84,15 @@ class BaseClient:
 
         self.response = None
 
-    def request(self, method: str, url: str, data=None, headers={}, cookies=[], **kwargs):
+    def request(self, method: str, url: str, data=None, headers={}, **kwargs):
 
         _headers = self.default_headers.copy()
         if headers:
             for key, value in headers.items():
                 _headers[key] = value
 
-        if cookies:
-            for c in cookies:
-                self.cookies.set_cookie(c)
+        if method == 'POST' and 'Content-Type' not in _headers:
+            _headers['Content-Type'] = 'application/x-www-form-urlencoded'
 
         connection = None
 
@@ -98,7 +110,7 @@ class BaseClient:
             connection = HTTPConnection(host, **kwargs) if scheme == 'http:' else HTTPSConnection(host, **kwargs)
 
         logger.debug('Creating request -----------------------------------------------------')
-        logger.debug("url='%s'", url)
+        logger.debug("%s %s", method, url)
 
         self.last_url = url
 
@@ -129,9 +141,13 @@ class BaseClient:
 
     def urlopen(self, url, redirect=True, data=None, headers={}, cookies=[], **kwargs):
 
+        if cookies:
+            for c in cookies:
+                self.cookies.set_cookie(c)
+
         self.last_url = None
 
-        self.response = self.request('POST' if data else 'GET', url, data, headers, cookies, **kwargs)
+        self.response = self.request('POST' if data else 'GET', url, data, headers, **kwargs)
         text = self.response.read()
 
         # Allow redirects - used by Yahoo for some cookie based consent
@@ -141,13 +157,14 @@ class BaseClient:
         # try mitigating by re-requesting straight away
         if 400 <= self.response.status < 500:
             if self.response.getheader('X-Cache') == 'Error from cloudfront':
-                self.response = self.request('POST' if data else 'GET', url, data, headers, cookies, **kwargs)
+                self.response = self.request('POST' if data else 'GET', url, data, headers, **kwargs)
                 text = self.response.read()
 
         while 300 <= self.response.status < 400 and self.redirect_count >= 0:
 
             self.redirect_count -= 1
-            location = self.response.getheader('Location')
+            location = str(self.response.getheader('Location'))
+            location = location.replace(' ', '%20')  # FT bug workaround - this should not be necessary
 
             if location and redirect:
 
@@ -155,16 +172,17 @@ class BaseClient:
                     scheme, _, host, path = url.split('/', 3)
                     location = '{}//{}{}'.format(scheme, host, location)
 
-                self.response = self.request('POST' if data else 'GET', location, data, headers, cookies, **kwargs)
+                self.response = self.request('GET', location, None, headers, **kwargs)
                 text = self.response.read()
 
             else:
                 raise RedirectException(location)
 
         if self.response.status >= 400:
-            logger.warning("last_url='%s' status=%s headers=%s", self.last_url, self.response.status,
+            logger.warning("last_url='%s' status=%s reason='%s' headers=%s", self.last_url, self.response.status,
+                           self.response.reason,
                            '\n'.join(sorted(self.response.headers.__str__().splitlines(), key=lambda l: l.lower())))
-            raise HttpException(url, self.response.status)
+            raise HttpException(url, self.response)
 
         if self.response.getheader('Content-Encoding') == 'gzip':
             text = gzip.decompress(text)
@@ -218,9 +236,6 @@ class BaseClient:
         tick[Datacode.TIMEZONE] = None
         tick[Datacode.VOLUME] = None
 
-        tick[Datacode.YAHOO_SUMMARY_RECEIVED] = False
-        tick[Datacode.YAHOO_STATISTIC_RECEIVED] = False
-        tick[Datacode.YAHOO_PROFILE_RECEIVED] = False
         tick[Datacode.TIMESTAMP] = None
 
         return tick
@@ -385,3 +400,11 @@ class BaseClient:
             pass
 
         return None
+
+    def close(self):
+        for connection in self.connections.values():
+            try:
+                connection.close()
+            except BaseException:
+                pass
+        self.connections = {}

src/datacode.py

@@ -60,9 +60,6 @@ class Datacode(Enum):
     NAME = 104
     TIMEZONE = 105
 
-    YAHOO_SUMMARY_RECEIVED = 996
-    YAHOO_STATISTIC_RECEIVED = 997
-    YAHOO_PROFILE_RECEIVED = 998
     TIMESTAMP = 999
 
     @classmethod

src/financials.py

@@ -54,7 +54,6 @@ import pyparsing
 import six
 
 from datacode import Datacode
-import financials_google as google
 import financials_yahoo as yahoo
 import financials_coinbase as coinbase
 import financials_ft as ft
@@ -95,7 +94,6 @@ class FinancialsImpl(unohelper.Base, Financials):
 
     def __init__(self, ctx):
         self.ctx = ctx
-        self.google = google.createInstance(ctx)
         self.yahoo = yahoo.createInstance(ctx)
         self.coinbase = coinbase.createInstance(ctx)
         self.ft = ft.createInstance(ctx)
@@ -136,9 +134,6 @@ class FinancialsImpl(unohelper.Base, Financials):
             ticker = str(ticker).strip()
             source = str(source).upper()
 
-            # if source == 'GOOGLE':
-            #     s = self.google.getRealtime(ticker, datacode)
-            # el
             if source == 'YAHOO':
                 s = self.yahoo.getRealtime(ticker, datacode)
             elif source == 'FT':
@@ -273,6 +268,11 @@ class FinancialsImpl(unohelper.Base, Financials):
 
         return s
 
+    def close(self):
+        if self.yahoo: self.yahoo.close()
+        if self.coinbase: self.coinbase.close()
+        if self.ft: self.ft.close()
+
 
 def createInstance(ctx):
     return FinancialsImpl(ctx)

src/financials_coinbase.py

@@ -79,7 +79,7 @@ class Coinbase(BaseClient):
         except BaseException as e:
             logger.exception("BaseException ticker=%s datacode=%s", ticker, datacode)
             del self.realtime[ticker]
-            return 'Coinbase.getRealtime({}, {}) - crumb: {}'.format(ticker, datacode, e)
+            return 'Coinbase.getRealtime({}, {}) - exception: {}'.format(ticker, datacode, e)
 
         try:
             price = results['last']

src/financials_google.py

@@ -1,248 +0,0 @@
-#  financials_google.py
-#
-#  license: GNU LGPL
-#
-#  This library is free software; you can redistribute it and/or
-#  modify it under the terms of the GNU Lesser General Public
-#  License as published by the Free Software Foundation; either
-#  version 3 of the License, or (at your option) any later version.
-
-
-import html
-import logging
-import os
-import re
-import time
-import xml.etree.ElementTree as ET
-
-import dateutil
-
-from baseclient import BaseClient, RedirectException
-from datacode import Datacode
-from naivehtmlparser import NaiveHTMLParser
-from tz import whois_timezone_info
-
-logger = logging.getLogger(__name__)
-
-
-# logger.setLevel(logging.DEBUG)
-
-
-def handle_abbreviations(s):
-    s = str(s).strip()
-    if s.endswith('M'):
-        return float(s[:-1]) * 1000000
-    elif s.endswith('B'):
-        return float(s[:-1]) * 1000000000
-    elif s.endswith('T'):
-        return float(s[:-1]) * 1000000000000
-    return float(s)
-
-
-def un_span(s):
-    return re.sub(r'<span [^>]*>', '', s).replace('</span>', '')
-
-
-class Google(BaseClient):
-    def __init__(self, ctx):
-        super().__init__()
-
-        self.realtime = {}
-        self.location = None
-
-    def getRealtime(self, ticker: str, datacode: int):
-
-        """
-        Retrieve realtime data for ticker from Google Finance and cache it for further lookups
-
-        :param ticker: the ticker symbol e.g. LON:VOD
-        :param datacode: the requested datacode
-        :return:
-        """
-
-        # remove white space
-        ticker = "".join(ticker.split())
-
-        # use cached value for up to 60 seconds
-        if ticker in self.realtime:
-            tick = self.realtime[ticker]
-            if Datacode.TIMESTAMP in tick and type(tick[Datacode.TIMESTAMP]) == float and time.time() - 60 < tick[Datacode.TIMESTAMP]:
-                return self._return_value(tick, datacode)
-            else:
-                del self.realtime[ticker]
-
-        q_param = 'q=' + ticker
-
-        if not self.location:
-            url = 'https://www.google.com/search?hl=en&tbm=fin&' + q_param
-
-            try:
-                self.urlopen(url, redirect=False)
-            except RedirectException as e:
-                self.location = e.location.replace('&' + q_param, '')
-            except BaseException as e:
-                logger.exception("BaseException ticker=%s datacode=%s", ticker, datacode)
-                return 'Google.getRealtime(\'{}\', {}) - location: {}'.format(ticker, datacode, e)
-
-        if not self.location:
-            url = 'https://www.google.com/search?tbm=fin&' + q_param
-        else:
-            url = self.location + '&' + q_param
-
-        try:
-            text = self.urlopen(url)
-        except BaseException as e:
-            logger.exception("BaseException ticker=%s datacode=%s", ticker, datacode)
-            return 'Google.getRealtime(\'{}\', {}) - urlopen: {} {}'.format(ticker, datacode, e, url)
-
-        try:
-            temp = ticker.replace(':', '_')  # Windows can't have ':' in file names
-            with open(os.path.join(self.basedir, 'google-{}.html'.format(temp)), "w", encoding="utf-8") as text_file:
-                print(f"<!-- '{url}' -->\r\n\r\n{text}", file=text_file)
-        except BaseException as e:
-            logger.exception("BaseException ticker=%s datacode=%s", ticker, datacode)
-
-        if ticker not in self.realtime:
-            self.realtime[ticker] =  self.get_ticker()
-
-        tick = self.realtime[ticker]
-
-        tick[Datacode.TIMESTAMP] = time.time()
-
-        try:
-            r = '<span[^>]+role="heading"[^>]+>(.*?)</span>'
-            pattern = re.compile(r)
-
-            match = pattern.search(text)
-            if not match:
-                return None
-            start = match.span(0)[1]
-
-            tick[Datacode.NAME] = self.save_wrapper(
-                lambda: html.unescape(un_span(match.group(1)).strip()))
-
-            # next div is TICKER
-            r = '<div [^>]*><div [^>]*>(.*?)</div></div>'
-            pattern = re.compile(r)
-
-            match = pattern.search(text, start)
-            if not match:
-                return 'Google.getRealtime({}, {}) - no match'.format(ticker, datacode)
-
-            ticker = self.save_wrapper(
-                lambda: html.unescape(match.group(1)).replace(' ', ''))
-
-            tick[Datacode.EXCHANGE] = self.save_wrapper(lambda: ticker.split(':')[0])
-            tick[Datacode.TICKER] = self.save_wrapper(lambda: ticker.split(':')[1])
-
-        except BaseException as e:
-            return 'Google.getRealtime({}, {}) - process: {}'.format(ticker, datacode, e)
-
-        try:
-            r = '<sticky-header [^>]*>(.*?)</sticky-header>'
-            pattern = re.compile(r, flags=re.DOTALL)
-            match = re.search(pattern, text)
-
-            if match:
-                text = match.group(1)
-            else:
-                return 'Data for \'{}\' not found'.format(ticker)
-
-            parser = NaiveHTMLParser()
-            root = parser.feed(text)
-            parser.close()
-
-            cards = root.findall('.//g-card-section')
-
-            if len(cards) < 4:
-                return 'Data for \'{}\' not found'.format(ticker)
-
-            header = cards[1]
-
-            tick[Datacode.LAST_PRICE] = self.save_wrapper(
-                lambda: float(
-                    html.unescape(header.find('./span[1]/span[1]/span[1]').text).replace(',', '').strip()))
-
-            tick[Datacode.CURRENCY] = self.save_wrapper(
-                lambda: html.unescape(header.find('./span[1]/span[1]/span[2]').text).strip())
-
-            tick[Datacode.CHANGE] = self.save_wrapper(
-                lambda: float(
-                    html.unescape(header.find('./span[2]/span[1]').text).replace('−', '-').replace(',', '').strip()))
-
-            # percentage is always wrapped in (...) and always positive even if there is a price drop
-            tick[Datacode.CHANGE_IN_PERCENT] = self.save_wrapper(
-                lambda: float(
-                    html.unescape(header.find('./span[2]/span[2]/span[1]').text).strip()
-                        .replace('(', '').replace(')', '').replace('%', '')))
-
-            try:
-                value = html.unescape(header.find('./div[1]/span[1]/span[2]').text).replace('·', '').strip()
-                logger.debug(value)
-                dt = dateutil.parser.parse(value, yearfirst=True, dayfirst=False, tzinfos=whois_timezone_info)
-                tick[Datacode.LAST_PRICE_DATE] = dt.date()
-                tick[Datacode.LAST_PRICE_TIME] = dt.time()
-
-                time_bits = value.split(' ')
-                if len(time_bits) >= 4:
-                    tick[Datacode.TIMEZONE] = time_bits[-1]
-
-            except BaseException as e:
-                pass
-
-            footer = cards[3]
-            logger.debug(ET.tostring(footer))
-
-            # parse 'footer' for remaining fields
-            table = footer.find('./div[1]/div[1]/div[1]/table[1]')
-
-            tick[Datacode.OPEN] = self.save_wrapper(
-                lambda: float(
-                    html.unescape(table.find('./tr[1]/td[2]').text).replace(',', '').strip()))
-
-            tick[Datacode.HIGH] = self.save_wrapper(
-                lambda: float(
-                    html.unescape(table.find('./tr[2]/td[2]').text).replace(',', '').strip()))
-
-            tick[Datacode.LOW] = self.save_wrapper(
-                lambda: float(
-                    html.unescape(table.find('./tr[3]/td[2]').text).replace(',', '').strip()))
-
-            tick[Datacode.MARKET_CAP] = self.save_wrapper(
-                lambda: handle_abbreviations(
-                    html.unescape(table.find('./tr[4]/td[2]').text).replace(',', '').replace('-', '').strip()))
-
-            table = footer.find('./div[1]/div[1]/div[2]/table[1]')
-
-            # for indices: first item on right side is LOW
-            if html.unescape(table.find('./tr[1]/td[1]').text).strip() == 'Low':
-                tick[Datacode.LOW] = self.save_wrapper(
-                    lambda: float(
-                        html.unescape(table.find('./tr[1]/td[2]').text).replace(',', '').strip()))
-
-            tick[Datacode.PREV_CLOSE] = self.save_wrapper(
-                lambda: float(
-                    html.unescape(table.find('./tr[2]/td[2]').text).replace(',', '').strip()))
-
-            tick[Datacode.HIGH_52_WEEK] = self.save_wrapper(
-                lambda: float(
-                    html.unescape(table.find('./tr[3]/td[2]').text).replace(',', '').strip()))
-
-            tick[Datacode.LOW_52_WEEK] = self.save_wrapper(
-                lambda: float(
-                    html.unescape(table.find('./tr[4]/td[2]').text).replace(',', '').strip()))
-
-            logger.info(tick)
-
-        except BaseException as e:
-            logger.exception("BaseException ticker=%s datacode=%s", ticker, datacode)
-            return 'Google.getRealtime({}, {}) - process: {}'.format(ticker, datacode, e)
-
-        return self._return_value(self.realtime[ticker], datacode)
-
-    def getHistoric(self, ticker, datacode, date):
-        return 'Google.getHistoric: Historic Data not implemented.'
-
-
-def createInstance(ctx):
-    return Google(ctx)

src/financials_yahoo.py

@@ -9,13 +9,14 @@
 
 
 import csv
-import html
+import datetime
+import json
 import logging
 import os
+import pytz
 import re
 import time
 import urllib.parse
-from http import cookiejar
 
 import dateutil.parser
 
@@ -57,25 +58,6 @@ def handle_abbreviations(s):
     return float(s)
 
 
-def cookie(name, value):
-    return cookiejar.Cookie(version=0, name=name, value=value,
-                            port=None, port_specified=False, domain=".yahoo.com", domain_specified=True,
-                            domain_initial_dot=True, path="/", path_specified=True, secure=True, expires=None,
-                            discard=False, comment=None, comment_url=None, rest=dict())
-
-
-def get_cookies():
-    return [
-        cookie("A1", "d=AQABBE6aomQCEJUamSIzqCl9UJ-spkNvMfkFEgABCAHqo2TKZPbPb2UBAiAAAAcIS5qiZF3wy-8&S=AQAAAnSEZV80kjd9J2RUh8TD5cY"),
-        cookie("A1S", "d=AQABBE6aomQCEJUamSIzqCl9UJ-spkNvMfkFEgABCAHqo2TKZPbPb2UBAiAAAAcIS5qiZF3wy-8&S=AQAAAnSEZV80kjd9J2RUh8TD5cY&j=GDPR"),
-        cookie("A3", "d=AQABBE6aomQCEJUamSIzqCl9UJ-spkNvMfkFEgABCAHqo2TKZPbPb2UBAiAAAAcIS5qiZF3wy-8&S=AQAAAnSEZV80kjd9J2RUh8TD5cY"),
-        cookie("EuConsent", "CPuVJsAPuVJsAAOACKENDICgAAAAAAAAACiQAAAAAABhoAMAAQSrEQAYAAglWKgAwABBKsA"),
-        cookie("GUC", "AQ81G6Da"),
-        cookie("maex", "{\"v2\":{}}"),
-        cookie("thamba", "1")
-    ]
-
-
 class Yahoo(BaseClient):
     def __init__(self, ctx):
         super().__init__()
@@ -85,61 +67,6 @@ class Yahoo(BaseClient):
         self.historicdata = {}
         self.js = jsonParser.jsonObject
 
-        self.create_cookies()
-
-    def create_cookies(self):
-
-        cookiejar_path = os.path.join(self.basedir, 'yahoo.cookiejar')
-        cookiejar_exists = os.path.isfile(cookiejar_path) and os.stat(cookiejar_path).st_size >= 0
-
-        if cookiejar_exists:
-            try:
-                self.cookies = cookiejar.LWPCookieJar()
-                self.cookies.load(cookiejar_path, ignore_discard=True)
-
-                required_cookie_names = ["A1", "A1S", "A3", "GUC", "maex", "thamba"]
-                for c in self.cookies:
-                    if c.name in required_cookie_names:
-                        required_cookie_names.remove(c.name)
-
-                if len(required_cookie_names) > 0:
-                    cookiejar_exists = False
-                    logger.info("Overriding cookiejar '%s'", cookiejar_path)
-
-            except BaseException:
-                cookiejar_exists = False
-                logger.exception("BaseException initial loading cookiejar_path=%s", cookiejar_path)
-
-        if not cookiejar_exists:
-            try:
-                lwp_cookiejar = cookiejar.LWPCookieJar()
-                for c in get_cookies():
-                    lwp_cookiejar.set_cookie(c)
-                lwp_cookiejar.save(cookiejar_path, ignore_discard=True)
-                logger.info("Created cookiejar '%s'", cookiejar_path)
-            except BaseException:
-                logger.exception("BaseException creating cookiejar_path=%s", cookiejar_path)
-
-        try:
-            self.cookies = cookiejar.LWPCookieJar()
-            self.cookies.load(cookiejar_path, ignore_discard=True)
-
-            logger.info("Loaded cookiejar '%s'", cookiejar_path)
-
-            for c in self.cookies:
-                logger.info("Cookie name'%s' value='%s' path='%s'", c.name, c.value, c.path)
-
-        except BaseException:
-            logger.exception("BaseException loading cookiejar_path=%s", cookiejar_path)
-
-    def save_cookies(self):
-        cookiejar_path = os.path.join(self.basedir, 'yahoo.cookiejar')
-        try:
-            self.cookies.save(cookiejar_path, ignore_discard=True)
-            logger.debug("Saved cookiejar '%s'", cookiejar_path)
-        except BaseException:
-            logger.exception("BaseException saving cookiejar_path=%s", cookiejar_path)
-
     def _read_ticker_csv_file(self, ticker):
 
         fn = os.path.join(self.basedir, 'yahoo-{}.csv'.format(ticker))
@@ -169,6 +96,58 @@ class Yahoo(BaseClient):
 
             self.historicdata[ticker] = ticks
 
+
+    def handleCookiesAndConsent(self, url, ticker, datacode, html_file):
+
+        try:
+            text = self.urlopen(url, redirect=True)
+        except BaseException as e:
+            logger.exception("BaseException (1) ticker=%s datacode=%s last_url=%s redirect_count=%s %s",
+                             ticker, datacode, self.last_url, self.redirect_count, e)
+            return None
+
+        try:
+            with open(os.path.join(self.basedir, html_file), "w", encoding="utf-8") as text_file:
+                print(f"<!-- '{self.last_url}' -->\r\n\r\n{text}", file=text_file)
+        except BaseException as e:
+            logger.exception("BaseException (2) ticker=%s datacode=%s %s", ticker, datacode, e)
+
+        if not text:
+            return None
+
+        try:
+            parser = NaiveHTMLParser()
+            root = parser.feed(text)
+            parser.close()
+        except BaseException as e:
+            logger.exception("BaseException (3) ticker=%s datacode=%s - HTML parsing - %s", ticker, datacode, e)
+            return None
+
+        form = root.find(f".//form[@class='consent-form']")
+
+        if form:
+            inputs = form.findall(f".//input")
+
+            if inputs:
+
+                data = {'reject': 'reject'}
+                for d in inputs:
+                    data[d.attrib['name']] = d.attrib['value']
+
+                try:
+                    text = self.urlopen(self.last_url, redirect=True, data=urllib.parse.urlencode(data))
+                except BaseException as e:
+                    logger.exception("BaseException (4) ticker=%s datacode=%s last_url=%s redirect_count=%s %s",
+                                     ticker, datacode, self.last_url, self.redirect_count, e)
+
+                try:
+                    with open(os.path.join(self.basedir, html_file), "w", encoding="utf-8") as text_file:
+                        print(f"<!-- '{self.last_url}' (after consent handling) -->\r\n\r\n{text}", file=text_file)
+                except BaseException as e:
+                    logger.exception("BaseException (5) ticker=%s datacode=%s %s", ticker, datacode, e)
+
+        return text
+
     def getRealtime(self, ticker, datacode):
 
         """
@@ -182,338 +161,175 @@ class Yahoo(BaseClient):
         # remove white space
         ticker = "".join(ticker.split())
 
-        needStatistics = datacode in [Datacode.SHARES_OUT.value, Datacode.FREE_FLOAT.value, Datacode.PAYOUT_RATIO.value]
-        needProfile = datacode in [Datacode.SECTOR.value, Datacode.INDUSTRY.value]
-
         # use cached value for up to 60 seconds
         if ticker in self.realtime:
             tick = self.realtime[ticker]
             if Datacode.TIMESTAMP in tick and type(tick[Datacode.TIMESTAMP]) == float and time.time() - 60 < tick[Datacode.TIMESTAMP]:
-                if (tick[Datacode.YAHOO_STATISTIC_RECEIVED] or not needStatistics) and (
-                        tick[Datacode.YAHOO_PROFILE_RECEIVED] or not needProfile) and (
-                        tick[Datacode.YAHOO_SUMMARY_RECEIVED]):
-                    return self._return_value(tick, datacode)
+                return self._return_value(tick, datacode)
             else:
                 del self.realtime[ticker]
 
         if ticker not in self.realtime:
             self.realtime[ticker] = self.get_ticker()
 
-        if needStatistics:
-            return self.getRealtimeStatistics(ticker, datacode)
-
-        if needProfile:
-            return self.getRealtimeProfile(ticker, datacode)
-
-        return self.getRealtimeSummary(ticker, datacode)
-
-    def getRealtimeSummary(self, ticker, datacode):
-
-        """
-        Retrieve realtime data from Yahoo Finance - Summary tab
-        """
-
         tick = self.realtime[ticker]
 
-        url = 'https://finance.yahoo.com/quote/{}?p={}'.format(ticker, ticker)
+        if not self.crumb:
 
-        try:
-            text = self.urlopen(url, redirect=True)
-            self.save_cookies()
-        except BaseException as e:
-            logger.exception("BaseException ticker=%s datacode=%s last_url=%s redirect_count=%s", ticker, datacode, self.last_url, self.redirect_count)
-            del self.realtime[ticker]
-            return 'Yahoo.getRealtimeSummary({}, {}) - urlopen: {}'.format(ticker, datacode, e)
+            url = 'https://finance.yahoo.com/quote/{}?p={}'.format(ticker, ticker)
+            text = self.handleCookiesAndConsent(url, ticker, datacode, f'yahoo-{ticker}.html')
 
-        try:
-            with open(os.path.join(self.basedir, 'yahoo-{}.html'.format(ticker)), "w", encoding="utf-8") as text_file:
-                print(f"<!-- '{self.last_url}' -->\r\n\r\n{text}", file=text_file)
-        except BaseException:
-            logger.exception("BaseException ticker=%s datacode=%s", ticker, datacode)
+            if text is None:
+                del self.realtime[ticker]
+                return 'Yahoo.getRealtime({}, {}) - handleCookiesAndConsent'.format(ticker, datacode)
 
-        try:
-            r = '"crumb":"([^"]{11})"'
-            pattern = re.compile(r)
-            match = pattern.search(text)
-            if match:
-                self.crumb = match.group(1)
-        except BaseException as e:
-            logger.exception("BaseException ticker=%s datacode=%s", ticker, datacode)
-            del self.realtime[ticker]
-            return 'Yahoo.getRealtimeSummary({}, {}) - crumb: {}'.format(ticker, datacode, e)
-
-        try:
-            parser = NaiveHTMLParser()
-            root = parser.feed(text)
-            parser.close()
-        except BaseException as e:
-            logger.exception("BaseException ticker=%s datacode=%s", ticker, datacode)
-            return 'Yahoo.getRealtimeSummary({}, {}) - HTML parsing: {}'.format(ticker, datacode, e)
-
-        try:
-            if not root:
+            # crumbs like 'TKkC\u002FZBwoUA' may contain unicode _text_ (not encoded code points)
+            try:
+                r = r'\bcrumb=([^"]{11,})"'
+                pattern = re.compile(r)
+                match = pattern.search(text)
+                if match:
+                    self.crumb = urllib.parse.unquote(match.group(1).encode('unicode-escape').decode('ascii'))
+                    logger.debug(f"crumb='{match.group(1)}' self.crumb='{self.crumb}'")
+                else:
+                    r = r'"crumb"\s*:\s*"([^"]{11,})"'
+                    pattern = re.compile(r)
+                    match = pattern.search(text)
+                    if match:
+                        self.crumb = urllib.parse.unquote(match.group(1).encode('unicode-escape').decode('ascii'))
+                        logger.debug(f"crumb='{match.group(1)}' self.crumb='{self.crumb}'")
+            except BaseException as e:
                 logger.exception("BaseException ticker=%s datacode=%s", ticker, datacode)
-                return 'Yahoo.getRealtimeSummary({}, {}) - root missing'.format(ticker, datacode)
+                del self.realtime[ticker]
+                return 'Yahoo.getRealtime({}, {}) - crumb: {}'.format(ticker, datacode, e)
+
+        if not self.crumb:
+            return 'Yahoo.getRealtime({}, {}) - crumb missing'.format(ticker, datacode)
+
+        try:
+
+            url = 'https://query1.finance.yahoo.com/v10/finance/quoteSummary/{}?formatted=true&' \
+                  'modules=summaryProfile,financialData,quoteType,recommendationTrend,earnings,equityPerformance,summaryDetail,defaultKeyStatistics,calendarEvents,esgScores,price,pageViews,financialsTemplate&' \
+                  'lang=en-US&region=US&crumb={}' \
+                .format(ticker, urllib.parse.quote_plus(self.crumb))
+
+            js = self.urlopen(url)
+
+        except HttpException as e:
+            logger.exception("HttpException querying ticker=%s datacode=%s", ticker, datacode)
+            del self.realtime[ticker]
+            return None
+
+        try:
+
+            with open(os.path.join(self.basedir, 'yahoo-{}.json'.format(ticker)), "w", encoding="utf-8") as json_file:
+                print(f"<!-- '{self.last_url}' -->\r\n\r\n{js}", file=json_file)
+
+            parsed = json.loads(js)
+            parsed = parsed['quoteSummary']['result'][0]
+
+            summaryDetail = dict(sorted(parsed['summaryDetail'].items()))
+            price = dict(sorted(parsed['price'].items()))
+
+            if 'defaultKeyStatistics' in parsed:
+                defaultKeyStatistics = dict(sorted(parsed['defaultKeyStatistics'].items()))
+            else:
+                defaultKeyStatistics = {}
+
+            if 'summaryProfile' in parsed:
+                summaryProfile = dict(sorted(parsed['summaryProfile'].items()))
+            else:
+                summaryProfile = {}
+
+            quoteType = dict(sorted(parsed['quoteType'].items()))
+
+        except BaseException as e:
+            logger.exception("BaseException parsing ticker=%s datacode=%s", ticker, datacode)
+            del self.realtime[ticker]
+            return 'Yahoo.getRealtimeSummary({}, {}) - exception: {}'.format(ticker, datacode, e)
+
+        try:
 
             tick[Datacode.TICKER] = ticker
             tick[Datacode.TIMESTAMP] = time.time()
-            tick[Datacode.YAHOO_SUMMARY_RECEIVED] = True
 
-            parsed = {}
-
-            found = root.findall(f".//fin-streamer[@data-symbol='{ticker}']")
-            for d in found:
-                if hasattr(d, 'attrib') and 'data-field' in d.attrib:
-                    parsed[d.attrib['data-field']] = default(d.attrib, 'value').replace('−', '-').replace(',', '').strip()
-
-            # for futures "regularMarketVolume" is from actual future ticker (potentially different to requested one)
-            if 'regularMarketVolume' not in parsed:
-                found = root.findall(f".//fin-streamer[@data-field='regularMarketVolume']")
-                for d in found:
-                    if hasattr(d, 'attrib') and 'data-field' in d.attrib and 'data-symbol' in d.attrib:
-                        parsed[d.attrib['data-field']] = default(d.attrib, 'value').replace('−', '-').replace(',', '').strip()
-                        tick[Datacode.TICKER] = default(d.attrib, 'data-symbol').strip()
-
-            found = root.findall(f".//td[@data-test]")
-            for d in found:
-                if d:
-                    span = d.find('./span')
-                    if hasattr(d, 'attrib') and hasattr(span, 'text'):
-                        parsed[d.attrib['data-test']] = default(span, 'text').replace('−', '-').replace(',', '').strip()
-                else:
-                    if hasattr(d, 'attrib') and hasattr(d, 'text'):
-                        parsed[d.attrib['data-test']] = default(d, 'text').replace('−', '-').replace(',', '').strip()
-
-            if 'regularMarketPrice' not in parsed:
+            if 'regularMarketPrice' not in price:
                 return None
 
-            tick[Datacode.PREV_CLOSE] = self.save_wrapper(lambda: float(parsed['PREV_CLOSE-value']))
-            tick[Datacode.OPEN] = self.save_wrapper(lambda: float(parsed['OPEN-value']))
-            tick[Datacode.CHANGE] = self.save_wrapper(lambda: float(parsed['regularMarketChange']))
-            tick[Datacode.CHANGE_IN_PERCENT] = self.save_wrapper(lambda: float(parsed['regularMarketChangePercent']))
+            tick[Datacode.PREV_CLOSE] = self.save_wrapper(lambda: float(price['regularMarketPreviousClose']['raw']))
+            tick[Datacode.OPEN] = self.save_wrapper(lambda: float(price['regularMarketOpen']['raw']))
+            tick[Datacode.CHANGE] = self.save_wrapper(lambda: float(price['regularMarketChange']['raw']))
+            tick[Datacode.CHANGE_IN_PERCENT] = self.save_wrapper(lambda: float(price['regularMarketChangePercent']['raw']))
 
-            t = default(parsed, 'DAYS_RANGE-value').split(' - ')
-            tick[Datacode.LOW] = self.save_wrapper(lambda: float(t[0]))
-            tick[Datacode.HIGH] = self.save_wrapper(lambda: float(t[1]))
+            tick[Datacode.LOW] = self.save_wrapper(lambda: float(price['regularMarketDayLow']['raw']))
+            tick[Datacode.HIGH] = self.save_wrapper(lambda: float(price['regularMarketDayHigh']['raw']))
 
-            tick[Datacode.LAST_PRICE] = self.save_wrapper(lambda: float(parsed['regularMarketPrice']))
-            tick[Datacode.VOLUME] = self.save_wrapper(lambda: float(parsed['regularMarketVolume']))
-            tick[Datacode.AVG_DAILY_VOL_3MONTH] = self.save_wrapper(lambda: float(parsed['AVERAGE_VOLUME_3MONTH-value']))
-            tick[Datacode.BETA] = self.save_wrapper(lambda: float(parsed['BETA_5Y-value']))
-            tick[Datacode.EPS] = self.save_wrapper(lambda: float(parsed['EPS_RATIO-value']))
-            tick[Datacode.PE_RATIO] = self.save_wrapper(lambda: float(parsed['PE_RATIO-value']))
-
-            t = default(parsed, 'DIVIDEND_AND_YIELD-value').replace('(', '').replace(')', '').replace('%', '').strip().split(' ')
-            tick[Datacode.DIV] = self.save_wrapper(lambda: float(t[0]))
-            tick[Datacode.DIV_YIELD] = self.save_wrapper(lambda: float(t[1])/100.0)
+            tick[Datacode.LAST_PRICE] = self.save_wrapper(lambda: float(price['regularMarketPrice']['raw']))
+            tick[Datacode.VOLUME] = self.save_wrapper(lambda: float(price['regularMarketVolume']['raw']))
+            tick[Datacode.AVG_DAILY_VOL_3MONTH] = self.save_wrapper(lambda: float(price['averageDailyVolume3Month']['raw']))
+            tick[Datacode.BETA] = self.save_wrapper(lambda: float(defaultKeyStatistics['beta']['raw']))
+            tick[Datacode.EPS] = self.save_wrapper(lambda: float(defaultKeyStatistics['trailingEps']['raw']))
+            tick[Datacode.PE_RATIO] = self.save_wrapper(lambda: float(summaryDetail['trailingPE']['raw']))
 
             tick[Datacode.EX_DIV_DATE] = self.save_wrapper(
-                lambda: dateutil.parser.parse(parsed['EX_DIVIDEND_DATE-value'], yearfirst=True, dayfirst=False).date())
+                lambda: dateutil.parser.parse(summaryDetail['exDividendDate']['fmt'], yearfirst=True, dayfirst=False).date())
 
-            t = default(parsed, 'FIFTY_TWO_WK_RANGE-value').split(' - ')
-            tick[Datacode.LOW_52_WEEK] = self.save_wrapper(lambda: float(t[0]))
-            tick[Datacode.HIGH_52_WEEK] = self.save_wrapper(lambda: float(t[1]))
+            tick[Datacode.LOW_52_WEEK] = self.save_wrapper(lambda: float(summaryDetail['fiftyTwoWeekLow']['raw']))
+            tick[Datacode.HIGH_52_WEEK] = self.save_wrapper(lambda: float(summaryDetail['fiftyTwoWeekHigh']['raw']))
 
-            tick[Datacode.MARKET_CAP] = self.save_wrapper(lambda: float(handle_abbreviations(parsed['MARKET_CAP-value'])))
+            tick[Datacode.MARKET_CAP] = self.save_wrapper(lambda: float(price['marketCap']['raw']))
 
-            t = default(parsed, 'BID-value').split(' x ')
-            tick[Datacode.BID] = self.save_wrapper(lambda: float(t[0]))
-            tick[Datacode.BIDSIZE] = self.save_wrapper(lambda: float(t[1]))
+            tick[Datacode.BID] = self.save_wrapper(lambda: float(summaryDetail['bid']['raw']))
+            tick[Datacode.BIDSIZE] = self.save_wrapper(lambda: float(summaryDetail['bidSize']['raw']))
 
-            t = default(parsed, 'ASK-value').split(' x ')
-            tick[Datacode.ASK] = self.save_wrapper(lambda: float(t[0]))
-            tick[Datacode.ASKSIZE] = self.save_wrapper(lambda: float(t[1]))
+            tick[Datacode.ASK] = self.save_wrapper(lambda: float(summaryDetail['ask']['raw']))
+            tick[Datacode.ASKSIZE] = self.save_wrapper(lambda: float(summaryDetail['askSize']['raw']))
 
-            tick[Datacode.EXPIRY_DATE] = self.save_wrapper(
-                lambda: dateutil.parser.parse(parsed['EXPIRE_DATE-value'], yearfirst=True, dayfirst=False).date())
+            if quoteType:
+                t = int(price['regularMarketTime'])
+                tz = pytz.timezone(quoteType['timeZoneFullName'])
 
-            tick[Datacode.SETTLEMENT_DATE] = self.save_wrapper(
-                lambda: dateutil.parser.parse(parsed['SETTLEMENT_DATE-value'], yearfirst=True, dayfirst=False).date())
+                tick[Datacode.TIMEZONE] = tz
+                dt = datetime.datetime.fromtimestamp(t, tz)
 
-            r = '<div id="quote-market-notice"[^>]*><span>([^>]*?)(. Market open.)?</span></div>'
-            match = re.compile(r, flags=re.DOTALL).search(text)
-            if match:
-                t = html.unescape(match.group(1)).strip().split(' ')
-                tick[Datacode.TIMEZONE] = self.save_wrapper(lambda: t[-1])
+                tick[Datacode.LAST_PRICE_DATE] = dt.date()
+                tick[Datacode.LAST_PRICE_TIME] = dt.time()
 
-            # if quoteType:
-            #     t = int(price['regularMarketTime'])
-            #     tz = pytz.timezone(quoteType['exchangeTimezoneName'])
-            #
-            #     tick[Datacode.TIMEZONE] = tz
-            #     dt = datetime.datetime.fromtimestamp(t, tz)
-            #
-            #     tick[Datacode.LAST_PRICE_DATE] = dt.date()
-            #     tick[Datacode.LAST_PRICE_TIME] = dt.time()
+            tick[Datacode.EXCHANGE] = self.save_wrapper(lambda: price['exchangeName'])
+            tick[Datacode.CURRENCY] = self.save_wrapper(lambda: price['currency'])
 
-            r = '<span>([ \\w]+?) - [^>]*Currency in ([\\w]+)[^>]*</span>'
-            match = re.compile(r, flags=re.DOTALL).search(text)
-            if match:
-                tick[Datacode.EXCHANGE] = self.save_wrapper(lambda: html.unescape(match.group(1)).strip())
-                tick[Datacode.CURRENCY] = self.save_wrapper(lambda: html.unescape(match.group(2)).strip())
+            tick[Datacode.DIV] = self.save_wrapper(lambda: float(summaryDetail['dividendRate']['raw']))
+            tick[Datacode.DIV_YIELD] = self.save_wrapper(lambda: float(summaryDetail['dividendYield']['raw']))
 
-            # fallback for dividend/yield on mutual funds and ETFs
+            # fallback to last dividend on mutual funds and ETFs
             if not tick[Datacode.DIV]:
-                tick[Datacode.DIV] = self.save_wrapper(lambda: float(parsed['LAST_DIVIDEND-value']))
-            if not tick[Datacode.DIV_YIELD]:
-                tick[Datacode.DIV_YIELD] = self.save_wrapper(lambda: float(parsed['TD_YIELD-value'].replace('%', '').strip())/100.0)
+                tick[Datacode.DIV] = self.save_wrapper(lambda: float(defaultKeyStatistics['lastDividendValue']['raw']))
 
-            tick[Datacode.NAME] = self.save_wrapper(
-                lambda: html.unescape(root.find('.//h1').text).strip())
+            if default(price, 'quoteType') == 'FUTURE':
+                tick[Datacode.TICKER] = self.save_wrapper(lambda: price['underlyingSymbol'])
+                tick[Datacode.NAME] = self.save_wrapper(lambda: price['shortName'])
+                tick[Datacode.SETTLEMENT_DATE] = self.save_wrapper(
+                    lambda: dateutil.parser.parse(summaryDetail['expireDate']['fmt'], yearfirst=True, dayfirst=False).date())
+            else:
+                tick[Datacode.NAME] = self.save_wrapper(lambda: price['longName'])
+                tick[Datacode.EXPIRY_DATE] = self.save_wrapper(
+                    lambda: dateutil.parser.parse(summaryDetail['expireDate']['fmt'], yearfirst=True, dayfirst=False).date())
+                tick[Datacode.SETTLEMENT_DATE] = None
 
             if not tick[Datacode.NAME]:
                 tick[Datacode.NAME] = tick[Datacode.TICKER]
 
-        except BaseException as e:
-            logger.exception("BaseException ticker=%s datacode=%s", ticker, datacode)
-            del self.realtime[ticker]
-            return 'Yahoo.getRealtimeSummary({}, {}) - process: {}'.format(ticker, datacode, e)
+            tick[Datacode.SECTOR] = self.save_wrapper(lambda: summaryProfile['sector'])
+            tick[Datacode.INDUSTRY] = self.save_wrapper(lambda: summaryProfile['industry'])
 
-        return self._return_value(self.realtime[ticker], datacode)
-
-    def getRealtimeStatistics(self, ticker, datacode):
-
-        """
-        Retrieve realtime data from Yahoo Finance - Statistics tab
-        """
-
-        tick = self.realtime[ticker]
-
-        url = 'https://finance.yahoo.com/quote/{}/key-statistics?p={}'.format(ticker, ticker)
-
-        try:
-            text = self.urlopen(url, redirect=True)
-            self.save_cookies()
-        except BaseException as e:
-            logger.exception("BaseException ticker=%s datacode=%s last_url=%s redirect_count=%s", ticker, datacode, self.last_url, self.redirect_count)
-            del self.realtime[ticker]
-            return 'Yahoo.getRealtimeStatistics({}, {}) - urlopen: {}'.format(ticker, datacode, e)
-
-        try:
-            with open(os.path.join(self.basedir, 'yahoo-{}-statistics.html'.format(ticker)), "w", encoding="utf-8") as text_file:
-                print(f"<!-- '{url}' -->\r\n\r\n{text}", file=text_file)
-        except BaseException:
-            logger.exception("BaseException open/write ticker=%s datacode=%s", ticker, datacode)
-
-        try:
-            parser = NaiveHTMLParser()
-            root = parser.feed(text)
-            parser.close()
-        except BaseException as e:
-            logger.exception("BaseException ticker=%s datacode=%s", ticker, datacode)
-            del self.realtime[ticker]
-            return 'Yahoo.getRealtimeStatistics({}, {}) - HTML parsing: {}'.format(ticker, datacode, e)
-
-        try:
-
-            parsed = {}
-
-            statistics = root.find(".//section[@data-test='qsp-statistics']")
-
-            if statistics is None:
-                return None
-
-            tick[Datacode.TICKER] = ticker
-            tick[Datacode.TIMESTAMP] = time.time()
-            tick[Datacode.YAHOO_STATISTIC_RECEIVED] = True
-
-            # Valuation Measures
-            found = statistics.find('./div[2]/div[1]//table')
-            if found:
-                for d in found.findall('.//tr'):
-                    key = d.find('./td[1]/span').text
-                    if key is not None:
-                        parsed[key] = d.find('./td[2]').text
-
-            # Stock Price History
-            found = statistics.find('./div[2]/div[2]/div[1]/div[1]//table')
-            if found:
-                for d in found.findall('.//tr'):
-                    key = d.find('./td[1]/span').text
-                    if key is not None:
-                        parsed[key] = d.find('./td[2]').text
-
-            # Share Statistics
-            found = statistics.find('./div[2]/div[2]/div[1]/div[2]//table')
-            if found:
-                for d in found.findall('.//tr'):
-                    key = d.find('./td[1]/span').text
-                    if key is not None:
-                        parsed[key] = d.find('./td[2]').text
-
-            # Dividends & Splits
-            found = statistics.find('./div[2]/div[2]/div[1]/div[3]//table')
-            if found:
-                for d in found.findall('.//tr'):
-                    key = d.find('./td[1]/span').text
-                    if key is not None:
-                        parsed[key] = d.find('./td[2]').text
-
-            tick[Datacode.SHARES_OUT] = self.save_wrapper(
-                lambda: float(handle_abbreviations(parsed['Shares Outstanding'])))
-            tick[Datacode.FREE_FLOAT] = self.save_wrapper(
-                lambda: float(handle_abbreviations(parsed['Float'])))
-            tick[Datacode.PAYOUT_RATIO] = self.save_wrapper(
-                lambda: float(handle_abbreviations(parsed['Payout Ratio'].replace('%', '').strip()))/100.0)
+            tick[Datacode.SHARES_OUT] = self.save_wrapper(lambda: float(defaultKeyStatistics['sharesOutstanding']['raw']))
+            tick[Datacode.FREE_FLOAT] = self.save_wrapper(lambda: float(defaultKeyStatistics['floatShares']['raw']))
+            tick[Datacode.PAYOUT_RATIO] = self.save_wrapper(lambda: float(summaryDetail['payoutRatio']['raw']))
 
         except BaseException as e:
             logger.exception("BaseException ticker=%s datacode=%s", ticker, datacode)
             del self.realtime[ticker]
-            return 'Yahoo.getRealtimeStatistics({}, {}) - process: {}'.format(ticker, datacode, e)
-
-        return self._return_value(self.realtime[ticker], datacode)
-
-    def getRealtimeProfile(self, ticker, datacode):
-
-        """
-        Retrieve realtime data from Yahoo Finance - Profile tab
-        """
-
-        tick = self.realtime[ticker]
-
-        url = 'https://finance.yahoo.com/quote/{}/profile?p={}'.format(ticker, ticker)
-
-        try:
-            text = self.urlopen(url, redirect=True)
-            self.save_cookies()
-        except BaseException as e:
-            logger.exception("BaseException ticker=%s datacode=%s last_url=%s redirect_count=%s", ticker, datacode, self.last_url, self.redirect_count)
-            del self.realtime[ticker]
-            return 'Yahoo.getRealtimeProfile({}, {}) - urlopen: {}'.format(ticker, datacode, e)
-
-        try:
-            with open(os.path.join(self.basedir, 'yahoo-{}-profile.html'.format(ticker)), "w", encoding="utf-8") as text_file:
-                print(f"<!-- '{url}' -->\r\n\r\n{text}", file=text_file)
-        except BaseException:
-            logger.exception("BaseException open/write ticker=%s datacode=%s", ticker, datacode)
-
-        try:
-            parser = NaiveHTMLParser()
-            root = parser.feed(text)
-            parser.close()
-        except BaseException as e:
-            logger.exception("BaseException ticker=%s datacode=%s", ticker, datacode)
-            del self.realtime[ticker]
-            return 'Yahoo.getRealtimeProfile({}, {}) - HTML parsing: {}'.format(ticker, datacode, e)
-
-        try:
-
-            p = root.find(".//*[span='Sector(s)']")
-
-            if p is None:
-                return None
-
-            tick[Datacode.TICKER] = ticker
-            tick[Datacode.TIMESTAMP] = time.time()
-            tick[Datacode.YAHOO_PROFILE_RECEIVED] = True
-
-            tick[Datacode.SECTOR] = self.save_wrapper(lambda: p.find("./span[2]").text)
-            tick[Datacode.INDUSTRY] = self.save_wrapper(lambda: p.find("./span[4]").text)
-
-        except BaseException as e:
-            logger.exception("BaseException ticker=%s datacode=%s", ticker, datacode)
-            del self.realtime[ticker]
-            return 'Yahoo.getRealtimeProfile({}, {}) - process: {}'.format(ticker, datacode, e)
+            return 'Yahoo.getRealtime({}, {}) - process: {}'.format(ticker, datacode, e)
 
         return self._return_value(self.realtime[ticker], datacode)
 

src/generate_metainfo.py

@@ -14,7 +14,7 @@ import os
 cur_dir = os.getcwd()
 
 addin_id = "com.financials.getinfo"
-addin_version = "3.3.2"
+addin_version = "3.6.3"
 addin_displayname = "Financial Market Extension"
 addin_publisher_link = "https://github.com/cmallwitz/Financials-Extension"
 addin_publisher_name = "The Publisher"

src/test_coinbase.py

@@ -25,6 +25,11 @@ financials = financials.createInstance(None)
 
 class Test(unittest.TestCase):
 
+    @classmethod
+    def tearDownClass(cls):
+        # this avoids "ResourceWarning: unclosed..." on cached socket connections
+        financials.close()
+
     def test_currency(self):
         s = financials.getRealtime('ETH-EUR', Datacode.LAST_PRICE.value, 'COINBASE')
         self.assertEqual(float, type(s), 'test_currency LAST_PRICE')

src/test_ft.py

@@ -23,6 +23,11 @@ financials = financials.createInstance(None)
 
 class Test(unittest.TestCase):
 
+    @classmethod
+    def tearDownClass(cls):
+        # this avoids "ResourceWarning: unclosed..." on cached socket connections
+        financials.close()
+
     def test_currency(self):
         s = financials.getRealtime('EURGBP', 'LAST_PRICE', 'FT')
         self.assertEqual(float, type(s), 'test_currency LAST_PRICE')
@@ -137,29 +142,31 @@ class Test(unittest.TestCase):
 
     def test_US_futures(self):
 
-        s = financials.getRealtime('ESU3:IOM', Datacode.NAME.value, 'FT')
-        self.assertEqual(str, type(s), 'test_realtime_US_futures NAME {}'.format(s))
-        self.assertEqual('EMINI S&P SEP3', s, 'test_US_futures NAME {}'.format(s))
+        # https://markets.ft.com/data/commodities/tearsheet/summary?s=775326843 ESH25:IOM
 
-        s = financials.getRealtime('ESU3:IOM', Datacode.LAST_PRICE.value, 'FT')
+        s = financials.getRealtime('775326843', Datacode.NAME.value, 'FT')
+        self.assertEqual(str, type(s), 'test_realtime_US_futures NAME {}'.format(s))
+        self.assertEqual('EMINI S&P MAR25', s, 'test_US_futures NAME {}'.format(s))
+
+        s = financials.getRealtime('775326843', Datacode.LAST_PRICE.value, 'FT')
         self.assertEqual(float, type(s), 'test_US_futures LAST_PRICE {}'.format(s))
 
-        # s = financials.getRealtime('ESH3:IOM', Datacode.OPEN.value, 'FT')
+        # s = financials.getRealtime('775326843', Datacode.OPEN.value, 'FT')
         # self.assertEqual(float, type(s), 'test_US_futures OPEN {}'.format(s))
 
-        s = financials.getRealtime('ESU3:IOM', Datacode.VOLUME.value, 'FT')
+        s = financials.getRealtime('775326843', Datacode.VOLUME.value, 'FT')
         self.assertEqual(float, type(s), 'test_US_futures VOLUME {}'.format(s))
 
-        s = financials.getRealtime('ESU3:IOM', Datacode.LOW_52_WEEK.value, 'FT')
+        s = financials.getRealtime('775326843', Datacode.LOW_52_WEEK.value, 'FT')
         self.assertEqual(float, type(s), 'test_US_futures LOW_52_WEEK {}'.format(s))
 
-        s = financials.getRealtime('ESU3:IOM', Datacode.HIGH_52_WEEK.value, 'FT')
+        s = financials.getRealtime('775326843', Datacode.HIGH_52_WEEK.value, 'FT')
         self.assertEqual(float, type(s), 'test_US_futures HIGH_52_WEEK {}'.format(s))
 
-        s = financials.getRealtime('ESU3:IOM', Datacode.CHANGE.value, 'FT')
+        s = financials.getRealtime('775326843', Datacode.CHANGE.value, 'FT')
         self.assertEqual(float, type(s), 'test_US_futures CHANGE {}'.format(s))
 
-        s = financials.getRealtime('ESU3:IOM', Datacode.CHANGE_IN_PERCENT.value, 'FT')
+        s = financials.getRealtime('775326843', Datacode.CHANGE_IN_PERCENT.value, 'FT')
         self.assertEqual(float, type(s), 'test_US_futures CHANGE_IN_PERCENT {}'.format(s))
 
     def test_UK_ETF(self):
@@ -268,7 +275,7 @@ class Test(unittest.TestCase):
         self.assertTrue(testutils.is_date(s), 'test_DE_equity EX_DIV_DATE {}'.format(s))
 
         s = financials.getRealtime('ISHAX:GER', 'NAME', 'FT')
-        self.assertEqual('INTERSHOP Communications AG', s, 'test_DE_equity NAME {}'.format(s))
+        self.assertEqual('Intershop Communications AG', s, 'test_DE_equity NAME {}'.format(s))
 
         s = financials.getRealtime('ISHAX:GER', 'BETA', 'FT')
         self.assertTrue(testutils.is_positive_float(s), 'test_DE_equity BETA {}'.format(s))
@@ -302,6 +309,13 @@ class Test(unittest.TestCase):
         self.assertEqual(str, type(s), 'test_DK_equity INDUSTRY {}'.format(s))
         self.assertEqual('Pharmaceuticals and Biotechnology', s, 'test_DK_equity INDUSTRY {}'.format(s))
 
+    def test_SE_equity(self):
+        s = financials.getRealtime('ACRI A:STO', 'name', 'FT')
+        self.assertEqual('Acrinova AB (publ)', s, 'test_SE_equity NAME {}'.format(s))
+
+        s = financials.getRealtime('SE0015660014', 'name', 'FT')
+        self.assertEqual('Acrinova AB (publ)', s, 'test_SE_equity NAME {}'.format(s))
+
     def test_TY_equity(self):
         s = financials.getRealtime('6503:TYO', 'OPEN', 'FT')
         self.assertEqual(float, type(s), 'test_TY_equity OPEN {}'.format(s))

src/test_google.py

@@ -1,259 +0,0 @@
-#  test_google.py
-#
-#  license: GNU LGPL
-#
-#  This library is free software; you can redistribute it and/or
-#  modify it under the terms of the GNU Lesser General Public
-#  License as published by the Free Software Foundation; either
-#  version 3 of the License, or (at your option) any later version.
-
-import argparse
-import logging
-import sys
-import unittest
-
-logging.basicConfig(level=logging.ERROR, format="%(asctime)s %(name)s %(levelname)s %(message)s")
-
-import financials
-from datacode import Datacode
-
-financials = financials.createInstance(None)
-
-
-# class Test(unittest.TestCase):
-class Test:
-
-    def test_currency(self):
-        s = financials.getRealtime('EURGBP', Datacode.LAST_PRICE.value, 'GOOGLE')
-        self.assertIsNone(s, 'test_currency LAST_PRICE')
-
-    def test_UK_equity(self):
-        s = financials.getRealtime('LON:VOD', Datacode.LAST_PRICE.value, 'GOOGLE')
-        self.assertEqual(float, type(s), 'test_UK_equity LAST_PRICE {}'.format(s))
-
-        s = financials.getRealtime('LON:VOD', Datacode.TICKER.value, 'GOOGLE')
-        self.assertEqual(s, 'VOD', 'test_UK_equity TICKER')
-
-        s = financials.getRealtime('LON:VOD', Datacode.NAME.value, 'GOOGLE')
-        self.assertEqual(type(s), str, 'test_UK_equity NAME')
-
-        s = financials.getRealtime('LON:VOD', Datacode.EXCHANGE.value, 'GOOGLE')
-        self.assertEqual(s, 'LON', 'test_UK_equity EXCHANGE')
-
-        s = financials.getRealtime('LON:VOD', Datacode.PREV_CLOSE.value, 'GOOGLE')
-        self.assertEqual(float, type(s), 'test_UK_equity PREV_CLOSE {}'.format(s))
-
-        # MARKET_CAP missing for UK stock but available for German stock - weekend issue (FX) ?
-        s = financials.getRealtime('LON:VOD', Datacode.MARKET_CAP.value, 'GOOGLE')
-        self.assertEqual(float, type(s), 'test_UK_equity MARKET_CAP {}'.format(s))
-
-    def test_UK_ETF(self):
-        s = financials.getRealtime('LON:CSP1', Datacode.LAST_PRICE.value, 'GOOGLE')
-        self.assertEqual(float, type(s), 'test_UK_ETF LAST_PRICE {}'.format(s))
-
-        s = financials.getRealtime('LON:CSP1', Datacode.CURRENCY.value, 'GOOGLE')
-        self.assertEqual(s, 'GBX', 'test_UK_ETF CURRENCY')
-
-        s = financials.getRealtime('LON:FTAL', Datacode.LAST_PRICE.value, 'GOOGLE')
-        self.assertEqual(float, type(s), 'test_UK_ETF LAST_PRICE {}'.format(s))
-
-        s = financials.getRealtime('LON:FTAL', Datacode.CURRENCY.value, 'GOOGLE')
-        self.assertEqual(s, 'GBP', 'test_UK_ETF CURRENCY')
-
-        s = financials.getRealtime('LON:FTAL', Datacode.NAME.value, 'GOOGLE')
-        self.assertEqual(type(s), str, 'test_UK_ETF NAME')
-
-    def test_DE_equity(self):
-        s = financials.getRealtime('FRA:SAP', 7, 'GOOGLE')
-        self.assertEqual(float, type(s), 'test_DE_equity 7')
-
-        s = financials.getRealtime('FRA:SAP', '7', 'GOOGLE')
-        self.assertEqual(float, type(s), 'test_DE_equity \'7\'')
-
-        s = financials.getRealtime('FRA:SAP', 8, 'GOOGLE')
-        self.assertEqual(type(s), str, 'test_DE_equity 8')
-
-        s = financials.getRealtime('FRA:SAP', 8.1, 'GOOGLE')
-        self.assertEqual(type(s), str, 'test_DE_equity 8.1')
-
-        s = financials.getRealtime('FRA:SAP', '8.1', 'GOOGLE')
-        self.assertEqual(type(s), str, 'test_DE_equity \'8.1\'')
-
-        s = financials.getRealtime('FRA:SAP', 10, 'GOOGLE')
-        self.assertEqual(type(s), str, 'test_DE_equity 10')
-
-        s = financials.getRealtime('FRA:SAP', '11', 'GOOGLE')
-        self.assertEqual(float, type(s), 'test_DE_equity \'11\'')
-
-        s = financials.getRealtime('FRA:SAP', '21', 'GOOGLE')
-        self.assertEqual(float, type(s), 'test_DE_equity \'21\'')
-
-        s = financials.getRealtime('FRA:SAP', Datacode.TIMEZONE.value, 'GOOGLE')
-        # self.assertEqual(s, 'Europe/Berlin', 'test_DE_equity TIMEZONE')
-        self.assertTrue(s == 'CET' or s == 'CEST', 'test_DE_equity TIMEZONE: {}'.format(s))
-
-    def test_DE_ETF(self):
-        s = financials.getRealtime('FRA:C060', Datacode.LAST_PRICE.value, 'GOOGLE')
-        self.assertEqual(float, type(s), 'test_DE_ETF LAST_PRICE {}'.format(s))
-
-        s = financials.getRealtime('FRA:C060', Datacode.CURRENCY.value, 'GOOGLE')
-        self.assertEqual('EUR', s, 'test_DE_ETF CURRENCY')
-
-        s = financials.getRealtime('FRA:C060', Datacode.TICKER.value, 'GOOGLE')
-        self.assertEqual('C060', s, 'test_DE_ETF TICKER')
-
-        s = financials.getRealtime('FRA:C060', Datacode.EXCHANGE.value, 'GOOGLE')
-        self.assertEqual('FRA', s, 'test_DE_ETF EXCHANGE')
-
-        s = financials.getRealtime('FRA:C060', Datacode.CURRENCY.value, 'GOOGLE')
-        self.assertEqual('EUR', s, 'test_DE_ETF CURRENCY')
-
-        s = financials.getRealtime('FRA:C060', Datacode.MARKET_CAP.value, 'GOOGLE')
-        self.assertIsNone(s, 'test_DE_ETF MARKET_CAP {}'.format(s))
-
-    def test_TY_equity(self):
-        s = financials.getRealtime('TYO:6503', Datacode.OPEN.value, 'GOOGLE')
-        self.assertEqual(float, type(s), 'test_TY_equity OPEN {}'.format(s))
-
-        s = financials.getRealtime('TYO:6503', Datacode.LOW.value, 'GOOGLE')
-        self.assertEqual(float, type(s), 'test_TY_equity LOW {}'.format(s))
-
-        s = financials.getRealtime('TYO:6503', Datacode.HIGH.value, 'GOOGLE')
-        self.assertEqual(float, type(s), 'test_TY_equity HIGH {}'.format(s))
-
-        s = financials.getRealtime('TYO:6503', Datacode.LOW_52_WEEK.value, 'GOOGLE')
-        self.assertEqual(float, type(s), 'test_TY_equity LOW_52_WEEK {}'.format(s))
-
-        s = financials.getRealtime('TYO:6503', Datacode.HIGH_52_WEEK.value, 'GOOGLE')
-        self.assertEqual(float, type(s), 'test_TY_equity HIGH_52_WEEK {}'.format(s))
-
-        s = financials.getRealtime('TYO:6503', Datacode.MARKET_CAP.value, 'GOOGLE')
-        self.assertEqual(float, type(s), 'test_TY_equity MARKET_CAP {}'.format(s))
-
-        # s = financials.getRealtime('TYO:6503', Datacode.VOLUME.value, 'GOOGLE')
-        # self.assertEqual(float, type(s), 'test_TY_equity VOLUME {}'.format(s))
-
-        s = financials.getRealtime('TYO:6503', Datacode.CURRENCY.value, 'GOOGLE')
-        self.assertEqual(s, 'JPY', 'test_TY_equity CURRENCY')
-
-    def test_US_equity(self):
-        s = financials.getRealtime(' NASDAQ : AAPL ', Datacode.LAST_PRICE.value, 'GOOGLE')
-        self.assertEqual(float, type(s), 'test_US_equity LAST_PRICE {}'.format(s))
-
-        s = financials.getRealtime(' NASDAQ : AAPL ', Datacode.TICKER.value, 'GOOGLE')
-        self.assertEqual(s, 'AAPL', 'test_US_equity TICKER')
-
-        s = financials.getRealtime(' NASDAQ : AAPL ', Datacode.EXCHANGE.value, 'GOOGLE')
-        self.assertEqual(s, 'NASDAQ', 'test_US_equity EXCHANGE')
-
-        s = financials.getRealtime(' NASDAQ : AAPL ', Datacode.CURRENCY.value, 'GOOGLE')
-        self.assertEqual(s, 'USD', 'test_US_equity CURRENCY')
-
-        s = financials.getRealtime('NYSE:IBM', Datacode.LAST_PRICE.value, 'GOOGLE')
-        self.assertEqual(float, type(s), 'test_US_equity LAST_PRICE {}'.format(s))
-
-        s = financials.getRealtime('NYSE:IBM', Datacode.TICKER.value, 'GOOGLE')
-        self.assertEqual(s, 'IBM', 'test_US_equity TICKER')
-
-        s = financials.getRealtime('NYSE:IBM', Datacode.EXCHANGE.value, 'GOOGLE')
-        self.assertEqual(s, 'NYSE', 'test_US_equity EXCHANGE')
-
-        s = financials.getRealtime('NYSE:IBM', Datacode.CURRENCY.value, 'GOOGLE')
-        self.assertEqual(s, 'USD', 'test_US_equity CURRENCY')
-
-        s = financials.getRealtime('NYSE:IBM', Datacode.NAME.value, 'GOOGLE')
-        self.assertEqual(type(s), str, 'test_US_equity NAME')
-        self.assertEqual(s, 'IBM Common Stock', 'test_US_equity NAME')
-
-        s = financials.getRealtime('NYSE:IBM', Datacode.LOW.value, 'GOOGLE')
-        self.assertEqual(float, type(s), 'test_US_equity LOW {}'.format(s))
-
-        s = financials.getRealtime('NYSE:IBM', Datacode.HIGH.value, 'GOOGLE')
-        self.assertEqual(float, type(s), 'test_US_equity HIGH {}'.format(s))
-
-        s = financials.getRealtime('NYSE:IBM', Datacode.LOW_52_WEEK.value, 'GOOGLE')
-        self.assertEqual(float, type(s), 'test_US_equity LOW_52_WEEK {}'.format(s))
-
-        s = financials.getRealtime('NYSE:IBM', Datacode.HIGH_52_WEEK.value, 'GOOGLE')
-        self.assertEqual(float, type(s), 'test_US_equity HIGH_52_WEEK {}'.format(s))
-
-        s = financials.getRealtime('NYSE:IBM', Datacode.MARKET_CAP.value, 'GOOGLE')
-        self.assertEqual(float, type(s), 'test_US_equity MARKET_CAP {}'.format(s))
-
-        # s = financials.getRealtime('NYSE:IBM', Datacode.VOLUME.value, 'GOOGLE')
-        # self.assertEqual(float, type(s), 'test_US_equity VOLUME {}'.format(s))
-
-        s = financials.getRealtime('NYSE:IBM', Datacode.TIMESTAMP.value, 'GOOGLE')
-        self.assertEqual(s, 'Data doesn\'t exist - 999', 'test_US_equity TIMESTAMP')
-
-        s = financials.getRealtime('NYSE:IBM', Datacode.TIMEZONE.value, 'GOOGLE')
-        # self.assertEqual('America/New_York', s, 'test_US_equity TIMEZONE')
-        # self.assertEqual('GMT-4', s, 'test_US_equity TIMEZONE')
-        self.assertEqual('GMT-5', s, 'test_US_equity TIMEZONE')
-
-    def test_US_mutuals(self):
-        s = financials.getRealtime('MUTF:VFIAX', Datacode.LAST_PRICE.value, 'GOOGLE')
-        self.assertEqual(float, type(s), 'test_US_mutuals LAST_PRICE - {}'.format(s))
-
-        s = financials.getRealtime('MUTF:VFIAX', Datacode.CURRENCY.value, 'GOOGLE')
-        self.assertEqual(s, 'USD', 'test_US_mutuals CURRENCY')
-
-        s = financials.getRealtime('MUTF:VFIAX', Datacode.TIMEZONE.value, 'GOOGLE')
-        self.assertIsNone(s, 'test_US_mutuals TIMEZONE')
-
-    def test_index(self):
-        s = financials.getRealtime('INDEXDB:DAX', Datacode.LAST_PRICE.value, 'GOOGLE')
-        self.assertEqual(float, type(s), 'test_index LAST_PRICE {}'.format(s))
-
-        s = financials.getRealtime('INDEXDB:DAX', Datacode.CHANGE_IN_PERCENT.value, 'GOOGLE')
-        self.assertEqual(float, type(s), 'test_index CHANGE_IN_PERCENT')
-
-        s = financials.getRealtime('INDEXDB:DAX', Datacode.CHANGE.value, 'GOOGLE')
-        self.assertEqual(float, type(s), 'test_index CHANGE')
-
-        s = financials.getRealtime('INDEXDB:DAX', Datacode.OPEN.value, 'GOOGLE')
-        self.assertEqual(float, type(s), 'test_index OPEN')
-
-        s = financials.getRealtime('INDEXDB:DAX', Datacode.LOW.value, 'GOOGLE')
-        self.assertEqual(float, type(s), 'test_index LOW')
-
-        s = financials.getRealtime('INDEXDB:DAX', Datacode.HIGH.value, 'GOOGLE')
-        self.assertEqual(float, type(s), 'test_index HIGH')
-
-    def test_errors(self):
-        s = financials.getRealtime(None, Datacode.LAST_PRICE.value, 'GOOGLE')
-        self.assertEqual(s, 'Ticker is empty', 'test_errors')
-
-        s = financials.getRealtime('NYS:IBM', None, 'GOOGLE')
-        self.assertEqual(s, 'Datacode is empty', 'test_errors')
-
-        s = financials.getRealtime('DOES_NOT_EXISTS', Datacode.LAST_PRICE.value, 'GOOGLE')
-        self.assertIsNone(s, 'test_errors')
-
-        s = financials.getRealtime('NYS:IBM', 'Foo', 'GOOGLE')
-        self.assertEqual(s, 'Datacode is invalid', 'test_errors')
-
-        # Historic data not supported on GOOGLE
-
-        s = financials.getHistoric('NYS:IBM', Datacode.LAST_PRICE.value, '2017-01-01', 'GOOGLE')
-        self.assertEqual(s, 'Source \'GOOGLE\' not supported', 'test_errors')
-
-    def test_errors_cell_range_passed(self):
-        cell_range = ((1, 2), ('3', '4'), (5.0, 6.0))
-
-        s = financials.getRealtime(cell_range, Datacode.LAST_PRICE.value, 'GOOGLE')
-        self.assertEqual(s, 'Cell range not allowed for ticker', 'test_errors')
-
-        s = financials.getRealtime('NYS:IBM', cell_range, 'GOOGLE')
-        self.assertEqual(s, 'Cell range not allowed for datacode', 'test_errors')
-
-        s = financials.getRealtime('NYS:IBM', Datacode.LAST_PRICE.value, cell_range)
-        self.assertEqual(s, 'Cell range not allowed for source', 'test_errors')
-
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser()
-    parser.add_argument('unittest_args', nargs='*')
-    args = parser.parse_args()
-    unit_argv = [sys.argv[0]] + args.unittest_args
-    unittest.main(argv=unit_argv)

src/test_support.py

@@ -1,4 +1,4 @@
-#  test_google.py
+#  test_support.py
 #
 #  license: GNU LGPL
 #

src/test_yahoo.py

@@ -25,11 +25,16 @@ financials = financials.createInstance(None)
 
 
 def urlopen_fail(self, url, redirect=True, data=None, headers={}, cookies=[], **kwargs):
-    raise baseclient.HttpException(url, 'simulated urlopen() failed')
+    raise baseclient.HttpException(url, 'ERROR: simulated urlopen() failed')
 
 
 class Test(unittest.TestCase):
 
+    @classmethod
+    def tearDownClass(cls):
+        # this avoids "ResourceWarning: unclosed..." on cached socket connections
+        financials.close()
+
     def test_recovery_from_urlopen_error_issue(self):
 
         financials.yahoo.last_url = 'blank'
@@ -100,19 +105,19 @@ class Test(unittest.TestCase):
 
         s = financials.getRealtime('IBM', Datacode.NAME.value, 'YAHOO')
         self.assertEqual(str, type(s), 'test_realtime_US_equity NAME {}'.format(s))
-        self.assertEqual(s, 'International Business Machines Corporation (IBM)',
+        self.assertEqual(s, 'International Business Machines Corporation',
                          'test_realtime_US_equity NAME {}'.format(s))
 
-        s = financials.getRealtime('IBM', Datacode.SECTOR.value, 'YAHOO')
+        s = financials.getRealtime('IBM', 'SECTOR', 'YAHOO')
         self.assertEqual(str, type(s), 'test_realtime_US_equity SECTOR {}'.format(s))
         self.assertEqual(s, 'Technology', 'test_realtime_US_equity SECTOR {}'.format(s))
 
-        s = financials.getRealtime('IBM', Datacode.INDUSTRY.value, 'YAHOO')
+        s = financials.getRealtime('IBM', 'INDUSTRY', 'YAHOO')
         self.assertEqual(str, type(s), 'test_realtime_US_equity INDUSTRY {}'.format(s))
         self.assertEqual(s, 'Information Technology Services', 'test_realtime_US_equity INDUSTRY {}'.format(s))
 
         s = financials.getRealtime('IBM', Datacode.TIMEZONE.value, 'YAHOO')
-        self.assertEqual(s, 'EDT', 'test_realtime_US_equity TIMEZONE {}'.format(s))
+        self.assertTrue(s == 'America/New_York', 'test_realtime_US_equity TIMEZONE: {}'.format(s))
 
         s = financials.getRealtime('IBM', Datacode.BETA.value, 'YAHOO')
         self.assertTrue(testutils.is_positive_float(s), 'test_realtime_US_equity BETA {}'.format(s))
@@ -133,14 +138,17 @@ class Test(unittest.TestCase):
         self.assertEqual(str, type(s), 'test_realtime_US_equity EX_DIV_DATE {}'.format(s))
         self.assertTrue(testutils.is_date(s), 'test_realtime_US_equity EX_DIV_DATE {}'.format(s))
 
-        s = financials.getRealtime('IBM', Datacode.PAYOUT_RATIO.value, 'YAHOO')
+        s = financials.getRealtime('IBM', 'PAYOUT_RATIO', 'YAHOO')
         self.assertTrue(testutils.is_positive_float(s), 'test_realtime_US_equity PAYOUT_RATIO {}'.format(s))
+        # self.assertEqual(None, s, 'test_realtime_US_equity PAYOUT_RATIO {}'.format(s))
 
-        s = financials.getRealtime('IBM', Datacode.SHARES_OUT.value, 'YAHOO')
+        s = financials.getRealtime('IBM', 'SHARES_OUT', 'YAHOO')
         self.assertTrue(testutils.is_positive_float(s), 'test_realtime_US_equity SHARES_OUT {}'.format(s))
+        # self.assertEqual(None, s, 'test_realtime_US_equity SHARES_OUT {}'.format(s))
 
-        s = financials.getRealtime('IBM', Datacode.FREE_FLOAT.value, 'YAHOO')
+        s = financials.getRealtime('IBM', 'FREE_FLOAT', 'YAHOO')
         self.assertTrue(testutils.is_positive_float(s), 'test_realtime_US_equity FREE_FLOAT {}'.format(s))
+        # self.assertEqual(None, s, 'test_realtime_US_equity FREE_FLOAT {}'.format(s))
 
         s = financials.getRealtime('IBM', Datacode.EXCHANGE.value, 'YAHOO')
         self.assertEqual(s, 'NYSE', 'test_realtime_US_equity EXCHANGE')
@@ -157,74 +165,71 @@ class Test(unittest.TestCase):
         self.assertEqual(float, type(s), 'test_realtime_US_mutuals DIV {}'.format(s))
 
         s = financials.getRealtime('VFIAX', Datacode.DIV_YIELD.value, 'YAHOO')
-        # self.assertIsNone(s, 'test_realtime_US_mutuals DIV_YIELD {}'.format(s)) # no yield
-        self.assertTrue(testutils.is_positive_float(s), 'test_realtime_US_mutuals DIV_YIELD {}'.format(s))
+        self.assertIsNone(s, 'test_realtime_US_mutuals DIV_YIELD {}'.format(s)) # no yield
 
         s = financials.getRealtime('SHRAX', Datacode.DIV.value, 'YAHOO')
         self.assertEqual(float, type(s), 'test_realtime_US_mutuals DIV {}'.format(s))
 
         s = financials.getRealtime('SHRAX', Datacode.DIV_YIELD.value, 'YAHOO')
-        # self.assertIsNone(s, 'test_realtime_US_mutuals DIV_YIELD {}'.format(s)) # no yield
-        self.assertEqual(float, type(s), 'test_realtime_US_mutuals DIV_YIELD {}'.format(s))
+        self.assertIsNone(s, 'test_realtime_US_mutuals DIV_YIELD {}'.format(s)) # no yield
 
-        # s = financials.getRealtime('VERX.L', Datacode.DIV.value, 'YAHOO')
-        # self.assertIsNone(s, 'test_realtime_US_mutuals DIV {}'.format(s)) # no dividend
+        s = financials.getRealtime('VERX.L', Datacode.DIV.value, 'YAHOO')
+        self.assertIsNone(s, 'test_realtime_US_mutuals DIV {}'.format(s)) # no dividend
 
         s = financials.getRealtime('VERX.L', Datacode.DIV_YIELD.value, 'YAHOO')
         self.assertIsNone(s, 'test_realtime_US_mutuals DIV_YIELD {}'.format(s)) # no yield
-        # self.assertEqual(float, type(s), 'test_realtime_US_mutuals DIV_YIELD {}'.format(s))
 
     def test_realtime_US_options(self):
 
         # symbol from https://finance.yahoo.com/quote/IBM/options?p=IBM
         
-        s = financials.getRealtime('IBM240119C00120000', Datacode.PREV_CLOSE.value, 'YAHOO')
+        s = financials.getRealtime('IBM250117C00165000', Datacode.PREV_CLOSE.value, 'YAHOO')
         self.assertEqual(float, type(s), 'test_realtime_US_options PREV_CLOSE {}'.format(s))
 
-        s = financials.getRealtime('IBM240119C00120000', Datacode.NAME.value, 'YAHOO')
+        s = financials.getRealtime('IBM250117C00165000', Datacode.NAME.value, 'YAHOO')
         self.assertEqual(str, type(s), 'test_realtime_US_options NAME {}'.format(s))
-        self.assertEqual('IBM Jan 2024 120.000 call', s, 'test_realtime_US_options NAME {}'.format(s))
+        self.assertEqual('IBM Jan 2025 165.000 call', s, 'test_realtime_US_options NAME {}'.format(s))
 
-        s = financials.getRealtime('IBM240119C00120000', Datacode.EXPIRY_DATE.value, 'YAHOO')
+        s = financials.getRealtime('IBM250117C00165000', Datacode.EXPIRY_DATE.value, 'YAHOO')
         self.assertEqual(str, type(s), 'test_realtime_US_options EXPIRY_DATE {}'.format(s))
         self.assertTrue(testutils.is_date(s), 'test_realtime_US_options EXPIRY_DATE {}'.format(s))
-        self.assertEqual("2024-01-19", s, 'test_realtime_US_options EXPIRY_DATE {}'.format(s))
+        self.assertEqual("2025-01-17", s, 'test_realtime_US_options EXPIRY_DATE {}'.format(s))
 
-        s = financials.getRealtime('IBM240119C00120000', Datacode.LAST_PRICE.value, 'YAHOO')
+        s = financials.getRealtime('IBM250117C00165000', Datacode.LAST_PRICE.value, 'YAHOO')
         self.assertEqual(float, type(s), 'test_realtime_US_options LAST_PRICE {}'.format(s))
 
-        s = financials.getRealtime('IBM240119C00120000', Datacode.OPEN.value, 'YAHOO')
+        s = financials.getRealtime('IBM250117C00165000', Datacode.OPEN.value, 'YAHOO')
         self.assertEqual(float, type(s), 'test_realtime_US_options OPEN {}'.format(s))
 
-        s = financials.getRealtime('IBM240119C00120000', Datacode.VOLUME.value, 'YAHOO')
+        s = financials.getRealtime('IBM250117C00165000', Datacode.VOLUME.value, 'YAHOO')
         self.assertEqual(float, type(s), 'test_realtime_US_options VOLUME {}'.format(s))
 
-        s = financials.getRealtime('IBM240119C00120000', Datacode.BID.value, 'YAHOO')
+        s = financials.getRealtime('IBM250117C00165000', Datacode.BID.value, 'YAHOO')
         self.assertEqual(float, type(s), 'test_realtime_US_options BID {}'.format(s))
 
-        s = financials.getRealtime('IBM240119C00120000', Datacode.ASK.value, 'YAHOO')
+        s = financials.getRealtime('IBM250117C00165000', Datacode.ASK.value, 'YAHOO')
         self.assertEqual(float, type(s), 'test_realtime_US_options ASK {}'.format(s))
 
-        s = financials.getRealtime('IBM240119C00120000', Datacode.PAYOUT_RATIO.value, 'YAHOO')
+        s = financials.getRealtime('IBM250117C00165000', Datacode.PAYOUT_RATIO.value, 'YAHOO')
         self.assertIsNone(s, 'test_realtime_US_options PAYOUT_RATIO {}'.format(s))
 
-        s = financials.getRealtime('IBM230120C00130000', Datacode.SECTOR.value, 'YAHOO')
+        s = financials.getRealtime('IBM250117C00165000', Datacode.SECTOR.value, 'YAHOO')
         self.assertIsNone(s, 'test_realtime_US_options SECTOR {}'.format(s))
 
     def test_realtime_US_futures(self):
 
         s = financials.getRealtime('ES=F', Datacode.NAME.value, 'YAHOO')
         self.assertEqual(str, type(s), 'test_realtime_US_futures NAME {}'.format(s))
-        self.assertEqual('E-Mini S&P 500 Sep 23 (ES=F)', s, 'test_realtime_US_futures NAME {}'.format(s))
+        self.assertEqual('E-Mini S&P 500 Sep 24', s, 'test_realtime_US_futures NAME {}'.format(s))
 
         s = financials.getRealtime('ES=F', Datacode.TICKER.value, 'YAHOO')
         self.assertEqual(str, type(s), 'test_realtime_US_futures TICKER {}'.format(s))
-        self.assertEqual('ESU23.CME', s, 'test_realtime_US_futures TICKER {}'.format(s))
+        self.assertEqual('ESU24.CME', s, 'test_realtime_US_futures TICKER {}'.format(s))
 
         s = financials.getRealtime('ES=F', Datacode.SETTLEMENT_DATE.value, 'YAHOO')
         self.assertEqual(str, type(s), 'test_realtime_US_futures SETTLEMENT_DATE {}'.format(s))
         self.assertTrue(testutils.is_date(s), 'test_realtime_US_futures SETTLEMENT_DATE {}'.format(s))
-        self.assertEqual("2023-09-15", s, 'test_realtime_US_futures SETTLEMENT_DATE {}'.format(s))
+        self.assertEqual("2024-09-20", s, 'test_realtime_US_futures SETTLEMENT_DATE {}'.format(s))
 
         s = financials.getRealtime('ES=F', Datacode.LAST_PRICE.value, 'YAHOO')
         self.assertEqual(float, type(s), 'test_realtime_US_futures LAST_PRICE {}'.format(s))
@@ -259,12 +264,23 @@ class Test(unittest.TestCase):
         self.assertEqual(float, type(s), 'test_realtime_UK_ETF LAST_PRICE {}'.format(s))
 
         s = financials.getRealtime('VERX.L', Datacode.TIMEZONE.value, 'YAHOO')
-        # self.assertEqual('GMT', s, 'test_realtime_UK_ETF TIMEZONE {}'.format(s))
-        self.assertEqual('BST', s, 'test_realtime_UK_ETF TIMEZONE {}'.format(s))
+        self.assertTrue(s == 'Europe/London', 'test_realtime_UK_ETF TIMEZONE: {}'.format(s))
 
         s = financials.getRealtime('CSP1.L', Datacode.NAME.value, 'YAHOO')
         self.assertEqual(str, type(s), 'test_realtime_UK_ETF NAME {}'.format(s))
-        self.assertEqual('iShares VII PLC - iShares Core S&P 500 UCITS ETF (CSP1.L)', s, 'test_realtime_UK_ETF NAME {}'.format(s))
+        self.assertEqual('iShares Core S&P 500 UCITS ETF USD (Acc)', s, 'test_realtime_UK_ETF NAME {}'.format(s))
+
+        s = financials.getRealtime('VERX.L', 'SECTOR', 'YAHOO')
+        self.assertIsNone(s, 'test_realtime_UK_ETF SECTOR {}'.format(s))
+
+        s = financials.getRealtime('VERX.L', 'INDUSTRY', 'YAHOO')
+        self.assertIsNone(s, 'test_realtime_UK_ETF INDUSTRY {}'.format(s))
+
+        s = financials.getRealtime('VERX.L', 'PAYOUT_RATIO', 'YAHOO')
+        self.assertIsNone(s, 'test_realtime_UK_ETF PAYOUT_RATIO {}'.format(s))
+
+        s = financials.getRealtime('VERX.L', 'SHARES_OUT', 'YAHOO')
+        self.assertIsNone(s, 'test_realtime_UK_ETF SHARES_OUT {}'.format(s))
 
     def test_realtime_DE_equity(self):
 
@@ -272,8 +288,7 @@ class Test(unittest.TestCase):
         self.assertEqual(float, type(s), 'test_realtime_DE_equity LAST_PRICE {}'.format(s))
 
         s = financials.getRealtime('SAP.DE', Datacode.TIMEZONE.value, 'YAHOO')
-        # self.assertEqual('CET', s, 'test_realtime_DE_equity TIMEZONE {}'.format(s))
-        self.assertEqual('CEST', s, 'test_realtime_DE_equity TIMEZONE {}'.format(s))
+        self.assertTrue(s == 'Europe/Berlin', 'test_realtime_DE_equity TIMEZONE: {}'.format(s))
 
         s = financials.getRealtime('SAP.DE', Datacode.SECTOR.value, 'YAHOO')
         self.assertEqual(str, type(s), 'test_realtime_DE_equity SECTOR {}'.format(s))
@@ -281,7 +296,7 @@ class Test(unittest.TestCase):
 
         s = financials.getRealtime('SAP.DE', Datacode.INDUSTRY.value, 'YAHOO')
         self.assertEqual(str, type(s), 'test_realtime_DE_equity INDUSTRY {}'.format(s))
-        self.assertEqual(s, 'Software—Application', 'test_realtime_DE_equity INDUSTRY {}'.format(s))
+        self.assertEqual(s, 'Software - Application', 'test_realtime_DE_equity INDUSTRY {}'.format(s))
 
         s = financials.getRealtime('LYY8.DE', Datacode.NAME.value, 'YAHOO')
         self.assertEqual(str, type(s), 'test_realtime_DE_equity NAME {}'.format(s))
@@ -324,14 +339,14 @@ class Test(unittest.TestCase):
 
         s = financials.getRealtime('LUMI.TA', 'INDUSTRY', 'YAHOO')
         self.assertEqual(str, type(s), 'test_TLV_equity INDUSTRY {}'.format(s))
-        self.assertEqual('Banks—Regional', s, 'test_TA_equity INDUSTRY {}'.format(s))
+        self.assertEqual('Banks - Regional', s, 'test_TA_equity INDUSTRY {}'.format(s))
 
     def test_DK_equity(self):
         s = financials.getRealtime('NOVO-B.CO', 'last_price', 'YAHOO')
         self.assertEqual(float, type(s), 'test_DK_equity LAST_PRICE {}'.format(s))
 
         s = financials.getRealtime('NOVO-B.CO', 'name', 'YAHOO')
-        self.assertEqual('Novo Nordisk A/S (NOVO-B.CO)', s, 'test_DK_equity NAME {}'.format(s))
+        self.assertEqual('Novo Nordisk A/S', s, 'test_DK_equity NAME {}'.format(s))
 
         s = financials.getRealtime('NOVO-B.CO', 'currency', 'YAHOO')
         self.assertEqual('DKK', s, 'test_DK_equity CURRENCY {}'.format(s))
@@ -377,7 +392,7 @@ class Test(unittest.TestCase):
         self.assertEqual(s, 'JPY', 'test_TY_equity CURRENCY')
 
         s = financials.getRealtime('6503.T', Datacode.TIMEZONE.value, 'YAHOO')
-        self.assertEqual(s, 'JST', 'test_TY_equity TIMEZONE')
+        self.assertEqual(s, 'Asia/Tokyo', 'test_TY_equity TIMEZONE')
 
     def test_historic_US_equity(self):