Yahoo: Don't re-fetch non-existing statistic/profile data

README.md

@@ -4,8 +4,9 @@ Version 3.3.0 includes improved cookie handling and somewhat improved logic to d
 
 ## Overview
 
-This is a Python based extension for LibreOffice Calc to make stock market, index and FX data available in Calc 
-spreadsheets - currently supporting Yahoo's and Financial Times' finance websites using old-fashioned web scraping. 
+This is a Python based extension for LibreOffice Calc to make market data available in Calc 
+spreadsheets - currently supporting Yahoo's (FX, crypto, equities, indices, futures, options) and Financial Times' 
+(FX, equities, indices, futures) websites using old-fashioned web scraping. 
 
 Starting with version 3.1.0, we received a contribution to get crypto data directly from Coinbase
 
@@ -32,7 +33,8 @@ Only Yahoo has historic data available.
 
 There is a file **examples.ods** there too with usage examples and possible arguments to functions.
 
-You have to check the respective websites to work out what symbol is the right one for you. If a website doesn't have 
+You have to check the respective websites to work out what symbol is the right one for you. Make sure today or the date 
+requested is a trading day (exchange is not closed). If a website doesn't have 
 the symbol/asset you want, this extension can't help you either. Having said that, I mostly look at US and West European 
 equities, ETFs and mutual funds and major FX rates - if you have issues with the data available for other assets or 
 assets in other regions, drop me a line (best to include full URLs and possibly the same asset listed on more than one 
@@ -135,13 +137,10 @@ python3 -m unittest discover src
 
 ### Tested with:
 - Windows 10 / LibreOffice Calc 7.1.2.2 / Python 3.8.8
-- Ubuntu 20.04.5 / LibreOffice Calc 6.4.7.2 / Python 3.8.10
+- Ubuntu 22.04.1 / LibreOffice Calc 7.3.7.2 / Python 3.10.6
 - MacOS 10.15.7 / LibreOffice Calc 7.2.0.4 / Python 3.8.10
 
 (Previous versions)
 - Debian 10.3  / LibreOffice Calc 6.1.5.2 / Python 3.7.3
-- Ubuntu 20.10 / LibreOffice Calc 7.0.3.1 / Python 3.8.6
-- Ubuntu 19.10 / LibreOffice Calc 6.3.5.2 / Python 3.7.6
-- Ubuntu 19.04 / LibreOffice Calc 6.2 / Python 3.7.3
+- Ubuntu 20.04.5 / LibreOffice Calc 6.4.7.2 / Python 3.8.10
 - Ubuntu 18.04.5 / LibreOffice Calc 6 / Python 3.6.9
-- Ubuntu 18.04 / LibreOffice Calc 6 / Python 3.6.7

compile.sh

@@ -29,13 +29,23 @@ mkdir "${PWD}"/build/META-INF/
 
 # Compile the binaries
 
-echo "Calling idlc..."
-idlc -w -verbose "${PWD}"/idl/XFinancials.idl
+if command -v idlc &> /dev/null
+then
+  echo "Calling idlc..."
+  idlc -w -verbose "${PWD}"/idl/XFinancials.idl
 
-echo "Calling regmerge..."
-regmerge -v "${PWD}"/build/XFinancials.rdb UCR "${PWD}"/idl/XFinancials.urd
+  echo "Calling regmerge..."
+  regmerge -v "${PWD}"/build/XFinancials.rdb UCR "${PWD}"/idl/XFinancials.urd
 
-rm "${PWD}"/idl/XFinancials.urd
+  rm "${PWD}"/idl/XFinancials.urd
+else
+  # LibreOffice 7.4+
+  export unoTypes=/usr/lib/libreoffice/program/types.rdb
+  export offTypes=/usr/lib/libreoffice/program/types/offapi.rdb
+
+  echo "Calling unoidl-write..."
+  unoidl-write $unoTypes $offTypes "${PWD}"/idl/XFinancials.idl "${PWD}"/build/XFinancials.rdb
+fi
 
 echo "Generating meta files..."
 python3 "${PWD}"/src/generate_metainfo.py
@@ -55,11 +65,11 @@ cp -f "${PWD}"/src/financials_coinbase.py "${PWD}"/build/
 
 TMPFILE=`mktemp`
 
-wget "https://files.pythonhosted.org/packages/d4/70/d60450c3dd48ef87586924207ae8907090de0b306af2bce5d134d78615cb/python_dateutil-2.8.1-py2.py3-none-any.whl" -O $TMPFILE
+wget "https://files.pythonhosted.org/packages/36/7a/87837f39d0296e723bb9b62bbb257d0355c7f6128853c78955f57342a56d/python_dateutil-2.8.2-py2.py3-none-any.whl" -O $TMPFILE
 unzip $TMPFILE dateutil/\* -d "${PWD}"/build/
 rm $TMPFILE
 
-wget "https://files.pythonhosted.org/packages/70/94/784178ca5dd892a98f113cdd923372024dc04b8d40abe77ca76b5fb90ca6/pytz-2021.1-py2.py3-none-any.whl" -O $TMPFILE
+wget "https://files.pythonhosted.org/packages/7f/99/ad6bd37e748257dd70d6f85d916cafe79c0b0f5e2e95b11f7fbc82bf3110/pytz-2023.3-py2.py3-none-any.whl" -O $TMPFILE
 unzip $TMPFILE pytz/\* -d "${PWD}"/build/
 rm $TMPFILE
 
@@ -68,7 +78,7 @@ unzip $TMPFILE pyparsing.py -d "${PWD}"/build/
 rm $TMPFILE
 
 # Windows LibreOffice 7.1 Python is missing this...
-wget "https://files.pythonhosted.org/packages/ee/ff/48bde5c0f013094d729fe4b0316ba2a24774b3ff1c52d924a8a4cb04078a/six-1.15.0-py2.py3-none-any.whl" -O $TMPFILE
+wget "https://files.pythonhosted.org/packages/d9/5a/e7c31adbe875f2abbb91bd84cf2dc52d792b5a01506781dbcf25c91daf11/six-1.16.0-py2.py3-none-any.whl" -O $TMPFILE
 unzip $TMPFILE six.py -d "${PWD}"/build/
 rm $TMPFILE
 

src/baseclient.py

@@ -49,15 +49,18 @@ class BaseClient:
         os.makedirs(self.basedir, exist_ok=True)
 
         user_agents = [
-            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:100.0) Gecko/20100101 Firefox/100.0',
-            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:101.0) Gecko/20100101 Firefox/101.0',
-            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:102.0) Gecko/20100101 Firefox/102.0',
-            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:103.0) Gecko/20100101 Firefox/103.0',
-            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:104.0) Gecko/20100101 Firefox/104.0',
-            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:105.0) Gecko/20100101 Firefox/105.0',
-            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:106.0) Gecko/20100101 Firefox/106.0',
-            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:107.0) Gecko/20100101 Firefox/107.0',
-            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:108.0) Gecko/20100101 Firefox/108.0'
+            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/110.0',
+            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/111.0',
+            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/112.0',
+            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/113.0',
+            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/114.0',
+            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/115.0',
+            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/116.0',
+            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/117.0',
+            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/118.0',
+            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/119.0',
+            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/120.0',
+            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/121.0'
         ]
 
         self.default_headers = {
@@ -71,16 +74,15 @@ class BaseClient:
 
         self.response = None
 
-    def request(self, method: str, url: str, data=None, headers={}, cookies=[], **kwargs):
+    def request(self, method: str, url: str, data=None, headers={}, **kwargs):
 
         _headers = self.default_headers.copy()
         if headers:
             for key, value in headers.items():
                 _headers[key] = value
 
-        if cookies:
-            for c in cookies:
-                self.cookies.set_cookie(c)
+        if method == 'POST' and 'Content-Type' not in _headers:
+            _headers['Content-Type'] = 'application/x-www-form-urlencoded'
 
         connection = None
 
@@ -98,7 +100,7 @@ class BaseClient:
             connection = HTTPConnection(host, **kwargs) if scheme == 'http:' else HTTPSConnection(host, **kwargs)
 
         logger.debug('Creating request -----------------------------------------------------')
-        logger.debug("url='%s'", url)
+        logger.debug("%s %s", method, url)
 
         self.last_url = url
 
@@ -129,9 +131,13 @@ class BaseClient:
 
     def urlopen(self, url, redirect=True, data=None, headers={}, cookies=[], **kwargs):
 
+        if cookies:
+            for c in cookies:
+                self.cookies.set_cookie(c)
+
         self.last_url = None
 
-        self.response = self.request('POST' if data else 'GET', url, data, headers, cookies, **kwargs)
+        self.response = self.request('POST' if data else 'GET', url, data, headers, **kwargs)
         text = self.response.read()
 
         # Allow redirects - used by Yahoo for some cookie based consent
@@ -141,7 +147,7 @@ class BaseClient:
         # try mitigating by re-requesting straight away
         if 400 <= self.response.status < 500:
             if self.response.getheader('X-Cache') == 'Error from cloudfront':
-                self.response = self.request('POST' if data else 'GET', url, data, headers, cookies, **kwargs)
+                self.response = self.request('POST' if data else 'GET', url, data, headers, **kwargs)
                 text = self.response.read()
 
         while 300 <= self.response.status < 400 and self.redirect_count >= 0:
@@ -155,7 +161,7 @@ class BaseClient:
                     scheme, _, host, path = url.split('/', 3)
                     location = '{}//{}{}'.format(scheme, host, location)
 
-                self.response = self.request('POST' if data else 'GET', location, data, headers, cookies, **kwargs)
+                self.response = self.request('GET', location, None, headers, **kwargs)
                 text = self.response.read()
 
             else:
@@ -385,3 +391,11 @@ class BaseClient:
             pass
 
         return None
+
+    def close(self):
+        for connection in self.connections.values():
+            try:
+                connection.close()
+            except BaseException:
+                pass
+        self.connections = {}

src/financials.py

@@ -95,7 +95,6 @@ class FinancialsImpl(unohelper.Base, Financials):
 
     def __init__(self, ctx):
         self.ctx = ctx
-        self.google = google.createInstance(ctx)
         self.yahoo = yahoo.createInstance(ctx)
         self.coinbase = coinbase.createInstance(ctx)
         self.ft = ft.createInstance(ctx)
@@ -136,9 +135,6 @@ class FinancialsImpl(unohelper.Base, Financials):
             ticker = str(ticker).strip()
             source = str(source).upper()
 
-            # if source == 'GOOGLE':
-            #     s = self.google.getRealtime(ticker, datacode)
-            # el
             if source == 'YAHOO':
                 s = self.yahoo.getRealtime(ticker, datacode)
             elif source == 'FT':
@@ -273,6 +269,11 @@ class FinancialsImpl(unohelper.Base, Financials):
 
         return s
 
+    def close(self):
+        if self.yahoo: self.yahoo.close()
+        if self.coinbase: self.coinbase.close()
+        if self.ft: self.ft.close()
+
 
 def createInstance(ctx):
     return FinancialsImpl(ctx)

src/financials_yahoo.py

@@ -15,7 +15,6 @@ import os
 import re
 import time
 import urllib.parse
-from http import cookiejar
 
 import dateutil.parser
 
@@ -57,24 +56,6 @@ def handle_abbreviations(s):
     return float(s)
 
 
-def cookie(name, value):
-    return cookiejar.Cookie(version=0, name=name, value=value,
-                            port=None, port_specified=False, domain=".yahoo.com", domain_specified=True,
-                            domain_initial_dot=True, path="/", path_specified=True, secure=True, expires=None,
-                            discard=False, comment=None, comment_url=None, rest=dict())
-
-
-def get_cookies():
-    return [
-        cookie("A1", "d=AQABBNAVmWICEEFBM1xh-RmAmPpJJIsAz3YFEgABBwFcmmJlY_bPb2UB9iMAAAcIzhWZYm7SAIg&S=AQAAAucqV1HMdCsRf6key1gdaFs"),
-        cookie("A1S", "d=AQABBNAVmWICEEFBM1xh-RmAmPpJJIsAz3YFEgABBwFcmmJlY_bPb2UB9iMAAAcIzhWZYm7SAIg&S=AQAAAucqV1HMdCsRf6key1gdaFs&j=GDPR"),
-        cookie("A3", "d=AQABBNAVmWICEEFBM1xh-RmAmPpJJIsAz3YFEgABBwFcmmJlY_bPb2UB9iMAAAcIzhWZYm7SAIg&S=AQAAAucqV1HMdCsRf6key1gdaFs"),
-        cookie("GUC", "AQABBwFimlxjZUIcxQRM"),
-        cookie("maex", "{\"v2\":{}}"),
-        cookie("thamba", "1")
-    ]
-
-
 class Yahoo(BaseClient):
     def __init__(self, ctx):
         super().__init__()
@@ -84,61 +65,6 @@ class Yahoo(BaseClient):
         self.historicdata = {}
         self.js = jsonParser.jsonObject
 
-        self.create_cookies()
-
-    def create_cookies(self):
-
-        cookiejar_path = os.path.join(self.basedir, 'yahoo.cookiejar')
-        cookiejar_exists = os.path.isfile(cookiejar_path) and os.stat(cookiejar_path).st_size >= 0
-
-        if cookiejar_exists:
-            try:
-                self.cookies = cookiejar.LWPCookieJar()
-                self.cookies.load(cookiejar_path, ignore_discard=True)
-
-                required_cookie_names = ["A1", "A1S", "A3", "GUC", "maex", "thamba"]
-                for c in self.cookies:
-                    if c.name in required_cookie_names:
-                        required_cookie_names.remove(c.name)
-
-                if len(required_cookie_names) > 0:
-                    cookiejar_exists = False
-                    logger.info("Overriding cookiejar '%s'", cookiejar_path)
-
-            except BaseException:
-                cookiejar_exists = False
-                logger.exception("BaseException initial loading cookiejar_path=%s", cookiejar_path)
-
-        if not cookiejar_exists:
-            try:
-                lwp_cookiejar = cookiejar.LWPCookieJar()
-                for c in get_cookies():
-                    lwp_cookiejar.set_cookie(c)
-                lwp_cookiejar.save(cookiejar_path, ignore_discard=True)
-                logger.info("Created cookiejar '%s'", cookiejar_path)
-            except BaseException:
-                logger.exception("BaseException creating cookiejar_path=%s", cookiejar_path)
-
-        try:
-            self.cookies = cookiejar.LWPCookieJar()
-            self.cookies.load(cookiejar_path, ignore_discard=True)
-
-            logger.info("Loaded cookiejar '%s'", cookiejar_path)
-
-            for c in self.cookies:
-                logger.info("Cookie name'%s' value='%s' path='%s'", c.name, c.value, c.path)
-
-        except BaseException:
-            logger.exception("BaseException loading cookiejar_path=%s", cookiejar_path)
-
-    def save_cookies(self):
-        cookiejar_path = os.path.join(self.basedir, 'yahoo.cookiejar')
-        try:
-            self.cookies.save(cookiejar_path, ignore_discard=True)
-            logger.debug("Saved cookiejar '%s'", cookiejar_path)
-        except BaseException:
-            logger.exception("BaseException saving cookiejar_path=%s", cookiejar_path)
-
     def _read_ticker_csv_file(self, ticker):
 
         fn = os.path.join(self.basedir, 'yahoo-{}.csv'.format(ticker))
@@ -206,6 +132,51 @@ class Yahoo(BaseClient):
 
         return self.getRealtimeSummary(ticker, datacode)
 
+    def getData(self, url, ticker, datacode, html_file):
+
+        try:
+            text = self.urlopen(url, redirect=True)
+        except BaseException as e:
+            logger.exception("BaseException (1) ticker=%s datacode=%s last_url=%s redirect_count=%s %s",
+                             ticker, datacode, self.last_url, self.redirect_count, e)
+            return None
+
+        try:
+            with open(os.path.join(self.basedir, html_file), "w", encoding="utf-8") as text_file:
+                print(f"<!-- '{self.last_url}' -->\r\n\r\n{text}", file=text_file)
+        except BaseException as e:
+            logger.exception("BaseException (2) ticker=%s datacode=%s %s", ticker, datacode, e)
+
+        if not text:
+            return None
+
+        try:
+            parser = NaiveHTMLParser()
+            root = parser.feed(text)
+            parser.close()
+        except BaseException as e:
+            logger.exception("BaseException (3) ticker=%s datacode=%s - HTML parsing - %s", ticker, datacode, e)
+            return None
+
+        form = root.find(f".//form[@class='consent-form']")
+
+        if form:
+            inputs = form.findall(f".//input")
+
+            if inputs:
+
+                data = {'reject': 'reject'}
+                for d in inputs:
+                    data[d.attrib['name']] = d.attrib['value']
+
+                try:
+                    text = self.urlopen(self.last_url, redirect=True, data=urllib.parse.urlencode(data))
+                except BaseException as e:
+                    logger.exception("BaseException (4) ticker=%s datacode=%s last_url=%s redirect_count=%s %s",
+                                     ticker, datacode, self.last_url, self.redirect_count, e)
+
+        return text
+
     def getRealtimeSummary(self, ticker, datacode):
 
         """
@@ -215,20 +186,11 @@ class Yahoo(BaseClient):
         tick = self.realtime[ticker]
 
         url = 'https://finance.yahoo.com/quote/{}?p={}'.format(ticker, ticker)
+        text = self.getData(url, ticker, datacode, f'yahoo-{ticker}.html')
 
-        try:
-            text = self.urlopen(url, redirect=True)
-            self.save_cookies()
-        except BaseException as e:
-            logger.exception("BaseException ticker=%s datacode=%s last_url=%s redirect_count=%s", ticker, datacode, self.last_url, self.redirect_count)
+        if text is None:
             del self.realtime[ticker]
-            return 'Yahoo.getRealtimeSummary({}, {}) - urlopen: {}'.format(ticker, datacode, e)
-
-        try:
-            with open(os.path.join(self.basedir, 'yahoo-{}.html'.format(ticker)), "w", encoding="utf-8") as text_file:
-                print(f"<!-- '{self.last_url}' -->\r\n\r\n{text}", file=text_file)
-        except BaseException:
-            logger.exception("BaseException ticker=%s datacode=%s", ticker, datacode)
+            return 'Yahoo.getRealtimeSummary({}, {}) - getData'.format(ticker, datacode)
 
         try:
             r = '"crumb":"([^"]{11})"'
@@ -250,6 +212,10 @@ class Yahoo(BaseClient):
             return 'Yahoo.getRealtimeSummary({}, {}) - HTML parsing: {}'.format(ticker, datacode, e)
 
         try:
+            if not root:
+                logger.exception("BaseException ticker=%s datacode=%s", ticker, datacode)
+                return 'Yahoo.getRealtimeSummary({}, {}) - root missing'.format(ticker, datacode)
+
             tick[Datacode.TICKER] = ticker
             tick[Datacode.TIMESTAMP] = time.time()
             tick[Datacode.YAHOO_SUMMARY_RECEIVED] = True
@@ -375,20 +341,11 @@ class Yahoo(BaseClient):
         tick = self.realtime[ticker]
 
         url = 'https://finance.yahoo.com/quote/{}/key-statistics?p={}'.format(ticker, ticker)
+        text = self.getData(url, ticker, datacode, f'yahoo-{ticker}-statistics.html')
 
-        try:
-            text = self.urlopen(url, redirect=True)
-            self.save_cookies()
-        except BaseException as e:
-            logger.exception("BaseException ticker=%s datacode=%s last_url=%s redirect_count=%s", ticker, datacode, self.last_url, self.redirect_count)
+        if text is None:
             del self.realtime[ticker]
-            return 'Yahoo.getRealtimeStatistics({}, {}) - urlopen: {}'.format(ticker, datacode, e)
-
-        try:
-            with open(os.path.join(self.basedir, 'yahoo-{}-statistics.html'.format(ticker)), "w", encoding="utf-8") as text_file:
-                print(f"<!-- '{url}' -->\r\n\r\n{text}", file=text_file)
-        except BaseException:
-            logger.exception("BaseException open/write ticker=%s datacode=%s", ticker, datacode)
+            return 'Yahoo.getRealtimeStatistics({}, {}) - getData'.format(ticker, datacode)
 
         try:
             parser = NaiveHTMLParser()
@@ -399,19 +356,23 @@ class Yahoo(BaseClient):
             del self.realtime[ticker]
             return 'Yahoo.getRealtimeStatistics({}, {}) - HTML parsing: {}'.format(ticker, datacode, e)
 
+        statistics = root.find(".//section[@data-test='qsp-statistics']")
+
+        tick[Datacode.TICKER] = ticker
+        tick[Datacode.TIMESTAMP] = time.time()
+        tick[Datacode.YAHOO_STATISTIC_RECEIVED] = True
+
+        tick[Datacode.SHARES_OUT] = None
+        tick[Datacode.FREE_FLOAT] = None
+        tick[Datacode.PAYOUT_RATIO] = None
+
+        if statistics is None:
+            return None
+
+        parsed = {}
+
         try:
 
-            parsed = {}
-
-            statistics = root.find(".//section[@data-test='qsp-statistics']")
-
-            if statistics is None:
-                return None
-
-            tick[Datacode.TICKER] = ticker
-            tick[Datacode.TIMESTAMP] = time.time()
-            tick[Datacode.YAHOO_STATISTIC_RECEIVED] = True
-
             # Valuation Measures
             found = statistics.find('./div[2]/div[1]//table')
             if found:
@@ -444,17 +405,15 @@ class Yahoo(BaseClient):
                     if key is not None:
                         parsed[key] = d.find('./td[2]').text
 
-            tick[Datacode.SHARES_OUT] = self.save_wrapper(
-                lambda: float(handle_abbreviations(parsed['Shares Outstanding'])))
-            tick[Datacode.FREE_FLOAT] = self.save_wrapper(
-                lambda: float(handle_abbreviations(parsed['Float'])))
-            tick[Datacode.PAYOUT_RATIO] = self.save_wrapper(
-                lambda: float(handle_abbreviations(parsed['Payout Ratio'].replace('%', '').strip()))/100.0)
+        except KeyError:
+            pass
 
-        except BaseException as e:
-            logger.exception("BaseException ticker=%s datacode=%s", ticker, datacode)
-            del self.realtime[ticker]
-            return 'Yahoo.getRealtimeStatistics({}, {}) - process: {}'.format(ticker, datacode, e)
+        tick[Datacode.SHARES_OUT] = self.save_wrapper(
+            lambda: float(handle_abbreviations(parsed['Shares Outstanding'])))
+        tick[Datacode.FREE_FLOAT] = self.save_wrapper(
+            lambda: float(handle_abbreviations(parsed['Float'])))
+        tick[Datacode.PAYOUT_RATIO] = self.save_wrapper(
+            lambda: float(handle_abbreviations(parsed['Payout Ratio'].replace('%', '').strip()))/100.0)
 
         return self._return_value(self.realtime[ticker], datacode)
 
@@ -467,20 +426,11 @@ class Yahoo(BaseClient):
         tick = self.realtime[ticker]
 
         url = 'https://finance.yahoo.com/quote/{}/profile?p={}'.format(ticker, ticker)
+        text = self.getData(url, ticker, datacode, f'yahoo-{ticker}-profile.html')
 
-        try:
-            text = self.urlopen(url, redirect=True)
-            self.save_cookies()
-        except BaseException as e:
-            logger.exception("BaseException ticker=%s datacode=%s last_url=%s redirect_count=%s", ticker, datacode, self.last_url, self.redirect_count)
+        if text is None:
             del self.realtime[ticker]
-            return 'Yahoo.getRealtimeProfile({}, {}) - urlopen: {}'.format(ticker, datacode, e)
-
-        try:
-            with open(os.path.join(self.basedir, 'yahoo-{}-profile.html'.format(ticker)), "w", encoding="utf-8") as text_file:
-                print(f"<!-- '{url}' -->\r\n\r\n{text}", file=text_file)
-        except BaseException:
-            logger.exception("BaseException open/write ticker=%s datacode=%s", ticker, datacode)
+            return 'Yahoo.getRealtimeProfile({}, {}) - getData'.format(ticker, datacode)
 
         try:
             parser = NaiveHTMLParser()
@@ -491,24 +441,17 @@ class Yahoo(BaseClient):
             del self.realtime[ticker]
             return 'Yahoo.getRealtimeProfile({}, {}) - HTML parsing: {}'.format(ticker, datacode, e)
 
-        try:
+        tick[Datacode.TICKER] = ticker
+        tick[Datacode.TIMESTAMP] = time.time()
+        tick[Datacode.YAHOO_PROFILE_RECEIVED] = True
 
+        p = None
+
+        if root:
             p = root.find(".//*[span='Sector(s)']")
 
-            if p is None:
-                return None
-
-            tick[Datacode.TICKER] = ticker
-            tick[Datacode.TIMESTAMP] = time.time()
-            tick[Datacode.YAHOO_PROFILE_RECEIVED] = True
-
-            tick[Datacode.SECTOR] = self.save_wrapper(lambda: p.find("./span[2]").text)
-            tick[Datacode.INDUSTRY] = self.save_wrapper(lambda: p.find("./span[4]").text)
-
-        except BaseException as e:
-            logger.exception("BaseException ticker=%s datacode=%s", ticker, datacode)
-            del self.realtime[ticker]
-            return 'Yahoo.getRealtimeProfile({}, {}) - process: {}'.format(ticker, datacode, e)
+        tick[Datacode.SECTOR] = self.save_wrapper(lambda: p.find("./span[2]").text)
+        tick[Datacode.INDUSTRY] = self.save_wrapper(lambda: p.find("./span[4]").text)
 
         return self._return_value(self.realtime[ticker], datacode)
 

src/generate_metainfo.py

@@ -14,7 +14,7 @@ import os
 cur_dir = os.getcwd()
 
 addin_id = "com.financials.getinfo"
-addin_version = "3.3.0"
+addin_version = "3.4.0"
 addin_displayname = "Financial Market Extension"
 addin_publisher_link = "https://github.com/cmallwitz/Financials-Extension"
 addin_publisher_name = "The Publisher"

src/test_coinbase.py

@@ -25,6 +25,11 @@ financials = financials.createInstance(None)
 
 class Test(unittest.TestCase):
 
+    @classmethod
+    def tearDownClass(cls):
+        # this avoids "ResourceWarning: unclosed..." on cached socket connections
+        financials.close()
+
     def test_currency(self):
         s = financials.getRealtime('ETH-EUR', Datacode.LAST_PRICE.value, 'COINBASE')
         self.assertEqual(float, type(s), 'test_currency LAST_PRICE')

src/test_ft.py

@@ -23,6 +23,11 @@ financials = financials.createInstance(None)
 
 class Test(unittest.TestCase):
 
+    @classmethod
+    def tearDownClass(cls):
+        # this avoids "ResourceWarning: unclosed..." on cached socket connections
+        financials.close()
+
     def test_currency(self):
         s = financials.getRealtime('EURGBP', 'LAST_PRICE', 'FT')
         self.assertEqual(float, type(s), 'test_currency LAST_PRICE')
@@ -137,29 +142,31 @@ class Test(unittest.TestCase):
 
     def test_US_futures(self):
 
-        s = financials.getRealtime('ESH3:IOM', Datacode.NAME.value, 'FT')
-        self.assertEqual(str, type(s), 'test_realtime_US_futures NAME {}'.format(s))
-        self.assertEqual('EMINI S&P MAR3', s, 'test_US_futures NAME {}'.format(s))
+        # https://markets.ft.com/data/commodities/tearsheet/summary?s=775326843 ESH25:IOM
 
-        s = financials.getRealtime('ESH3:IOM', Datacode.LAST_PRICE.value, 'FT')
+        s = financials.getRealtime('775326843', Datacode.NAME.value, 'FT')
+        self.assertEqual(str, type(s), 'test_realtime_US_futures NAME {}'.format(s))
+        self.assertEqual('EMINI S&P MAR5', s, 'test_US_futures NAME {}'.format(s))
+
+        s = financials.getRealtime('775326843', Datacode.LAST_PRICE.value, 'FT')
         self.assertEqual(float, type(s), 'test_US_futures LAST_PRICE {}'.format(s))
 
-        # s = financials.getRealtime('ESH3:IOM', Datacode.OPEN.value, 'FT')
+        # s = financials.getRealtime('775326843', Datacode.OPEN.value, 'FT')
         # self.assertEqual(float, type(s), 'test_US_futures OPEN {}'.format(s))
 
-        s = financials.getRealtime('ESH3:IOM', Datacode.VOLUME.value, 'FT')
+        s = financials.getRealtime('775326843', Datacode.VOLUME.value, 'FT')
         self.assertEqual(float, type(s), 'test_US_futures VOLUME {}'.format(s))
 
-        s = financials.getRealtime('ESH3:IOM', Datacode.LOW_52_WEEK.value, 'FT')
+        s = financials.getRealtime('775326843', Datacode.LOW_52_WEEK.value, 'FT')
         self.assertEqual(float, type(s), 'test_US_futures LOW_52_WEEK {}'.format(s))
 
-        s = financials.getRealtime('ESH3:IOM', Datacode.HIGH_52_WEEK.value, 'FT')
+        s = financials.getRealtime('775326843', Datacode.HIGH_52_WEEK.value, 'FT')
         self.assertEqual(float, type(s), 'test_US_futures HIGH_52_WEEK {}'.format(s))
 
-        s = financials.getRealtime('ESH3:IOM', Datacode.CHANGE.value, 'FT')
+        s = financials.getRealtime('775326843', Datacode.CHANGE.value, 'FT')
         self.assertEqual(float, type(s), 'test_US_futures CHANGE {}'.format(s))
 
-        s = financials.getRealtime('ESH3:IOM', Datacode.CHANGE_IN_PERCENT.value, 'FT')
+        s = financials.getRealtime('775326843', Datacode.CHANGE_IN_PERCENT.value, 'FT')
         self.assertEqual(float, type(s), 'test_US_futures CHANGE_IN_PERCENT {}'.format(s))
 
     def test_UK_ETF(self):
@@ -167,9 +174,6 @@ class Test(unittest.TestCase):
         self.assertEqual(str, type(s), 'test_UK_ETF NAME {}'.format(s))
         self.assertEqual('iShares Core S&P 500 UCITS ETF USD (Acc)', s, 'test_UK_ETF NAME {}'.format(s))
 
-        s = financials.getRealtime('C060:GER:EUR', 'NAME', 'FT')
-        self.assertEqual(str, type(s), 'test_UK_ETF NAME {}'.format(s))
-
         s = financials.getRealtime('VERX:LSE:GBP', 'LAST_PRICE', 'FT')
         self.assertEqual(float, type(s), 'test_UK_ETF LAST_PRICE {}'.format(s))
 

src/test_yahoo.py

@@ -25,11 +25,16 @@ financials = financials.createInstance(None)
 
 
 def urlopen_fail(self, url, redirect=True, data=None, headers={}, cookies=[], **kwargs):
-    raise baseclient.HttpException(url, 'simulated urlopen() failed')
+    raise baseclient.HttpException(url, 'ERROR: simulated urlopen() failed')
 
 
 class Test(unittest.TestCase):
 
+    @classmethod
+    def tearDownClass(cls):
+        # this avoids "ResourceWarning: unclosed..." on cached socket connections
+        financials.close()
+
     def test_recovery_from_urlopen_error_issue(self):
 
         financials.yahoo.last_url = 'blank'
@@ -103,16 +108,16 @@ class Test(unittest.TestCase):
         self.assertEqual(s, 'International Business Machines Corporation (IBM)',
                          'test_realtime_US_equity NAME {}'.format(s))
 
-        s = financials.getRealtime('IBM', Datacode.SECTOR.value, 'YAHOO')
+        s = financials.getRealtime('IBM', 'SECTOR', 'YAHOO')
         self.assertEqual(str, type(s), 'test_realtime_US_equity SECTOR {}'.format(s))
         self.assertEqual(s, 'Technology', 'test_realtime_US_equity SECTOR {}'.format(s))
 
-        s = financials.getRealtime('IBM', Datacode.INDUSTRY.value, 'YAHOO')
+        s = financials.getRealtime('IBM', 'INDUSTRY', 'YAHOO')
         self.assertEqual(str, type(s), 'test_realtime_US_equity INDUSTRY {}'.format(s))
         self.assertEqual(s, 'Information Technology Services', 'test_realtime_US_equity INDUSTRY {}'.format(s))
 
         s = financials.getRealtime('IBM', Datacode.TIMEZONE.value, 'YAHOO')
-        self.assertEqual(s, 'EST', 'test_realtime_US_equity TIMEZONE {}'.format(s))
+        self.assertTrue(s == 'EST' or s == 'EDT', 'test_realtime_US_equity TIMEZONE: {}'.format(s))
 
         s = financials.getRealtime('IBM', Datacode.BETA.value, 'YAHOO')
         self.assertTrue(testutils.is_positive_float(s), 'test_realtime_US_equity BETA {}'.format(s))
@@ -133,13 +138,13 @@ class Test(unittest.TestCase):
         self.assertEqual(str, type(s), 'test_realtime_US_equity EX_DIV_DATE {}'.format(s))
         self.assertTrue(testutils.is_date(s), 'test_realtime_US_equity EX_DIV_DATE {}'.format(s))
 
-        s = financials.getRealtime('IBM', Datacode.PAYOUT_RATIO.value, 'YAHOO')
+        s = financials.getRealtime('IBM', 'PAYOUT_RATIO', 'YAHOO')
         self.assertTrue(testutils.is_positive_float(s), 'test_realtime_US_equity PAYOUT_RATIO {}'.format(s))
 
-        s = financials.getRealtime('IBM', Datacode.SHARES_OUT.value, 'YAHOO')
+        s = financials.getRealtime('IBM', 'SHARES_OUT', 'YAHOO')
         self.assertTrue(testutils.is_positive_float(s), 'test_realtime_US_equity SHARES_OUT {}'.format(s))
 
-        s = financials.getRealtime('IBM', Datacode.FREE_FLOAT.value, 'YAHOO')
+        s = financials.getRealtime('IBM', 'FREE_FLOAT', 'YAHOO')
         self.assertTrue(testutils.is_positive_float(s), 'test_realtime_US_equity FREE_FLOAT {}'.format(s))
 
         s = financials.getRealtime('IBM', Datacode.EXCHANGE.value, 'YAHOO')
@@ -171,59 +176,60 @@ class Test(unittest.TestCase):
         # self.assertIsNone(s, 'test_realtime_US_mutuals DIV {}'.format(s)) # no dividend
 
         s = financials.getRealtime('VERX.L', Datacode.DIV_YIELD.value, 'YAHOO')
-        self.assertEqual(float, type(s), 'test_realtime_US_mutuals DIV_YIELD {}'.format(s))
+        self.assertIsNone(s, 'test_realtime_US_mutuals DIV_YIELD {}'.format(s)) # no yield
+        # self.assertEqual(float, type(s), 'test_realtime_US_mutuals DIV_YIELD {}'.format(s))
 
     def test_realtime_US_options(self):
 
         # symbol from https://finance.yahoo.com/quote/IBM/options?p=IBM
         
-        s = financials.getRealtime('IBM240119C00120000', Datacode.PREV_CLOSE.value, 'YAHOO')
+        s = financials.getRealtime('IBM250117C00165000', Datacode.PREV_CLOSE.value, 'YAHOO')
         self.assertEqual(float, type(s), 'test_realtime_US_options PREV_CLOSE {}'.format(s))
 
-        s = financials.getRealtime('IBM240119C00120000', Datacode.NAME.value, 'YAHOO')
+        s = financials.getRealtime('IBM250117C00165000', Datacode.NAME.value, 'YAHOO')
         self.assertEqual(str, type(s), 'test_realtime_US_options NAME {}'.format(s))
-        self.assertEqual('IBM Jan 2024 120.000 call', s, 'test_realtime_US_options NAME {}'.format(s))
+        self.assertEqual('IBM Jan 2025 165.000 call', s, 'test_realtime_US_options NAME {}'.format(s))
 
-        s = financials.getRealtime('IBM240119C00120000', Datacode.EXPIRY_DATE.value, 'YAHOO')
+        s = financials.getRealtime('IBM250117C00165000', Datacode.EXPIRY_DATE.value, 'YAHOO')
         self.assertEqual(str, type(s), 'test_realtime_US_options EXPIRY_DATE {}'.format(s))
         self.assertTrue(testutils.is_date(s), 'test_realtime_US_options EXPIRY_DATE {}'.format(s))
-        self.assertEqual("2024-01-19", s, 'test_realtime_US_options EXPIRY_DATE {}'.format(s))
+        self.assertEqual("2025-01-17", s, 'test_realtime_US_options EXPIRY_DATE {}'.format(s))
 
-        s = financials.getRealtime('IBM240119C00120000', Datacode.LAST_PRICE.value, 'YAHOO')
+        s = financials.getRealtime('IBM250117C00165000', Datacode.LAST_PRICE.value, 'YAHOO')
         self.assertEqual(float, type(s), 'test_realtime_US_options LAST_PRICE {}'.format(s))
 
-        s = financials.getRealtime('IBM240119C00120000', Datacode.OPEN.value, 'YAHOO')
+        s = financials.getRealtime('IBM250117C00165000', Datacode.OPEN.value, 'YAHOO')
         self.assertEqual(float, type(s), 'test_realtime_US_options OPEN {}'.format(s))
 
-        s = financials.getRealtime('IBM240119C00120000', Datacode.VOLUME.value, 'YAHOO')
+        s = financials.getRealtime('IBM250117C00165000', Datacode.VOLUME.value, 'YAHOO')
         self.assertEqual(float, type(s), 'test_realtime_US_options VOLUME {}'.format(s))
 
-        s = financials.getRealtime('IBM240119C00120000', Datacode.BID.value, 'YAHOO')
+        s = financials.getRealtime('IBM250117C00165000', Datacode.BID.value, 'YAHOO')
         self.assertEqual(float, type(s), 'test_realtime_US_options BID {}'.format(s))
 
-        s = financials.getRealtime('IBM240119C00120000', Datacode.ASK.value, 'YAHOO')
+        s = financials.getRealtime('IBM250117C00165000', Datacode.ASK.value, 'YAHOO')
         self.assertEqual(float, type(s), 'test_realtime_US_options ASK {}'.format(s))
 
-        s = financials.getRealtime('IBM240119C00120000', Datacode.PAYOUT_RATIO.value, 'YAHOO')
+        s = financials.getRealtime('IBM250117C00165000', Datacode.PAYOUT_RATIO.value, 'YAHOO')
         self.assertIsNone(s, 'test_realtime_US_options PAYOUT_RATIO {}'.format(s))
 
-        s = financials.getRealtime('IBM230120C00130000', Datacode.SECTOR.value, 'YAHOO')
+        s = financials.getRealtime('IBM250117C00165000', Datacode.SECTOR.value, 'YAHOO')
         self.assertIsNone(s, 'test_realtime_US_options SECTOR {}'.format(s))
 
     def test_realtime_US_futures(self):
 
         s = financials.getRealtime('ES=F', Datacode.NAME.value, 'YAHOO')
         self.assertEqual(str, type(s), 'test_realtime_US_futures NAME {}'.format(s))
-        self.assertEqual('E-Mini S&P 500 Mar 23 (ES=F)', s, 'test_realtime_US_futures NAME {}'.format(s))
+        self.assertEqual('E-Mini S&P 500 Mar 24 (ES=F)', s, 'test_realtime_US_futures NAME {}'.format(s))
 
         s = financials.getRealtime('ES=F', Datacode.TICKER.value, 'YAHOO')
         self.assertEqual(str, type(s), 'test_realtime_US_futures TICKER {}'.format(s))
-        self.assertEqual('ESH23.CME', s, 'test_realtime_US_futures TICKER {}'.format(s))
+        self.assertEqual('ESH24.CME', s, 'test_realtime_US_futures TICKER {}'.format(s))
 
         s = financials.getRealtime('ES=F', Datacode.SETTLEMENT_DATE.value, 'YAHOO')
         self.assertEqual(str, type(s), 'test_realtime_US_futures SETTLEMENT_DATE {}'.format(s))
         self.assertTrue(testutils.is_date(s), 'test_realtime_US_futures SETTLEMENT_DATE {}'.format(s))
-        self.assertEqual("2023-03-17", s, 'test_realtime_US_futures SETTLEMENT_DATE {}'.format(s))
+        self.assertEqual("2024-03-15", s, 'test_realtime_US_futures SETTLEMENT_DATE {}'.format(s))
 
         s = financials.getRealtime('ES=F', Datacode.LAST_PRICE.value, 'YAHOO')
         self.assertEqual(float, type(s), 'test_realtime_US_futures LAST_PRICE {}'.format(s))
@@ -258,11 +264,23 @@ class Test(unittest.TestCase):
         self.assertEqual(float, type(s), 'test_realtime_UK_ETF LAST_PRICE {}'.format(s))
 
         s = financials.getRealtime('VERX.L', Datacode.TIMEZONE.value, 'YAHOO')
-        self.assertEqual('GMT', s, 'test_realtime_UK_ETF TIMEZONE {}'.format(s))
+        self.assertTrue(s == 'GMT' or s == 'BST', 'test_realtime_UK_ETF TIMEZONE: {}'.format(s))
 
         s = financials.getRealtime('CSP1.L', Datacode.NAME.value, 'YAHOO')
         self.assertEqual(str, type(s), 'test_realtime_UK_ETF NAME {}'.format(s))
-        self.assertEqual('iShares Core S&P 500 UCITS ETF USD (Acc) (CSP1.L)', s, 'test_realtime_UK_ETF NAME {}'.format(s))
+        self.assertEqual('iShares VII PLC - iShares Core S&P 500 UCITS ETF (CSP1.L)', s, 'test_realtime_UK_ETF NAME {}'.format(s))
+
+        s = financials.getRealtime('VERX.L', 'SECTOR', 'YAHOO')
+        self.assertIsNone(s, 'test_realtime_UK_ETF SECTOR {}'.format(s))
+
+        s = financials.getRealtime('VERX.L', 'INDUSTRY', 'YAHOO')
+        self.assertIsNone(s, 'test_realtime_UK_ETF INDUSTRY {}'.format(s))
+
+        s = financials.getRealtime('VERX.L', 'PAYOUT_RATIO', 'YAHOO')
+        self.assertIsNone(s, 'test_realtime_UK_ETF PAYOUT_RATIO {}'.format(s))
+
+        s = financials.getRealtime('VERX.L', 'SHARES_OUT', 'YAHOO')
+        self.assertIsNone(s, 'test_realtime_UK_ETF SHARES_OUT {}'.format(s))
 
     def test_realtime_DE_equity(self):
 
@@ -270,7 +288,7 @@ class Test(unittest.TestCase):
         self.assertEqual(float, type(s), 'test_realtime_DE_equity LAST_PRICE {}'.format(s))
 
         s = financials.getRealtime('SAP.DE', Datacode.TIMEZONE.value, 'YAHOO')
-        self.assertEqual('CET', s, 'test_realtime_DE_equity TIMEZONE {}'.format(s))
+        self.assertTrue(s == 'CET' or s == 'CEST', 'test_realtime_DE_equity TIMEZONE: {}'.format(s))
 
         s = financials.getRealtime('SAP.DE', Datacode.SECTOR.value, 'YAHOO')
         self.assertEqual(str, type(s), 'test_realtime_DE_equity SECTOR {}'.format(s))
@@ -278,12 +296,12 @@ class Test(unittest.TestCase):
 
         s = financials.getRealtime('SAP.DE', Datacode.INDUSTRY.value, 'YAHOO')
         self.assertEqual(str, type(s), 'test_realtime_DE_equity INDUSTRY {}'.format(s))
-        self.assertEqual(s, 'Software—Application', 'test_realtime_DE_equity INDUSTRY {}'.format(s))
+        self.assertEqual(s, 'Software - Application', 'test_realtime_DE_equity INDUSTRY {}'.format(s))
 
-        s = financials.getRealtime('C060.DE', Datacode.NAME.value, 'YAHOO')
+        s = financials.getRealtime('LYY8.DE', Datacode.NAME.value, 'YAHOO')
         self.assertEqual(str, type(s), 'test_realtime_DE_equity NAME {}'.format(s))
 
-        s = financials.getRealtime('C060.DE', Datacode.EXCHANGE.value, 'YAHOO')
+        s = financials.getRealtime('LYY8.DE', Datacode.EXCHANGE.value, 'YAHOO')
         self.assertEqual(s, 'XETRA', 'test_realtime_DE_equity EXCHANGE')
 
     def test_TA_equity(self):
@@ -321,7 +339,7 @@ class Test(unittest.TestCase):
 
         s = financials.getRealtime('LUMI.TA', 'INDUSTRY', 'YAHOO')
         self.assertEqual(str, type(s), 'test_TLV_equity INDUSTRY {}'.format(s))
-        self.assertEqual('Banks—Regional', s, 'test_TA_equity INDUSTRY {}'.format(s))
+        self.assertEqual('Banks - Regional', s, 'test_TA_equity INDUSTRY {}'.format(s))
 
     def test_DK_equity(self):
         s = financials.getRealtime('NOVO-B.CO', 'last_price', 'YAHOO')
@@ -455,8 +473,8 @@ class Test(unittest.TestCase):
         s = financials.getHistoric('SAP.DE', Datacode.CLOSE.value, '2017-01-03', 'YAHOO')
         self.assertEqual(s, 82.889999, 'test_historic_DE_equity CLOSE {}'.format(s))
 
-        s = financials.getHistoric('C060.DE', Datacode.CLOSE.value, '2017-01-03', 'YAHOO')
-        self.assertEqual(s, 72.870003, 'test_historic_DE_equity CLOSE {}'.format(s))
+        s = financials.getHistoric('LYY8.DE', Datacode.CLOSE.value, '2017-01-03', 'YAHOO')
+        self.assertEqual(s, 96.010002, 'test_historic_DE_equity CLOSE {}'.format(s))
 
     def test_realtime_errors(self):
 
@@ -469,7 +487,7 @@ class Test(unittest.TestCase):
         s = financials.getRealtime('NO_NAME', Datacode.SECTOR.value, 'YAHOO')
         self.assertIsNone(s, 'test_realtime_errors SECTOR {}'.format(s))
 
-        s = financials.getRealtime('C060.DE', -1, 'YAHOO')
+        s = financials.getRealtime('LYY8.DE', -1, 'YAHOO')
         self.assertEqual('Datacode -1 not supported', s, 'test_realtime_errors -1 {}'.format(s))
 
     def test_historic_errors(self):