Added new source FT

README.md

@@ -1,6 +1,7 @@
 # Financials-Extension
 
-Extension for LibreOffice Calc to make stock, index and FX (Yahoo only) market data available in a Calc spread sheet - currently supports Yahoo and Google using web scraping.
+Extension for LibreOffice Calc to make stock market, index and FX data available in a Calc spread 
+sheet - currently supports Yahoo's, Financial Times' and Google's (without FX data) finance web sites using web scraping.
 
 ### System dependencies:
 
@@ -14,10 +15,28 @@ The following steps have worked for me on a fresh Debian installation (as root)
 
 ### Usage:
 
-Have a look at the 'releases' tab above: download a prebuild **Financials-Extension.oxt** file and load it into Calc under menu item: Tools, Extension Manager...
+Have a look at the 'releases' tab above: download a prebuild **Financials-Extension.oxt** file and load it into Calc 
+under menu item: Tools, Extension Manager...
 
 There is a file **examples.ods** there too with usage examples and possible arguments to functions.
 
+You have to check the respective web sites to work out what symbol is the right one for you. If a web site doesn't have 
+the symbol/asset you want, this extension can't help you either. Having said that, I mostly look at US and West European 
+equities, ETFs and mutual funds and major FX rates - if you have issues with the data available for other asset or 
+assets in other regions, drop me a line (best to include full URLs and possibly the same asset listed on more than one 
+site for comparison). While data for last price is most likely consistent across sites, they will differ when it comes to 
+less well defined data points.     
+
+### Dealing with missing data:
+
+First of all, a hint for using LibreCalc: if you want to refresh data you can press SHIFT-CTRL-F9 - this will force a 
+recalculation of all formulas in all sheets. 
+
+Secondly the extension saves some debug information under your user directory in a directory ".financials-extension": 
+the HTML for each stock symbol is saved in a separate file (depending on source and ticker symbol). You can open it 
+your favorite web browser (or other tools) to check if the page actually contained the information you are looking for. 
+If it does the file trace.log has a record of all calls to the extension and the value returned to LibreOffice.  
+
 ### Build:
 
 You will need the LibreOffice SDK installed. 

compile.sh

@@ -29,7 +29,8 @@ cp -f "${PWD}"/src/baseclient.py "${PWD}"/build/
 cp -f "${PWD}"/src/jsonParser.py "${PWD}"/build/
 cp -f "${PWD}"/src/naivehtmlparser.py "${PWD}"/build/
 cp -f "${PWD}"/src/tz.py "${PWD}"/build/
-cp -f "${PWD}"/src/google2.py "${PWD}"/build/
+cp -f "${PWD}"/src/ft.py "${PWD}"/build/
+cp -f "${PWD}"/src/google.py "${PWD}"/build/
 cp -f "${PWD}"/src/yahoo.py "${PWD}"/build/
 
 echo "Package into oxt file..."

src/baseclient.py

@@ -11,6 +11,8 @@
 import codecs
 import gzip
 import logging
+import os
+import pathlib
 import random
 import select
 
@@ -40,31 +42,26 @@ class BaseClient:
     def __init__(self):
         self.connections = {}
         self.cookies = cookiejar.CookieJar()
+        self.last_url = None
+
+        self.basedir = os.path.join(str(pathlib.Path.home()), '.financials-extension')
+        os.makedirs(self.basedir, exist_ok=True)
 
         user_agents = [
-            'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:55.0) Gecko/20100101 Firefox/55.0'
-            'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:56.0) Gecko/20100101 Firefox/56.0',
-            'Mozilla/5.0 (X11; Linux x86_64; rv:57.0) Gecko/20100101 Firefox/57.0',
+            'Mozilla/5.0 (Windows NT 6.1; rv:73.0) Gecko/20100101 Firefox/73.0',
+            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:74.0) Gecko/20100101 Firefox/74.0',
+            'Mozilla/5.0 (Windows NT 6.3; Win64; x64; rv:74.0) Gecko/20100101 Firefox/74.0'
 
-            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0',
-            'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.13; rv:59.0) Gecko/20100101 Firefox/59.0',
-            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.13; rv:59.0) Gecko/20100101 Firefox/59.0',
-            'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0) Gecko/20100101 Firefox/59.0'
-
-            'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.36',
-            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36',
-            'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36',
-
-            'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.162 Safari/537.36',
-            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.146 Safari/537.36',
-            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.146 Safari/537.36'
+            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36',
+            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36',
+            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36'
         ]
 
         self.default_headers = {
             'User-Agent': random.sample(user_agents, 1)[0],
-            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
+            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
             'Accept-Encoding': 'gzip, deflate',
-            'Accept-Language': 'en-GB,en-US;q=0.9,en;q=0.8'
+            'Accept-Language': 'en-US,en;q=0.5'
         }
 
     def request(self, method: str, url: str, data=None, headers={}, cookies=[], **kwargs):
@@ -96,6 +93,8 @@ class BaseClient:
         logger.debug('Creating request -----------------------------------------------------')
         logger.info('url=%s', url)
 
+        self.last_url = url
+
         # generate and add cookie headers
         request = urllib.request.Request(url)
 
@@ -180,6 +179,9 @@ class BaseClient:
         :return: value or None
         """
 
+        if data is None:
+            return None
+
         try:
             if datacode == Datacode.PREV_CLOSE.value and Datacode.PREV_CLOSE in data:
                 return data[Datacode.PREV_CLOSE]
@@ -191,10 +193,16 @@ class BaseClient:
                 return data[Datacode.CHANGE]
 
             elif datacode == Datacode.LAST_PRICE_DATE.value and Datacode.LAST_PRICE_DATE in data:
-                return data[Datacode.LAST_PRICE_DATE].isoformat()
+                if data[Datacode.LAST_PRICE_DATE]:
+                    return data[Datacode.LAST_PRICE_DATE].isoformat()
+                else:
+                    return data[Datacode.LAST_PRICE_DATE]
 
             elif datacode == Datacode.LAST_PRICE_TIME.value and Datacode.LAST_PRICE_TIME in data:
-                return data[Datacode.LAST_PRICE_TIME].isoformat()
+                if data[Datacode.LAST_PRICE_TIME]:
+                    return data[Datacode.LAST_PRICE_TIME].isoformat()
+                else:
+                    return data[Datacode.LAST_PRICE_TIME]
 
             elif datacode == Datacode.CHANGE_IN_PERCENT.value and Datacode.CHANGE_IN_PERCENT in data:
                 return data[Datacode.CHANGE_IN_PERCENT]
@@ -214,7 +222,7 @@ class BaseClient:
             elif datacode == Datacode.HIGH_52_WEEK.value and Datacode.HIGH_52_WEEK in data:
                 return data[Datacode.HIGH_52_WEEK]
 
-            elif datacode == Datacode.MARKET_CAP.value and Datacode.MARKET_CAP in data and data[Datacode.MARKET_CAP]:
+            elif datacode == Datacode.MARKET_CAP.value and Datacode.MARKET_CAP in data:
                 return data[Datacode.MARKET_CAP]
 
             elif datacode == Datacode.VOLUME.value and Datacode.VOLUME in data:
@@ -232,17 +240,20 @@ class BaseClient:
             elif datacode == Datacode.TICKER.value and Datacode.TICKER in data:
                 return data[Datacode.TICKER]
 
-            elif datacode == Datacode.EXCHANGE.value and data[Datacode.EXCHANGE]:
+            elif datacode == Datacode.EXCHANGE.value and Datacode.EXCHANGE in data:
                 return data[Datacode.EXCHANGE]
 
             elif datacode == Datacode.CURRENCY.value and Datacode.CURRENCY in data:
                 return data[Datacode.CURRENCY]
 
-            elif datacode == Datacode.NAME.value and data[Datacode.NAME]:
+            elif datacode == Datacode.NAME.value and Datacode.NAME in data:
                 return data[Datacode.NAME]
 
-            elif datacode == Datacode.TIMEZONE.value and Datacode.TIMEZONE in data and data[Datacode.TIMEZONE]:
-                return str(data[Datacode.TIMEZONE])
+            elif datacode == Datacode.TIMEZONE.value and Datacode.TIMEZONE in data:
+                if data[Datacode.TIMEZONE] is not None and type(data[Datacode.TIMEZONE]) != str:
+                    return str(data[Datacode.TIMEZONE])
+                else:
+                    return data[Datacode.TIMEZONE]
 
         except BaseException as e:
             return 'BaseClient.return_value(\'{}\', {}) - {}'.format(data, datacode, e)

src/datacode.py

@@ -42,8 +42,6 @@ class Datacode(Enum):
 
     TIMESTAMP = 999
 
-    # TODO YAHOO fundInceptionDate
-
     @classmethod
     def has_value(cls, value):
         return any(value == item.value for item in cls)

src/financials.py

@@ -15,9 +15,10 @@ import sys
 
 import pathlib
 import platform
+import time
+from functools import wraps
 
 import unohelper
-
 from com.financials.getinfo import Financials
 
 # Add current directory to import path
@@ -26,12 +27,34 @@ if current_dir not in sys.path:
     sys.path.insert(0, current_dir)
 
 from datacode import Datacode
-import google2 as google
+import google
 import yahoo
+import ft
+from version import version
 
 implementation_name = "com.financials.getinfo.python.FinancialsImpl"  # as defined in Financials.xcu
 implementation_services = ("com.sun.star.sheet.AddIn",)
 
+basedir = os.path.join(str(pathlib.Path.home()), '.financials-extension')
+os.makedirs(basedir, exist_ok=True)
+
+
+def profile(fn):
+    @wraps(fn)
+    def with_profiling(*args, **kwargs):
+        start = time.perf_counter()
+        r = fn(*args, **kwargs)
+        elapsed = time.perf_counter() - start
+
+        with open(os.path.join(basedir, 'trace.log'), "a+") as text_file:
+            print(
+                f"{datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')} {fn.__name__} *args={args[1:]} r='{r}' {(1000 * elapsed):.3f} ms",
+                file=text_file)
+
+        return r
+
+    return with_profiling
+
 
 class FinancialsImpl(unohelper.Base, Financials):
     """Define the main class for the Financials extension """
@@ -40,7 +63,9 @@ class FinancialsImpl(unohelper.Base, Financials):
         self.ctx = ctx
         self.google = google.createInstance(ctx)
         self.yahoo = yahoo.createInstance(ctx)
+        self.ft = ft.createInstance(ctx)
 
+    @profile
     def getRealtime(self, ticker, datacode=None, source=None):
 
         if ticker == 'SUPPORT':
@@ -77,6 +102,8 @@ class FinancialsImpl(unohelper.Base, Financials):
                 s = self.google.getRealtime(ticker, datacode)
             elif source == 'YAHOO':
                 s = self.yahoo.getRealtime(ticker, datacode)
+            elif source == 'FT':
+                s = self.ft.getRealtime(ticker, datacode)
             else:
                 s = 'Source \'{}\' not supported'.format(source)
 
@@ -90,6 +117,7 @@ class FinancialsImpl(unohelper.Base, Financials):
 
         return x
 
+    @profile
     def getHistoric(self, ticker, datacode=None, date=None, source=None):
 
         if ticker == 'SUPPORT':
@@ -128,7 +156,7 @@ class FinancialsImpl(unohelper.Base, Financials):
             if type(date) == float or type(date) == int:
 
                 try:
-                    offset = int(date) # offset for 1899-12-30
+                    offset = int(date)  # offset for 1899-12-30
                     d = dateutil.parser.parse('1899-12-30') + datetime.timedelta(days=offset)
                     d = d.date().isoformat()
                 except:
@@ -163,16 +191,20 @@ class FinancialsImpl(unohelper.Base, Financials):
 
         return x
 
+    @profile
     def support(self, datacode):
 
-        s = 'ctx={}\nid(self)={}\npid={}\nuname={}\nsys.executable={}\nsys.version={}\nhome={}'.format(
+        s = 'ctx={}\nid(self)={}\nversion={}\nfile={}\ncwd={}\nhome={}\nuname={}\npid={}\nsys.executable={}\nsys.version={}'.format(
             self.ctx,
             id(self),
-            os.getpid(),
+            version,
+            os.path.realpath(__file__),
+            os.path.realpath(os.getcwd()),
+            str(pathlib.Path.home()),
             ' '.join(platform.uname()),
+            os.getpid(),
             sys.executable,
-            sys.version.replace("\n", " "),
-            str(pathlib.Path.home()))
+            sys.version.replace("\n", " "))
 
         if datacode:
             s = '{}\ntype(datacode)={}\nstr(datacode)={}'.format(
@@ -182,6 +214,7 @@ class FinancialsImpl(unohelper.Base, Financials):
 
         return s
 
+
 def createInstance(ctx):
     return FinancialsImpl(ctx)
 

src/ft.py

@@ -0,0 +1,271 @@
+#  ft.py
+#
+#  license: GNU LGPL
+#
+#  This library is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 3 of the License, or (at your option) any later version.
+
+import dateutil
+import html
+import logging
+import os
+import re
+import time
+import traceback
+
+import jsonParser
+from baseclient import BaseClient
+from datacode import Datacode
+from tz import whois_timezone_info
+
+logger = logging.getLogger(__name__)
+# logger.setLevel(logging.DEBUG)
+
+
+def handle_abbreviations(s):
+    s = str(s).strip().replace(',', '')
+    if s.endswith('k'):
+        return float(s[:-1]) * 1000
+    elif s.endswith('m'):
+        return float(s[:-1]) * 1000000
+    elif s.endswith('bn'):
+        return float(s[:-2]) * 1000000000
+    elif s.endswith('tn'):
+        return float(s[:-2]) * 1000000000000
+    return float(s)
+
+
+class FT(BaseClient):
+
+    def __init__(self, ctx):
+        super().__init__()
+
+        self.crumb = None
+        self.realtime = {}
+        self.historicdata = {}
+        self.js = jsonParser.jsonObject
+
+    def getRealtime(self, ticker: str, datacode: int):
+
+        """
+        Retrieve data for ticker from Alpha Vantage and cache it for further lookups
+
+        :param ticker: the ticker symbol e.g. VOD.LON
+        :param datacode: the requested datacode
+        :return:
+        """
+
+        # remove white space
+        ticker = "".join(ticker.split())
+
+        # use cached value for up to 60 seconds
+        if ticker in self.realtime:
+            tick = self.realtime[ticker]
+            if time.time() - 60 < tick[Datacode.TIMESTAMP]:
+                return self._return_value(tick, datacode)
+            else:
+                del self.realtime[ticker]
+
+        if ticker not in self.realtime:
+            self.realtime[ticker] = {}
+
+        tick = self.realtime[ticker]
+
+        asset_class = self.guess_asset_class(ticker)
+
+        url = f'https://markets.ft.com/data/{asset_class}/tearsheet/summary?s={ticker}'
+
+        try:
+            text = self.urlopen(url, redirect=True, data=None, headers=None)
+            with open(os.path.join(self.basedir, f'ft-{ticker}.html'), "w") as text_file:
+                print(f"<!-- '{self.last_url}' -->\r\n\r\n{text}", file=text_file)
+        except BaseException as e:
+            logger.error(traceback.format_exc())
+            return f'FT.getRealtime({ticker}, {datacode}) - urlopen endpoint: {str(e)}'
+
+        tick[Datacode.TIMESTAMP] = time.time()
+
+        tick[Datacode.NAME] = None
+        tick[Datacode.TICKER] = None
+        tick[Datacode.CURRENCY] = None
+        tick[Datacode.LAST_PRICE] = None
+        tick[Datacode.CHANGE] = None
+        tick[Datacode.CHANGE_IN_PERCENT] = None
+        tick[Datacode.VOLUME] = None
+        tick[Datacode.LOW_52_WEEK] = None
+        tick[Datacode.HIGH_52_WEEK] = None
+        tick[Datacode.LAST_PRICE_DATE] = None
+        tick[Datacode.LAST_PRICE_TIME] = None
+        tick[Datacode.TIMEZONE] = None
+
+        tick[Datacode.OPEN] = None
+        tick[Datacode.HIGH] = None
+        tick[Datacode.LOW] = None
+        tick[Datacode.PREV_CLOSE] = None
+        tick[Datacode.MARKET_CAP] = None
+
+        tick[Datacode.EXCHANGE] = None
+        tick[Datacode.AVG_DAILY_VOL_3MOMTH] = None
+
+        try:
+            r = '<div class="mod-tearsheet-overview__header"><h1 [^>]*>(.*?)</h1>'
+            match = re.compile(r, flags=re.DOTALL).search(text)
+            if not match:
+                return None
+            start = match.span(0)[1]
+
+            tick[Datacode.NAME] = self.save_wrapper(
+                lambda: html.unescape(match.group(1)).strip())
+
+            r = '<div class="mod-tearsheet-overview__header__symbol">(?:<div [^>]*>)?<span *[^>]*>(.*?)<'
+            match = re.compile(r, flags=re.DOTALL).search(text, start)
+            if match:
+                start = match.span(0)[1]
+                tick[Datacode.TICKER] = self.save_wrapper(
+                    lambda: html.unescape(match.group(1)).strip())
+
+            r = '<span [^>]*>Price \(([A-Z]+)\)</span><span [^>]*>([0-9,\.]+)</span>'
+            match = re.compile(r, flags=re.DOTALL).search(text, start)
+            if match:
+                start = match.span(0)[1]
+                tick[Datacode.CURRENCY] = self.save_wrapper(
+                    lambda: html.unescape(match.group(1)).strip())
+                tick[Datacode.LAST_PRICE] = self.save_wrapper(
+                    lambda: float(html.unescape(match.group(2)).replace(',', '').strip()))
+
+            r = '<span[^>]*>Today\'s Change</span><span[^>]*><span[^>]*>(?:<i[^>]*></i>)?([0-9,\.-]+) */ *([0-9,\.-]+)%</span>'
+            match = re.compile(r, flags=re.DOTALL).search(text, start)
+            if match:
+                start = match.span(0)[1]
+                tick[Datacode.CHANGE] = self.save_wrapper(
+                    lambda: float(html.unescape(match.group(1)).replace(',', '').strip()))
+                tick[Datacode.CHANGE_IN_PERCENT] = self.save_wrapper(
+                    lambda: float(html.unescape(match.group(2)).replace(',', '').strip()))
+
+            r = '<span[^>]*>Shares traded</span><span[^>]*>([0-9mk,\.]+)</span>'
+            match = re.compile(r, flags=re.DOTALL).search(text, start)
+            if match:
+                start = match.span(0)[1]
+                tick[Datacode.VOLUME] = self.save_wrapper(
+                    lambda: handle_abbreviations(html.unescape(match.group(1))))
+
+            r = '<span[^>]*>52 week range</span><span[^>]*>([0-9,\.]+) *- *([0-9,\.]+)</span>'
+            match = re.compile(r, flags=re.DOTALL).search(text, start)
+            if match:
+                start = match.span(0)[1]
+                tick[Datacode.LOW_52_WEEK] = self.save_wrapper(
+                    lambda: float(html.unescape(match.group(1)).replace(',', '').strip()))
+                tick[Datacode.HIGH_52_WEEK] = self.save_wrapper(
+                    lambda: float(html.unescape(match.group(2)).replace(',', '').strip()))
+
+            # <div class="mod-disclaimer">Data delayed at least 15 minutes, as of Apr 13 2020 15:01 BST.</div>
+            # <div class="mod-disclaimer">.+?as of (.+)\.?</div>
+
+            r = '<div class="mod-disclaimer">.+?as of (.+?)\.?</div>'
+            match = re.compile(r, flags=re.DOTALL).search(text, start)
+            if match:
+                start = match.span(0)[1]
+
+                try:
+                    value = html.unescape(match.group(1)).strip()
+                    dt = dateutil.parser.parse(value, tzinfos=whois_timezone_info)
+                    tick[Datacode.LAST_PRICE_DATE] = dt.date()
+                    tick[Datacode.LAST_PRICE_TIME] = dt.time()
+
+                    time_bits = value.split(' ')
+                    if len(time_bits) >= 4:
+                        tick[Datacode.TIMEZONE] = time_bits[-1]
+
+                except BaseException as e:
+                    pass
+
+            # second attempt at 52 week range
+            if Datacode.LOW_52_WEEK not in tick or not tick[Datacode.LOW_52_WEEK]:
+                r = '<span class="mod-ui-range-bar__container__label--lo"><span[^>]*>([0-9,\.]+)</span>'
+                match = re.compile(r, flags=re.DOTALL).search(text, start)
+                if match:
+                    tick[Datacode.LOW_52_WEEK] = self.save_wrapper(
+                        lambda: float(html.unescape(match.group(1)).replace(',', '').strip()))
+
+            if Datacode.HIGH_52_WEEK not in tick or not tick[Datacode.HIGH_52_WEEK]:
+                r = '<span class="mod-ui-range-bar__container__label--hi"><span[^>]*>([0-9,\.]+)</span>'
+                match = re.compile(r, flags=re.DOTALL).search(text, start)
+                if match:
+                    tick[Datacode.HIGH_52_WEEK] = self.save_wrapper(
+                        lambda: float(html.unescape(match.group(1)).replace(',', '').strip()))
+
+            # just moving forward to data table
+            r = '<div class="mod-tearsheet-key-stats__data__table">'
+            match = re.compile(r, flags=re.DOTALL).search(text, start)
+            if match:
+                start = match.span(0)[1]
+
+            r = '<th>Open</th><td>([0-9,\.]+)</td>'
+            match = re.compile(r, flags=re.DOTALL).search(text, start)
+            if match:
+                tick[Datacode.OPEN] = self.save_wrapper(
+                    lambda: float(html.unescape(match.group(1)).replace(',', '').strip()))
+
+            r = '<th>High</th><td>([0-9,\.]+)</td>'
+            match = re.compile(r, flags=re.DOTALL).search(text, start)
+            if match:
+                tick[Datacode.HIGH] = self.save_wrapper(
+                    lambda: float(html.unescape(match.group(1)).replace(',', '').strip()))
+
+            r = '<th>Low</th><td>([0-9,\.]+)</td>'
+            match = re.compile(r, flags=re.DOTALL).search(text, start)
+            if match:
+                tick[Datacode.LOW] = self.save_wrapper(
+                    lambda: float(html.unescape(match.group(1)).replace(',', '').strip()))
+
+            r = '<th>\s*Previous close\s*</th><td>\s*([0-9,\.]+)\s*</td>'
+            match = re.compile(r, flags=re.DOTALL).search(text, start)
+            if match:
+                tick[Datacode.PREV_CLOSE] = self.save_wrapper(
+                    lambda: float(html.unescape(match.group(1)).replace(',', '').strip()))
+
+            r = '<th>\s*Average volume\s*</th><td>\s*([0-9,\.btnm]+)\s*</td>'
+            match = re.compile(r, flags=re.DOTALL).search(text, start)
+            if match:
+                tick[Datacode.AVG_DAILY_VOL_3MOMTH] = self.save_wrapper(
+                    lambda: handle_abbreviations(html.unescape(match.group(1))))
+
+            r = '<th>\s*Market cap\s*</th><td>\s*([0-9,\.btnm]+)\s*<'
+            match = re.compile(r, flags=re.DOTALL).search(text, start)
+            if match:
+                tick[Datacode.MARKET_CAP] = self.save_wrapper(
+                    lambda: handle_abbreviations(html.unescape(match.group(1))))
+
+        except BaseException as e:
+            logger.error(traceback.format_exc())
+            return f'FT.getRealtime({ticker}, {datacode}) - process: {str(e)}'
+
+        logger.info(tick)
+
+        return self._return_value(self.realtime[ticker], datacode)
+
+    def getHistoric(self, ticker, datacode, date):
+        return 'FT.getHistoric: Historic Data not implemented.'
+
+    def guess_asset_class(self, ticker):
+
+        if len(ticker) == 6:
+            if ticker[0:2] in ['USD', 'EUR', 'GBP']:
+                return 'currencies'
+            if ticker[3:5] in ['USD', 'EUR', 'GBP']:
+                return 'currencies'
+
+        colon_count = ticker.count(':')
+
+        if colon_count == 0:
+            return 'funds'
+        elif colon_count == 3:
+            return 'etfs'
+
+        return 'equities'
+
+
+def createInstance(ctx):
+    return FT(ctx)

src/generate_metainfo.py

@@ -10,15 +10,23 @@
 #
 
 import os
+
 cur_dir = os.getcwd()
 
 addin_id = "com.financials.getinfo"
-addin_version = "1.0.5"
+addin_version = "2.0.0"
 addin_displayname = "Financial Market Extension"
 addin_publisher_link = "https://github.com/cmallwitz/Financials-Extension"
 addin_publisher_name = "The Publisher"
 
-print ("Generating extension files for version", addin_version)
+print("Generating extension files for version", addin_version)
+
+################################################################################
+# version.py
+
+version_file = open(cur_dir + '/build/version.py', 'w')
+version_file.write(f"version = '{addin_version}'")
+version_file.close()
 
 ################################################################################
 # description.xml
@@ -35,15 +43,17 @@ desc_xml.write('    <l:LibreOffice-minimal-version value="5.0" d:name="LibreOffi
 desc_xml.write('</dependencies> \n')
 desc_xml.write('\n')
 desc_xml.write('<identifier value="' + addin_id + '" /> \n')
-desc_xml.write('<version value="' + addin_version + '" />\n')   
+desc_xml.write('<version value="' + addin_version + '" />\n')
 desc_xml.write('<display-name><name lang="en">' + addin_displayname + '</name></display-name>\n')
-desc_xml.write('<publisher><name xlink:href="' + addin_publisher_link + '" lang="en">' + addin_publisher_name + '</name></publisher>\n')
+desc_xml.write(
+    '<publisher><name xlink:href="' + addin_publisher_link + '" lang="en">' + addin_publisher_name + '</name></publisher>\n')
 desc_xml.write('<extension-description><src xlink:href="description-en-US.txt" lang="en" /></extension-description>')
 desc_xml.write('\n \n')
 desc_xml.write('</description> \n')
 
 desc_xml.close()
 
+
 ################################################################################
 # manifest.xml
 # List of files in package and their types.
@@ -53,9 +63,10 @@ def add_manifest_entry(xml_file, file_type, file_name):
     xml_file.write('<manifest:file-entry manifest:media-type="application/vnd.sun.star.' + file_type + '" \n')
     xml_file.write('    manifest:full-path="' + file_name + '"/> \n')
 
+
 manifest_xml = open(cur_dir + '/build/META-INF/manifest.xml', 'w')
 
-manifest_xml.write('<manifest:manifest>\n');
+manifest_xml.write('<manifest:manifest>\n')
 add_manifest_entry(manifest_xml, 'uno-typelibrary;type=RDB', 'XFinancials.rdb')
 add_manifest_entry(manifest_xml, 'configuration-data', 'Financials.xcu')
 add_manifest_entry(manifest_xml, 'uno-component;type=Python', 'financials.py')
@@ -63,6 +74,7 @@ manifest_xml.write('</manifest:manifest> \n')
 
 manifest_xml.close()
 
+
 ################################################################################
 # Financials.xcu
 
@@ -71,12 +83,13 @@ def define_function(xml_file, function_name, description, parameters):
     xml_file.write('    <prop oor:name="DisplayName"><value xml:lang="en">' + function_name + '</value></prop>\n')
     xml_file.write('    <prop oor:name="Description"><value xml:lang="en">' + description + '</value></prop>\n')
     xml_file.write('    <prop oor:name="Category"><value>Add-In</value></prop>\n')
-    xml_file.write('    <prop oor:name="CompatibilityName"><value xml:lang="en">AutoAddIn.Financials.' + function_name + '</value></prop>\n')
+    xml_file.write(
+        '    <prop oor:name="CompatibilityName"><value xml:lang="en">AutoAddIn.Financials.' + function_name + '</value></prop>\n')
     xml_file.write('    <node oor:name="Parameters">\n')
 
     for p, desc in parameters:
-        # Optional parameters will have a displayname enclosed in square brackets.
-        p_name = p.strip("[]")        
+        # Optional parameters will have a display name enclosed in square brackets.
+        p_name = p.strip("[]")
         xml_file.write('      <node oor:name="' + p_name + '" oor:op="replace">\n')
         xml_file.write('        <prop oor:name="DisplayName"><value xml:lang="en">' + p_name + '</value></prop>\n')
         xml_file.write('        <prop oor:name="Description"><value xml:lang="en">' + desc + '</value></prop>\n')
@@ -85,6 +98,7 @@ def define_function(xml_file, function_name, description, parameters):
     xml_file.write('    </node>\n')
     xml_file.write('  </node>\n')
 
+
 # instance_id references the named UNO component instantiated by Python code (that is my understanding at least).
 implementation_name = "com.financials.getinfo.python.FinancialsImpl"
 
@@ -94,17 +108,19 @@ excel_addin_name = "Financials.xlam"
 financials_xml = open(cur_dir + '/build/Financials.xcu', 'w')
 
 financials_xml.write('<?xml version="1.0" encoding="UTF-8"?>\n')
-financials_xml.write('<oor:component-data xmlns:oor="http://openoffice.org/2001/registry" xmlns:xs="http://www.w3.org/2001/XMLSchema" oor:name="CalcAddIns" oor:package="org.openoffice.Office">\n')
+financials_xml.write(
+    '<oor:component-data xmlns:oor="http://openoffice.org/2001/registry" xmlns:xs="http://www.w3.org/2001/XMLSchema" oor:name="CalcAddIns" oor:package="org.openoffice.Office">\n')
 financials_xml.write('<node oor:name="AddInInfo">\n')
 financials_xml.write('<node oor:name="' + implementation_name + '" oor:op="replace">\n')
 financials_xml.write('<node oor:name="AddInFunctions">\n')
 
-define_function(financials_xml, \
-    'getRealtime', 'Fetches Realtime Financial Data.', \
+define_function(financials_xml,
+                'getRealtime', 'Fetches Realtime Financial Data.',
                 [('ticker', 'The ticker symbol.'), ('datacode', 'The data code.'), ('source', 'The source.')])
-define_function(financials_xml, \
-    'getHistoric', 'Fetches Historic Financial Data.', \
-                [('ticker', 'The ticker symbol.'), ('datacode', 'The data code.'), ('date', 'The date.'), ('source', 'The source.')])
+define_function(financials_xml,
+                'getHistoric', 'Fetches Historic Financial Data.',
+                [('ticker', 'The ticker symbol.'), ('datacode', 'The data code.'), ('date', 'The date.'),
+                 ('source', 'The source.')])
 
 financials_xml.write('</node>\n')
 financials_xml.write('</node>\n')

src/google.py

@@ -8,46 +8,54 @@
 #  version 3 of the License, or (at your option) any later version.
 
 
-import datetime
-import locale
-import logging
+import dateutil
 import html
+import logging
+import os
 import re
 import time
 import traceback
+import xml.etree.ElementTree as ET
 
-import urllib.parse
-
+from baseclient import BaseClient, RedirectException
 from datacode import Datacode
-from baseclient import BaseClient
+from naivehtmlparser import NaiveHTMLParser
+from tz import whois_timezone_info
 
 logger = logging.getLogger(__name__)
+
+
 # logger.setLevel(logging.DEBUG)
 
 
 def handle_abbreviations(s):
     s = str(s).strip()
-    if s.endswith('T'):
-        return float(s.replace('T', ''))*1000
     if s.endswith('M'):
-        return float(s.replace('M', ''))*1000000
-    if s.endswith('B'):
-        return float(s.replace('B', ''))*1000000000
+        return float(s[:-1]) * 1000000
+    elif s.endswith('B'):
+        return float(s[:-1]) * 1000000000
+    elif s.endswith('T'):
+        return float(s[:-1]) * 1000000000000
     return float(s)
 
 
+def un_span(s):
+    return re.sub(r'<span [^>]*>', '', s).replace('</span>', '')
+
+
 class Google(BaseClient):
     def __init__(self, ctx):
         super().__init__()
 
         self.realtime = {}
+        self.location = None
 
     def getRealtime(self, ticker: str, datacode: int):
 
         """
         Retrieve realtime data for ticker from Google Finance and cache it for further lookups
 
-        :param ticker: the ticker symbol e.g. VOD.L or LON:VOD
+        :param ticker: the ticker symbol e.g. LON:VOD
         :param datacode: the requested datacode
         :return:
         """
@@ -55,148 +63,191 @@ class Google(BaseClient):
         # remove white space
         ticker = "".join(ticker.split())
 
-        # use cached value for up to 5 minutes
+        # use cached value for up to 60 seconds
         if ticker in self.realtime:
             tick = self.realtime[ticker]
-            if time.time() - 5*60 < tick[Datacode.TIMESTAMP]:
+            if time.time() - 60 < tick[Datacode.TIMESTAMP]:
                 return self._return_value(tick, datacode)
             else:
                 del self.realtime[ticker]
 
-        url = 'https://finance.google.com/finance?{}'.format(urllib.parse.urlencode({'q': ticker}))
+        q_param = 'q=' + ticker
+
+        if not self.location:
+            url = 'https://www.google.com/search?hl=en&tbm=fin&' + q_param
+
+            try:
+                self.urlopen(url, redirect=False)
+            except RedirectException as e:
+                self.location = e.location.replace('&' + q_param, '')
+            except BaseException as e:
+                logger.error(traceback.format_exc())
+                return 'Google.getRealtime(\'{}\', {}) - location: {}'.format(ticker, datacode, e)
+
+        if not self.location:
+            url = 'https://www.google.com/search?tbm=fin&' + q_param
+        else:
+            url = self.location + '&' + q_param
 
         try:
             text = self.urlopen(url)
+            with open(os.path.join(self.basedir, 'google-{}.html'.format(ticker)), "w") as text_file:
+                print(f"<!-- '{url}' -->\r\n\r\n{text}", file=text_file)
         except BaseException as e:
             logger.error(traceback.format_exc())
-            return 'Google.getRealtime(\'{}\', {}) - read: {}'.format(ticker, datacode, e)
+            return 'Google.getRealtime(\'{}\', {}) - urlopen: {} {}'.format(ticker, datacode, e, url)
+
+        if ticker not in self.realtime:
+            self.realtime[ticker] = {}
+
+        tick = self.realtime[ticker]
+
+        tick[Datacode.TIMESTAMP] = time.time()
+
+        tick[Datacode.NAME] = None
+        tick[Datacode.TICKER] = None
+        tick[Datacode.CURRENCY] = None
+        tick[Datacode.LAST_PRICE] = None
+        tick[Datacode.CHANGE] = None
+        tick[Datacode.CHANGE_IN_PERCENT] = None
+        tick[Datacode.VOLUME] = None
+        tick[Datacode.LOW_52_WEEK] = None
+        tick[Datacode.HIGH_52_WEEK] = None
+        tick[Datacode.LAST_PRICE_DATE] = None
+        tick[Datacode.LAST_PRICE_TIME] = None
+        tick[Datacode.TIMEZONE] = None
+
+        tick[Datacode.OPEN] = None
+        tick[Datacode.HIGH] = None
+        tick[Datacode.LOW] = None
+        tick[Datacode.PREV_CLOSE] = None
+        tick[Datacode.MARKET_CAP] = None
+
+        tick[Datacode.EXCHANGE] = None
+        tick[Datacode.AVG_DAILY_VOL_3MOMTH] = None
 
         try:
-            r = '<meta\s*itemprop="([^"]+)"\s*content="([^"]+)"\s*/>'
+            r = '<span[^>]+role="heading"[^>]+>(.*?)</span>'
             pattern = re.compile(r)
-            result = pattern.findall(text)
 
-            if len(result) == 0:
+            match = pattern.search(text)
+            if not match:
                 return None
+            start = match.span(0)[1]
 
-            if ticker not in self.realtime:
-                self.realtime[ticker] = {}
+            tick[Datacode.NAME] = self.save_wrapper(
+                lambda: html.unescape(un_span(match.group(1)).strip()))
 
-            tick = self.realtime[ticker]
+            r = '<div [^>]*>(.*?)</div>'
+            pattern = re.compile(r)
 
-            for key, value in result:
-
-                if key == 'exchangeTimezone':
-                    tick[Datacode.TIMEZONE] = self.save_wrapper(lambda: str(value))
-
-                elif key == 'priceChange':
-                    tick[Datacode.CHANGE] = self.save_wrapper(lambda: float(value))
-
-                elif key == 'quoteTime':
-                    try:
-                        dt = datetime.datetime.strptime(value, "%Y-%m-%dT%H:%M:%SZ")
-                        tick[Datacode.LAST_PRICE_DATE] = dt.date()
-                        tick[Datacode.LAST_PRICE_TIME] = dt.time()
-                    except:
-                        pass
-
-                elif key == 'priceChangePercent':
-                    tick[Datacode.CHANGE_IN_PERCENT] = self.save_wrapper(lambda: (float(value)))
-
-                elif key == 'price':
-                    locale.setlocale(locale.LC_ALL, 'en_US.UTF-8')
-                    tick[Datacode.LAST_PRICE] = self.save_wrapper(lambda: locale.atof(str(value)))
-
-                elif key == 'priceCurrency':
-                    tick[Datacode.CURRENCY] = self.save_wrapper(lambda: str(value))
-
-                elif key == 'exchange':
-                    tick[Datacode.EXCHANGE] = self.save_wrapper(lambda: str(value))
-
-                elif key == 'name':
-                    tick[Datacode.NAME] = self.save_wrapper(lambda: html.unescape(str(value)))
-
-                elif key == 'tickerSymbol':
-                    tick[Datacode.TICKER] = self.save_wrapper(lambda: str(value))
-
-                else:
-                    logger.info('ignored key=%s value=%s', key, value)
-
-            start = 0
-
-            r = '<td[^>]+data-snapfield="range">[^<]+</td>\s*<td class="val">\s*([^<]+)\s*</td>'
-            pattern = re.compile(r, flags=re.DOTALL)
+            # first div is TICKER
             match = pattern.search(text, start)
+            if not match:
+                return 'Google.getRealtime({}, {}) - no match'.format(ticker, datacode)
+
+            ticker = self.save_wrapper(
+                lambda: html.unescape(match.group(1)).replace(' ', ''))
+
+            tick[Datacode.EXCHANGE] = self.save_wrapper(lambda: ticker.split(':')[0])
+            tick[Datacode.TICKER] = self.save_wrapper(lambda: ticker.split(':')[1])
+
+        except BaseException as e:
+            return 'Google.getRealtime({}, {}) - process: {}'.format(ticker, datacode, e)
+
+        try:
+            r = '<sticky-header [^>]*>(.*?)</sticky-header>'
+            pattern = re.compile(r, flags=re.DOTALL)
+            match = re.search(pattern, text)
 
             if match:
-                lowhigh = self.save_wrapper(
-                    lambda: list(map(
-                        lambda s: float(s),
-                        html.unescape(match.group(1))
-                            .replace('-', '').replace(',', '').strip().split())))
+                text = match.group(1)
+            else:
+                return 'Data for \'{}\' not found'.format(ticker)
 
-                if lowhigh and len(lowhigh) == 2:
-                    tick[Datacode.LOW] = lowhigh[0]
-                    tick[Datacode.HIGH] = lowhigh[1]
-                    start = match.span(0)[1]
+            parser = NaiveHTMLParser()
+            root = parser.feed(text)
+            parser.close()
 
-            r = '<td[^>]+data-snapfield="range_52week">[^<]+</td>\s*<td class="val">\s*([^<]+)\s*</td>'
-            pattern = re.compile(r, flags=re.DOTALL)
-            match = pattern.search(text, start)
+            cards = root.findall('.//g-card-section')
 
-            if match:
-                lowhigh = self.save_wrapper(
-                    lambda: list(map(
-                        lambda s: float(s),
-                        html.unescape(match.group(1))
-                            .replace('-', '').replace(',', '').strip().split())))
+            if len(cards) < 4:
+                return 'Data for \'{}\' not found'.format(ticker)
 
-                if lowhigh and len(lowhigh) == 2:
-                    tick[Datacode.LOW_52_WEEK] = lowhigh[0]
-                    tick[Datacode.HIGH_52_WEEK] = lowhigh[1]
-                    start = match.span(0)[1]
+            header = cards[1]
 
-            r = '<td[^>]+data-snapfield="open">[^<]+</td>\s*<td class="val">\s*([^<]+)\s*</td>'
-            pattern = re.compile(r, flags=re.DOTALL)
-            match = pattern.search(text, start)
+            tick[Datacode.LAST_PRICE] = self.save_wrapper(
+                lambda: float(
+                    html.unescape(header.find('./span[1]/span[1]/span[1]').text).replace(',', '').strip()))
 
-            if match:
-                tick[Datacode.OPEN] = self.save_wrapper(
-                    lambda: float(html.unescape(match.group(1)).replace(',', '').strip()))
-                start = match.span(0)[1]
+            tick[Datacode.CURRENCY] = self.save_wrapper(
+                lambda: html.unescape(header.find('./span[1]/span[1]/span[2]').text).strip())
 
-            r = '<td[^>]+data-snapfield="vol_and_avg">[^<]+</td>\s*<td class="val">\s*([^<]+)\s*</td>'
-            pattern = re.compile(r, flags=re.DOTALL)
-            match = pattern.search(text, start)
+            tick[Datacode.CHANGE] = self.save_wrapper(
+                lambda: float(
+                    html.unescape(header.find('./span[2]/span[1]').text).replace('−', '-').replace(',', '').strip()))
 
-            if match:
-                volavg = self.save_wrapper(
-                    lambda: list(map(
-                        lambda s: handle_abbreviations(s),
-                        html.unescape(match.group(1)).replace('/', ' ').strip().split())))
+            # percentage is always wrapped in (...) and always positive even if there is a price drop
+            tick[Datacode.CHANGE_IN_PERCENT] = self.save_wrapper(
+                lambda: float(
+                    html.unescape(header.find('./span[2]/span[2]/span[1]').text).strip()
+                        .replace('(', '').replace(')', '').replace('%', '')))
 
-                if volavg:
-                    if len(volavg) > 0:
-                        tick[Datacode.VOLUME] = volavg[0]
-                    start = match.span(0)[1]
+            try:
+                value = html.unescape(header.find('./div[1]/span[1]/span[2]').text).replace('·', '').strip()
+                logger.debug(value)
+                dt = dateutil.parser.parse(value, tzinfos=whois_timezone_info)
+                tick[Datacode.LAST_PRICE_DATE] = dt.date()
+                tick[Datacode.LAST_PRICE_TIME] = dt.time()
 
-            r = '<td[^>]+data-snapfield="market_cap">[^<]+</td>\s*<td class="val">\s*([^<]+)'
-            pattern = re.compile(r, flags=re.DOTALL)
-            match = pattern.search(text, start)
+                time_bits = value.split(' ')
+                if len(time_bits) >= 4:
+                    tick[Datacode.TIMEZONE] = time_bits[-1]
 
-            if match:
-                mcap = self.save_wrapper(
-                    lambda: handle_abbreviations(html.unescape(match.group(1)).replace('-', ' ').strip()))
+            except BaseException as e:
+                pass
 
-                if mcap:
-                    tick[Datacode.MARKET_CAP] = mcap
+            footer = cards[3]
+            logger.debug(ET.tostring(footer))
 
-                # start = match.span(0)[1]
+            # parse 'footer' for remaining fields
+            table = footer.find('./div[1]/div[1]/div[1]/table[1]')
 
-            tick[Datacode.TIMESTAMP] = time.time()
+            tick[Datacode.OPEN] = self.save_wrapper(
+                lambda: float(
+                    html.unescape(table.find('./tr[1]/td[2]').text).replace(',', '').strip()))
 
-            if tick[Datacode.EXCHANGE] == 'CURRENCY' and Datacode.CURRENCY not in tick:
-                tick[Datacode.CURRENCY] = ''
+            tick[Datacode.HIGH] = self.save_wrapper(
+                lambda: float(
+                    html.unescape(table.find('./tr[2]/td[2]').text).replace(',', '').strip()))
+
+            tick[Datacode.LOW] = self.save_wrapper(
+                lambda: float(
+                    html.unescape(table.find('./tr[3]/td[2]').text).replace(',', '').strip()))
+
+            tick[Datacode.MARKET_CAP] = self.save_wrapper(
+                lambda: handle_abbreviations(
+                    html.unescape(table.find('./tr[4]/td[2]').text).replace(',', '').replace('-', '').strip()))
+
+            table = footer.find('./div[1]/div[1]/div[2]/table[1]')
+
+            # for indices: first item on right side is LOW
+            if html.unescape(table.find('./tr[1]/td[1]').text).strip() == 'Low':
+                tick[Datacode.LOW] = self.save_wrapper(
+                    lambda: float(
+                        html.unescape(table.find('./tr[1]/td[2]').text).replace(',', '').strip()))
+
+            tick[Datacode.PREV_CLOSE] = self.save_wrapper(
+                lambda: float(
+                    html.unescape(table.find('./tr[2]/td[2]').text).replace(',', '').strip()))
+
+            tick[Datacode.HIGH_52_WEEK] = self.save_wrapper(
+                lambda: float(
+                    html.unescape(table.find('./tr[3]/td[2]').text).replace(',', '').strip()))
+
+            tick[Datacode.LOW_52_WEEK] = self.save_wrapper(
+                lambda: float(
+                    html.unescape(table.find('./tr[4]/td[2]').text).replace(',', '').strip()))
 
             logger.info(tick)
 

src/google2.py

@@ -1,268 +0,0 @@
-#  google.py
-#
-#  license: GNU LGPL
-#
-#  This library is free software; you can redistribute it and/or
-#  modify it under the terms of the GNU Lesser General Public
-#  License as published by the Free Software Foundation; either
-#  version 3 of the License, or (at your option) any later version.
-
-
-import dateutil
-import locale
-import logging
-import html
-import os
-import pathlib
-import re
-import time
-import traceback
-
-import xml.etree.ElementTree as ET
-from naivehtmlparser import NaiveHTMLParser
-
-from datacode import Datacode
-from baseclient import BaseClient, RedirectException
-from tz import whois_timezone_info
-
-logger = logging.getLogger(__name__)
-
-
-# logger.setLevel(logging.DEBUG)
-
-
-def handle_abbreviations(s):
-    s = str(s).strip()
-    if s.endswith('T'):
-        return float(s.replace('T', '')) * 1000
-    if s.endswith('M'):
-        return float(s.replace('M', '')) * 1000000
-    if s.endswith('B'):
-        return float(s.replace('B', '')) * 1000000000
-    return float(s)
-
-
-def un_span(s):
-    return re.sub(r'<span [^>]*>', '', s).replace('</span>', '')
-
-
-class Google(BaseClient):
-    def __init__(self, ctx):
-        super().__init__()
-
-        self.realtime = {}
-        self.location = None
-
-        self.basedir = os.path.join(str(pathlib.Path.home()), '.financials-extension')
-        os.makedirs(self.basedir, exist_ok=True)
-
-    def getRealtime(self, ticker: str, datacode: int):
-
-        """
-        Retrieve realtime data for ticker from Google Finance and cache it for further lookups
-
-        :param ticker: the ticker symbol e.g. VOD.L or LON:VOD
-        :param datacode: the requested datacode
-        :return:
-        """
-
-        # remove white space
-        ticker = "".join(ticker.split())
-
-        # use cached value for up to 60 seconds
-        if ticker in self.realtime:
-            tick = self.realtime[ticker]
-            if time.time() - 60 < tick[Datacode.TIMESTAMP]:
-                return self._return_value(tick, datacode)
-            else:
-                del self.realtime[ticker]
-
-        q_param = 'q=' + ticker
-
-        if not self.location:
-            url = 'https://www.google.com/search?hl=en&tbm=fin&' + q_param
-
-            try:
-                self.urlopen(url, redirect=False)
-            except RedirectException as e:
-                self.location = e.location.replace('&' + q_param, '')
-            except BaseException as e:
-                logger.error(traceback.format_exc())
-                return 'Google.getRealtime(\'{}\', {}) - location: {}'.format(ticker, datacode, e)
-
-        if not self.location:
-            url = 'https://www.google.com/search?tbm=fin&' + q_param
-        else:
-            url = self.location + '&' + q_param
-
-        try:
-            text = self.urlopen(url)
-            with open(os.path.join(self.basedir, 'google-{}.html'.format(ticker)), "w") as text_file:
-                print(text, file=text_file)
-        except BaseException as e:
-            logger.error(traceback.format_exc())
-            return 'Google.getRealtime(\'{}\', {}) - urlopen: {} {}'.format(ticker, datacode, e, url)
-
-        if ticker not in self.realtime:
-            self.realtime[ticker] = {}
-
-        tick = self.realtime[ticker]
-
-        try:
-            r = '<span[^>]+role="heading"[^>]+>(.*?)</span>'
-            pattern = re.compile(r)
-
-            match = pattern.search(text)
-            if not match:
-                return 'Google.getRealtime({}, {}) - no match'.format(ticker, datacode)
-            start = match.span(0)[1]
-
-            tick[Datacode.NAME] = self.save_wrapper(
-                lambda: html.unescape(un_span(match.group(1)).strip()))
-
-            # match = pattern.search(text, start)
-            # if not match:
-            #     return 'Google.getRealtime({}, {}) - no match'.format(ticker, datacode)
-            # start = match.span(0)[1]
-
-            r = '<div [^>]*>(.*?)</div>'
-            pattern = re.compile(r)
-
-            # # first div ignored
-            # match = pattern.search(text, start)
-            # if not match:
-            #     return 'Google.getRealtime({}, {}) - no match'.format(ticker, datacode)
-            # start = match.span(0)[1]
-
-            # # second div is NAME
-            # match = pattern.search(text, start)
-            # if not match:
-            #     return 'Google.getRealtime({}, {}) - no match'.format(ticker, datacode)
-            # start = match.span(0)[1]
-            #
-            # tick[Datacode.NAME] = self.save_wrapper(
-            #     lambda: html.unescape(un_span(match.group(1)).strip()))
-
-            # first div is TICKER
-            match = pattern.search(text, start)
-            if not match:
-                return 'Google.getRealtime({}, {}) - no match'.format(ticker, datacode)
-
-            ticker = self.save_wrapper(
-                lambda: html.unescape(match.group(1)).replace(' ', ''))
-
-            tick[Datacode.EXCHANGE] = self.save_wrapper(lambda: ticker.split(':')[0])
-            tick[Datacode.TICKER] = self.save_wrapper(lambda: ticker.split(':')[1])
-
-        except BaseException as e:
-            return 'Google.getRealtime({}, {}) - process: {}'.format(ticker, datacode, e)
-
-        try:
-            r = '<sticky-header [^>]*>(.*?)</sticky-header>'
-            pattern = re.compile(r, flags=re.DOTALL)
-            match = re.search(pattern, text)
-
-            if match:
-                text = match.group(1)
-            else:
-                return 'Data for \'{}\' not found'.format(ticker)
-
-            parser = NaiveHTMLParser()
-            root = parser.feed(text)
-            parser.close()
-
-            cards = root.findall('.//g-card-section')
-
-            if len(cards) < 4:
-                return 'Data for \'{}\' not found'.format(ticker)
-
-            header = cards[1]
-
-            locale.setlocale(locale.LC_ALL, 'en_US.UTF-8')
-
-            tick[Datacode.LAST_PRICE] = self.save_wrapper(
-                lambda: locale.atof(
-                    html.unescape(header.find('./span[1]/span[1]/span[1]').text).strip()))
-
-            tick[Datacode.CURRENCY] = self.save_wrapper(
-                lambda: html.unescape(header.find('./span[1]/span[1]/span[2]').text).strip())
-
-            tick[Datacode.CHANGE] = self.save_wrapper(
-                lambda: locale.atof(
-                    html.unescape(header.find('./span[2]/span[1]').text).replace('−', '-').strip()))
-
-            tick[Datacode.CHANGE_IN_PERCENT] = self.save_wrapper(
-                lambda: float(
-                    html.unescape(header.find('./span[2]/span[2]/span[1]').text).strip()
-                        .replace('(', '').replace(')', '').replace('%', '')))
-
-            if tick[Datacode.CHANGE_IN_PERCENT] != 0.0:
-                if html.unescape(header.find('./span[2]/span[2]/span[1]').text).strip().startswith('('):
-                    tick[Datacode.CHANGE_IN_PERCENT] = -tick[Datacode.CHANGE_IN_PERCENT]
-
-            try:
-                value = html.unescape(header.find('./div[1]/span[1]/span[2]').text).replace('·', '').strip()
-                logger.debug(value)
-                dt = dateutil.parser.parse(value, tzinfos=whois_timezone_info)
-                tick[Datacode.LAST_PRICE_DATE] = dt.date()
-                tick[Datacode.LAST_PRICE_TIME] = dt.time()
-
-                time_bits = value.split(' ')
-                if len(time_bits) >= 4:
-                    tick[Datacode.TIMEZONE] = time_bits[-1]
-
-            except BaseException as e:
-                pass
-
-            footer = cards[3]
-            logger.debug(ET.tostring(footer))
-
-            # parse 'footer' for remaining fields
-            table = footer.find('./div[1]/div[1]/div[1]/table[1]')
-
-            tick[Datacode.OPEN] = self.save_wrapper(
-                lambda: float(
-                    html.unescape(table.find('./tr[1]/td[2]').text).replace(',', '').strip()))
-
-            tick[Datacode.HIGH] = self.save_wrapper(
-                lambda: float(
-                    html.unescape(table.find('./tr[2]/td[2]').text).replace(',', '').strip()))
-
-            tick[Datacode.LOW] = self.save_wrapper(
-                lambda: float(
-                    html.unescape(table.find('./tr[3]/td[2]').text).replace(',', '').strip()))
-
-            tick[Datacode.MARKET_CAP] = self.save_wrapper(
-                lambda: handle_abbreviations(
-                    html.unescape(table.find('./tr[4]/td[2]').text).replace(',', '').replace('-', '').strip()))
-
-            table = footer.find('./div[1]/div[1]/div[2]/table[1]')
-
-            tick[Datacode.PREV_CLOSE] = self.save_wrapper(
-                lambda: float(
-                    html.unescape(table.find('./tr[2]/td[2]').text).replace(',', '').strip()))
-
-            tick[Datacode.HIGH_52_WEEK] = self.save_wrapper(
-                lambda: float(
-                    html.unescape(table.find('./tr[3]/td[2]').text).replace(',', '').strip()))
-
-            tick[Datacode.LOW_52_WEEK] = self.save_wrapper(
-                lambda: float(
-                    html.unescape(table.find('./tr[4]/td[2]').text).replace(',', '').strip()))
-
-            tick[Datacode.TIMESTAMP] = time.time()
-
-            logger.info(tick)
-
-        except BaseException as e:
-            logger.warning(traceback.format_exc())
-            return 'Google.getRealtime({}, {}) - process: {}'.format(ticker, datacode, e)
-
-        return self._return_value(self.realtime[ticker], datacode)
-
-    def getHistoric(self, ticker, datacode, date):
-        return 'Google.getHistoric: Historic Data not implemented.'
-
-
-def createInstance(ctx):
-    return Google(ctx)

src/test_ft.py

@@ -0,0 +1,234 @@
+#  test_yahoo.py
+#
+#  license: GNU LGPL
+#
+#  This library is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 3 of the License, or (at your option) any later version.
+
+import argparse
+import logging
+import sys
+import unittest
+
+import financials
+from datacode import Datacode
+
+financials = financials.createInstance(None)
+
+logging.basicConfig(level=logging.ERROR)
+
+
+class Test(unittest.TestCase):
+
+    def test_currency(self):
+        s = financials.getRealtime('EURGBP', Datacode.CURRENCY.value, 'FT')
+        self.assertEqual(type(s), str, 'test_currency CURRENCY')
+
+        s = financials.getRealtime('EURGBP', Datacode.LAST_PRICE.value, 'FT')
+        self.assertEqual(type(s), float, 'test_currency LAST_PRICE')
+
+    def test_US_equity(self):
+
+        s = financials.getRealtime('INTC:NSQ', Datacode.CHANGE.value, 'FT')
+        self.assertEqual(type(s), float, 'test_realtime_US_equity CHANGE {}'.format(s))
+
+        s = financials.getRealtime('INTC:NSQ', Datacode.CHANGE_IN_PERCENT.value, 'FT')
+        self.assertEqual(type(s), float, 'test_realtime_US_equity CHANGE_IN_PERCENT {}'.format(s))
+
+        s = financials.getRealtime('INTC:NSQ', Datacode.AVG_DAILY_VOL_3MOMTH.value, 'FT')
+        self.assertEqual(type(s), float, 'test_realtime_US_equity AVG_DAILY_VOL_3MOMTH {}'.format(s))
+
+        s = financials.getRealtime('INTC:NSQ', Datacode.MARKET_CAP.value, 'FT')
+        self.assertEqual(type(s), float, 'test_realtime_US_equity MARKET_CAP {}'.format(s))
+
+        s = financials.getRealtime('IBM:NYQ', Datacode.TICKER.value, 'FT')
+        self.assertEqual(type(s), str, 'test_realtime_US_equity TICKER {}'.format(s))
+        self.assertEqual(s, 'IBM:NYQ', 'test_realtime_US_equity TICKER {}'.format(s))
+
+        s = financials.getRealtime('IBM:NYQ', Datacode.PREV_CLOSE.value, 'FT')
+        self.assertEqual(type(s), float, 'test_realtime_US_equity PREV_CLOSE {}'.format(s))
+
+        s = financials.getRealtime('IBM:NYQ', Datacode.OPEN.value, 'FT')
+        self.assertEqual(type(s), float, 'test_realtime_US_equity OPEN {}'.format(s))
+
+        s = financials.getRealtime('IBM:NYQ', Datacode.LAST_PRICE.value, 'FT')
+        self.assertEqual(type(s), float, 'test_realtime_US_equity LAST_PRICE {}'.format(s))
+
+        s = financials.getRealtime('IBM:NYQ', Datacode.LOW.value, 'FT')
+        self.assertEqual(type(s), float, 'test_realtime_US_equity LOW {}'.format(s))
+
+        s = financials.getRealtime('IBM:NYQ', Datacode.HIGH.value, 'FT')
+        self.assertEqual(type(s), float, 'test_realtime_US_equity HIGH {}'.format(s))
+
+        s = financials.getRealtime('IBM:NYQ', Datacode.VOLUME.value, 'FT')
+        self.assertEqual(type(s), float, 'test_realtime_US_equity VOLUME {}'.format(s))
+
+        s = financials.getRealtime('IBM:NYQ', Datacode.NAME.value, 'FT')
+        self.assertEqual(type(s), str, 'test_realtime_US_equity NAME {}'.format(s))
+        self.assertEqual(s, 'International Business Machines Corp',
+                         'test_realtime_US_equity NAME {}'.format(s))
+
+        s = financials.getRealtime('IBM:NYQ', Datacode.TIMEZONE.value, 'FT')
+        self.assertEqual(type(s), str, 'test_realtime_US_equity TIMEZONE {}'.format(s))
+
+    def test_US_mutuals(self):
+
+        s = financials.getRealtime('VGSLX', Datacode.LAST_PRICE.value, 'FT')
+        self.assertEqual(type(s), float, 'test_realtime_US_mutuals LAST_PRICE {}'.format(s))
+
+        s = financials.getRealtime('VGSLX', Datacode.NAME.value, 'FT')
+        self.assertEqual(s, "Vanguard Real Estate Index Fund Admiral Shares",
+                         'test_realtime_US_mutuals NAME {}'.format(s))
+
+        s = financials.getRealtime('VGSLX', Datacode.CURRENCY.value, 'FT')
+        self.assertEqual(s, "USD", 'test_realtime_US_mutuals CURRENCY {}'.format(s))
+
+        s = financials.getRealtime('VGSLX', Datacode.CHANGE.value, 'FT')
+        self.assertEqual(type(s), float, 'test_realtime_US_mutuals CHANGE {}'.format(s))
+
+        s = financials.getRealtime('VGSLX', Datacode.CHANGE_IN_PERCENT.value, 'FT')
+        self.assertEqual(type(s), float, 'test_realtime_US_mutuals CHANGE_IN_PERCENT {}'.format(s))
+
+        s = financials.getRealtime('VGSLX', Datacode.TIMEZONE.value, 'FT')
+        self.assertEqual(type(s), str, 'test_realtime_US_mutuals TIMEZONE {}'.format(s))
+
+        s = financials.getRealtime('VFIAX', Datacode.LAST_PRICE.value, 'FT')
+        self.assertEqual(type(s), float, 'test_realtime_US_mutuals LAST_PRICE {}'.format(s))
+
+        s = financials.getRealtime('VFIAX', Datacode.LAST_PRICE_DATE.value, 'FT')
+        self.assertEqual(type(s), str, 'test_realtime_US_mutuals LAST_PRICE_DATE {}'.format(s))
+
+        s = financials.getRealtime('VFIAX', Datacode.LAST_PRICE_TIME.value, 'FT')
+        self.assertEqual(type(s), str, 'test_realtime_US_mutuals LAST_PRICE_TIME {}'.format(s))
+
+    def test_UK_ETF(self):
+
+        s = financials.getRealtime('VERX:LSE:GBP', Datacode.LAST_PRICE.value, 'FT')
+        self.assertEqual(type(s), float, 'test_realtime_UK_ETF LAST_PRICE {}'.format(s))
+
+        s = financials.getRealtime('VERX:LSE:GBP', Datacode.TIMEZONE.value, 'FT')
+        self.assertEqual(type(s), str, 'test_realtime_UK_ETF TIMEZONE {}'.format(s))
+
+        s = financials.getRealtime('CSP1:LSE:GBX', Datacode.NAME.value, 'FT')
+        self.assertEqual(type(s), str, 'test_realtime_UK_ETF NAME {}'.format(s))
+        self.assertEqual(s, 'iShares Core S&P 500 UCITS ETF USD (Acc)',
+                         'test_realtime_UK_ETF NAME {}'.format(s))
+
+        s = financials.getRealtime('C060:GER:EUR', Datacode.NAME.value, 'FT')
+        self.assertEqual(type(s), str, 't_realtime_UK_ETF NAME {}'.format(s))
+
+    def test_DE_equity(self):
+
+        s = financials.getRealtime('SAPX:GER', Datacode.NAME.value, 'FT')
+        self.assertEqual(s, "SAP SE", 'test_realtime_DE_equity NAME {}'.format(s))
+
+        s = financials.getRealtime('SAPX:GER', Datacode.TICKER.value, 'FT')
+        self.assertEqual(s, "SAPX:GER", 'test_realtime_DE_equity TICKER {}'.format(s))
+
+        s = financials.getRealtime('SAPX:GER', Datacode.CURRENCY.value, 'FT')
+        self.assertEqual(s, 'EUR', 'test_realtime_DE_equity CURRENCY {}'.format(s))
+
+        s = financials.getRealtime('SAPX:GER', Datacode.LAST_PRICE.value, 'FT')
+        self.assertEqual(type(s), float, 'test_realtime_DE_equity LAST_PRICE {}'.format(s))
+
+        s = financials.getRealtime('SAPX:GER', Datacode.TIMEZONE.value, 'FT')
+        self.assertEqual(type(s), str, 'test_realtime_DE_equity TIMEZONE {}'.format(s))
+
+        s = financials.getRealtime('SAPX:GER', Datacode.CHANGE.value, 'FT')
+        self.assertEqual(type(s), float, 'test_realtime_DE_equity CHANGE {}'.format(s))
+
+        s = financials.getRealtime('SAPX:GER', Datacode.CHANGE_IN_PERCENT.value, 'FT')
+        self.assertEqual(type(s), float, 'test_realtime_DE_equity CHANGE_IN_PERCENT {}'.format(s))
+
+        s = financials.getRealtime('SAPX:GER', Datacode.VOLUME.value, 'FT')
+        self.assertEqual(type(s), float, 'test_realtime_DE_equity VOLUME {}'.format(s))
+
+        s = financials.getRealtime('SAPX:GER', Datacode.OPEN.value, 'FT')
+        self.assertEqual(type(s), float, 'test_realtime_DE_equity OPEN {}'.format(s))
+
+        s = financials.getRealtime('SAPX:GER', Datacode.HIGH.value, 'FT')
+        self.assertEqual(type(s), float, 'test_realtime_DE_equity HIGH {}'.format(s))
+
+        s = financials.getRealtime('SAPX:GER', Datacode.LOW.value, 'FT')
+        self.assertEqual(type(s), float, 'test_realtime_DE_equity LOW {}'.format(s))
+
+        s = financials.getRealtime('SAPX:GER', Datacode.PREV_CLOSE.value, 'FT')
+        self.assertEqual(type(s), float, 'test_realtime_DE_equity PREV_CLOSE {}'.format(s))
+
+        s = financials.getRealtime('SAPX:GER', Datacode.MARKET_CAP.value, 'FT')
+        self.assertEqual(type(s), float, 'test_realtime_DE_equity MARKET_CAP {}'.format(s))
+
+    def test_TY_equity(self):
+        s = financials.getRealtime('6503:TYO', Datacode.OPEN.value, 'FT')
+        self.assertEqual(type(s), float, 'test_TY_equity OPEN {}'.format(s))
+
+        s = financials.getRealtime('6503:TYO', Datacode.LOW.value, 'FT')
+        self.assertEqual(type(s), float, 'test_TY_equity LOW {}'.format(s))
+
+        s = financials.getRealtime('6503:TYO', Datacode.HIGH.value, 'FT')
+        self.assertEqual(type(s), float, 'test_TY_equity HIGH {}'.format(s))
+
+        s = financials.getRealtime('6503:TYO', Datacode.LOW_52_WEEK.value, 'FT')
+        self.assertEqual(type(s), float, 'test_TY_equity LOW_52_WEEK {}'.format(s))
+
+        s = financials.getRealtime('6503:TYO', Datacode.HIGH_52_WEEK.value, 'FT')
+        self.assertEqual(type(s), float, 'test_TY_equity HIGH_52_WEEK {}'.format(s))
+
+        s = financials.getRealtime('6503:TYO', Datacode.MARKET_CAP.value, 'FT')
+        self.assertEqual(type(s), float, 'test_TY_equity MARKET_CAP {}'.format(s))
+
+        s = financials.getRealtime('6503:TYO', Datacode.VOLUME.value, 'FT')
+        self.assertEqual(type(s), float, 'test_TY_equity VOLUME {}'.format(s))
+
+        s = financials.getRealtime('6503:TYO', Datacode.CURRENCY.value, 'FT')
+        self.assertEqual(s, 'JPY', 'test_TY_equity CURRENCY')
+
+    def test_index(self):
+
+        s = financials.getRealtime('INX:IOM', Datacode.TICKER.value, 'FT')
+        self.assertEqual(s, "INX:IOM", 'test_index TICKER {}'.format(s))
+
+        s = financials.getRealtime('DAXX:GER', Datacode.TICKER.value, 'FT')
+        self.assertEqual(s, "DAXX:GER", 'test_index TICKER {}'.format(s))
+
+        s = financials.getRealtime('DAXX:GER', Datacode.LAST_PRICE.value, 'FT')
+        self.assertEqual(type(s), float, 'test_index LAST_PRICE {}'.format(s))
+
+        s = financials.getRealtime('DAXX:GER', Datacode.VOLUME.value, 'FT')
+        self.assertEqual(type(s), float, 'test_realtime_DE_equity VOLUME {}'.format(s))
+
+        s = financials.getRealtime('DAXX:GER', Datacode.LOW_52_WEEK.value, 'FT')
+        self.assertEqual(type(s), float, 'test_realtime_DE_equity LOW_52_WEEK {}'.format(s))
+
+        s = financials.getRealtime('DAXX:GER', Datacode.HIGH_52_WEEK.value, 'FT')
+        self.assertEqual(type(s), float, 'test_realtime_DE_equity HIGH_52_WEEK {}'.format(s))
+
+        s = financials.getRealtime('DAXX:GER', Datacode.OPEN.value, 'FT')
+        self.assertIsNone(s, 'test_realtime_DE_equity OPEN {}'.format(s))
+
+        s = financials.getRealtime('DAXX:GER', Datacode.HIGH.value, 'FT')
+        self.assertIsNone(s, 'test_realtime_DE_equity HIGH {}'.format(s))
+
+        s = financials.getRealtime('DAXX:GER', Datacode.LOW.value, 'FT')
+        self.assertIsNone(s, 'test_realtime_DE_equity LOW {}'.format(s))
+
+        s = financials.getRealtime('DAXX:GER', Datacode.PREV_CLOSE.value, 'FT')
+        self.assertIsNone(s, 'test_realtime_DE_equity PREV_CLOSE {}'.format(s))
+
+        s = financials.getRealtime('DAXX:GER', Datacode.MARKET_CAP.value, 'FT')
+        self.assertIsNone(s, 'test_realtime_DE_equity MARKET_CAP {}'.format(s))
+
+    def test_errors(self):
+
+        s = financials.getRealtime('NO_NAME', Datacode.LAST_PRICE.value, 'FT')
+        self.assertIsNone(s, 'test_realtime_errors LAST_PRICE {}'.format(s))
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('unittest_args', nargs='*')
+    args = parser.parse_args()
+    unit_argv = [sys.argv[0]] + args.unittest_args
+    unittest.main(argv=unit_argv)

src/test_google.py

@@ -24,11 +24,7 @@ class Test(unittest.TestCase):
 
     def test_currency(self):
         s = financials.getRealtime('EURGBP', Datacode.LAST_PRICE.value, 'GOOGLE')
-        self.assertEqual('Google.getRealtime(EURGBP, 21) - no match', s, 'test_currency LAST_PRICE')
-
-        # s = financials.getRealtime('EURGBP', Datacode.CURRENCY.value, 'GOOGLE')
-        # self.assertEqual(type(s), str, 'test_currency CURRENCY')
-        # self.assertEqual(s, '', 'test_currency CURRENCY')
+        self.assertIsNone(s, 'test_currency LAST_PRICE')
 
     def test_UK_equity(self):
         s = financials.getRealtime('LON:VOD', Datacode.LAST_PRICE.value, 'GOOGLE')
@@ -112,7 +108,7 @@ class Test(unittest.TestCase):
         self.assertEqual(s, 'EUR', 'test_DE_ETF CURRENCY')
 
         s = financials.getRealtime('FRA:C060', Datacode.MARKET_CAP.value, 'GOOGLE')
-        self.assertEqual(s, 'Data doesn\'t exist - 27', 'test_DE_ETF TIMESTAMP {}'.format(s))
+        self.assertIsNone(s, 'test_DE_ETF MARKET_CAP {}'.format(s))
 
     def test_TY_equity(self):
         s = financials.getRealtime('TYO:6503', Datacode.OPEN.value, 'GOOGLE')
@@ -202,7 +198,7 @@ class Test(unittest.TestCase):
         self.assertEqual(s, 'USD', 'test_US_mutuals CURRENCY')
 
         s = financials.getRealtime('MUTF:VFIAX', Datacode.TIMEZONE.value, 'GOOGLE')
-        self.assertEqual(s, 'Data doesn\'t exist - 105', 'test_US_mutuals')
+        self.assertIsNone(s, 'test_US_mutuals TIMEZONE')
 
     def test_index(self):
         s = financials.getRealtime('INDEXDB:DAX', Datacode.LAST_PRICE.value, 'GOOGLE')
@@ -214,6 +210,15 @@ class Test(unittest.TestCase):
         s = financials.getRealtime('INDEXDB:DAX', Datacode.CHANGE.value, 'GOOGLE')
         self.assertEqual(type(s), float, 'test_index CHANGE')
 
+        s = financials.getRealtime('INDEXDB:DAX', Datacode.OPEN.value, 'GOOGLE')
+        self.assertEqual(type(s), float, 'test_index OPEN')
+
+        s = financials.getRealtime('INDEXDB:DAX', Datacode.LOW.value, 'GOOGLE')
+        self.assertEqual(type(s), float, 'test_index LOW')
+
+        s = financials.getRealtime('INDEXDB:DAX', Datacode.HIGH.value, 'GOOGLE')
+        self.assertEqual(type(s), float, 'test_index HIGH')
+
     def test_errors(self):
         s = financials.getRealtime(None, Datacode.LAST_PRICE.value, 'GOOGLE')
         self.assertEqual(s, 'Ticker is empty', 'test_errors')
@@ -222,7 +227,7 @@ class Test(unittest.TestCase):
         self.assertEqual(s, 'Datacode is empty', 'test_errors')
 
         s = financials.getRealtime('DOES_NOT_EXISTS', Datacode.LAST_PRICE.value, 'GOOGLE')
-        self.assertEqual(s, 'Google.getRealtime(DOES_NOT_EXISTS, 21) - no match', 'test_errors')
+        self.assertIsNone(s, 'test_errors')
 
         s = financials.getRealtime('NYS:IBM', 'Foo', 'GOOGLE')
         self.assertEqual(s, 'Datacode is not a number', 'test_errors')

src/test_yahoo.py

@@ -102,8 +102,8 @@ class Test(unittest.TestCase):
         self.assertEqual(s, 'iShares VII Public Limited Company - iShares Core S&P 500 UCITS ETF',
                          'test_realtime_UK_ETF NAME {}'.format(s))
 
-        s = financials.getRealtime('C060.DE', 104, 'YAHOO')
-        self.assertEqual(type(s), str, 't_realtime_UK_ETF AME {}'.format(s))
+        s = financials.getRealtime('C060.DE', Datacode.NAME.value, 'YAHOO')
+        self.assertEqual(type(s), str, 't_realtime_UK_ETF NAME {}'.format(s))
 
     def test_realtime_DE_equity(self):
 
@@ -202,7 +202,7 @@ class Test(unittest.TestCase):
         self.assertEqual(s, 82.889999, 'test_historic_DE_equity CLOSE {}'.format(s))
 
         s = financials.getHistoric('C060.DE', Datacode.CLOSE.value, '2017-01-03', 'YAHOO')
-        self.assertEqual(s, 72.870003, 'test_historic_DE_equity CLOSE {}'.format(s))
+        self.assertEqual(s, 68.209999, 'test_historic_DE_equity CLOSE {}'.format(s))
 
     def test_realtime_errors(self):
 

src/version.py

@@ -0,0 +1,15 @@
+#  version.py
+#
+#  license: GNU LGPL
+#
+#  This library is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 3 of the License, or (at your option) any later version.
+
+# This file will not be actually used as it is regenerated with the correct build version
+# by generate_metainfo.py in the "compile" step
+
+# This is a bit of a hack but I can't find out how to access the version in description.xml once deployed
+
+version = '0.0.0'

src/yahoo.py

@@ -14,7 +14,6 @@ import dateutil.parser
 import html
 import logging
 import os
-import pathlib
 import pprint
 import pytz
 import re
@@ -50,9 +49,6 @@ class Yahoo(BaseClient):
         self.historicdata = {}
         self.js = jsonParser.jsonObject
 
-        self.basedir = os.path.join(str(pathlib.Path.home()), '.financials-extension')
-        os.makedirs(self.basedir, exist_ok=True)
-
     def _read_ticker_csv_file(self, ticker):
 
         fn = os.path.join(self.basedir, 'yahoo-{}.csv'.format(ticker))
@@ -87,7 +83,7 @@ class Yahoo(BaseClient):
         """
         Retrieve realtime data for ticker from Yahoo Finance and cache it for further lookups
 
-        :param ticker: the ticker symbol e.g. VOD.L or LON:VOD
+        :param ticker: the ticker symbol e.g. VOD.L
         :param datacode: the requested datacode
         :return:
         """
@@ -95,10 +91,10 @@ class Yahoo(BaseClient):
         # remove white space
         ticker = "".join(ticker.split())
 
-        # use cached value for up to 5 minutes
+        # use cached value for up to 60 seconds
         if ticker in self.realtime:
             tick = self.realtime[ticker]
-            if time.time() - 5*60 < tick[Datacode.TIMESTAMP]:
+            if time.time() - 60 < tick[Datacode.TIMESTAMP]:
                 return self._return_value(tick, datacode)
             else:
                 del self.realtime[ticker]
@@ -121,6 +117,8 @@ class Yahoo(BaseClient):
 
         try:
             text = self.urlopen(url, redirect=True, data=None, headers=None, cookies=cookies)
+            with open(os.path.join(self.basedir, 'yahoo-{}.html'.format(ticker)), "w") as text_file:
+                print(f"<!-- '{url}' -->\r\n\r\n{text}", file=text_file)
         except BaseException as e:
             logger.error(traceback.format_exc())
             return 'Yahoo.getRealtime({}, {}) - urlopen: {}'.format(ticker, datacode, e)
@@ -135,9 +133,6 @@ class Yahoo(BaseClient):
 
             if match:
                 self.crumb = match.group(1)
-            else:
-                with open(os.path.join(self.basedir, 'yahoo-{}.html'.format(ticker)), "w") as text_file:
-                    print(text, file=text_file)
 
         except BaseException as e:
             logger.error(traceback.format_exc())
@@ -147,16 +142,12 @@ class Yahoo(BaseClient):
             start = text.find('"QuoteSummaryStore":{')
 
             if start < 0:
-                with open(os.path.join(self.basedir, 'yahoo-{}.html'.format(ticker)), "w") as text_file:
-                    print(text, file=text_file)
                 return None
 
             start = start + len('"QuoteSummaryStore":')
             results = self.js.parseString(text[start:])
 
             if not results:
-                with open(os.path.join(self.basedir, 'yahoo-{}.html'.format(ticker)), "w") as text_file:
-                    print(text, file=text_file)
                 return None
 
         except BaseException as e:
@@ -176,6 +167,8 @@ class Yahoo(BaseClient):
 
             tick = self.realtime[ticker]
 
+            tick[Datacode.TIMESTAMP] = time.time()
+
             tick[Datacode.PREV_CLOSE] = float(raw(price, 'regularMarketPreviousClose'))
             tick[Datacode.OPEN] = float(raw(price, 'regularMarketOpen'))
             tick[Datacode.CHANGE] = float(raw(price, 'regularMarketChange'))
@@ -210,11 +203,9 @@ class Yahoo(BaseClient):
             else:
                 tick[Datacode.NAME] = tick[Datacode.TICKER]
 
-            tick[Datacode.TIMESTAMP] = time.time()
-
         except BaseException as e:
             with open(os.path.join(self.basedir, 'yahoo-{}.js'.format(ticker)), "w") as text_file:
-                pprint.pprint(results.asList(), stream=text_file)
+                pprint.pprint(f"// '{url}'\r\n\r\n{results.asList()}", stream=text_file)
 
             logger.error(traceback.format_exc())
             return 'Yahoo.getRealtime({}, {}) - process: {}'.format(ticker, datacode, e)
@@ -226,7 +217,7 @@ class Yahoo(BaseClient):
         """
         Retrieve historic data for ticker from Yahoo Finance and cache it for further lookups
 
-        :param ticker: the ticker symbol e.g. VOD.L or LON:VOD
+        :param ticker: the ticker symbol e.g. VOD.L
         :param datacode: the requested datacode
         :param date: the requested date
         :return: