Remove locale

README.md

@@ -1,10 +1,30 @@
 # Financials-Extension
 
-Extension for LibreOffice Calc to access stock market data. Currently supports Yahoo and Google.
+Extension for LibreOffice Calc to make stock, index and FX (Yahoo only) market data available in a Calc spread sheet - currently supports Yahoo and Google using web scraping.
 
-Only tested this with Ubuntu 16.04 and LibreOffice 5
+### System dependencies:
 
-To Build:
+Requires the following Python 3 (used by LibreOffice) packages (on top of standard libs): dateutil, pytz, pyparsing 
+
+The following steps have worked for me on a fresh Debian installation (as root)
+- apt install python3-pip
+- pip3 install python-dateutil
+- pip3 install pytz
+- pip3 install pyparsing
+
+### Usage:
+
+Have a look at the 'releases' tab above: download a prebuild **Financials-Extension.oxt** file and load it into Calc under menu item: Tools, Extension Manager...
+
+There is a file **examples.ods** there too with usage examples and possible arguments to functions.
+
+### Build:
+
+You will need the LibreOffice SDK installed. 
+
+On my system I installed packages: libreoffice-dev libreoffice-java-common libreoffice-script-provider-python
+
+\# depending on your location...
 
 cd ~/tech/IdeaProjects/Financials-Extension/
 
@@ -12,5 +32,13 @@ python3 src/test_yahoo.py
 
 python3 src/test_google.py
 
+\# This builds file **Financials-Extension.oxt**
+
 ./compile.sh
 
+### Tested with:
+- Debian 10.3  / LibreOffice Calc 6.1.5.2 / Python 3.7.3
+- Ubuntu 19.10 / LibreOffice Calc 6.3.5.2 / Python 3.7.6
+- Ubuntu 19.04 / LibreOffice Calc 6.2 / Python 3.7.3
+- Ubuntu 18.04 / LibreOffice Calc 6 / Python 3.6.7
+- Ubuntu 16.04 / LibreOffice Calc 5 (previous versions)

compile.sh

@@ -27,6 +27,8 @@ cp -f "${PWD}"/src/financials.py "${PWD}"/build/
 cp -f "${PWD}"/src/datacode.py "${PWD}"/build/
 cp -f "${PWD}"/src/baseclient.py "${PWD}"/build/
 cp -f "${PWD}"/src/jsonParser.py "${PWD}"/build/
+cp -f "${PWD}"/src/naivehtmlparser.py "${PWD}"/build/
+cp -f "${PWD}"/src/tz.py "${PWD}"/build/
 cp -f "${PWD}"/src/google.py "${PWD}"/build/
 cp -f "${PWD}"/src/yahoo.py "${PWD}"/build/
 

src/baseclient.py

@@ -10,21 +10,32 @@
 
 import codecs
 import gzip
+import logging
+import os
+import pathlib
 import random
-import sys
 import select
 
-from http.client import HTTPConnection, HTTPSConnection
+from http.client import HTTPConnection, HTTPSConnection, HTTPException
 from http import cookiejar
 
 import urllib.request
 
 from datacode import Datacode
 
+logger = logging.getLogger(__name__)
+# logger.setLevel(logging.DEBUG)
 
-def log(str):
-    # print(str, file=sys.stderr)
-    pass
+
+class RedirectException(HTTPException):
+    def __init__(self, location):
+        self.location = location
+
+
+class HttpException(HTTPException):
+    def __init__(self, url, status):
+        self.url = url
+        self.status = status
 
 
 class BaseClient:
@@ -32,27 +43,36 @@ class BaseClient:
         self.connections = {}
         self.cookies = cookiejar.CookieJar()
 
+        self.basedir = os.path.join(str(pathlib.Path.home()), '.financials-extension')
+        os.makedirs(self.basedir, exist_ok=True)
+
         user_agents = [
-            'Mozilla/5.0 (X11; Linux x86_64; rv:57.0) Gecko/20100101 Firefox/57.0',
-            'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:56.0) Gecko/20100101 Firefox/56.0',
-            'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:55.0) Gecko/20100101 Firefox/55.0'
-            'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36',
-            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36',
-            'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.36'
+            'Mozilla/5.0 (Windows NT 6.1; rv:73.0) Gecko/20100101 Firefox/73.0',
+            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:74.0) Gecko/20100101 Firefox/74.0',
+            'Mozilla/5.0 (Windows NT 6.3; Win64; x64; rv:74.0) Gecko/20100101 Firefox/74.0'
+
+            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36',
+            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36',
+            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36'
         ]
 
         self.default_headers = {
             'User-Agent': random.sample(user_agents, 1)[0],
-            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
+            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
             'Accept-Encoding': 'gzip, deflate',
-            'Accept-Language': 'en-GB,en-US;q=0.9,en;q=0.8'
+            'Accept-Language': 'en-US,en;q=0.5'
         }
 
-    def request(self, method: str, url: str, data=None, headers={}, **kwargs):
+    def request(self, method: str, url: str, data=None, headers={}, cookies=[], **kwargs):
 
         _headers = self.default_headers.copy()
-        for key, value in headers.items():
-            _headers[key] = value
+        if headers:
+            for key, value in headers.items():
+                _headers[key] = value
+
+        if cookies:
+            for c in cookies:
+                self.cookies.set_cookie(c)
 
         connection = None
 
@@ -66,11 +86,11 @@ class BaseClient:
             connection = None
 
         if not connection:
-            log('Creating HTTP connection --------- ----------------------------------------')
+            logger.debug('Creating connection --------------------------------------------------')
             connection = HTTPConnection(host, **kwargs) if scheme == 'http:' else HTTPSConnection(host, **kwargs)
 
-        log('Creating HTTP request ------------ ----------------------------------------')
-        log(url)
+        logger.debug('Creating request -----------------------------------------------------')
+        logger.info('url=%s', url)
 
         # generate and add cookie headers
         request = urllib.request.Request(url)
@@ -80,41 +100,61 @@ class BaseClient:
             _headers['Cookie'] = request.get_header('Cookie')
 
         for key, value in _headers.items():
-            log('{}: {}'.format(key, value))
+            logger.debug('Header: %s=%s', key, value)
 
         # request
         connection.request(method, '/' + path, data, _headers)
         response = connection.getresponse()
 
-        log('Processing HTTP response --------- ----------------------------------------')
+        logger.debug('Processing response --------------------------------------------------')
 
-        # log('response.status={}'.format(response.status))
+        # logger.debug('response.status={}'.format(response.status))
         for key, value in response.getheaders():
-            log('{}: {}'.format(key, value))
+            logger.debug('Header: %s=%s', key, value)
 
         self.cookies.extract_cookies(response, request)
         self.connections[(scheme, host)] = connection
 
         return response
 
-    def urlopen(self, url, data=None, headers={}, **kwargs):
+    def urlopen(self, url, redirect=True, data=None, headers={}, cookies=[], **kwargs):
 
-        response = self.request('POST' if data else 'GET', url, data, headers, **kwargs)
+        response = self.request('POST' if data else 'GET', url, data, headers, cookies, **kwargs)
         text = response.read()
 
+        # Allow two redirects: used by Yahoo for some cookie based consent
+
         if 300 <= response.status < 400:
+            location = response.getheader('Location')
 
-            scheme, _, host, path = url.split('/', 3)
-            redirect_to = response.getheader('Location')
-            if host not in redirect_to:
-                redirect_to = scheme + '//' + host + redirect_to
+            if location and redirect:
 
-            if response.getheader('Location'):
-                response = self.request('POST' if data else 'GET', redirect_to, data, headers, **kwargs)
+                if location.startswith('/'):
+                    scheme, _, host, path = url.split('/', 3)
+                    location = '{}//{}{}'.format(scheme, host, location)
+
+                response = self.request('POST' if data else 'GET', location, data, headers, cookies, **kwargs)
                 text = response.read()
 
-        assert response.status < 400, \
-            'HTTP Status={} Reason={} url={}'.format(response.status, response.reason, url)
+                if 300 <= response.status < 400:
+                    location = response.getheader('Location')
+
+                    if location and redirect:
+
+                        if location.startswith('/'):
+                            scheme, _, host, path = url.split('/', 3)
+                            location = '{}//{}{}'.format(scheme, host, location)
+
+                        response = self.request('POST' if data else 'GET', location, data, headers, cookies, **kwargs)
+                        text = response.read()
+                    else:
+                        raise RedirectException(location)
+
+            else:
+                raise RedirectException(location)
+
+        if response.status >= 400:
+            raise HttpException(url, response.status)
 
         if response.getheader('Content-Encoding') == 'gzip':
             text = gzip.decompress(text)
@@ -164,6 +204,15 @@ class BaseClient:
             elif datacode == Datacode.LAST_PRICE.value and Datacode.LAST_PRICE in data:
                 return data[Datacode.LAST_PRICE]
 
+            elif datacode == Datacode.LOW_52_WEEK.value and Datacode.LOW_52_WEEK in data:
+                return data[Datacode.LOW_52_WEEK]
+
+            elif datacode == Datacode.HIGH_52_WEEK.value and Datacode.HIGH_52_WEEK in data:
+                return data[Datacode.HIGH_52_WEEK]
+
+            elif datacode == Datacode.MARKET_CAP.value and Datacode.MARKET_CAP in data and data[Datacode.MARKET_CAP]:
+                return data[Datacode.MARKET_CAP]
+
             elif datacode == Datacode.VOLUME.value and Datacode.VOLUME in data:
                 return data[Datacode.VOLUME]
 
@@ -188,10 +237,20 @@ class BaseClient:
             elif datacode == Datacode.NAME.value and data[Datacode.NAME]:
                 return data[Datacode.NAME]
 
-            elif datacode == Datacode.TIMEZONE.value and data[Datacode.TIMEZONE]:
+            elif datacode == Datacode.TIMEZONE.value and Datacode.TIMEZONE in data and data[Datacode.TIMEZONE]:
                 return str(data[Datacode.TIMEZONE])
 
         except BaseException as e:
             return 'BaseClient.return_value(\'{}\', {}) - {}'.format(data, datacode, e)
 
         return "Data doesn't exist - {}".format(datacode)
+
+    def save_wrapper(self, f):
+        try:
+            value = f()
+            logger.debug(value)
+            return value
+        except BaseException as e:
+            pass
+
+        return None

src/datacode.py

@@ -24,6 +24,10 @@ class Datacode(Enum):
 
     LAST_PRICE = 21
 
+    HIGH_52_WEEK = 24
+    LOW_52_WEEK = 26
+    MARKET_CAP = 27
+
     VOLUME = 35
     AVG_DAILY_VOL_3MOMTH = 39
 
@@ -42,4 +46,4 @@ class Datacode(Enum):
 
     @classmethod
     def has_value(cls, value):
-        return (any(value == item.value for item in cls))
+        return any(value == item.value for item in cls)

src/financials.py

@@ -15,9 +15,10 @@ import sys
 
 import pathlib
 import platform
+import time
+from functools import wraps
 
 import unohelper
-
 from com.financials.getinfo import Financials
 
 # Add current directory to import path
@@ -28,10 +29,31 @@ if current_dir not in sys.path:
 from datacode import Datacode
 import google
 import yahoo
+from version import version
 
 implementation_name = "com.financials.getinfo.python.FinancialsImpl"  # as defined in Financials.xcu
 implementation_services = ("com.sun.star.sheet.AddIn",)
 
+basedir = os.path.join(str(pathlib.Path.home()), '.financials-extension')
+os.makedirs(basedir, exist_ok=True)
+
+
+def profile(fn):
+    @wraps(fn)
+    def with_profiling(*args, **kwargs):
+        start = time.perf_counter()
+        r = fn(*args, **kwargs)
+        elapsed = time.perf_counter() - start
+
+        with open(os.path.join(basedir, 'trace.log'), "a+") as text_file:
+            print(
+                f"{datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')} {fn.__name__} *args={args} r='{r}' {(1000 * elapsed):.3f} ms",
+                file=text_file)
+
+        return r
+
+    return with_profiling
+
 
 class FinancialsImpl(unohelper.Base, Financials):
     """Define the main class for the Financials extension """
@@ -41,6 +63,7 @@ class FinancialsImpl(unohelper.Base, Financials):
         self.google = google.createInstance(ctx)
         self.yahoo = yahoo.createInstance(ctx)
 
+    @profile
     def getRealtime(self, ticker, datacode=None, source=None):
 
         if ticker == 'SUPPORT':
@@ -70,14 +93,15 @@ class FinancialsImpl(unohelper.Base, Financials):
             if not Datacode.has_value(datacode):
                 return 'Datacode {} not supported'.format(datacode)
 
-            source = source.upper()
+            ticker = str(ticker).strip()
+            source = str(source).upper()
 
             if source == 'GOOGLE':
-                s = self.google.getRealtime(str(ticker).strip(), datacode)
+                s = self.google.getRealtime(ticker, datacode)
             elif source == 'YAHOO':
-                s = self.yahoo.getRealtime(str(ticker).strip(), datacode)
+                s = self.yahoo.getRealtime(ticker, datacode)
             else:
-                s = 'getRealtime:Source \'{}\' not supported'.format(source)
+                s = 'Source \'{}\' not supported'.format(source)
 
         except Exception as ex:
             return str(ex)
@@ -89,6 +113,7 @@ class FinancialsImpl(unohelper.Base, Financials):
 
         return x
 
+    @profile
     def getHistoric(self, ticker, datacode=None, date=None, source=None):
 
         if ticker == 'SUPPORT':
@@ -127,7 +152,7 @@ class FinancialsImpl(unohelper.Base, Financials):
             if type(date) == float or type(date) == int:
 
                 try:
-                    offset = int(date) # offset for 1899-12-30
+                    offset = int(date)  # offset for 1899-12-30
                     d = dateutil.parser.parse('1899-12-30') + datetime.timedelta(days=offset)
                     d = d.date().isoformat()
                 except:
@@ -144,12 +169,13 @@ class FinancialsImpl(unohelper.Base, Financials):
             else:
                 return 'Date type not supported: {} \'{}\''.format(type(date), date)
 
-            source = source.upper()
+            ticker = str(ticker).strip()
+            source = str(source).upper()
 
             if source == 'YAHOO':
                 s = self.yahoo.getHistoric(str(ticker).strip(), datacode, date)
             else:
-                s = 'getHistoric: Source \'{}\' not supported'.format(source)
+                s = 'Source \'{}\' not supported'.format(source)
 
         except Exception as ex:
             return str(ex)
@@ -161,18 +187,20 @@ class FinancialsImpl(unohelper.Base, Financials):
 
         return x
 
+    @profile
     def support(self, datacode):
 
-        s = 'ctx={}\nid(self)={}\npid={}\nuname={}\nsys.executable={}\nsys.version={}\nhome={}'.format(
+        s = 'ctx={}\nid(self)={}\nversion={}\nfile={}\ncwd={}\nhome={}\nuname={}\npid={}\nsys.executable={}\nsys.version={}'.format(
             self.ctx,
             id(self),
-            os.getpid(),
-            ' '.join(platform.uname()),
-            sys.executable,
-            sys.version.replace("\n", " "),
+            version,
+            os.path.realpath(__file__),
+            os.path.realpath(os.getcwd()),
             str(pathlib.Path.home()),
-            type(datacode),
-            str(datacode))
+            ' '.join(platform.uname()),
+            os.getpid(),
+            sys.executable,
+            sys.version.replace("\n", " "))
 
         if datacode:
             s = '{}\ntype(datacode)={}\nstr(datacode)={}'.format(
@@ -182,6 +210,7 @@ class FinancialsImpl(unohelper.Base, Financials):
 
         return s
 
+
 def createInstance(ctx):
     return FinancialsImpl(ctx)
 

src/generate_metainfo.py

@@ -10,15 +10,23 @@
 #
 
 import os
+
 cur_dir = os.getcwd()
 
 addin_id = "com.financials.getinfo"
-addin_version = "0.0.3"
+addin_version = "1.0.6"
 addin_displayname = "Financial Market Extension"
 addin_publisher_link = "https://github.com/cmallwitz/Financials-Extension"
 addin_publisher_name = "The Publisher"
 
-print ("Generating extension files for version", addin_version)
+print("Generating extension files for version", addin_version)
+
+################################################################################
+# version.py
+
+version_file = open(cur_dir + '/build/version.py', 'w')
+version_file.write(f"version = '{addin_version}'")
+version_file.close()
 
 ################################################################################
 # description.xml
@@ -35,15 +43,17 @@ desc_xml.write('    <l:LibreOffice-minimal-version value="5.0" d:name="LibreOffi
 desc_xml.write('</dependencies> \n')
 desc_xml.write('\n')
 desc_xml.write('<identifier value="' + addin_id + '" /> \n')
-desc_xml.write('<version value="' + addin_version + '" />\n')   
+desc_xml.write('<version value="' + addin_version + '" />\n')
 desc_xml.write('<display-name><name lang="en">' + addin_displayname + '</name></display-name>\n')
-desc_xml.write('<publisher><name xlink:href="' + addin_publisher_link + '" lang="en">' + addin_publisher_name + '</name></publisher>\n')
+desc_xml.write(
+    '<publisher><name xlink:href="' + addin_publisher_link + '" lang="en">' + addin_publisher_name + '</name></publisher>\n')
 desc_xml.write('<extension-description><src xlink:href="description-en-US.txt" lang="en" /></extension-description>')
 desc_xml.write('\n \n')
 desc_xml.write('</description> \n')
 
 desc_xml.close()
 
+
 ################################################################################
 # manifest.xml
 # List of files in package and their types.
@@ -53,9 +63,10 @@ def add_manifest_entry(xml_file, file_type, file_name):
     xml_file.write('<manifest:file-entry manifest:media-type="application/vnd.sun.star.' + file_type + '" \n')
     xml_file.write('    manifest:full-path="' + file_name + '"/> \n')
 
+
 manifest_xml = open(cur_dir + '/build/META-INF/manifest.xml', 'w')
 
-manifest_xml.write('<manifest:manifest>\n');
+manifest_xml.write('<manifest:manifest>\n')
 add_manifest_entry(manifest_xml, 'uno-typelibrary;type=RDB', 'XFinancials.rdb')
 add_manifest_entry(manifest_xml, 'configuration-data', 'Financials.xcu')
 add_manifest_entry(manifest_xml, 'uno-component;type=Python', 'financials.py')
@@ -63,6 +74,7 @@ manifest_xml.write('</manifest:manifest> \n')
 
 manifest_xml.close()
 
+
 ################################################################################
 # Financials.xcu
 
@@ -71,12 +83,13 @@ def define_function(xml_file, function_name, description, parameters):
     xml_file.write('    <prop oor:name="DisplayName"><value xml:lang="en">' + function_name + '</value></prop>\n')
     xml_file.write('    <prop oor:name="Description"><value xml:lang="en">' + description + '</value></prop>\n')
     xml_file.write('    <prop oor:name="Category"><value>Add-In</value></prop>\n')
-    xml_file.write('    <prop oor:name="CompatibilityName"><value xml:lang="en">AutoAddIn.Financials.' + function_name + '</value></prop>\n')
+    xml_file.write(
+        '    <prop oor:name="CompatibilityName"><value xml:lang="en">AutoAddIn.Financials.' + function_name + '</value></prop>\n')
     xml_file.write('    <node oor:name="Parameters">\n')
 
     for p, desc in parameters:
-        # Optional parameters will have a displayname enclosed in square brackets.
-        p_name = p.strip("[]")        
+        # Optional parameters will have a display name enclosed in square brackets.
+        p_name = p.strip("[]")
         xml_file.write('      <node oor:name="' + p_name + '" oor:op="replace">\n')
         xml_file.write('        <prop oor:name="DisplayName"><value xml:lang="en">' + p_name + '</value></prop>\n')
         xml_file.write('        <prop oor:name="Description"><value xml:lang="en">' + desc + '</value></prop>\n')
@@ -85,6 +98,7 @@ def define_function(xml_file, function_name, description, parameters):
     xml_file.write('    </node>\n')
     xml_file.write('  </node>\n')
 
+
 # instance_id references the named UNO component instantiated by Python code (that is my understanding at least).
 implementation_name = "com.financials.getinfo.python.FinancialsImpl"
 
@@ -94,17 +108,19 @@ excel_addin_name = "Financials.xlam"
 financials_xml = open(cur_dir + '/build/Financials.xcu', 'w')
 
 financials_xml.write('<?xml version="1.0" encoding="UTF-8"?>\n')
-financials_xml.write('<oor:component-data xmlns:oor="http://openoffice.org/2001/registry" xmlns:xs="http://www.w3.org/2001/XMLSchema" oor:name="CalcAddIns" oor:package="org.openoffice.Office">\n')
+financials_xml.write(
+    '<oor:component-data xmlns:oor="http://openoffice.org/2001/registry" xmlns:xs="http://www.w3.org/2001/XMLSchema" oor:name="CalcAddIns" oor:package="org.openoffice.Office">\n')
 financials_xml.write('<node oor:name="AddInInfo">\n')
 financials_xml.write('<node oor:name="' + implementation_name + '" oor:op="replace">\n')
 financials_xml.write('<node oor:name="AddInFunctions">\n')
 
-define_function(financials_xml, \
-    'getRealtime', 'Fetches Realtime Financial Data.', \
+define_function(financials_xml,
+                'getRealtime', 'Fetches Realtime Financial Data.',
                 [('ticker', 'The ticker symbol.'), ('datacode', 'The data code.'), ('source', 'The source.')])
-define_function(financials_xml, \
-    'getHistoric', 'Fetches Historic Financial Data.', \
-                [('ticker', 'The ticker symbol.'), ('datacode', 'The data code.'), ('date', 'The date.'), ('source', 'The source.')])
+define_function(financials_xml,
+                'getHistoric', 'Fetches Historic Financial Data.',
+                [('ticker', 'The ticker symbol.'), ('datacode', 'The data code.'), ('date', 'The date.'),
+                 ('source', 'The source.')])
 
 financials_xml.write('</node>\n')
 financials_xml.write('</node>\n')

src/google.py

@@ -8,30 +8,40 @@
 #  version 3 of the License, or (at your option) any later version.
 
 
-import datetime
-import locale
 import html
+import logging
+import os
 import re
-import sys
 import time
 import traceback
+import xml.etree.ElementTree as ET
 
-import urllib.parse
+import dateutil
 
+from baseclient import BaseClient, RedirectException
 from datacode import Datacode
-from baseclient import BaseClient
+from naivehtmlparser import NaiveHTMLParser
+from tz import whois_timezone_info
+
+logger = logging.getLogger(__name__)
 
 
-def log(str):
-    # print(str, file=sys.stderr)
-    pass
+# logger.setLevel(logging.DEBUG)
 
-# TODO migrate to:
-# https://www.google.com/search?q=NYSE:IBM&tbm=fin
-# https://www.google.com/search?q=NASDAQ:INTC&tbm=fin
-# https://www.google.com/search?q=LON:VOD&tbm=fin
-# https://www.google.com/search?q=EURGBP
-# https://www.google.com/search?q=INDEXSP:.INX
+
+def handle_abbreviations(s):
+    s = str(s).strip()
+    if s.endswith('T'):
+        return float(s.replace('T', '')) * 1000
+    if s.endswith('M'):
+        return float(s.replace('M', '')) * 1000000
+    if s.endswith('B'):
+        return float(s.replace('B', '')) * 1000000000
+    return float(s)
+
+
+def un_span(s):
+    return re.sub(r'<span [^>]*>', '', s).replace('</span>', '')
 
 
 class Google(BaseClient):
@@ -39,6 +49,7 @@ class Google(BaseClient):
         super().__init__()
 
         self.realtime = {}
+        self.location = None
 
     def getRealtime(self, ticker: str, datacode: int):
 
@@ -61,101 +72,166 @@ class Google(BaseClient):
             else:
                 del self.realtime[ticker]
 
-        url = 'https://finance.google.com/finance?{}'.format(urllib.parse.urlencode({'q': ticker}))
+        q_param = 'q=' + ticker
+
+        if not self.location:
+            url = 'https://www.google.com/search?hl=en&tbm=fin&' + q_param
+
+            try:
+                self.urlopen(url, redirect=False)
+            except RedirectException as e:
+                self.location = e.location.replace('&' + q_param, '')
+            except BaseException as e:
+                logger.error(traceback.format_exc())
+                return 'Google.getRealtime(\'{}\', {}) - location: {}'.format(ticker, datacode, e)
+
+        if not self.location:
+            url = 'https://www.google.com/search?tbm=fin&' + q_param
+        else:
+            url = self.location + '&' + q_param
 
         try:
             text = self.urlopen(url)
+            with open(os.path.join(self.basedir, 'google-{}.html'.format(ticker)), "w") as text_file:
+                print(f"<!-- '{url}' -->\r\n\r\n{text}", file=text_file)
         except BaseException as e:
-            log(traceback.format_exc())
-            return 'Google.getRealtime(\'{}\', {}) - read: {}'.format(ticker, datacode, e)
+            logger.error(traceback.format_exc())
+            return 'Google.getRealtime(\'{}\', {}) - urlopen: {} {}'.format(ticker, datacode, e, url)
+
+        if ticker not in self.realtime:
+            self.realtime[ticker] = {}
+
+        tick = self.realtime[ticker]
 
         try:
-            r = '<meta\s*itemprop="([^"]+)"\s*content="([^"]+)"\s*/>'
+            r = '<span[^>]+role="heading"[^>]+>(.*?)</span>'
             pattern = re.compile(r)
-            result = re.findall(pattern, text)
 
-            if len(result) == 0:
+            match = pattern.search(text)
+            if not match:
+                return 'Google.getRealtime({}, {}) - no match'.format(ticker, datacode)
+            start = match.span(0)[1]
+
+            tick[Datacode.NAME] = self.save_wrapper(
+                lambda: html.unescape(un_span(match.group(1)).strip()))
+
+            r = '<div [^>]*>(.*?)</div>'
+            pattern = re.compile(r)
+
+            # first div is TICKER
+            match = pattern.search(text, start)
+            if not match:
+                return 'Google.getRealtime({}, {}) - no match'.format(ticker, datacode)
+
+            ticker = self.save_wrapper(
+                lambda: html.unescape(match.group(1)).replace(' ', ''))
+
+            tick[Datacode.EXCHANGE] = self.save_wrapper(lambda: ticker.split(':')[0])
+            tick[Datacode.TICKER] = self.save_wrapper(lambda: ticker.split(':')[1])
+
+        except BaseException as e:
+            return 'Google.getRealtime({}, {}) - process: {}'.format(ticker, datacode, e)
+
+        try:
+            r = '<sticky-header [^>]*>(.*?)</sticky-header>'
+            pattern = re.compile(r, flags=re.DOTALL)
+            match = re.search(pattern, text)
+
+            if match:
+                text = match.group(1)
+            else:
                 return 'Data for \'{}\' not found'.format(ticker)
 
-            if ticker not in self.realtime:
-                self.realtime[ticker] = {}
+            parser = NaiveHTMLParser()
+            root = parser.feed(text)
+            parser.close()
 
-            tick = self.realtime[ticker]
+            cards = root.findall('.//g-card-section')
 
-            for key, value in result:
+            if len(cards) < 4:
+                return 'Data for \'{}\' not found'.format(ticker)
 
-                if key == 'exchangeTimezone':
-                    try:
-                        tick[Datacode.TIMEZONE] = str(value)
-                    except:
-                        pass
+            header = cards[1]
 
-                elif key == 'priceChange':
-                    try:
-                        tick[Datacode.CHANGE] = float(value)
-                    except:
-                        pass
+            tick[Datacode.LAST_PRICE] = self.save_wrapper(
+                lambda: float(
+                    html.unescape(header.find('./span[1]/span[1]/span[1]').text).replace(',', '').strip()))
 
-                elif key == 'quoteTime':
-                    try:
-                        dt = datetime.datetime.strptime(value, "%Y-%m-%dT%H:%M:%SZ")
-                        tick[Datacode.LAST_PRICE_DATE] = dt.date()
-                        tick[Datacode.LAST_PRICE_TIME] = dt.time()
-                    except:
-                        pass
+            tick[Datacode.CURRENCY] = self.save_wrapper(
+                lambda: html.unescape(header.find('./span[1]/span[1]/span[2]').text).strip())
 
-                elif key == 'priceChangePercent':
-                    try:
-                        tick[Datacode.CHANGE_IN_PERCENT] = float(value)
-                    except:
-                        pass
+            tick[Datacode.CHANGE] = self.save_wrapper(
+                lambda: float(
+                    html.unescape(header.find('./span[2]/span[1]').text).replace('−', '-').replace(',', '').strip()))
 
-                elif key == 'price':
-                    try:
-                        locale.setlocale(locale.LC_ALL, 'en_US.UTF-8')
-                        tick[Datacode.LAST_PRICE] = locale.atof(str(value))
-                    except:
-                        pass
+            # percentage is always wrapped in (...) and always positive even if there is a price drop
+            tick[Datacode.CHANGE_IN_PERCENT] = self.save_wrapper(
+                lambda: float(
+                    html.unescape(header.find('./span[2]/span[2]/span[1]').text).strip()
+                        .replace('(', '').replace(')', '').replace('%', '')))
 
-                elif key == 'priceCurrency':
-                    try:
-                        tick[Datacode.CURRENCY] = str(value)
-                    except:
-                        pass
+            try:
+                value = html.unescape(header.find('./div[1]/span[1]/span[2]').text).replace('·', '').strip()
+                logger.debug(value)
+                dt = dateutil.parser.parse(value, tzinfos=whois_timezone_info)
+                tick[Datacode.LAST_PRICE_DATE] = dt.date()
+                tick[Datacode.LAST_PRICE_TIME] = dt.time()
 
-                elif key == 'priceCurrency':
-                    pass
+                time_bits = value.split(' ')
+                if len(time_bits) >= 4:
+                    tick[Datacode.TIMEZONE] = time_bits[-1]
 
-                elif key == 'exchange':
-                    try:
-                        tick[Datacode.EXCHANGE] = str(value)
-                    except:
-                        pass
+            except BaseException as e:
+                pass
 
-                elif key == 'name':
-                    try:
-                        tick[Datacode.NAME] = html.unescape(str(value))
-                    except:
-                        pass
+            footer = cards[3]
+            logger.debug(ET.tostring(footer))
 
-                elif key == 'tickerSymbol':
-                    try:
-                        tick[Datacode.TICKER] = str(value)
-                    except:
-                        pass
+            # parse 'footer' for remaining fields
+            table = footer.find('./div[1]/div[1]/div[1]/table[1]')
 
-                else:
-                    log('ignored {} {}'.format(key, value))
+            tick[Datacode.OPEN] = self.save_wrapper(
+                lambda: float(
+                    html.unescape(table.find('./tr[1]/td[2]').text).replace(',', '').strip()))
+
+            tick[Datacode.HIGH] = self.save_wrapper(
+                lambda: float(
+                    html.unescape(table.find('./tr[2]/td[2]').text).replace(',', '').strip()))
+
+            tick[Datacode.LOW] = self.save_wrapper(
+                lambda: float(
+                    html.unescape(table.find('./tr[3]/td[2]').text).replace(',', '').strip()))
+
+            tick[Datacode.MARKET_CAP] = self.save_wrapper(
+                lambda: handle_abbreviations(
+                    html.unescape(table.find('./tr[4]/td[2]').text).replace(',', '').replace('-', '').strip()))
+
+            table = footer.find('./div[1]/div[1]/div[2]/table[1]')
+
+            # for indices: first item on right side is LOW
+            if html.unescape(table.find('./tr[1]/td[1]').text).strip() == 'Low':
+                tick[Datacode.LOW] = self.save_wrapper(
+                    lambda: float(
+                        html.unescape(table.find('./tr[1]/td[2]').text).replace(',', '').strip()))
+
+            tick[Datacode.PREV_CLOSE] = self.save_wrapper(
+                lambda: float(
+                    html.unescape(table.find('./tr[2]/td[2]').text).replace(',', '').strip()))
+
+            tick[Datacode.HIGH_52_WEEK] = self.save_wrapper(
+                lambda: float(
+                    html.unescape(table.find('./tr[3]/td[2]').text).replace(',', '').strip()))
+
+            tick[Datacode.LOW_52_WEEK] = self.save_wrapper(
+                lambda: float(
+                    html.unescape(table.find('./tr[4]/td[2]').text).replace(',', '').strip()))
 
             tick[Datacode.TIMESTAMP] = time.time()
 
-            if tick[Datacode.EXCHANGE] == 'CURRENCY' and Datacode.CURRENCY not in tick:
-                tick[Datacode.CURRENCY] = ''
-
-            log(tick)
+            logger.info(tick)
 
         except BaseException as e:
-            log(traceback.format_exc())
+            logger.warning(traceback.format_exc())
             return 'Google.getRealtime({}, {}) - process: {}'.format(ticker, datacode, e)
 
         return self._return_value(self.realtime[ticker], datacode)

src/naivehtmlparser.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python
+
+# https://github.com/marmelo/python-htmlparser - revision cbe9633 on 25 Dec 2013
+# Copyright by Rafael Marmelo
+
+"""
+Python 3.x HTMLParser extension with ElementTree support.
+"""
+
+from html.parser import HTMLParser
+from xml.etree import ElementTree
+
+
+class NaiveHTMLParser(HTMLParser):
+    """
+    Python 3.x HTMLParser extension with ElementTree support.
+    @see https://github.com/marmelo/python-htmlparser
+    """
+
+    def __init__(self):
+        self.root = None
+        self.tree = []
+        HTMLParser.__init__(self)
+
+    def feed(self, data):
+        HTMLParser.feed(self, data)
+        return self.root
+
+    def handle_starttag(self, tag, attrs):
+        if len(self.tree) == 0:
+            element = ElementTree.Element(tag, dict(self.__filter_attrs(attrs)))
+            self.tree.append(element)
+            self.root = element
+        else:
+            element = ElementTree.SubElement(self.tree[-1], tag, dict(self.__filter_attrs(attrs)))
+            self.tree.append(element)
+
+    def handle_endtag(self, tag):
+        self.tree.pop()
+
+    def handle_startendtag(self, tag, attrs):
+        self.handle_starttag(tag, attrs)
+        self.handle_endtag(tag)
+        pass
+
+    def handle_data(self, data):
+        if self.tree:
+            self.tree[-1].text = data
+
+    def get_root_element(self):
+        return self.root
+
+    def __filter_attrs(self, attrs):
+        return filter(lambda x: x[0] and x[1], attrs) if attrs else []
+
+
+# example usage
+if __name__ == "__main__":
+
+    html = """
+    <html>
+      <head>
+        <title>GitHub</title>
+      </head>
+      <body>
+        <a href="https://github.com/marmelo">GitHub</a>
+        <a href="https://github.com/marmelo/python-htmlparser">GitHub Project</a>
+      </body>
+    </html>
+    """
+
+    parser = NaiveHTMLParser()
+    root = parser.feed(html)
+    parser.close()
+
+    # root is an xml.etree.Element and supports the ElementTree API
+    # (e.g. you may use its limited support for XPath expressions)
+
+    # get title
+    print(root.find('head/title').text)
+
+    # get all anchors
+    for a in root.findall('.//a'):
+        print(a.get('href'))
+
+    # for more information, see:
+    # http://docs.python.org/2/library/xml.etree.elementtree.html
+    # http://docs.python.org/2/library/xml.etree.elementtree.html#xpath-support

src/test_google.py

@@ -7,6 +7,9 @@
 #  License as published by the Free Software Foundation; either
 #  version 3 of the License, or (at your option) any later version.
 
+import argparse
+import logging
+import sys
 import unittest
 
 import financials
@@ -14,42 +17,48 @@ from datacode import Datacode
 
 financials = financials.createInstance(None)
 
+logging.basicConfig(level=logging.ERROR)
 
-class TestGoogle(unittest.TestCase):
+
+class Test(unittest.TestCase):
 
     def test_currency(self):
         s = financials.getRealtime('EURGBP', Datacode.LAST_PRICE.value, 'GOOGLE')
-        self.assertEqual(type(s), float, 'test_currency LAST_PRICE')
+        self.assertEqual('Google.getRealtime(EURGBP, 21) - no match', s, 'test_currency LAST_PRICE')
 
-        s = financials.getRealtime('EURGBP', Datacode.CURRENCY.value, 'GOOGLE')
-        self.assertEqual(type(s), str, 'test_currency CURRENCY')
-        self.assertEqual(s, '', 'test_currency CURRENCY')
+        # s = financials.getRealtime('EURGBP', Datacode.CURRENCY.value, 'GOOGLE')
+        # self.assertEqual(type(s), str, 'test_currency CURRENCY')
+        # self.assertEqual(s, '', 'test_currency CURRENCY')
 
     def test_UK_equity(self):
-        s = financials.getRealtime('EURGBP', Datacode.LAST_PRICE.value, 'GOOGLE')
-        self.assertEqual(type(s), float, 'test_UK_equity LAST_PRICE')
-
         s = financials.getRealtime('LON:VOD', Datacode.LAST_PRICE.value, 'GOOGLE')
-        self.assertEqual(type(s), float, 'test_UK_equity LAST_PRICE')
+        self.assertEqual(type(s), float, 'test_UK_equity LAST_PRICE {}'.format(s))
 
-        s = financials.getRealtime('VOD.L', Datacode.LAST_PRICE.value, 'GOOGLE')
-        self.assertEqual(type(s), float, 'test_UK_equity LAST_PRICE')
-
-        s = financials.getRealtime('VOD.L', Datacode.TICKER.value, 'GOOGLE')
+        s = financials.getRealtime('LON:VOD', Datacode.TICKER.value, 'GOOGLE')
         self.assertEqual(s, 'VOD', 'test_UK_equity TICKER')
 
-        s = financials.getRealtime('VOD.L', Datacode.NAME.value, 'GOOGLE')
+        s = financials.getRealtime('LON:VOD', Datacode.NAME.value, 'GOOGLE')
         self.assertEqual(type(s), str, 'test_UK_equity NAME')
 
+        s = financials.getRealtime('LON:VOD', Datacode.EXCHANGE.value, 'GOOGLE')
+        self.assertEqual(s, 'LON', 'test_UK_equity EXCHANGE')
+
+        s = financials.getRealtime('LON:VOD', Datacode.PREV_CLOSE.value, 'GOOGLE')
+        self.assertEqual(type(s), float, 'test_UK_equity PREV_CLOSE {}'.format(s))
+
+        # MARKET_CAP missing for UK stock but available for German stock - weekend issue (FX) ?
+        s = financials.getRealtime('LON:VOD', Datacode.MARKET_CAP.value, 'GOOGLE')
+        self.assertEqual(type(s), float, 'test_UK_equity MARKET_CAP {}'.format(s))
+
     def test_UK_ETF(self):
         s = financials.getRealtime('LON:CSP1', Datacode.LAST_PRICE.value, 'GOOGLE')
-        self.assertEqual(type(s), float, 'test_UK_ETF LAST_PRICE')
+        self.assertEqual(type(s), float, 'test_UK_ETF LAST_PRICE {}'.format(s))
 
         s = financials.getRealtime('LON:CSP1', Datacode.CURRENCY.value, 'GOOGLE')
         self.assertEqual(s, 'GBX', 'test_UK_ETF CURRENCY')
 
         s = financials.getRealtime('LON:FTAL', Datacode.LAST_PRICE.value, 'GOOGLE')
-        self.assertEqual(type(s), float, 'test_UK_ETF LAST_PRICE')
+        self.assertEqual(type(s), float, 'test_UK_ETF LAST_PRICE {}'.format(s))
 
         s = financials.getRealtime('LON:FTAL', Datacode.CURRENCY.value, 'GOOGLE')
         self.assertEqual(s, 'GBP', 'test_UK_ETF CURRENCY')
@@ -83,30 +92,56 @@ class TestGoogle(unittest.TestCase):
         self.assertEqual(type(s), float, 'test_DE_equity \'21\'')
 
         s = financials.getRealtime('FRA:SAP', Datacode.TIMEZONE.value, 'GOOGLE')
-        self.assertEqual(s, 'Europe/Berlin', 'test_DE_equity TIMEZONE')
+        # self.assertEqual(s, 'Europe/Berlin', 'test_DE_equity TIMEZONE')
+        self.assertTrue(s == 'CET' or s == 'CEST', 'test_DE_equity TIMEZONE: {}'.format(s))
 
     def test_DE_ETF(self):
         s = financials.getRealtime('FRA:C060', Datacode.LAST_PRICE.value, 'GOOGLE')
-        self.assertEqual(type(s), float, 'test_DE_ETF LAST_PRICE')
+        self.assertEqual(type(s), float, 'test_DE_ETF LAST_PRICE {}'.format(s))
 
         s = financials.getRealtime('FRA:C060', Datacode.CURRENCY.value, 'GOOGLE')
         self.assertEqual(s, 'EUR', 'test_DE_ETF CURRENCY')
 
-        s = financials.getRealtime('C060.de', Datacode.LAST_PRICE.value, 'GOOGLE')
-        self.assertEqual(type(s), float, 'test_DE_ETF LAST_PRICE')
+        s = financials.getRealtime('FRA:C060', Datacode.TICKER.value, 'GOOGLE')
+        self.assertEqual(s, 'C060', 'test_DE_ETF TICKER')
 
-        s = financials.getRealtime('C060.de', Datacode.TICKER.value, 'GOOGLE')
-        self.assertEqual(s, 'C060', 'test_DE_ETF CURRENCY')
+        s = financials.getRealtime('FRA:C060', Datacode.EXCHANGE.value, 'GOOGLE')
+        self.assertEqual(s, 'FRA', 'test_DE_ETF EXCHANGE')
 
-        s = financials.getRealtime('C060.de', Datacode.EXCHANGE.value, 'GOOGLE')
-        self.assertEqual(s, 'FRA', 'test_DE_ETF CURRENCY')
-
-        s = financials.getRealtime('C060.de', Datacode.CURRENCY.value, 'GOOGLE')
+        s = financials.getRealtime('FRA:C060', Datacode.CURRENCY.value, 'GOOGLE')
         self.assertEqual(s, 'EUR', 'test_DE_ETF CURRENCY')
 
+        s = financials.getRealtime('FRA:C060', Datacode.MARKET_CAP.value, 'GOOGLE')
+        self.assertEqual(s, 'Data doesn\'t exist - 27', 'test_DE_ETF TIMESTAMP {}'.format(s))
+
+    def test_TY_equity(self):
+        s = financials.getRealtime('TYO:6503', Datacode.OPEN.value, 'GOOGLE')
+        self.assertEqual(type(s), float, 'test_TY_equity OPEN {}'.format(s))
+
+        s = financials.getRealtime('TYO:6503', Datacode.LOW.value, 'GOOGLE')
+        self.assertEqual(type(s), float, 'test_TY_equity LOW {}'.format(s))
+
+        s = financials.getRealtime('TYO:6503', Datacode.HIGH.value, 'GOOGLE')
+        self.assertEqual(type(s), float, 'test_TY_equity HIGH {}'.format(s))
+
+        s = financials.getRealtime('TYO:6503', Datacode.LOW_52_WEEK.value, 'GOOGLE')
+        self.assertEqual(type(s), float, 'test_TY_equity LOW_52_WEEK {}'.format(s))
+
+        s = financials.getRealtime('TYO:6503', Datacode.HIGH_52_WEEK.value, 'GOOGLE')
+        self.assertEqual(type(s), float, 'test_TY_equity HIGH_52_WEEK {}'.format(s))
+
+        s = financials.getRealtime('TYO:6503', Datacode.MARKET_CAP.value, 'GOOGLE')
+        self.assertEqual(type(s), float, 'test_TY_equity MARKET_CAP {}'.format(s))
+
+        # s = financials.getRealtime('TYO:6503', Datacode.VOLUME.value, 'GOOGLE')
+        # self.assertEqual(type(s), float, 'test_TY_equity VOLUME {}'.format(s))
+
+        s = financials.getRealtime('TYO:6503', Datacode.CURRENCY.value, 'GOOGLE')
+        self.assertEqual(s, 'JPY', 'test_TY_equity CURRENCY')
+
     def test_US_equity(self):
         s = financials.getRealtime(' NASDAQ : AAPL ', Datacode.LAST_PRICE.value, 'GOOGLE')
-        self.assertEqual(type(s), float, 'test_US_equity LAST_PRICE')
+        self.assertEqual(type(s), float, 'test_US_equity LAST_PRICE {}'.format(s))
 
         s = financials.getRealtime(' NASDAQ : AAPL ', Datacode.TICKER.value, 'GOOGLE')
         self.assertEqual(s, 'AAPL', 'test_US_equity TICKER')
@@ -118,7 +153,7 @@ class TestGoogle(unittest.TestCase):
         self.assertEqual(s, 'USD', 'test_US_equity CURRENCY')
 
         s = financials.getRealtime('NYSE:IBM', Datacode.LAST_PRICE.value, 'GOOGLE')
-        self.assertEqual(type(s), float, 'test_US_equity LAST_PRICE')
+        self.assertEqual(type(s), float, 'test_US_equity LAST_PRICE {}'.format(s))
 
         s = financials.getRealtime('NYSE:IBM', Datacode.TICKER.value, 'GOOGLE')
         self.assertEqual(s, 'IBM', 'test_US_equity TICKER')
@@ -131,23 +166,63 @@ class TestGoogle(unittest.TestCase):
 
         s = financials.getRealtime('NYSE:IBM', Datacode.NAME.value, 'GOOGLE')
         self.assertEqual(type(s), str, 'test_US_equity NAME')
+        self.assertEqual(s, 'IBM Common Stock', 'test_US_equity NAME')
 
-        s = financials.getRealtime('NYSE:IBM', Datacode.NAME.value, 'GOOGLE')
-        self.assertEqual(type(s), str, 'test_US_equity NAME')
+        s = financials.getRealtime('NYSE:IBM', Datacode.LOW.value, 'GOOGLE')
+        self.assertEqual(type(s), float, 'test_US_equity LOW {}'.format(s))
+
+        s = financials.getRealtime('NYSE:IBM', Datacode.HIGH.value, 'GOOGLE')
+        self.assertEqual(type(s), float, 'test_US_equity HIGH {}'.format(s))
+
+        s = financials.getRealtime('NYSE:IBM', Datacode.LOW_52_WEEK.value, 'GOOGLE')
+        self.assertEqual(type(s), float, 'test_US_equity LOW_52_WEEK {}'.format(s))
+
+        s = financials.getRealtime('NYSE:IBM', Datacode.HIGH_52_WEEK.value, 'GOOGLE')
+        self.assertEqual(type(s), float, 'test_US_equity HIGH_52_WEEK {}'.format(s))
+
+        s = financials.getRealtime('NYSE:IBM', Datacode.MARKET_CAP.value, 'GOOGLE')
+        self.assertEqual(type(s), float, 'test_US_equity MARKET_CAP {}'.format(s))
+
+        # s = financials.getRealtime('NYSE:IBM', Datacode.VOLUME.value, 'GOOGLE')
+        # self.assertEqual(type(s), float, 'test_US_equity VOLUME {}'.format(s))
 
         s = financials.getRealtime('NYSE:IBM', Datacode.TIMESTAMP.value, 'GOOGLE')
         self.assertEqual(s, 'Data doesn\'t exist - 999', 'test_US_equity TIMESTAMP')
 
         s = financials.getRealtime('NYSE:IBM', Datacode.TIMEZONE.value, 'GOOGLE')
-        self.assertEqual(s, 'America/New_York', 'test_US_equity TIMEZONE')
+        # self.assertEqual(s, 'America/New_York', 'test_US_equity TIMEZONE')
+        self.assertEqual(s, 'GMT-4', 'test_US_equity TIMEZONE')
+        # self.assertEqual(s, 'GMT-5', 'test_US_equity TIMEZONE')
 
     def test_US_mutuals(self):
         s = financials.getRealtime('MUTF:VFIAX', Datacode.LAST_PRICE.value, 'GOOGLE')
-        self.assertEqual(type(s), float, 'test_US_mutuals LAST_PRICE')
+        self.assertEqual(type(s), float, 'test_US_mutuals LAST_PRICE - {}'.format(s))
 
         s = financials.getRealtime('MUTF:VFIAX', Datacode.CURRENCY.value, 'GOOGLE')
         self.assertEqual(s, 'USD', 'test_US_mutuals CURRENCY')
 
+        s = financials.getRealtime('MUTF:VFIAX', Datacode.TIMEZONE.value, 'GOOGLE')
+        self.assertEqual(s, 'Data doesn\'t exist - 105', 'test_US_mutuals')
+
+    def test_index(self):
+        s = financials.getRealtime('INDEXDB:DAX', Datacode.LAST_PRICE.value, 'GOOGLE')
+        self.assertEqual(type(s), float, 'test_index LAST_PRICE {}'.format(s))
+
+        s = financials.getRealtime('INDEXDB:DAX', Datacode.CHANGE_IN_PERCENT.value, 'GOOGLE')
+        self.assertEqual(type(s), float, 'test_index CHANGE_IN_PERCENT')
+
+        s = financials.getRealtime('INDEXDB:DAX', Datacode.CHANGE.value, 'GOOGLE')
+        self.assertEqual(type(s), float, 'test_index CHANGE')
+
+        s = financials.getRealtime('INDEXDB:DAX', Datacode.OPEN.value, 'GOOGLE')
+        self.assertEqual(type(s), float, 'test_index OPEN')
+
+        s = financials.getRealtime('INDEXDB:DAX', Datacode.LOW.value, 'GOOGLE')
+        self.assertEqual(type(s), float, 'test_index LOW')
+
+        s = financials.getRealtime('INDEXDB:DAX', Datacode.HIGH.value, 'GOOGLE')
+        self.assertEqual(type(s), float, 'test_index HIGH')
+
     def test_errors(self):
         s = financials.getRealtime(None, Datacode.LAST_PRICE.value, 'GOOGLE')
         self.assertEqual(s, 'Ticker is empty', 'test_errors')
@@ -156,7 +231,7 @@ class TestGoogle(unittest.TestCase):
         self.assertEqual(s, 'Datacode is empty', 'test_errors')
 
         s = financials.getRealtime('DOES_NOT_EXISTS', Datacode.LAST_PRICE.value, 'GOOGLE')
-        self.assertEqual(s, 'Data for \'DOES_NOT_EXISTS\' not found', 'test_errors')
+        self.assertEqual(s, 'Google.getRealtime(DOES_NOT_EXISTS, 21) - no match', 'test_errors')
 
         s = financials.getRealtime('NYS:IBM', 'Foo', 'GOOGLE')
         self.assertEqual(s, 'Datacode is not a number', 'test_errors')
@@ -164,7 +239,7 @@ class TestGoogle(unittest.TestCase):
         # Historic data not supported on GOOGLE
 
         s = financials.getHistoric('NYS:IBM', Datacode.LAST_PRICE.value, '2017-01-01', 'GOOGLE')
-        self.assertEqual(s, 'getHistoric: Source \'GOOGLE\' not supported', 'test_errors')
+        self.assertEqual(s, 'Source \'GOOGLE\' not supported', 'test_errors')
 
     def test_errors_cell_range_passed(self):
         cell_range = ((1, 2), ('3', '4'), (5.0, 6.0))
@@ -206,4 +281,8 @@ class TestGoogle(unittest.TestCase):
 
 
 if __name__ == '__main__':
-    unittest.main()
+    parser = argparse.ArgumentParser()
+    parser.add_argument('unittest_args', nargs='*')
+    args = parser.parse_args()
+    unit_argv = [sys.argv[0]] + args.unittest_args
+    unittest.main(argv=unit_argv)

src/test_yahoo.py

@@ -7,8 +7,11 @@
 #  License as published by the Free Software Foundation; either
 #  version 3 of the License, or (at your option) any later version.
 
+import argparse
+import logging
 import os
 import pathlib
+import sys
 import unittest
 
 import financials
@@ -16,8 +19,18 @@ from datacode import Datacode
 
 financials = financials.createInstance(None)
 
+logging.basicConfig(level=logging.ERROR)
+
+
+class Test(unittest.TestCase):
+
+    def test_currency(self):
+        s = financials.getRealtime('EURGBP=X', Datacode.CURRENCY.value, 'YAHOO')
+        self.assertEqual(type(s), str, 'test_currency CURRENCY')
+
+        s = financials.getRealtime('EURGBP=X', Datacode.LAST_PRICE.value, 'YAHOO')
+        self.assertEqual(type(s), float, 'test_currency LAST_PRICE')
 
-class TestYahoo(unittest.TestCase):
     def test_realtime_US_equity(self):
 
         s = financials.getRealtime('^GSPC', Datacode.NAME.value, 'YAHOO')
@@ -39,6 +52,15 @@ class TestYahoo(unittest.TestCase):
         s = financials.getRealtime('IBM', Datacode.HIGH.value, 'YAHOO')
         self.assertEqual(type(s), float, 'test_realtime_US_equity HIGH {}'.format(s))
 
+        s = financials.getRealtime('IBM', Datacode.HIGH_52_WEEK.value, 'YAHOO')
+        self.assertEqual(type(s), float, 'test_realtime_US_equity HIGH_52_WEEK {}'.format(s))
+
+        s = financials.getRealtime('IBM', Datacode.LOW_52_WEEK.value, 'YAHOO')
+        self.assertEqual(type(s), float, 'test_realtime_US_equity LOW_52_WEEK {}'.format(s))
+
+        s = financials.getRealtime('IBM', Datacode.MARKET_CAP.value, 'YAHOO')
+        self.assertEqual(type(s), float, 'test_realtime_US_equity MARKET_CAP {}'.format(s))
+
         s = financials.getRealtime('IBM', Datacode.VOLUME.value, 'YAHOO')
         self.assertEqual(type(s), float, 'test_realtime_US_equity VOLUME {}'.format(s))
 
@@ -47,12 +69,17 @@ class TestYahoo(unittest.TestCase):
 
         s = financials.getRealtime('IBM', Datacode.NAME.value, 'YAHOO')
         self.assertEqual(type(s), str, 'test_realtime_US_equity NAME {}'.format(s))
+        self.assertEqual(s, 'International Business Machines Corporation',
+                         'test_realtime_US_equity NAME {}'.format(s))
 
         s = financials.getRealtime('IBM', Datacode.TIMEZONE.value, 'YAHOO')
         self.assertEqual(s, 'America/New_York', 'test_realtime_US_equity TIMEZONE {}'.format(s))
 
     def test_realtime_US_mutuals(self):
 
+        s = financials.getRealtime('VGSLX', Datacode.LAST_PRICE.value, 'YAHOO')
+        self.assertEqual(type(s), float, 'test_realtime_US_mutuals LAST_PRICE {}'.format(s))
+
         s = financials.getRealtime('VFIAX', Datacode.LAST_PRICE.value, 'YAHOO')
         self.assertEqual(type(s), float, 'test_realtime_US_mutuals LAST_PRICE {}'.format(s))
 
@@ -75,6 +102,9 @@ class TestYahoo(unittest.TestCase):
         self.assertEqual(s, 'iShares VII Public Limited Company - iShares Core S&P 500 UCITS ETF',
                          'test_realtime_UK_ETF NAME {}'.format(s))
 
+        s = financials.getRealtime('C060.DE', 104, 'YAHOO')
+        self.assertEqual(type(s), str, 't_realtime_UK_ETF AME {}'.format(s))
+
     def test_realtime_DE_equity(self):
 
         s = financials.getRealtime('SAP.DE', Datacode.LAST_PRICE.value, 'YAHOO')
@@ -114,8 +144,9 @@ class TestYahoo(unittest.TestCase):
         s = financials.getHistoric('IBM', Datacode.CLOSE.value, '2017-01-03', 'YAHOO')
         self.assertEqual(s, 167.190002, 'test_historic_US_equity CLOSE {}'.format(s))
 
+        # Note: quarterly dividend and splits will change past adjusted prices - will fail after the next dividend
         s = financials.getHistoric('IBM', Datacode.ADJ_CLOSE.value, '2017-01-03', 'YAHOO')
-        self.assertEqual(s, 160.947433, 'test_historic_US_equity ADJ_CLOSE {}'.format(s))
+        self.assertEqual(s, 145.416626, 'test_historic_US_equity ADJ_CLOSE {}'.format(s))
 
     def test_historic_UK_ETF(self):
 
@@ -129,8 +160,8 @@ class TestYahoo(unittest.TestCase):
         financials.yahoo.historicdata = {}
 
         #  Inception Date 2014-09-30
-        s = financials.getHistoric('VERX.L', Datacode.CLOSE.value, '2014-01-06', 'YAHOO')
-        self.assertEqual(s, 'Not a trading day \'2014-01-06\'', 'test_historic_UK_ETF CLOSE {}'.format(s))
+        s = financials.getHistoric('VERX.L', Datacode.CLOSE.value, '2018-04-02', 'YAHOO') # Easter Monday
+        self.assertEqual(s, 'Not a trading day \'2018-04-02\'', 'test_historic_UK_ETF CLOSE {}'.format(s))
 
         #  Inception Date 2014-09-30
         s = financials.getHistoric('VERX.L', Datacode.CLOSE.value, '2015-01-01', 'YAHOO')
@@ -146,8 +177,8 @@ class TestYahoo(unittest.TestCase):
         self.assertEqual(s, 22.26, 'test_historic_UK_ETF CLOSE {}'.format(s))
 
         #  Inception Date 2014-09-30
-        s = financials.getHistoric('VERX.L', Datacode.CLOSE.value, '2014-01-06', 'YAHOO')
-        self.assertEqual(s, 'Not a trading day \'2014-01-06\'', 'test_historic_UK_ETF CLOSE {}'.format(s))
+        s = financials.getHistoric('VERX.L', Datacode.CLOSE.value, '2018-04-02', 'YAHOO')
+        self.assertEqual(s, 'Not a trading day \'2018-04-02\'', 'test_historic_UK_ETF CLOSE {}'.format(s))
 
         #  Inception Date 2014-09-30
         s = financials.getHistoric('VERX.L', Datacode.CLOSE.value, '2015-01-01', 'YAHOO')
@@ -173,31 +204,39 @@ class TestYahoo(unittest.TestCase):
         s = financials.getHistoric('C060.DE', Datacode.CLOSE.value, '2017-01-03', 'YAHOO')
         self.assertEqual(s, 72.870003, 'test_historic_DE_equity CLOSE {}'.format(s))
 
-    def test_errors(self):
+    def test_realtime_errors(self):
+
+        s = financials.getRealtime('NO_NAME', Datacode.LAST_PRICE.value, 'YAHOO')
+        self.assertIsNone(s, 'test_realtime_errors LAST_PRICE {}'.format(s))
+
+    def test_historic_errors(self):
+
+        s = financials.getHistoric('NO_NAME', Datacode.LAST_PRICE.value, '2018-01-08', 'YAHOO')
+        self.assertIsNone(s, 'test_historic_errors LAST_PRICE {}'.format(s))
 
         s = financials.getHistoric('IBM', Datacode.CLOSE.value, '2030-01-01', 'YAHOO')
-        self.assertEqual(s, 'Future date \'2030-01-01\'', 'test_errors CLOSE {}'.format(s))
+        self.assertEqual(s, 'Future date \'2030-01-01\'', 'test_historic_errors CLOSE {}'.format(s))
 
         s = financials.getRealtime('IBM', 9999, 'YAHOO')
-        self.assertEqual(s, 'Datacode 9999 not supported', 'test_errors 9999')
+        self.assertEqual(s, 'Datacode 9999 not supported', 'test_historic_errors 9999')
 
         s = financials.getRealtime('IBM', Datacode.ADJ_CLOSE.value, 'YAHOO')
-        self.assertEqual(s, 'Data doesn\'t exist - 91', 'test_errors ADJ_CLOSE {}'.format(s))
+        self.assertEqual(s, 'Data doesn\'t exist - 91', 'test_historic_errors ADJ_CLOSE {}'.format(s))
 
         s = financials.getHistoric('IBM', Datacode.CLOSE.value, '2030-01-01', 'YAHOO')
-        self.assertEqual(s, 'Future date \'2030-01-01\'', 'test_errors CLOSE {}'.format(s))
+        self.assertEqual(s, 'Future date \'2030-01-01\'', 'test_historic_errors CLOSE {}'.format(s))
 
         s = financials.getHistoric('IBM', Datacode.CLOSE.value, '1990-01-01', 'YAHOO')
-        self.assertEqual(s, 'Date before 2000 \'1990-01-01\'', 'test_errors CLOSE {}'.format(s))
+        self.assertEqual(s, 'Date before 2000 \'1990-01-01\'', 'test_historic_errors CLOSE {}'.format(s))
 
         s = financials.getHistoric('IBM', Datacode.CLOSE.value, 'abcdef', 'YAHOO')
-        self.assertEqual(s, 'Date format not supported: \'abcdef\'', 'test_errors CLOSE {}'.format(s))
+        self.assertEqual(s, 'Date format not supported: \'abcdef\'', 'test_historic_errors CLOSE {}'.format(s))
 
         s = financials.getHistoric('IBM', Datacode.CLOSE.value, True, 'YAHOO')
-        self.assertEqual(s, 'Date type not supported: <class \'bool\'> \'True\'', 'test_errors CLOSE {}'.format(s))
+        self.assertEqual(s, 'Date type not supported: <class \'bool\'> \'True\'', 'test_historic_errors CLOSE {}'.format(s))
 
         s = financials.getHistoric('IBM', Datacode.CLOSE.value, -1000000, 'YAHOO')
-        self.assertEqual(s, 'Date format not supported: -1000000', 'test_errors CLOSE {}'.format(s))
+        self.assertEqual(s, 'Date format not supported: -1000000', 'test_historic_errors CLOSE {}'.format(s))
 
     def test_errors_cell_range_passed(self):
         cell_range = ((1, 2), ('3', '4'), (5.0, 6.0))
@@ -216,4 +255,8 @@ class TestYahoo(unittest.TestCase):
 
 
 if __name__ == '__main__':
-    unittest.main()
+    parser = argparse.ArgumentParser()
+    parser.add_argument('unittest_args', nargs='*')
+    args = parser.parse_args()
+    unit_argv = [sys.argv[0]] + args.unittest_args
+    unittest.main(argv=unit_argv)

src/tz.py

@@ -0,0 +1,232 @@
+
+# Copied from https://gist.github.com/h-j-13/e3a585796510b59601e34a07e99b386d
+
+whois_timezone_info = {
+        "A": 1 * 3600,
+        "ACDT": 10.5 * 3600,
+        "ACST": 9.5 * 3600,
+        "ACT": -5 * 3600,
+        "ACWST": 8.75 * 3600,
+        "ADT": 4 * 3600,
+        "AEDT": 11 * 3600,
+        "AEST": 10 * 3600,
+        "AET": 10 * 3600,
+        "AFT": 4.5 * 3600,
+        "AKDT": -8 * 3600,
+        "AKST": -9 * 3600,
+        "ALMT": 6 * 3600,
+        "AMST": -3 * 3600,
+        "AMT": -4 * 3600,
+        "ANAST": 12 * 3600,
+        "ANAT": 12 * 3600,
+        "AQTT": 5 * 3600,
+        "ART": -3 * 3600,
+        "AST": 3 * 3600,
+        "AT": -4 * 3600,
+        "AWDT": 9 * 3600,
+        "AWST": 8 * 3600,
+        "AZOST": 0 * 3600,
+        "AZOT": -1 * 3600,
+        "AZST": 5 * 3600,
+        "AZT": 4 * 3600,
+        "AoE": -12 * 3600,
+        "B": 2 * 3600,
+        "BNT": 8 * 3600,
+        "BOT": -4 * 3600,
+        "BRST": -2 * 3600,
+        "BRT": -3 * 3600,
+        "BST": 6 * 3600,
+        "BTT": 6 * 3600,
+        "C": 3 * 3600,
+        "CAST": 8 * 3600,
+        "CAT": 2 * 3600,
+        "CCT": 6.5 * 3600,
+        "CDT": -5 * 3600,
+        "CEST": 2 * 3600,
+        "CET": 1 * 3600,
+        "CHADT": 13.75 * 3600,
+        "CHAST": 12.75 * 3600,
+        "CHOST": 9 * 3600,
+        "CHOT": 8 * 3600,
+        "CHUT": 10 * 3600,
+        "CIDST": -4 * 3600,
+        "CIST": -5 * 3600,
+        "CKT": -10 * 3600,
+        "CLST": -3 * 3600,
+        "CLT": -4 * 3600,
+        "COT": -5 * 3600,
+        "CST": -6 * 3600,
+        "CT": -6 * 3600,
+        "CVT": -1 * 3600,
+        "CXT": 7 * 3600,
+        "ChST": 10 * 3600,
+        "D": 4 * 3600,
+        "DAVT": 7 * 3600,
+        "DDUT": 10 * 3600,
+        "E": 5 * 3600,
+        "EASST": -5 * 3600,
+        "EAST": -6 * 3600,
+        "EAT": 3 * 3600,
+        "ECT": -5 * 3600,
+        "EDT": -4 * 3600,
+        "EEST": 3 * 3600,
+        "EET": 2 * 3600,
+        "EGST": 0 * 3600,
+        "EGT": -1 * 3600,
+        "EST": -5 * 3600,
+        "ET": -5 * 3600,
+        "F": 6 * 3600,
+        "FET": 3 * 3600,
+        "FJST": 13 * 3600,
+        "FJT": 12 * 3600,
+        "FKST": -3 * 3600,
+        "FKT": -4 * 3600,
+        "FNT": -2 * 3600,
+        "G": 7 * 3600,
+        "GALT": -6 * 3600,
+        "GAMT": -9 * 3600,
+        "GET": 4 * 3600,
+        "GFT": -3 * 3600,
+        "GILT": 12 * 3600,
+        "GMT": 0 * 3600,
+        "GST": 4 * 3600,
+        "GYT": -4 * 3600,
+        "H": 8 * 3600,
+        "HDT": -9 * 3600,
+        "HKT": 8 * 3600,
+        "HOVST": 8 * 3600,
+        "HOVT": 7 * 3600,
+        "HST": -10 * 3600,
+        "I": 9 * 3600,
+        "ICT": 7 * 3600,
+        "IDT": 3 * 3600,
+        "IOT": 6 * 3600,
+        "IRDT": 4.5 * 3600,
+        "IRKST": 9 * 3600,
+        "IRKT": 8 * 3600,
+        "IRST": 3.5 * 3600,
+        "IST": 5.5 * 3600,
+        "JST": 9 * 3600,
+        "K": 10 * 3600,
+        "KGT": 6 * 3600,
+        "KOST": 11 * 3600,
+        "KRAST": 8 * 3600,
+        "KRAT": 7 * 3600,
+        "KST": 9 * 3600,
+        "KUYT": 4 * 3600,
+        "L": 11 * 3600,
+        "LHDT": 11 * 3600,
+        "LHST": 10.5 * 3600,
+        "LINT": 14 * 3600,
+        "M": 12 * 3600,
+        "MAGST": 12 * 3600,
+        "MAGT": 11 * 3600,
+        "MART": 9.5 * 3600,
+        "MAWT": 5 * 3600,
+        "MDT": -6 * 3600,
+        "MHT": 12 * 3600,
+        "MMT": 6.5 * 3600,
+        "MSD": 4 * 3600,
+        "MSK": 3 * 3600,
+        "MST": -7 * 3600,
+        "MT": -7 * 3600,
+        "MUT": 4 * 3600,
+        "MVT": 5 * 3600,
+        "MYT": 8 * 3600,
+        "N": -1 * 3600,
+        "NCT": 11 * 3600,
+        "NDT": 2.5 * 3600,
+        "NFT": 11 * 3600,
+        "NOVST": 7 * 3600,
+        "NOVT": 7 * 3600,
+        "NPT": 5.5 * 3600,
+        "NRT": 12 * 3600,
+        "NST": 3.5 * 3600,
+        "NUT": -11 * 3600,
+        "NZDT": 13 * 3600,
+        "NZST": 12 * 3600,
+        "O": -2 * 3600,
+        "OMSST": 7 * 3600,
+        "OMST": 6 * 3600,
+        "ORAT": 5 * 3600,
+        "P": -3 * 3600,
+        "PDT": -7 * 3600,
+        "PET": -5 * 3600,
+        "PETST": 12 * 3600,
+        "PETT": 12 * 3600,
+        "PGT": 10 * 3600,
+        "PHOT": 13 * 3600,
+        "PHT": 8 * 3600,
+        "PKT": 5 * 3600,
+        "PMDT": -2 * 3600,
+        "PMST": -3 * 3600,
+        "PONT": 11 * 3600,
+        "PST": -8 * 3600,
+        "PT": -8 * 3600,
+        "PWT": 9 * 3600,
+        "PYST": -3 * 3600,
+        "PYT": -4 * 3600,
+        "Q": -4 * 3600,
+        "QYZT": 6 * 3600,
+        "R": -5 * 3600,
+        "RET": 4 * 3600,
+        "ROTT": -3 * 3600,
+        "S": -6 * 3600,
+        "SAKT": 11 * 3600,
+        "SAMT": 4 * 3600,
+        "SAST": 2 * 3600,
+        "SBT": 11 * 3600,
+        "SCT": 4 * 3600,
+        "SGT": 8 * 3600,
+        "SRET": 11 * 3600,
+        "SRT": -3 * 3600,
+        "SST": -11 * 3600,
+        "SYOT": 3 * 3600,
+        "T": -7 * 3600,
+        "TAHT": -10 * 3600,
+        "TFT": 5 * 3600,
+        "TJT": 5 * 3600,
+        "TKT": 13 * 3600,
+        "TLT": 9 * 3600,
+        "TMT": 5 * 3600,
+        "TOST": 14 * 3600,
+        "TOT": 13 * 3600,
+        "TRT": 3 * 3600,
+        "TVT": 12 * 3600,
+        "U": -8 * 3600,
+        "ULAST": 9 * 3600,
+        "ULAT": 8 * 3600,
+        "UTC": 0 * 3600,
+        "UYST": -2 * 3600,
+        "UYT": -3 * 3600,
+        "UZT": 5 * 3600,
+        "V": -9 * 3600,
+        "VET": -4 * 3600,
+        "VLAST": 11 * 3600,
+        "VLAT": 10 * 3600,
+        "VOST": 6 * 3600,
+        "VUT": 11 * 3600,
+        "W": -10 * 3600,
+        "WAKT": 12 * 3600,
+        "WARST": -3 * 3600,
+        "WAST": 2 * 3600,
+        "WAT": 1 * 3600,
+        "WEST": 1 * 3600,
+        "WET": 0 * 3600,
+        "WFT": 12 * 3600,
+        "WGST": -2 * 3600,
+        "WGT": -3 * 3600,
+        "WIB": 7 * 3600,
+        "WIT": 9 * 3600,
+        "WITA": 8 * 3600,
+        "WST": 14 * 3600,
+        "WT": 0 * 3600,
+        "X": -11 * 3600,
+        "Y": -12 * 3600,
+        "YAKST": 10 * 3600,
+        "YAKT": 9 * 3600,
+        "YAPT": 10 * 3600,
+        "YEKST": 6 * 3600,
+        "YEKT": 5 * 3600,
+        "Z": 0 * 3600,
+}

src/version.py

@@ -0,0 +1,15 @@
+#  version.py
+#
+#  license: GNU LGPL
+#
+#  This library is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 3 of the License, or (at your option) any later version.
+
+# This file will not be actually used as it is regenerated with the correct build version
+# by generate_metainfo.py in the "compile" step
+
+# This is a bit of a hack but I can't find out how to access the version in description.xml once deployed
+
+version = '0.0.0'

src/yahoo.py

@@ -12,36 +12,35 @@ import csv
 import datetime
 import dateutil.parser
 import html
+import logging
 import os
-import pathlib
 import pprint
 import pytz
 import re
-import sys
 import time
 import traceback
 import urllib.parse
 
 from datacode import Datacode
-import baseclient
+from baseclient import BaseClient, HttpException
+from http import cookiejar
 import jsonParser
 
 
-def log(str):
-    # print(str, file=sys.stderr)
-    pass
+logger = logging.getLogger(__name__)
+# logger.setLevel(logging.DEBUG)
 
 
-def raw(price, key, default=0.0):
+def raw(m, key, default=0.0):
     try:
-        return price[key]['raw']
+        return m[key]['raw']
     except:
         pass
 
     return default
 
 
-class Yahoo(baseclient.BaseClient):
+class Yahoo(BaseClient):
     def __init__(self, ctx):
         super().__init__()
 
@@ -50,9 +49,6 @@ class Yahoo(baseclient.BaseClient):
         self.historicdata = {}
         self.js = jsonParser.jsonObject
 
-        self.basedir = os.path.join(str(pathlib.Path.home()), '.financials-extension')
-        os.makedirs(self.basedir, exist_ok=True)
-
     def _read_ticker_csv_file(self, ticker):
 
         fn = os.path.join(self.basedir, 'yahoo-{}.csv'.format(ticker))
@@ -105,10 +101,24 @@ class Yahoo(baseclient.BaseClient):
 
         url = 'https://finance.yahoo.com/quote/{}?p={}'.format(ticker, ticker)
 
+        cookies = [cookiejar.Cookie(version=0,
+                                    name="B",
+                                    value="9898htldgiar5&b=3&s=gt",
+                                    port=None, port_specified=None,
+                                    domain=".yahoo.com", domain_specified=True, domain_initial_dot=True,
+                                    path="/", path_specified=True,
+                                    secure=True,
+                                    expires=None,
+                                    discard=False,
+                                    comment=None,
+                                    comment_url=None,
+                                    rest=None)
+                   ]
+
         try:
-            text = self.urlopen(url)
+            text = self.urlopen(url, redirect=True, data=None, headers=None, cookies=cookies)
         except BaseException as e:
-            log(traceback.format_exc())
+            logger.error(traceback.format_exc())
             return 'Yahoo.getRealtime({}, {}) - urlopen: {}'.format(ticker, datacode, e)
 
         try:
@@ -117,16 +127,16 @@ class Yahoo(baseclient.BaseClient):
 
             r = '"CrumbStore":{"crumb":"([^"]{11})"'
             pattern = re.compile(r)
-            match = re.search(pattern, text)
+            match = pattern.search(text)
 
             if match:
                 self.crumb = match.group(1)
             else:
                 with open(os.path.join(self.basedir, 'yahoo-{}.html'.format(ticker)), "w") as text_file:
-                    print(text, file=text_file)
+                    print(f"<!-- '{url}' -->\r\n\r\n{text}", file=text_file)
 
         except BaseException as e:
-            log(traceback.format_exc())
+            logger.error(traceback.format_exc())
             return 'Yahoo.getRealtime({}, {}) - crumb: {}'.format(ticker, datacode, e)
 
         try:
@@ -134,20 +144,25 @@ class Yahoo(baseclient.BaseClient):
 
             if start < 0:
                 with open(os.path.join(self.basedir, 'yahoo-{}.html'.format(ticker)), "w") as text_file:
-                    print(text, file=text_file)
-
-                return 'Could not find QuoteSummaryStore for \'{}\''.format(ticker)
+                    print(f"<!-- '{url}' -->\r\n\r\n{text}", file=text_file)
+                return None
 
             start = start + len('"QuoteSummaryStore":')
             results = self.js.parseString(text[start:])
 
             if not results:
                 with open(os.path.join(self.basedir, 'yahoo-{}.html'.format(ticker)), "w") as text_file:
-                    print(text, file=text_file)
+                    print(f"<!-- '{url}' -->\r\n\r\n{text}", file=text_file)
                 return None
 
+        except BaseException as e:
+            logger.error(traceback.format_exc())
+            return 'Yahoo.getRealtime({}, {}) - parsing: {}'.format(ticker, datacode, e)
+
+        try:
             price = results['price']
             quoteType = results['quoteType']
+            summaryDetail = results['summaryDetail']
 
             if not price:
                 return 'Could not find price for \'{}\''.format(ticker)
@@ -167,6 +182,10 @@ class Yahoo(baseclient.BaseClient):
             tick[Datacode.VOLUME] = float(raw(price, 'regularMarketVolume'))
             tick[Datacode.AVG_DAILY_VOL_3MOMTH] = float(raw(price, 'averageDailyVolume3Month'))
 
+            tick[Datacode.LOW_52_WEEK] = float(raw(summaryDetail, 'fiftyTwoWeekLow'))
+            tick[Datacode.HIGH_52_WEEK] = float(raw(summaryDetail, 'fiftyTwoWeekHigh'))
+            tick[Datacode.MARKET_CAP] = float(raw(summaryDetail, 'marketCap'))
+
             if quoteType:
                 t = int(price['regularMarketTime'])
                 tz = pytz.timezone(quoteType['exchangeTimezoneName'])
@@ -185,15 +204,15 @@ class Yahoo(baseclient.BaseClient):
             if name:
                 tick[Datacode.NAME] = html.unescape(str(name))
             else:
-                tick[Datacode.NAME] = ''
+                tick[Datacode.NAME] = tick[Datacode.TICKER]
 
             tick[Datacode.TIMESTAMP] = time.time()
 
         except BaseException as e:
             with open(os.path.join(self.basedir, 'yahoo-{}.js'.format(ticker)), "w") as text_file:
-                pprint.pprint(results.asList(), stream=text_file)
+                pprint.pprint(f"// '{url}'\r\n\r\n{results.asList()}", stream=text_file)
 
-            log(traceback.format_exc())
+            logger.error(traceback.format_exc())
             return 'Yahoo.getRealtime({}, {}) - process: {}'.format(ticker, datacode, e)
 
         return self._return_value(self.realtime[ticker], datacode)
@@ -213,7 +232,10 @@ class Yahoo(baseclient.BaseClient):
         ticker = "".join(ticker.split())
         min_tick_date = None
 
-        if ticker not in self.historicdata:
+        # dividend and splits will change past adjusted prices
+        # the moment we are asked for ADJ_CLOSE we ignore the ticker cache to refresh
+
+        if Datacode.ADJ_CLOSE != datacode and ticker not in self.historicdata:
             self._read_ticker_csv_file(ticker)
 
         if ticker in self.historicdata:
@@ -222,7 +244,7 @@ class Yahoo(baseclient.BaseClient):
             if date in ticks:
                 return self._return_value(ticks[date], datacode)
 
-            # weekend, trading holiday or as yet unfetched
+            # weekend, trading holiday or as yet un-fetched
             if min(ticks) <= date <= max(ticks):
                 return 'Not a trading day \'{}\''.format(date)
 
@@ -257,7 +279,7 @@ class Yahoo(baseclient.BaseClient):
             t1 = t1 - 2682000  # pad with extra month
 
         except BaseException as e:
-            log(traceback.format_exc())
+            logger.error(traceback.format_exc())
             return 'Yahoo.getHistoric({}, {}, {}) - date: {}'.format(ticker, datacode, date, e)
 
         try:
@@ -273,8 +295,12 @@ class Yahoo(baseclient.BaseClient):
 
             self._read_ticker_csv_file(ticker)
 
+        except HttpException:
+            logger.error(traceback.format_exc())
+            return None
+
         except BaseException as e:
-            log(traceback.format_exc())
+            logger.error(traceback.format_exc())
             return 'Yahoo.getHistoric({}, {}, {}) - read: {}'.format(ticker, datacode, date, e)
 
         try:
@@ -292,7 +318,7 @@ class Yahoo(baseclient.BaseClient):
                 return 'Not a trading day \'{}\''.format(date)
 
         except BaseException as e:
-            log(traceback.format_exc())
+            logger.error(traceback.format_exc())
             return 'Yahoo.getHistoric({}, {}, {}) - process: {}'.format(ticker, datacode, date, e)
 
         return None