Fix currency issue with Yahoo / Moscow symbols

README.md

@@ -6,7 +6,7 @@ using old-fashioned web scraping.
 
 ### Feedback requested:
 
-Please provide about using the extension here: https://github.com/cmallwitz/Financials-Extension/issues/10
+Please provide feedback about using the extension here: https://github.com/cmallwitz/Financials-Extension/issues/10
 
 ### System requirements:
 
@@ -19,13 +19,23 @@ On Ubuntu 18 and 20, dateutil and pytz may already be installed, but you can get
 
 On Debian 10, the following steps have worked for me (as root)
 - apt install python3-pip
-- pip3 install python-dateutil
-- pip3 install pytz
-- pip3 install pyparsing
+- pip3 install python-dateutil pytz pyparsing
+
+Users on Windows 10 have reported this to work - as LibreOffice on Windows ships with its own, but minimal 
+Python runtime a few more steps are required
+- Download the script https://bootstrap.pypa.io/get-pip.py to your computer
+- Start a Command Prompt (CMD) as Administrator
+- on this command prompt run (change path as required)
+  
+  "c:\Program Files\LibreOffice\program\python.exe" c:\temp\get-pip.py
+
+- and then
+
+   "c:\Program Files\LibreOffice\program\python.exe" -m pip install python-dateutil pytz pyparsing
 
 ### Usage:
 
-Have a look at the 'releases' tab on GitHub and download a prebuilt **Financials-Extension.oxt** file and load it into Calc 
+Under 'Releases' on GitHub there is downloadable **Financials-Extension.oxt** file - load it into Calc 
 under menu item: Tools, Extension Manager...
 
 Getting data should be a simple as having this in a cell: 
@@ -46,6 +56,14 @@ assets in other regions, drop me a line (best to include full URLs and possibly
 site for comparison). While data for last price is most likely consistent across sites, they may differ for other data 
 points.     
 
+### LibreOffice: using , (comma) vs ; (semicolon) to separate arguments in formula
+
+There is a setting in "Tools" / "Options..." / "LibreOffice Calc" / "Formula" called "Functions". Here the user can specify the character used to separate arguments in formula.
+
+Mine is set to , (comma) - when I enter ; (semicolon) in a formula (Ubuntu / UK English), no error is reported but the semicolon is converted to , (comma)
+
+Depending on your system's language and default LibreOffice settings, you maybe better off using ; instead of , in your formulas.
+
 ### List of supported data points
 
 You can either specify numbers or names (lower or upper case) - not all bits are available from all sources and they are not necessarily consistent across sources either.
@@ -104,9 +122,7 @@ On my system (Ubuntu) I installed packages: libreoffice-dev libreoffice-java-com
 
 cd ~/tech/IdeaProjects/Financials-Extension/
 
-python3 src/test_yahoo.py
-
-python3 src/test_google.py
+python3 -m unittest discover src
 
 \# This builds file **Financials-Extension.oxt**
 

src/financials.py

@@ -19,7 +19,6 @@ import time
 from functools import wraps
 from importlib import util
 
-import dateutil.parser
 import unohelper
 from com.financials.getinfo import Financials
 
@@ -47,6 +46,8 @@ if dateutil_missing or pyparsing_missing or pytz_missing:
     msg += ' pytz' if pytz_missing else ''
     raise Exception("THIS EXTENSION NEEDS THE FOLLOWING PYTHON 3 LIBRARIES INSTALLED:" + msg)
 
+import dateutil.parser
+
 from datacode import Datacode
 import financials_google as google
 import financials_yahoo as yahoo
@@ -181,7 +182,7 @@ class FinancialsImpl(unohelper.Base, Financials):
 
                 try:
                     offset = int(date)  # offset for 1899-12-30
-                    d = dateutil.parser.parse('1899-12-30') + datetime.timedelta(days=offset)
+                    d = dateutil.parser.parse('1899-12-30', yearfirst=True, dayfirst=False) + datetime.timedelta(days=offset)
                     d = d.date().isoformat()
                 except:
                     return 'Date format not supported: {}'.format(date)
@@ -190,7 +191,7 @@ class FinancialsImpl(unohelper.Base, Financials):
             elif type(date) == str:
 
                 try:
-                    int(dateutil.parser.parse(date).strftime('%s'))
+                    int(dateutil.parser.parse(date, yearfirst=True, dayfirst=False).timestamp())
                 except:
                     return 'Date format not supported: \'{}\''.format(date)
 
@@ -218,7 +219,7 @@ class FinancialsImpl(unohelper.Base, Financials):
     @profile
     def support(self, datacode):
 
-        s = 'ctx={}\nid(self)={}\nversion={}\nfile={}\ncwd={}\nhome={}\nuname={}\npid={}\nsys.executable={}\nsys.version={}\nlocale={}'.format(
+        s = 'ctx={}\nid(self)={}\nversion={}\nfile={}\ncwd={}\nhome={}\nuname={}\npid={}\nsys.executable={}\nsys.version={}\nlocale={}\ndefaultlocale={}'.format(
             self.ctx,
             id(self),
             version,
@@ -229,7 +230,8 @@ class FinancialsImpl(unohelper.Base, Financials):
             os.getpid(),
             sys.executable,
             sys.version.replace("\n", " "),
-            locale.getlocale())
+            locale.getlocale(),
+            locale.getdefaultlocale())
 
         if datacode:
             s = '{}\ntype(datacode)={}\nstr(datacode)={}'.format(

src/financials_ft.py

@@ -197,7 +197,7 @@ class FT(BaseClient):
 
                 try:
                     value = html.unescape(match.group(1)).strip()
-                    dt = dateutil.parser.parse(value, tzinfos=whois_timezone_info)
+                    dt = dateutil.parser.parse(value, yearfirst=True, dayfirst=False, tzinfos=whois_timezone_info)
                     tick[Datacode.LAST_PRICE_DATE] = dt.date()
                     tick[Datacode.LAST_PRICE_TIME] = dt.time()
 
@@ -298,7 +298,7 @@ class FT(BaseClient):
             if match:
                 try:
                     value = html.unescape(match.group(1)).strip()
-                    dt = dateutil.parser.parse(value, tzinfos=whois_timezone_info)
+                    dt = dateutil.parser.parse(value, yearfirst=True, dayfirst=False, tzinfos=whois_timezone_info)
                     tick[Datacode.EX_DIV_DATE] = dt.date()
                 except BaseException as e:
                     pass

src/financials_google.py

@@ -200,7 +200,7 @@ class Google(BaseClient):
             try:
                 value = html.unescape(header.find('./div[1]/span[1]/span[2]').text).replace('·', '').strip()
                 logger.debug(value)
-                dt = dateutil.parser.parse(value, tzinfos=whois_timezone_info)
+                dt = dateutil.parser.parse(value, yearfirst=True, dayfirst=False, tzinfos=whois_timezone_info)
                 tick[Datacode.LAST_PRICE_DATE] = dt.date()
                 tick[Datacode.LAST_PRICE_TIME] = dt.time()
 

src/financials_yahoo.py

@@ -202,7 +202,7 @@ class Yahoo(BaseClient):
             tick[Datacode.DIV] = float(raw(summaryDetail, 'dividendRate'))
             tick[Datacode.DIV_YIELD] = float(raw(summaryDetail, 'dividendYield'))
             tick[Datacode.EX_DIV_DATE] = self.save_wrapper(
-                lambda: dateutil.parser.parse(str(fmt(summaryDetail, 'exDividendDate'))).date())
+                lambda: dateutil.parser.parse(str(fmt(summaryDetail, 'exDividendDate')), yearfirst=True, dayfirst=False).date())
             tick[Datacode.PAYOUT_RATIO] = float(raw(summaryDetail, 'payoutRatio'))
             tick[Datacode.LOW_52_WEEK] = float(raw(summaryDetail, 'fiftyTwoWeekLow'))
             tick[Datacode.HIGH_52_WEEK] = float(raw(summaryDetail, 'fiftyTwoWeekHigh'))
@@ -222,9 +222,16 @@ class Yahoo(BaseClient):
                 tick[Datacode.LAST_PRICE_DATE] = dt.date()
                 tick[Datacode.LAST_PRICE_TIME] = dt.time()
 
-            tick[Datacode.TICKER] = str(price['symbol'])
-            tick[Datacode.EXCHANGE] = str(price['exchange'])
-            tick[Datacode.CURRENCY] = str(price['currency'])
+            tick[Datacode.TICKER] = self.save_wrapper(lambda: str(price['symbol']))
+            tick[Datacode.EXCHANGE] = self.save_wrapper(lambda: str(price['exchange']))
+            tick[Datacode.CURRENCY] = self.save_wrapper(lambda: str(price['currency']))
+
+            # some Moscow symbols miss currency in data block but show it in text e.g. VTBBA.ME, TBIOA.ME
+            if not tick[Datacode.CURRENCY]:
+                r = r'Currency in ([A-Z]{3})\b'
+                match = re.compile(r, flags=re.DOTALL).search(text)
+                if match:
+                    tick[Datacode.CURRENCY] = match.group(1)
 
             name = price['longName'] or price['shortName']
             if name:
@@ -262,6 +269,12 @@ class Yahoo(BaseClient):
         if Datacode.ADJ_CLOSE != datacode and ticker not in self.historicdata:
             self._read_ticker_csv_file(ticker)
 
+        try:
+            date_as_dt = dateutil.parser.parse(date, yearfirst=True, dayfirst=False)
+        except BaseException as e:
+            logger.exception("BaseException ticker=%s datacode=%s", ticker, datacode)
+            return 'Yahoo.getHistoric({}, {}, {}) - date_as_dt: {}'.format(ticker, datacode, date, e)
+
         if ticker in self.historicdata:
             ticks = self.historicdata[ticker]
 
@@ -274,12 +287,12 @@ class Yahoo(BaseClient):
 
             # (potentially) future date
             if date > max(ticks):
-                t1 = int(dateutil.parser.parse(date).strftime('%s'))
+                t1 = int(date_as_dt.timestamp())
                 t2 = int(time.time())
                 if t1 > t2:
                     return 'Future date \'{}\''.format(date)
 
-                min_tick_date = int(dateutil.parser.parse(min(ticks)).strftime('%s'))  # remember current earliest date
+                min_tick_date = int(dateutil.parser.parse(min(ticks), yearfirst=True, dayfirst=False).timestamp())  # remember current earliest date
 
         if not self.crumb:
             self.getRealtime(ticker, datacode)
@@ -288,7 +301,7 @@ class Yahoo(BaseClient):
             return 'Yahoo.getHistoric({}, {}, {}) - crumb'.format(ticker, datacode, date)
 
         try:
-            t1 = int(dateutil.parser.parse(date).strftime('%s'))
+            t1 = int(date_as_dt.timestamp())
             t2 = int(time.time())
 
             if min_tick_date:
@@ -297,7 +310,7 @@ class Yahoo(BaseClient):
             if t1 >= t2:
                 return 'Future date \'{}\''.format(date)
 
-            if t1 < int(dateutil.parser.parse('2000-01-01').strftime('%s')):
+            if t1 < int(dateutil.parser.parse('2000-01-01', yearfirst=True, dayfirst=False).timestamp()):
                 return 'Date before 2000 \'{}\''.format(date)
 
             t1 = t1 - 2682000  # pad with extra month

src/generate_metainfo.py

@@ -14,7 +14,7 @@ import os
 cur_dir = os.getcwd()
 
 addin_id = "com.financials.getinfo"
-addin_version = "2.4.0"
+addin_version = "2.5.1"
 addin_displayname = "Financial Market Extension"
 addin_publisher_link = "https://github.com/cmallwitz/Financials-Extension"
 addin_publisher_name = "The Publisher"