Reworked Yahoo cookie

README.md

@@ -4,8 +4,9 @@ Version 3.3.0 includes improved cookie handling and somewhat improved logic to d
 
 ## Overview
 
-This is a Python based extension for LibreOffice Calc to make stock market, index and FX data available in Calc 
-spreadsheets - currently supporting Yahoo's and Financial Times' finance websites using old-fashioned web scraping. 
+This is a Python based extension for LibreOffice Calc to make market data available in Calc 
+spreadsheets - currently supporting Yahoo's (FX, crypto, equities, indices, futures, options) and Financial Times' 
+(FX, equities, indices, futures) websites using old-fashioned web scraping. 
 
 Starting with version 3.1.0, we received a contribution to get crypto data directly from Coinbase
 
@@ -32,7 +33,8 @@ Only Yahoo has historic data available.
 
 There is a file **examples.ods** there too with usage examples and possible arguments to functions.
 
-You have to check the respective websites to work out what symbol is the right one for you. If a website doesn't have 
+You have to check the respective websites to work out what symbol is the right one for you. Make sure today or the date 
+requested is a trading day (exchange is not closed). If a website doesn't have 
 the symbol/asset you want, this extension can't help you either. Having said that, I mostly look at US and West European 
 equities, ETFs and mutual funds and major FX rates - if you have issues with the data available for other assets or 
 assets in other regions, drop me a line (best to include full URLs and possibly the same asset listed on more than one 
@@ -135,13 +137,10 @@ python3 -m unittest discover src
 
 ### Tested with:
 - Windows 10 / LibreOffice Calc 7.1.2.2 / Python 3.8.8
-- Ubuntu 20.04.5 / LibreOffice Calc 6.4.7.2 / Python 3.8.10
+- Ubuntu 22.04.1 / LibreOffice Calc 7.3.7.2 / Python 3.10.6
 - MacOS 10.15.7 / LibreOffice Calc 7.2.0.4 / Python 3.8.10
 
 (Previous versions)
 - Debian 10.3  / LibreOffice Calc 6.1.5.2 / Python 3.7.3
-- Ubuntu 20.10 / LibreOffice Calc 7.0.3.1 / Python 3.8.6
-- Ubuntu 19.10 / LibreOffice Calc 6.3.5.2 / Python 3.7.6
-- Ubuntu 19.04 / LibreOffice Calc 6.2 / Python 3.7.3
+- Ubuntu 20.04.5 / LibreOffice Calc 6.4.7.2 / Python 3.8.10
 - Ubuntu 18.04.5 / LibreOffice Calc 6 / Python 3.6.9
-- Ubuntu 18.04 / LibreOffice Calc 6 / Python 3.6.7

src/financials_yahoo.py

@@ -66,10 +66,11 @@ def cookie(name, value):
 
 def get_cookies():
     return [
-        cookie("A1", "d=AQABBNAVmWICEEFBM1xh-RmAmPpJJIsAz3YFEgABBwFcmmJlY_bPb2UB9iMAAAcIzhWZYm7SAIg&S=AQAAAucqV1HMdCsRf6key1gdaFs"),
-        cookie("A1S", "d=AQABBNAVmWICEEFBM1xh-RmAmPpJJIsAz3YFEgABBwFcmmJlY_bPb2UB9iMAAAcIzhWZYm7SAIg&S=AQAAAucqV1HMdCsRf6key1gdaFs&j=GDPR"),
-        cookie("A3", "d=AQABBNAVmWICEEFBM1xh-RmAmPpJJIsAz3YFEgABBwFcmmJlY_bPb2UB9iMAAAcIzhWZYm7SAIg&S=AQAAAucqV1HMdCsRf6key1gdaFs"),
-        cookie("GUC", "AQABBwFimlxjZUIcxQRM"),
+        cookie("A1", "d=AQABBE6aomQCEJUamSIzqCl9UJ-spkNvMfkFEgABCAHqo2TKZPbPb2UBAiAAAAcIS5qiZF3wy-8&S=AQAAAnSEZV80kjd9J2RUh8TD5cY"),
+        cookie("A1S", "d=AQABBE6aomQCEJUamSIzqCl9UJ-spkNvMfkFEgABCAHqo2TKZPbPb2UBAiAAAAcIS5qiZF3wy-8&S=AQAAAnSEZV80kjd9J2RUh8TD5cY&j=GDPR"),
+        cookie("A3", "d=AQABBE6aomQCEJUamSIzqCl9UJ-spkNvMfkFEgABCAHqo2TKZPbPb2UBAiAAAAcIS5qiZF3wy-8&S=AQAAAnSEZV80kjd9J2RUh8TD5cY"),
+        cookie("EuConsent", "CPuVJsAPuVJsAAOACKENDICgAAAAAAAAACiQAAAAAABhoAMAAQSrEQAYAAglWKgAwABBKsA"),
+        cookie("GUC", "AQ81G6Da"),
         cookie("maex", "{\"v2\":{}}"),
         cookie("thamba", "1")
     ]
@@ -250,6 +251,10 @@ class Yahoo(BaseClient):
             return 'Yahoo.getRealtimeSummary({}, {}) - HTML parsing: {}'.format(ticker, datacode, e)
 
         try:
+            if not root:
+                logger.exception("BaseException ticker=%s datacode=%s", ticker, datacode)
+                return 'Yahoo.getRealtimeSummary({}, {}) - root missing'.format(ticker, datacode)
+
             tick[Datacode.TICKER] = ticker
             tick[Datacode.TIMESTAMP] = time.time()
             tick[Datacode.YAHOO_SUMMARY_RECEIVED] = True

src/generate_metainfo.py

@@ -14,7 +14,7 @@ import os
 cur_dir = os.getcwd()
 
 addin_id = "com.financials.getinfo"
-addin_version = "3.3.0"
+addin_version = "3.3.1"
 addin_displayname = "Financial Market Extension"
 addin_publisher_link = "https://github.com/cmallwitz/Financials-Extension"
 addin_publisher_name = "The Publisher"

src/test_ft.py

@@ -137,29 +137,29 @@ class Test(unittest.TestCase):
 
     def test_US_futures(self):
 
-        s = financials.getRealtime('ESH3:IOM', Datacode.NAME.value, 'FT')
+        s = financials.getRealtime('ESU3:IOM', Datacode.NAME.value, 'FT')
         self.assertEqual(str, type(s), 'test_realtime_US_futures NAME {}'.format(s))
-        self.assertEqual('EMINI S&P MAR3', s, 'test_US_futures NAME {}'.format(s))
+        self.assertEqual('EMINI S&P SEP3', s, 'test_US_futures NAME {}'.format(s))
 
-        s = financials.getRealtime('ESH3:IOM', Datacode.LAST_PRICE.value, 'FT')
+        s = financials.getRealtime('ESU3:IOM', Datacode.LAST_PRICE.value, 'FT')
         self.assertEqual(float, type(s), 'test_US_futures LAST_PRICE {}'.format(s))
 
         # s = financials.getRealtime('ESH3:IOM', Datacode.OPEN.value, 'FT')
         # self.assertEqual(float, type(s), 'test_US_futures OPEN {}'.format(s))
 
-        s = financials.getRealtime('ESH3:IOM', Datacode.VOLUME.value, 'FT')
+        s = financials.getRealtime('ESU3:IOM', Datacode.VOLUME.value, 'FT')
         self.assertEqual(float, type(s), 'test_US_futures VOLUME {}'.format(s))
 
-        s = financials.getRealtime('ESH3:IOM', Datacode.LOW_52_WEEK.value, 'FT')
+        s = financials.getRealtime('ESU3:IOM', Datacode.LOW_52_WEEK.value, 'FT')
         self.assertEqual(float, type(s), 'test_US_futures LOW_52_WEEK {}'.format(s))
 
-        s = financials.getRealtime('ESH3:IOM', Datacode.HIGH_52_WEEK.value, 'FT')
+        s = financials.getRealtime('ESU3:IOM', Datacode.HIGH_52_WEEK.value, 'FT')
         self.assertEqual(float, type(s), 'test_US_futures HIGH_52_WEEK {}'.format(s))
 
-        s = financials.getRealtime('ESH3:IOM', Datacode.CHANGE.value, 'FT')
+        s = financials.getRealtime('ESU3:IOM', Datacode.CHANGE.value, 'FT')
         self.assertEqual(float, type(s), 'test_US_futures CHANGE {}'.format(s))
 
-        s = financials.getRealtime('ESH3:IOM', Datacode.CHANGE_IN_PERCENT.value, 'FT')
+        s = financials.getRealtime('ESU3:IOM', Datacode.CHANGE_IN_PERCENT.value, 'FT')
         self.assertEqual(float, type(s), 'test_US_futures CHANGE_IN_PERCENT {}'.format(s))
 
     def test_UK_ETF(self):
@@ -167,9 +167,6 @@ class Test(unittest.TestCase):
         self.assertEqual(str, type(s), 'test_UK_ETF NAME {}'.format(s))
         self.assertEqual('iShares Core S&P 500 UCITS ETF USD (Acc)', s, 'test_UK_ETF NAME {}'.format(s))
 
-        s = financials.getRealtime('C060:GER:EUR', 'NAME', 'FT')
-        self.assertEqual(str, type(s), 'test_UK_ETF NAME {}'.format(s))
-
         s = financials.getRealtime('VERX:LSE:GBP', 'LAST_PRICE', 'FT')
         self.assertEqual(float, type(s), 'test_UK_ETF LAST_PRICE {}'.format(s))
 

src/test_yahoo.py

@@ -112,7 +112,7 @@ class Test(unittest.TestCase):
         self.assertEqual(s, 'Information Technology Services', 'test_realtime_US_equity INDUSTRY {}'.format(s))
 
         s = financials.getRealtime('IBM', Datacode.TIMEZONE.value, 'YAHOO')
-        self.assertEqual(s, 'EST', 'test_realtime_US_equity TIMEZONE {}'.format(s))
+        self.assertEqual(s, 'EDT', 'test_realtime_US_equity TIMEZONE {}'.format(s))
 
         s = financials.getRealtime('IBM', Datacode.BETA.value, 'YAHOO')
         self.assertTrue(testutils.is_positive_float(s), 'test_realtime_US_equity BETA {}'.format(s))
@@ -171,7 +171,8 @@ class Test(unittest.TestCase):
         # self.assertIsNone(s, 'test_realtime_US_mutuals DIV {}'.format(s)) # no dividend
 
         s = financials.getRealtime('VERX.L', Datacode.DIV_YIELD.value, 'YAHOO')
-        self.assertEqual(float, type(s), 'test_realtime_US_mutuals DIV_YIELD {}'.format(s))
+        self.assertIsNone(s, 'test_realtime_US_mutuals DIV_YIELD {}'.format(s)) # no yield
+        # self.assertEqual(float, type(s), 'test_realtime_US_mutuals DIV_YIELD {}'.format(s))
 
     def test_realtime_US_options(self):
 
@@ -214,16 +215,16 @@ class Test(unittest.TestCase):
 
         s = financials.getRealtime('ES=F', Datacode.NAME.value, 'YAHOO')
         self.assertEqual(str, type(s), 'test_realtime_US_futures NAME {}'.format(s))
-        self.assertEqual('E-Mini S&P 500 Mar 23 (ES=F)', s, 'test_realtime_US_futures NAME {}'.format(s))
+        self.assertEqual('E-Mini S&P 500 Sep 23 (ES=F)', s, 'test_realtime_US_futures NAME {}'.format(s))
 
         s = financials.getRealtime('ES=F', Datacode.TICKER.value, 'YAHOO')
         self.assertEqual(str, type(s), 'test_realtime_US_futures TICKER {}'.format(s))
-        self.assertEqual('ESH23.CME', s, 'test_realtime_US_futures TICKER {}'.format(s))
+        self.assertEqual('ESU23.CME', s, 'test_realtime_US_futures TICKER {}'.format(s))
 
         s = financials.getRealtime('ES=F', Datacode.SETTLEMENT_DATE.value, 'YAHOO')
         self.assertEqual(str, type(s), 'test_realtime_US_futures SETTLEMENT_DATE {}'.format(s))
         self.assertTrue(testutils.is_date(s), 'test_realtime_US_futures SETTLEMENT_DATE {}'.format(s))
-        self.assertEqual("2023-03-17", s, 'test_realtime_US_futures SETTLEMENT_DATE {}'.format(s))
+        self.assertEqual("2023-09-15", s, 'test_realtime_US_futures SETTLEMENT_DATE {}'.format(s))
 
         s = financials.getRealtime('ES=F', Datacode.LAST_PRICE.value, 'YAHOO')
         self.assertEqual(float, type(s), 'test_realtime_US_futures LAST_PRICE {}'.format(s))
@@ -258,11 +259,12 @@ class Test(unittest.TestCase):
         self.assertEqual(float, type(s), 'test_realtime_UK_ETF LAST_PRICE {}'.format(s))
 
         s = financials.getRealtime('VERX.L', Datacode.TIMEZONE.value, 'YAHOO')
-        self.assertEqual('GMT', s, 'test_realtime_UK_ETF TIMEZONE {}'.format(s))
+        # self.assertEqual('GMT', s, 'test_realtime_UK_ETF TIMEZONE {}'.format(s))
+        self.assertEqual('BST', s, 'test_realtime_UK_ETF TIMEZONE {}'.format(s))
 
         s = financials.getRealtime('CSP1.L', Datacode.NAME.value, 'YAHOO')
         self.assertEqual(str, type(s), 'test_realtime_UK_ETF NAME {}'.format(s))
-        self.assertEqual('iShares Core S&P 500 UCITS ETF USD (Acc) (CSP1.L)', s, 'test_realtime_UK_ETF NAME {}'.format(s))
+        self.assertEqual('iShares VII PLC - iShares Core S&P 500 UCITS ETF (CSP1.L)', s, 'test_realtime_UK_ETF NAME {}'.format(s))
 
     def test_realtime_DE_equity(self):
 
@@ -270,7 +272,8 @@ class Test(unittest.TestCase):
         self.assertEqual(float, type(s), 'test_realtime_DE_equity LAST_PRICE {}'.format(s))
 
         s = financials.getRealtime('SAP.DE', Datacode.TIMEZONE.value, 'YAHOO')
-        self.assertEqual('CET', s, 'test_realtime_DE_equity TIMEZONE {}'.format(s))
+        # self.assertEqual('CET', s, 'test_realtime_DE_equity TIMEZONE {}'.format(s))
+        self.assertEqual('CEST', s, 'test_realtime_DE_equity TIMEZONE {}'.format(s))
 
         s = financials.getRealtime('SAP.DE', Datacode.SECTOR.value, 'YAHOO')
         self.assertEqual(str, type(s), 'test_realtime_DE_equity SECTOR {}'.format(s))
@@ -280,10 +283,10 @@ class Test(unittest.TestCase):
         self.assertEqual(str, type(s), 'test_realtime_DE_equity INDUSTRY {}'.format(s))
         self.assertEqual(s, 'Software—Application', 'test_realtime_DE_equity INDUSTRY {}'.format(s))
 
-        s = financials.getRealtime('C060.DE', Datacode.NAME.value, 'YAHOO')
+        s = financials.getRealtime('LYY8.DE', Datacode.NAME.value, 'YAHOO')
         self.assertEqual(str, type(s), 'test_realtime_DE_equity NAME {}'.format(s))
 
-        s = financials.getRealtime('C060.DE', Datacode.EXCHANGE.value, 'YAHOO')
+        s = financials.getRealtime('LYY8.DE', Datacode.EXCHANGE.value, 'YAHOO')
         self.assertEqual(s, 'XETRA', 'test_realtime_DE_equity EXCHANGE')
 
     def test_TA_equity(self):
@@ -455,8 +458,8 @@ class Test(unittest.TestCase):
         s = financials.getHistoric('SAP.DE', Datacode.CLOSE.value, '2017-01-03', 'YAHOO')
         self.assertEqual(s, 82.889999, 'test_historic_DE_equity CLOSE {}'.format(s))
 
-        s = financials.getHistoric('C060.DE', Datacode.CLOSE.value, '2017-01-03', 'YAHOO')
-        self.assertEqual(s, 72.870003, 'test_historic_DE_equity CLOSE {}'.format(s))
+        s = financials.getHistoric('LYY8.DE', Datacode.CLOSE.value, '2017-01-03', 'YAHOO')
+        self.assertEqual(s, 96.010002, 'test_historic_DE_equity CLOSE {}'.format(s))
 
     def test_realtime_errors(self):
 
@@ -469,7 +472,7 @@ class Test(unittest.TestCase):
         s = financials.getRealtime('NO_NAME', Datacode.SECTOR.value, 'YAHOO')
         self.assertIsNone(s, 'test_realtime_errors SECTOR {}'.format(s))
 
-        s = financials.getRealtime('C060.DE', -1, 'YAHOO')
+        s = financials.getRealtime('LYY8.DE', -1, 'YAHOO')
         self.assertEqual('Datacode -1 not supported', s, 'test_realtime_errors -1 {}'.format(s))
 
     def test_historic_errors(self):