transparently split large time range calls to ws2 therapytimeline

tconnectsync/api/common.py

@@ -1,6 +1,8 @@
 import datetime
+from typing import List, Tuple
 import requests
 import random
+import arrow
 
 from tconnectsync import secret
 
@@ -9,6 +11,9 @@ def parse_date(date):
         return date
     return (date or datetime.datetime.now()).strftime('%m-%d-%Y')
 
+def parsed_date_to_arrow(date):
+    return arrow.get(datetime.datetime.strptime(date, '%m-%d-%Y'))
+
 USER_AGENTS = [
     'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/537.36',
     'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36',
@@ -97,6 +102,28 @@ def base_session():
         s.request = wrapped_request.__get__(s, requests.Session)
     return s
 
+def days_between(start, end) -> int:
+    diff = arrow.get(end) - arrow.get(start)
+    return diff.days
+
+# both inclusive
+def split_days_range(start_a, end_a, days: int = 5) -> List[Tuple[str, str]]:
+    ranges = []
+    start = arrow.get(start_a)
+    end = arrow.get(end_a)
+    cur_s = start
+    cur = start
+    while cur <= end:
+        if (cur - cur_s).days >= days-1:
+            ranges.append((cur_s, cur))
+            cur_s = cur + datetime.timedelta(days=1)
+        
+        cur += datetime.timedelta(days=1)
+    if len(ranges) > 0 and (end - ranges[-1][-1]).days > 0:
+        ranges.append((cur_s, end))
+    
+    return ranges
+
 class ApiException(Exception):
     def __init__(self, status_code, text, *args, **kwargs):
         self.status_code = status_code

tconnectsync/api/ws2.py

@@ -5,7 +5,7 @@ import logging
 import time
 import json
 
-from .common import base_session, parse_date, base_headers, ApiException
+from .common import base_session, parse_date, parsed_date_to_arrow, base_headers, days_between, split_days_range, ApiException
 
 logger = logging.getLogger(__name__)
 
@@ -75,10 +75,34 @@ class WS2Api:
     This has its own built-in retry logic because Tandem's frontend serving
     the API returns 500s when its backend times out.
     """
+    MAX_THERAPY_TIMELINE_DAYS = 2
     def therapy_timeline_csv(self, start=None, end=None, tries=0):
         startDate = parse_date(start)
         endDate = parse_date(end)
 
+        pStart = parsed_date_to_arrow(startDate)
+        pEnd = parsed_date_to_arrow(endDate)
+        if days_between(pStart, pEnd) > self.MAX_THERAPY_TIMELINE_DAYS:
+            ranges = split_days_range(pStart, pEnd, self.MAX_THERAPY_TIMELINE_DAYS)
+            logger.debug("Splitting call to therapy_timeline_csv(%s, %s) into: %s", start, end, ranges)
+            outputs = []
+            for rng in ranges:
+                rStart, rEnd = rng
+                logger.debug("split therapy_timeline_csv(%s, %s)", rStart, rEnd)
+                output = self.therapy_timeline_csv(rStart, rEnd, tries=tries)
+                logger.debug("split therapy_timeline_csv(%s, %s) = %s", rStart, rEnd, ["%s: %s items" % (key, len(val)) for key, val in output.items()])
+                outputs.append(output)
+            full = {}
+            for o in outputs:
+                for key, val in o.items():
+                    if key not in full:
+                        full[key] = val
+                    elif val is not None:
+                        full[key] += val
+            
+            logger.debug("therapy_timeline_csv merge: %s", ["%s: %s items" % (key, len(val)) for key, val in full.items()])
+            return full
+
         try:
             req_text = self.get('therapytimeline2csv/%s/%s/%s?format=csv' % (self.userGuid, startDate, endDate), timeout=10)
         except ApiException as e:
@@ -92,6 +116,7 @@ class WS2Api:
                     return self.therapy_timeline_csv(start, end, tries+1)
             raise e
 
+        logger.debug('req_text: %s', req_text)
         sections = self._split_empty_sections(req_text)
 
         readingData = None

tests/api/test_ws2.py

@@ -2,6 +2,7 @@
 
 import unittest
 import itertools
+import copy
 
 from .fake import WS2Api
 
@@ -28,7 +29,7 @@ class TestWS2Api(unittest.TestCase):
         ws2.get = self.fake_get_with_http_500(2)
 
         self.assertEqual(
-            ws2.therapy_timeline_csv('2021-04-01', '2021-04-02'),
+            ws2.therapy_timeline_csv('04-01-2021', '04-02-2021'),
             {
                 "readingData": [],
                 "iobData": [],
@@ -41,7 +42,7 @@ class TestWS2Api(unittest.TestCase):
 
         ws2.get = self.fake_get_with_http_500(3)
 
-        self.assertRaises(ApiException, ws2.therapy_timeline_csv, '2021-04-01', '2021-04-02')
+        self.assertRaises(ApiException, ws2.therapy_timeline_csv, '04-01-2021', '04-02-2021')
 
     RAW_DATA_HEADER = """Tandem Diabetes Care Inc.
 t:connect Therapy Timeline Data Export
@@ -96,12 +97,12 @@ Report Generated On, 4/24/2021 7:50:04 PM
 
         def fake_get(endpoint, **kwargs):
             nonlocal rawData
-            if endpoint == 'therapytimeline2csv/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/2021-04-01/2021-04-02?format=csv':
+            if endpoint == 'therapytimeline2csv/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/04-01-2021/04-02-2021?format=csv':
                 return rawData
 
         ws2.get = fake_get
 
-        tt = ws2.therapy_timeline_csv('2021-04-01', '2021-04-02')
+        tt = ws2.therapy_timeline_csv('04-01-2021', '04-02-2021')
 
         self.assertDictEqual(tt, self.PARSED_DATA)
 
@@ -113,7 +114,7 @@ Report Generated On, 4/24/2021 7:50:04 PM
 
         def fake_get(endpoint, **kwargs):
             nonlocal rawData
-            if endpoint == 'therapytimeline2csv/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/2021-04-01/2021-04-02?format=csv':
+            if endpoint == 'therapytimeline2csv/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/04-01-2021/04-02-2021?format=csv':
                 return rawData
 
         ws2.get = fake_get
@@ -122,8 +123,57 @@ Report Generated On, 4/24/2021 7:50:04 PM
         for i in itertools.permutations([self.RAW_DATA_HEADER, self.RAW_DATA_CGM, self.RAW_DATA_IOB, self.RAW_DATA_BOLUS], 4):
             rawData = "\n".join(i)
 
-            tt = ws2.therapy_timeline_csv('2021-04-01', '2021-04-02')
+            tt = ws2.therapy_timeline_csv('04-01-2021', '04-02-2021')
             self.assertDictEqual(tt, self.PARSED_DATA)
+    
+    def test_therapy_timeline_csv_split_past_max_days(self):
+        ws2 = WS2Api()
+        ws2.userGuid = 'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee'
+
+
+        def replace_str(raw, one, two):
+            return raw.replace('04-01-2021', one).replace('04-02-2021', two)
+        rawData1 = self.RAW_DATA_FULL
+        rawData2 = replace_str(self.RAW_DATA_FULL, '04-03-2021', '04-04-2021')
+        rawData3 = replace_str(self.RAW_DATA_FULL, '04-05-2021', '04-06-2021')
+        rawData4 = replace_str(self.RAW_DATA_FULL, '04-07-2021', '04-07-2021')
+
+        def replace_parsed(one, two):
+            parsedData = copy.deepcopy(self.PARSED_DATA)
+            for typ in parsedData.keys():
+                for i in range(len(parsedData[typ])):
+                    for f in parsedData[typ][i].keys():
+                        if 'datetime' in f.lower():
+                            parsedData[typ][i][f] = replace_str(parsedData[typ][i][f], one, two)
+            return parsedData
+
+        parsedData1 = self.PARSED_DATA
+        parsedData2 = replace_parsed('04-03-2021', '04-04-2021')
+        parsedData3 = replace_parsed('04-05-2021', '04-06-2021')
+        parsedData4 = replace_parsed('04-07-2021', '04-07-2021')
+
+        fullParsedData = parsedData1
+        for d in [parsedData2, parsedData3, parsedData4]:
+            for typ in d.keys():
+                fullParsedData[typ] += d[typ]
+
+        def fake_get(endpoint, **kwargs):
+            nonlocal rawData1, rawData2, rawData3, rawData4
+            print('fake_get call %s' % endpoint)
+            if endpoint == 'therapytimeline2csv/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/04-01-2021/04-02-2021?format=csv':
+                return rawData1
+            elif endpoint == 'therapytimeline2csv/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/04-03-2021/04-04-2021?format=csv':
+                return rawData2
+            elif endpoint == 'therapytimeline2csv/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/04-05-2021/04-06-2021?format=csv':
+                return rawData3
+            elif endpoint == 'therapytimeline2csv/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/04-07-2021/04-07-2021?format=csv':
+                return rawData4
+
+        ws2.get = fake_get
+
+        tt = ws2.therapy_timeline_csv('04-01-2021', '04-07-2021')
+
+        self.assertDictEqual(tt, fullParsedData)
 
 if __name__ == '__main__':
     unittest.main()