bump v0.9.1

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = tconnectsync
-version = 0.9.0
+version = 0.9.1
 author = James Woglom 
 author_email = j@wogloms.net
 description = Syncs Tandem t:connect pump data to Nightscout for the t:slim X2

tconnectsync/api/common.py

@@ -1,6 +1,8 @@
 import datetime
+from typing import List, Tuple
 import requests
 import random
+import arrow
 
 from tconnectsync import secret
 
@@ -9,6 +11,9 @@ def parse_date(date):
         return date
     return (date or datetime.datetime.now()).strftime('%m-%d-%Y')
 
+def parsed_date_to_arrow(date):
+    return arrow.get(datetime.datetime.strptime(date, '%m-%d-%Y'))
+
 USER_AGENTS = [
     'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/537.36',
     'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36',
@@ -97,6 +102,28 @@ def base_session():
         s.request = wrapped_request.__get__(s, requests.Session)
     return s
 
+def days_between(start, end) -> int:
+    diff = arrow.get(end) - arrow.get(start)
+    return diff.days
+
+# both inclusive
+def split_days_range(start_a, end_a, days: int = 5) -> List[Tuple[str, str]]:
+    ranges = []
+    start = arrow.get(start_a)
+    end = arrow.get(end_a)
+    cur_s = start
+    cur = start
+    while cur <= end:
+        if (cur - cur_s).days >= days-1:
+            ranges.append((cur_s, cur))
+            cur_s = cur + datetime.timedelta(days=1)
+        
+        cur += datetime.timedelta(days=1)
+    if len(ranges) > 0 and (end - ranges[-1][-1]).days > 0:
+        ranges.append((cur_s, end))
+    
+    return ranges
+
 class ApiException(Exception):
     def __init__(self, status_code, text, *args, **kwargs):
         self.status_code = status_code

tconnectsync/api/ws2.py

@@ -5,7 +5,7 @@ import logging
 import time
 import json
 
-from .common import base_session, parse_date, base_headers, ApiException
+from .common import base_session, parse_date, parsed_date_to_arrow, base_headers, days_between, split_days_range, ApiException
 
 logger = logging.getLogger(__name__)
 
@@ -75,10 +75,34 @@ class WS2Api:
     This has its own built-in retry logic because Tandem's frontend serving
     the API returns 500s when its backend times out.
     """
+    MAX_THERAPY_TIMELINE_DAYS = 2
     def therapy_timeline_csv(self, start=None, end=None, tries=0):
         startDate = parse_date(start)
         endDate = parse_date(end)
 
+        pStart = parsed_date_to_arrow(startDate)
+        pEnd = parsed_date_to_arrow(endDate)
+        if days_between(pStart, pEnd) > self.MAX_THERAPY_TIMELINE_DAYS:
+            ranges = split_days_range(pStart, pEnd, self.MAX_THERAPY_TIMELINE_DAYS)
+            logger.debug("Splitting call to therapy_timeline_csv(%s, %s) into: %s", start, end, ranges)
+            outputs = []
+            for rng in ranges:
+                rStart, rEnd = rng
+                logger.debug("split therapy_timeline_csv(%s, %s)", rStart, rEnd)
+                output = self.therapy_timeline_csv(rStart, rEnd, tries=tries)
+                logger.debug("split therapy_timeline_csv(%s, %s) = %s", rStart, rEnd, ["%s: %s items" % (key, len(val)) for key, val in output.items()])
+                outputs.append(output)
+            full = {}
+            for o in outputs:
+                for key, val in o.items():
+                    if key not in full:
+                        full[key] = val
+                    elif val is not None:
+                        full[key] += val
+            
+            logger.debug("therapy_timeline_csv merge: %s", ["%s: %s items" % (key, len(val)) for key, val in full.items()])
+            return full
+
         try:
             req_text = self.get('therapytimeline2csv/%s/%s/%s?format=csv' % (self.userGuid, startDate, endDate), timeout=10)
         except ApiException as e:
@@ -92,6 +116,7 @@ class WS2Api:
                     return self.therapy_timeline_csv(start, end, tries+1)
             raise e
 
+        logger.debug('req_text: %s', req_text)
         sections = self._split_empty_sections(req_text)
 
         readingData = None

tconnectsync/features.py

@@ -7,11 +7,13 @@ IOB = "IOB"
 BOLUS_BG = "BOLUS_BG"
 CGM = "CGM"
 PUMP_EVENTS = "PUMP_EVENTS"
+PUMP_EVENTS_BASAL_SUSPENSION = "PUMP_EVENTS_BASAL_SUSPENSION"
 PROFILES = "PROFILES"
 
 DEFAULT_FEATURES = [
     BASAL,
-    BOLUS
+    BOLUS,
+    PUMP_EVENTS
 ]
 
 ALL_FEATURES = [
@@ -19,6 +21,7 @@ ALL_FEATURES = [
     BOLUS,
     IOB,
     PUMP_EVENTS,
+    PUMP_EVENTS_BASAL_SUSPENSION,
     PROFILES
 ]
 

tconnectsync/parser/nightscout.py

@@ -114,8 +114,8 @@ class NightscoutEntry:
     @staticmethod
     def profile_store(profile: Profile, device_settings: DeviceSettings) -> dict:
         return {
-            # insulin duration in hours
-            "dia": (profile.insulin_duration_min / 60),
+            # insulin duration in hours; Nightscout JS bug requires all top-level fields to be strings
+            "dia": "%s" % (profile.insulin_duration_min / 60),
             "carbratio": [
                 {
                     "time": tandem_to_ns_time(segment.time),

tconnectsync/process.py

@@ -31,7 +31,7 @@ from .sync.pump_events import (
 )
 from .sync.profile import process_profiles
 from .parser.tconnect import TConnectEntry
-from .features import BASAL, BOLUS, IOB, BOLUS_BG, CGM, DEFAULT_FEATURES, PUMP_EVENTS, PROFILES
+from .features import BASAL, BOLUS, IOB, BOLUS_BG, CGM, DEFAULT_FEATURES, PUMP_EVENTS, PROFILES, PUMP_EVENTS_BASAL_SUSPENSION
 from tconnectsync.sync import basal
 
 logger = logging.getLogger(__name__)
@@ -146,19 +146,22 @@ def process_time_range(tconnect, nightscout, time_start, time_end, pretend, feat
         logger.debug("Writing basal events")
         added += ns_write_basal_events(nightscout, basalEvents, pretend=pretend, time_start=time_start, time_end=time_end)
         logger.debug("Finished writing basal events")
-    
-    if PUMP_EVENTS in features:
-        pumpEvents = process_ciq_activity_events(ciqTherapyTimelineData)
-        logger.debug("CIQ activity events: %s" % pumpEvents)
 
-        logger.warning("Using WS2 data source for basalsuspension because PUMP_EVENTS is an enabled feature")
+    if PUMP_EVENTS_BASAL_SUSPENSION in features:
+        logger.warning("Using WS2 data source for basalsuspension because PUMP_EVENTS_BASAL_SUSPENSION is an enabled feature")
         logger.warning("<!!> The WS2 data source is unreliable and may prevent timely synchronization")
         ws2BasalSuspension = tconnect.ws2.basalsuspension(time_start, time_end)
 
         bsPumpEvents = process_basalsuspension_events(ws2BasalSuspension)
         logger.debug("basalsuspension events: %s" % bsPumpEvents)
 
-        pumpEvents += bsPumpEvents
+        logger.debug("Writing pump basalsuspension events")
+        added += ns_write_pump_events(nightscout, bsPumpEvents, pretend=pretend, time_start=time_start, time_end=time_end)
+        logger.debug("Finished writing basal events")
+
+    if PUMP_EVENTS in features:
+        pumpEvents = process_ciq_activity_events(ciqTherapyTimelineData)
+        logger.debug("CIQ activity events: %s" % pumpEvents)
 
         logger.debug("Writing pump events")
         added += ns_write_pump_events(nightscout, pumpEvents, pretend=pretend, time_start=time_start, time_end=time_end)
@@ -192,5 +195,8 @@ def process_time_range(tconnect, nightscout, time_start, time_end, pretend, feat
         if process_profiles(tconnect, nightscout, pretend=pretend):
             added += 1
 
-    logger.info("Wrote %d events to Nightscout this process cycle" % added)
+    if pretend:
+        logger.info("Would have written %d events to Nightscout this process cycle (in pretend mode)" % added)
+    else:
+        logger.info("Wrote %d events to Nightscout this process cycle" % added)
     return added

tconnectsync/sync/profile.py

@@ -88,6 +88,10 @@ def compare_profiles(device_profiles: List[Profile], device_settings: DeviceSett
         if profile.active:
             current_pump_profile = profile.title
 
+    if not current_pump_profile:
+        logger.error('No current pump profile, so skipping profile update: device: %s', device_profiles)
+        return False, current_ns_profile
+
     current_ns_profile = ns_profile_obj.get('defaultProfile')
     if current_pump_profile != current_ns_profile:
         logger.info("Current profile changed: pump: %s nightscout: %s", current_pump_profile, current_ns_profile)

tests/api/test_ws2.py

@@ -2,6 +2,7 @@
 
 import unittest
 import itertools
+import copy
 
 from .fake import WS2Api
 
@@ -28,7 +29,7 @@ class TestWS2Api(unittest.TestCase):
         ws2.get = self.fake_get_with_http_500(2)
 
         self.assertEqual(
-            ws2.therapy_timeline_csv('2021-04-01', '2021-04-02'),
+            ws2.therapy_timeline_csv('04-01-2021', '04-02-2021'),
             {
                 "readingData": [],
                 "iobData": [],
@@ -41,7 +42,7 @@ class TestWS2Api(unittest.TestCase):
 
         ws2.get = self.fake_get_with_http_500(3)
 
-        self.assertRaises(ApiException, ws2.therapy_timeline_csv, '2021-04-01', '2021-04-02')
+        self.assertRaises(ApiException, ws2.therapy_timeline_csv, '04-01-2021', '04-02-2021')
 
     RAW_DATA_HEADER = """Tandem Diabetes Care Inc.
 t:connect Therapy Timeline Data Export
@@ -96,12 +97,12 @@ Report Generated On, 4/24/2021 7:50:04 PM
 
         def fake_get(endpoint, **kwargs):
             nonlocal rawData
-            if endpoint == 'therapytimeline2csv/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/2021-04-01/2021-04-02?format=csv':
+            if endpoint == 'therapytimeline2csv/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/04-01-2021/04-02-2021?format=csv':
                 return rawData
 
         ws2.get = fake_get
 
-        tt = ws2.therapy_timeline_csv('2021-04-01', '2021-04-02')
+        tt = ws2.therapy_timeline_csv('04-01-2021', '04-02-2021')
 
         self.assertDictEqual(tt, self.PARSED_DATA)
 
@@ -113,7 +114,7 @@ Report Generated On, 4/24/2021 7:50:04 PM
 
         def fake_get(endpoint, **kwargs):
             nonlocal rawData
-            if endpoint == 'therapytimeline2csv/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/2021-04-01/2021-04-02?format=csv':
+            if endpoint == 'therapytimeline2csv/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/04-01-2021/04-02-2021?format=csv':
                 return rawData
 
         ws2.get = fake_get
@@ -122,8 +123,57 @@ Report Generated On, 4/24/2021 7:50:04 PM
         for i in itertools.permutations([self.RAW_DATA_HEADER, self.RAW_DATA_CGM, self.RAW_DATA_IOB, self.RAW_DATA_BOLUS], 4):
             rawData = "\n".join(i)
 
-            tt = ws2.therapy_timeline_csv('2021-04-01', '2021-04-02')
+            tt = ws2.therapy_timeline_csv('04-01-2021', '04-02-2021')
             self.assertDictEqual(tt, self.PARSED_DATA)
+    
+    def test_therapy_timeline_csv_split_past_max_days(self):
+        ws2 = WS2Api()
+        ws2.userGuid = 'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee'
+
+
+        def replace_str(raw, one, two):
+            return raw.replace('04-01-2021', one).replace('04-02-2021', two)
+        rawData1 = self.RAW_DATA_FULL
+        rawData2 = replace_str(self.RAW_DATA_FULL, '04-03-2021', '04-04-2021')
+        rawData3 = replace_str(self.RAW_DATA_FULL, '04-05-2021', '04-06-2021')
+        rawData4 = replace_str(self.RAW_DATA_FULL, '04-07-2021', '04-07-2021')
+
+        def replace_parsed(one, two):
+            parsedData = copy.deepcopy(self.PARSED_DATA)
+            for typ in parsedData.keys():
+                for i in range(len(parsedData[typ])):
+                    for f in parsedData[typ][i].keys():
+                        if 'datetime' in f.lower():
+                            parsedData[typ][i][f] = replace_str(parsedData[typ][i][f], one, two)
+            return parsedData
+
+        parsedData1 = self.PARSED_DATA
+        parsedData2 = replace_parsed('04-03-2021', '04-04-2021')
+        parsedData3 = replace_parsed('04-05-2021', '04-06-2021')
+        parsedData4 = replace_parsed('04-07-2021', '04-07-2021')
+
+        fullParsedData = parsedData1
+        for d in [parsedData2, parsedData3, parsedData4]:
+            for typ in d.keys():
+                fullParsedData[typ] += d[typ]
+
+        def fake_get(endpoint, **kwargs):
+            nonlocal rawData1, rawData2, rawData3, rawData4
+            print('fake_get call %s' % endpoint)
+            if endpoint == 'therapytimeline2csv/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/04-01-2021/04-02-2021?format=csv':
+                return rawData1
+            elif endpoint == 'therapytimeline2csv/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/04-03-2021/04-04-2021?format=csv':
+                return rawData2
+            elif endpoint == 'therapytimeline2csv/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/04-05-2021/04-06-2021?format=csv':
+                return rawData3
+            elif endpoint == 'therapytimeline2csv/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/04-07-2021/04-07-2021?format=csv':
+                return rawData4
+
+        ws2.get = fake_get
+
+        tt = ws2.therapy_timeline_csv('04-01-2021', '04-07-2021')
+
+        self.assertDictEqual(tt, fullParsedData)
 
 if __name__ == '__main__':
     unittest.main()

tests/test_process.py

@@ -8,7 +8,7 @@ from unittest.mock import patch
 
 from tconnectsync.process import process_time_range
 from tconnectsync.parser.nightscout import EXERCISE_EVENTTYPE, IOB_ACTIVITYTYPE, SLEEP_EVENTTYPE, NightscoutEntry
-from tconnectsync.features import BASAL, BOLUS, IOB, PROFILES, PUMP_EVENTS
+from tconnectsync.features import BASAL, BOLUS, IOB, PROFILES, PUMP_EVENTS, PUMP_EVENTS_BASAL_SUSPENSION
 from tconnectsync.domain.device_settings import Device
 from tests.secrets import build_secrets
 from tests.sync.test_profile import DEVICE_PROFILE_A, DEVICE_PROFILE_B, DEVICE_SETTINGS, NS_PROFILE_A, NS_PROFILE_B, build_ns_profile
@@ -649,7 +649,7 @@ class TestProcessTimeRange(unittest.TestCase):
         nightscout.last_uploaded_entry = self.stub_last_uploaded_entry
         nightscout.last_uploaded_activity = self.stub_last_uploaded_activity
 
-        count = process_time_range(tconnect, nightscout, start, end, pretend=False, features=[PUMP_EVENTS])
+        count = process_time_range(tconnect, nightscout, start, end, pretend=False, features=[PUMP_EVENTS, PUMP_EVENTS_BASAL_SUSPENSION])
 
         self.assertEqual(len(nightscout.uploaded_entries["treatments"]), 3)
         self.assertDictEqual(dict(nightscout.uploaded_entries), {
@@ -715,7 +715,7 @@ class TestProcessTimeRange(unittest.TestCase):
         nightscout.last_uploaded_entry = fake_last_uploaded_entry
         nightscout.last_uploaded_activity = self.stub_last_uploaded_activity
 
-        count = process_time_range(tconnect, nightscout, start, end, pretend=False, features=[PUMP_EVENTS])
+        count = process_time_range(tconnect, nightscout, start, end, pretend=False, features=[PUMP_EVENTS, PUMP_EVENTS_BASAL_SUSPENSION])
 
         self.assertEqual(len(nightscout.uploaded_entries["treatments"]), 1)
         self.assertDictEqual(dict(nightscout.uploaded_entries), {