parser/nightscout: add rate field for temp basals for autotune compatibility (fixes #7)

.codecov.yml

@@ -0,0 +1,11 @@
+  status:
+    patch: no
+    changes: no
+    project:
+      default: false
+      tconnectsync:
+        paths: "tconnectsync/"
+        target: 75%
+      tests:
+        paths: "tests/"
+        target: 95%

.github/workflows/python-package.yml

@@ -40,3 +40,11 @@ jobs:
     - name: Test with pytest
       run: |
         pytest
+    - name: Generate Coverage Report
+      run: |
+        pip install coverage
+        coverage run -m unittest
+    - name: Upload Coverage to Codecov
+      uses: codecov/codecov-action@v1
+      with:
+        fail_ci_if_error: true

Pipfile

@@ -11,7 +11,8 @@ bs4 = "*"
 arrow = "*"
 lxml = "*"
 python-dotenv = "*"
+requests-mock = "*"
 
 [scripts]
 tconnectsync = "python3 main.py"
-test = "python3 -m unittest discover -vv"
+test = "python3 -m unittest discover -vv"

Pipfile.lock

@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "071afba23de7f532f99a0611caed63951c6f7ba968ba877ad5c76c244e78577e"
+            "sha256": "979feca3aba1b7f43890a94862347738158af5847959c14dc1d5f23ce5db8cdc"
         },
         "pipfile-spec": 6,
         "requires": {},
@@ -126,6 +126,14 @@
             "index": "pypi",
             "version": "==2.25.1"
         },
+        "requests-mock": {
+            "hashes": [
+                "sha256:11215c6f4df72702aa357f205cf1e537cffd7392b3e787b58239bde5fb3db53b",
+                "sha256:e68f46844e4cee9d447150343c9ae875f99fa8037c6dcf5f15bf1fe9ab43d226"
+            ],
+            "index": "pypi",
+            "version": "==1.8.0"
+        },
         "six": {
             "hashes": [
                 "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259",

README.md

@@ -1,6 +1,7 @@
 # tconnectsync
 
 ![Python Package workflow](https://github.com/jwoglom/tconnectsync/actions/workflows/python-package.yml/badge.svg)
+[![codecov](https://codecov.io/gh/jwoglom/tconnectsync/branch/master/graph/badge.svg)](https://codecov.io/gh/jwoglom/tconnectsync)
 
 Tconnectsync synchronizes data one-way from the Tandem Diabetes t:connect web/mobile application to Nightscout.
 

main.py

@@ -4,6 +4,7 @@ import sys
 import datetime
 import arrow
 import argparse
+import logging
 
 from tconnectsync.api import TConnectApi
 from tconnectsync.process import process_time_range
@@ -26,6 +27,7 @@ except Exception:
 def parse_args():
     parser = argparse.ArgumentParser(description="Syncs bolus, basal, and IOB data from Tandem Diabetes t:connect to Nightscout.")
     parser.add_argument('--pretend', dest='pretend', action='store_const', const=True, default=False, help='Pretend mode: do not upload any data to Nightscout.')
+    parser.add_argument('-v', '--verbose', dest='verbose', action='store_const', const=True, default=False, help='Verbose mode: show extra logging details')
     parser.add_argument('--start-date', dest='start_date', type=str, default=None, help='The oldest date to process data from. Must be specified with --end-date.')
     parser.add_argument('--end-date', dest='end_date', type=str, default=None, help='The newest date to process data until (inclusive). Must be specified with --start-date.')
     parser.add_argument('--days', dest='days', type=int, default=1, help='The number of days of t:connect data to read in. Cannot be used with --from-date and --until-date.')
@@ -37,6 +39,12 @@ def parse_args():
 def main():
     args = parse_args()
 
+    if args.verbose:
+        logging.basicConfig(level=logging.DEBUG)
+        logging.root.debug("Set logging level to DEBUG")
+    else:
+        logging.basicConfig(level=logging.INFO)
+
     if args.auto_update and (args.start_date or args.end_date):
         raise Exception('Auto-update cannot be used with start/end date')
 

tconnectsync/api/__init__.py

@@ -1,7 +1,11 @@
+import logging
+
 from .android import AndroidApi
 from .controliq import ControlIQApi
 from .ws2 import WS2Api
 
+logger = logging.getLogger(__name__)
+
 """A wrapper for the three different t:connect API types."""
 class TConnectApi:
     email = None
@@ -21,6 +25,8 @@ class TConnectApi:
         if self._ciq and not self._ciq.needs_relogin():
             return self._ciq
 
+        logger.debug("Instantiating new ControlIQApi")
+
         self._ciq = ControlIQApi(self.email, self.password)
         return self._ciq
 
@@ -29,6 +35,12 @@ class TConnectApi:
         if self._ws2:
             return self._ws2
 
+        logger.debug("Instantiating new WS2Api")
+
+        # Trigger login or re-login via controliq api if necessary
+        # so userGuid can be accessed from it
+        self.controliq
+
         self._ws2 = WS2Api(self._ciq.userGuid)
         return self._ws2
 
@@ -37,6 +49,8 @@ class TConnectApi:
         if self._android and not self._android.needs_relogin():
             return self._android
 
+        logger.debug("Instantiating new AndroidApi")
+
         self._android = AndroidApi(self.email, self.password)
         return self._android
 

tconnectsync/api/android.py

@@ -5,11 +5,16 @@ import datetime
 import csv
 import base64
 import arrow
+import time
+import logging
 
 from bs4 import BeautifulSoup
 
+from ..util import timeago
 from .common import ApiException, ApiLoginException
 
+logger = logging.getLogger(__name__)
+
 """
 The AndroidApi class contains methods which are queried in the t:connect
 Android application. These methods are a part of the tdc API which require
@@ -34,6 +39,8 @@ class AndroidApi:
 
     def __init__(self, email, password):
         self.login(email, password)
+        self._email = email
+        self._password = password
 
     def login(self, email, password):
         r = requests.post(
@@ -64,6 +71,8 @@ class AndroidApi:
         self.userId = j["user"]["id"]
         self.patientObjectId = j["user"]["patientObjectId"]
 
+        logger.info("Logged in to AndroidApi successfully (expiration: %s, %s)" % (self.accessTokenExpiresAt, timeago(self.accessTokenExpiresAt)))
+
     def needs_relogin(self):
         diff = (arrow.get(self.accessTokenExpiresAt) - arrow.get())
         return (diff.seconds <= 5 * 60)
@@ -73,12 +82,33 @@ class AndroidApi:
             raise Exception('No access token')
         return {'Authorization': 'Bearer %s' % self.accessToken}
 
-    def get(self, endpoint, query={}, **kwargs):
+    def _get(self, endpoint, query={}, **kwargs):
         r = requests.get(self.BASE_URL + endpoint, query, headers=self.api_headers(), **kwargs)
+
         if r.status_code != 200:
-            raise ApiException(r.status_code, "Internal API HTTP %s response: %s" % (str(r.status_code), r.text))
+            raise ApiException(r.status_code, "Android API HTTP %s response: %s" % (str(r.status_code), r.text))
         return r.json()
 
+    def get(self, endpoint, query={}, tries=0, **kwargs):
+        try:
+            return self._get(endpoint, query, **kwargs)
+        except ApiException as e:
+            if tries > 0:
+                raise ApiException(e.status_code, "Android API HTTP %s on retry #%d: %s" % (e.status_code, tries, e))
+
+            # Trigger automatic re-login, and try again once
+            if e.status_code == 401:
+                self.accessTokenExpiresAt = time.time()
+                self.login(self._email, self._password)
+
+                return self.get(endpoint, query, tries=tries+1, **kwargs)
+
+            if e.status_code == 500:
+                return self.get(endpoint, query, tries=tries+1, **kwargs)
+
+            raise e
+
+
     def post(self, endpoint, query={}, **kwargs):
         r = requests.post(self.BASE_URL + endpoint, query, headers=self.api_headers(), **kwargs)
         if r.status_code != 200:

tconnectsync/api/common.py

@@ -11,7 +11,7 @@ def base_headers():
 class ApiException(Exception):
     def __init__(self, status_code, text, *args, **kwargs):
         self.status_code = status_code
-        super().__init__(text, *args, **kwargs)
+        super().__init__('%s (HTTP %s)' % (text, status_code), *args, **kwargs)
 
 class ApiLoginException(ApiException):
     pass

tconnectsync/api/controliq.py

@@ -2,11 +2,16 @@ import requests
 import urllib
 import datetime
 import arrow
+import time
+import logging
 
 from bs4 import BeautifulSoup
 
+from ..util import timeago
 from .common import parse_date, base_headers, ApiException, ApiLoginException
 
+logger = logging.getLogger(__name__)
+
 class ControlIQApi:
     BASE_URL = 'https://tdcservices.tandemdiabetes.com/tconnect/controliq/api/'
     LOGIN_URL = 'https://tconnect.tandemdiabetes.com/login.aspx?ReturnUrl=%2f'
@@ -17,23 +22,16 @@ class ControlIQApi:
 
     def __init__(self, email, password):
         self.login(email, password)
+        self._email = email
+        self._password = password
 
     def login(self, email, password):
+        logger.info("Logging in to ControlIQApi...")
         with requests.Session() as s:
             initial = s.get(self.LOGIN_URL, headers=base_headers())
             soup = BeautifulSoup(initial.content, features='lxml')
-            data = {
-                "__LASTFOCUS": "",
-                "__EVENTTARGET": "ctl00$ContentBody$LoginControl$linkLogin",
-                "__EVENTARGUMENT": "",
-                "__VIEWSTATE": soup.select_one("#__VIEWSTATE")["value"],
-                "__VIEWSTATEGENERATOR": soup.select_one("#__VIEWSTATEGENERATOR")["value"],
-                "__EVENTVALIDATION": soup.select_one("#__EVENTVALIDATION")["value"],
-                "ctl00$ContentBody$LoginControl$txtLoginEmailAddress": email,
-                "txtLoginEmailAddress_ClientState": '{"enabled":true,"emptyMessage":"","validationText":"%s","valueAsString":"%s","lastSetTextBoxValue":"%s"}' % (email, email, email),
-                "ctl00$ContentBody$LoginControl$txtLoginPassword": password,
-                "txtLoginPassword_ClientState": '{"enabled":true,"emptyMessage":"","validationText":"%s","valueAsString":"%s","lastSetTextBoxValue":"%s"}' % (password, password, password)
-            }
+            data = self._build_login_data(email, password, soup)
+
             req = s.post(self.LOGIN_URL, data=data, headers={'Referer': self.LOGIN_URL, **base_headers()}, allow_redirects=False)
             if req.status_code != 302:
                 raise ApiLoginException(req.status_code, 'Error logging in to t:connect. Check your login credentials.')
@@ -46,8 +44,23 @@ class ControlIQApi:
             self.userGuid = req.cookies['UserGUID']
             self.accessToken = req.cookies['accessToken']
             self.accessTokenExpiresAt = req.cookies['accessTokenExpiresAt']
+            logger.info("Logged in to ControlIQApi successfully (expiration: %s, %s)" % (self.accessTokenExpiresAt, timeago(self.accessTokenExpiresAt)))
             return True
 
+    def _build_login_data(self, email, password, soup):
+        return {
+            "__LASTFOCUS": "",
+            "__EVENTTARGET": "ctl00$ContentBody$LoginControl$linkLogin",
+            "__EVENTARGUMENT": "",
+            "__VIEWSTATE": soup.select_one("#__VIEWSTATE")["value"],
+            "__VIEWSTATEGENERATOR": soup.select_one("#__VIEWSTATEGENERATOR")["value"],
+            "__EVENTVALIDATION": soup.select_one("#__EVENTVALIDATION")["value"],
+            "ctl00$ContentBody$LoginControl$txtLoginEmailAddress": email,
+            "txtLoginEmailAddress_ClientState": '{"enabled":true,"emptyMessage":"","validationText":"%s","valueAsString":"%s","lastSetTextBoxValue":"%s"}' % (email, email, email),
+            "ctl00$ContentBody$LoginControl$txtLoginPassword": password,
+            "txtLoginPassword_ClientState": '{"enabled":true,"emptyMessage":"","validationText":"%s","valueAsString":"%s","lastSetTextBoxValue":"%s"}' % (password, password, password)
+        }
+
     def needs_relogin(self):
         diff = (arrow.get(self.accessTokenExpiresAt) - arrow.get())
         return (diff.seconds <= 5 * 60)
@@ -57,12 +70,35 @@ class ControlIQApi:
             raise Exception('No access token provided')
         return {'Authorization': 'Bearer %s' % self.accessToken, **base_headers()}
 
-    def get(self, endpoint, query):
+    def _get(self, endpoint, query):
         r = requests.get(self.BASE_URL + endpoint, query, headers=self.api_headers())
+
         if r.status_code != 200:
             raise ApiException(r.status_code, "ControlIQ API HTTP %s response: %s" % (str(r.status_code), r.text))
         return r.json()
 
+
+    def get(self, endpoint, query, tries=0):
+        try:
+            return self._get(endpoint, query)
+        except ApiException as e:
+            logger.warning("Received ApiException in ControlIQApi with endpoint '%s' (tries %d): %s" % (endpoint, tries, e))
+            if tries > 0:
+                raise ApiException(e.status_code, "ControlIQ API HTTP %d on retry #%d: %s", e.status_code, tries, e)
+
+            # Trigger automatic re-login, and try again once
+            if e.status_code == 401:
+                logger.info("Performing automatic re-login after HTTP 401 for ControlIQApi")
+                self.accessTokenExpiresAt = time.time()
+                self.login(self._email, self._password)
+
+                return self.get(endpoint, query, tries=tries+1)
+
+            if e.status_code == 500:
+                return self.get(endpoint, query, tries=tries+1)
+
+            raise e
+
     """
     Returns detailed basal event information and reasons for delivery suspension.
     """

tconnectsync/api/ws2.py

@@ -1,12 +1,18 @@
 import requests
 import datetime
 import csv
+import logging
+import time
 
 from .common import parse_date, base_headers, ApiException
 
+logger = logging.getLogger(__name__)
+
 class WS2Api:
     BASE_URL = 'https://tconnectws2.tandemdiabetes.com/'
 
+    MAX_RETRIES = 2
+
     userGuid = None
 
     def __init__(self, userGuid):
@@ -54,11 +60,22 @@ class WS2Api:
         return data
 
 
-    def therapy_timeline_csv(self, start=None, end=None):
+    def therapy_timeline_csv(self, start=None, end=None, tries=0):
         startDate = parse_date(start)
         endDate = parse_date(end)
 
-        req_text = self.get('therapytimeline2csv/%s/%s/%s?format=csv' % (self.userGuid, startDate, endDate), {})
+        try:
+            req_text = self.get('therapytimeline2csv/%s/%s/%s?format=csv' % (self.userGuid, startDate, endDate), {})
+        except ApiException as e:
+            # This seems to occur as some kind of soft rate-limit.
+            logger.warning("Received ApiException in therapy_timeline_csv: (retry count %d) %s" % (tries, e))
+            if e.status_code == 500:
+                sleep_seconds = (tries+1) * 60
+                logger.error("Retrying in %d seconds after HTTP 500 in therapy_timeline_csv (retry count %d): %s" % (sleep_seconds, tries, e))
+                time.sleep(sleep_seconds)
+                if tries < self.MAX_RETRIES:
+                    return self.therapy_timeline_csv(start, end, tries+1)
+            raise e
 
         sections = self._split_empty_sections(req_text)
 

tconnectsync/autoupdate.py

@@ -1,13 +1,18 @@
 import time
+import logging
 
 from .process import process_time_range
 from .secret import (
     PUMP_SERIAL_NUMBER,
     AUTOUPDATE_DEFAULT_SLEEP_SECONDS,
     AUTOUPDATE_MAX_SLEEP_SECONDS,
-    AUTOUPDATE_USE_FIXED_SLEEP
+    AUTOUPDATE_USE_FIXED_SLEEP,
+    AUTOUPDATE_FAILURE_MINUTES,
+    AUTOUPDATE_RESTART_ON_FAILURE
 )
 
+logger = logging.getLogger(__name__)
+
 """
 Performs the auto-update functionality. Runs indefinitely in a loop
 until stopped (ctrl+c).
@@ -18,30 +23,53 @@ def process_auto_update(tconnect, nightscout, time_start, time_end, pretend):
 
     last_event_index = None
     last_event_time = None
+    last_process_time_range = None
     time_diffs = []
     while True:
         last_event = tconnect.android.last_event_uploaded(PUMP_SERIAL_NUMBER)
         if not last_event_index or last_event['maxPumpEventIndex'] > last_event_index:
             now = time.time()
-            print('New event index:', last_event['maxPumpEventIndex'], 'last:', last_event_index)
+            logger.info('New reported t:connect data. (event index: %d last: %d)' % (last_event['maxPumpEventIndex'], last_event_index))
 
             if pretend:
-                print('Would update now')
+                logger.info('Would update now if not in pretend mode')
             else:
                 added = process_time_range(tconnect, nightscout, time_start, time_end, pretend)
-                print('Added', added, 'items')
+                logger.info('Added %d items from process_time_range' % added)
+                if len(added) == 0:
+                    if last_event_index:
+                        logger.error('An event index change was recorded, but no new data was found via the API. ' +
+                                     'If this error reoccurs, try restarting tconnectsync.')
+                else:
+                    last_process_time_range = now
+
 
             if last_event_index:
                 time_diffs.append(now - last_event_time)
-                print('Time diffs:', time_diffs)
+                logger.debug('Updating tracking of time since last update: %s' % time_diffs)
 
             last_event_index = last_event['maxPumpEventIndex']
             last_event_time = now
         else:
-            print('No event index change:', last_event['maxPumpEventIndex'])
+            logger.info('No new reported t:connect data. (last event index: %d)' % last_event['maxPumpEventIndex'])
+            now = time.time()
+
+            if (now - last_event_time) >= 60 * AUTOUPDATE_FAILURE_MINUTES:
+                logger.error("No new data event indexes have been detected for over %d minutes. " % AUTOUPDATE_FAILURE_MINUTES +
+                             "The t:connect app might no longer be functioning.")
+
+                if AUTOUPDATE_RESTART_ON_FAILURE:
+                    raise AutoupdateFailureException
+            elif (now - last_process_time_range) >= 60 * AUTOUPDATE_FAILURE_MINUTES:
+                logger.error("No new data has been found via the API for over %d minutes. " % AUTOUPDATE_FAILURE_MINUTES +
+                             "tconnectsync might not be functioning properly.")
+
+                if AUTOUPDATE_RESTART_ON_FAILURE:
+                    raise AutoupdateFailureException
+
 
             if len(time_diffs) > 2:
-                print('Sleeping 60 seconds after unexpected no index change')
+                logger.info('Sleeping 60 seconds after unexpected no index change. (New data might be delayed.)')
                 time.sleep(60)
                 continue
 
@@ -57,5 +85,8 @@ def process_auto_update(tconnect, nightscout, time_start, time_end, pretend):
                 sleep_secs = AUTOUPDATE_MAX_SLEEP_SECONDS
 
         # Sleep for a rolling average of time between updates
-        print('Sleeping for', sleep_secs, 'sec')
-        time.sleep(sleep_secs)
+        logger.info('Sleeping for %d sec' % sleep_secs)
+        time.sleep(sleep_secs)
+
+class AutoupdateFailureException(RuntimeError):
+    pass

tconnectsync/parser/nightscout.py

@@ -14,6 +14,7 @@ class NightscoutEntry:
             "reason": reason,
             "duration": float(duration_mins) if duration_mins else None,
             "absolute": float(value),
+            "rate": float(value),
             "created_at": created_at,
             "carbs": None,
             "insulin": None,

tconnectsync/process.py

@@ -1,5 +1,9 @@
-from datetime import datetime
+import logging
+import datetime
+import arrow
+import time
 
+from .util import timeago
 from .api.common import ApiException
 from .sync.basal import (
     process_ciq_basal_events,
@@ -14,6 +18,9 @@ from .sync.iob import (
     process_iob_events,
     ns_write_iob_events
 )
+from .parser.tconnect import TConnectEntry
+
+logger = logging.getLogger(__name__)
 
 """
 Given a TConnectApi object and start/end range, performs a single
@@ -21,7 +28,7 @@ cycle of synchronizing data within the time range.
 If pretend is true, then doesn't actually write data to Nightscout.
 """
 def process_time_range(tconnect, nightscout, time_start, time_end, pretend):
-    print("Downloading t:connect ControlIQ data")
+    logger.info("Downloading t:connect ControlIQ data")
     try:
         ciqTherapyTimelineData = tconnect.controliq.therapy_timeline(time_start, time_end)
     except ApiException as e:
@@ -29,12 +36,12 @@ def process_time_range(tconnect, nightscout, time_start, time_end, pretend):
         # device in the time range which is queried. Since it launched in early 2020,
         # ignore 404's before February.
         if e.status_code == 404 and time_start.date() < datetime.date(2020, 2, 1):
-            print("Ignoring HTTP 404 for ControlIQ API request before Feb 2020")
+            logger.warning("Ignoring HTTP 404 for ControlIQ API request before Feb 2020")
             ciqTherapyTimelineData = None
         else:
             raise e
 
-    print("Downloading t:connect CSV data")
+    logger.info("Downloading t:connect CSV data")
     csvdata = tconnect.ws2.therapy_timeline_csv(time_start, time_end)
 
     readingData = csvdata["readingData"]
@@ -43,13 +50,21 @@ def process_time_range(tconnect, nightscout, time_start, time_end, pretend):
     bolusData = csvdata["bolusData"]
 
     if readingData and len(readingData) > 0:
-        print("Last CGM reading from t:connect:", readingData[-1]['EventDateTime'] if 'EventDateTime' in readingData[-1] else readingData)
+        lastReading = readingData[-1]['EventDateTime'] if 'EventDateTime' in readingData[-1] else 0
+        lastReading = TConnectEntry._datetime_parse(lastReading)
+        logger.debug(readingData[-1])
+        logger.info("Last CGM reading from t:connect: %s (%s)" % (lastReading, timeago(lastReading)))
+    else:
+        logger.warning("No last CGM reading is able to be determined")
 
     added = 0
 
     basalEvents = process_ciq_basal_events(ciqTherapyTimelineData)
     if csvBasalData:
+        logger.debug("CSV basal data found: processing it")
         add_csv_basal_events(basalEvents, csvBasalData)
+    else:
+        logger.debug("No CSV basal data found")
 
     added += ns_write_basal_events(nightscout, basalEvents, pretend=pretend)
 
@@ -59,4 +74,5 @@ def process_time_range(tconnect, nightscout, time_start, time_end, pretend):
     iobEvents = process_iob_events(iobData)
     added += ns_write_iob_events(nightscout, iobEvents, pretend=pretend)
 
+    logger.info("Wrote %d events to Nightscout this process cycle")
     return added

tconnectsync/secret.py

@@ -15,6 +15,9 @@ def get_number(name, default):
         print("Current value: %s" % val)
         sys.exit(1)
 
+def get_bool(name, default):
+    return str(get(name, default) or '').lower() in ('true', '1')
+
 TCONNECT_EMAIL = get('TCONNECT_EMAIL', 'email@email.com')
 TCONNECT_PASSWORD = get('TCONNECT_PASSWORD', 'password')
 
@@ -27,14 +30,17 @@ TIMEZONE_NAME = get('TIMEZONE_NAME', 'America/New_York')
 
 # Optional configuration
 
-AUTOUPDATE_DEFAULT_SLEEP_SECONDS = get_number('AUTOUPDATE_DEFAULT_SLEEP_SECONDS', '60')
-AUTOUPDATE_MAX_SLEEP_SECONDS = get_number('AUTOUPDATE_MAX_SLEEP_SECONDS', '600')
-AUTOUPDATE_USE_FIXED_SLEEP = get_number('AUTOUPDATE_USE_FIXED_SLEEP', '0')
+AUTOUPDATE_DEFAULT_SLEEP_SECONDS = get_number('AUTOUPDATE_DEFAULT_SLEEP_SECONDS', '300') # 5 minutes
+AUTOUPDATE_MAX_SLEEP_SECONDS = get_number('AUTOUPDATE_MAX_SLEEP_SECONDS', '1500') # 25 minutes
+AUTOUPDATE_USE_FIXED_SLEEP = get_bool('AUTOUPDATE_USE_FIXED_SLEEP', 'false')
+AUTOUPDATE_FAILURE_MINUTES = get_number('AUTOUPDATE_FAILURE_MINUTES', '180') # 3 hours
+AUTOUPDATE_RESTART_ON_FAILURE = get_bool('AUTOUPDATE_RESTART_ON_FAILURE', 'false')
 
 _config = ['TCONNECT_EMAIL', 'TCONNECT_PASSWORD', 'PUMP_SERIAL_NUMBER',
           'NS_URL', 'NS_SECRET', 'TIMEZONE_NAME',
           'AUTOUPDATE_DEFAULT_SLEEP_SECONDS', 'AUTOUPDATE_MAX_SLEEP_SECONDS',
-          'AUTOUPDATE_USE_FIXED_SLEEP']
+          'AUTOUPDATE_USE_FIXED_SLEEP', 'AUTOUPDATE_FAILURE_MINUTES',
+          'AUTOUPDATE_RESTART_ON_FAILURE']
 
 if __name__ == '__main__':
     for k in locals():

tconnectsync/sync/basal.py

@@ -1,4 +1,5 @@
 import arrow
+import logging
 
 from ..parser.nightscout import (
     BASAL_EVENTTYPE,
@@ -6,6 +7,7 @@ from ..parser.nightscout import (
 )
 from ..parser.tconnect import TConnectEntry
 
+logger = logging.getLogger(__name__)
 
 """
 Merges together input from the therapy timeline API
@@ -60,17 +62,18 @@ def add_csv_basal_events(basalEvents, data):
 Given processed basal data, adds basal events to Nightscout.
 """
 def ns_write_basal_events(nightscout, basalEvents, pretend=False):
+    logger.debug("ns_write_basal_events: querying for last uploaded entry")
     last_upload = nightscout.last_uploaded_entry(BASAL_EVENTTYPE)
     last_upload_time = None
     if last_upload:
         last_upload_time = arrow.get(last_upload["created_at"])
-    print("Last Nightscout basal upload:", last_upload_time)
+    logger.info("Last Nightscout basal upload: %s" % last_upload_time)
 
     add_count = 0
     for event in basalEvents:
         if last_upload_time and arrow.get(event["time"]) < last_upload_time:
             if pretend:
-                print("Skipping basal event before last upload time:", event)
+                logger.info("Skipping basal event before last upload time: %s" % event)
             continue
 
         recent_needs_update = False
@@ -97,13 +100,14 @@ def ns_write_basal_events(nightscout, basalEvents, pretend=False):
 
         add_count += 1
 
-        print("  Processing basal:", event, "entry:", entry)
+        logger.info("  Processing basal: %s entry: %s" % (event, entry))
         if recent_needs_update:
-            print("Replacing last uploaded entry:", last_upload)
+            logger.info("Replacing last uploaded entry: %s" % last_upload)
             if not pretend:
                 entry['_id'] = last_upload['_id']
                 nightscout.put_entry(entry, entity='treatments')
         elif not pretend:
             nightscout.upload_entry(entry)
 
+    logger.debug("ns_write_basal_events: added %d events" % add_count)
     return add_count

tconnectsync/sync/bolus.py

@@ -1,4 +1,5 @@
 import arrow
+import logging
 
 from ..parser.nightscout import (
     BOLUS_EVENTTYPE,
@@ -6,6 +7,8 @@ from ..parser.nightscout import (
 )
 from ..parser.tconnect import TConnectEntry
 
+logger = logging.getLogger(__name__)
+
 """
 Given bolus data input from the therapy timeline CSV, converts it into a digestable format.
 """
@@ -19,7 +22,7 @@ def process_bolus_events(bolusdata):
                 # Count non-completed bolus if any insulin was delivered (vs. the amount of insulin requested)
                 parsed["description"] += " (%s)" % parsed["completion"]
             else:
-                print("Skipping non-completed bolus data:", b, "parsed:", parsed)
+                logger.warning("Skipping non-completed bolus data (was a bolus in progress?): %s parsed: %s" % (b, parsed))
                 continue
         bolusEvents.append(parsed)
 
@@ -31,17 +34,18 @@ def process_bolus_events(bolusdata):
 Given processed bolus data, adds bolus events to Nightscout.
 """
 def ns_write_bolus_events(nightscout, bolusEvents, pretend=False):
+    logger.debug("ns_write_bolus_events: querying for last uploaded entry")
     last_upload = nightscout.last_uploaded_entry(BOLUS_EVENTTYPE)
     last_upload_time = None
     if last_upload:
         last_upload_time = arrow.get(last_upload["created_at"])
-    print("Last Nightscout bolus upload:", last_upload_time)
+    logger.info("Last Nightscout bolus upload: %s" % last_upload_time)
 
     add_count = 0
     for event in bolusEvents:
         if last_upload_time and arrow.get(event["completion_time"]) <= last_upload_time:
             if pretend:
-                print("Skipping basal event before last upload time:", event)
+                logger.info("Skipping basal event before last upload time: %s" % event)
             continue
 
         entry = NightscoutEntry.bolus(
@@ -53,7 +57,7 @@ def ns_write_bolus_events(nightscout, bolusEvents, pretend=False):
 
         add_count += 1
 
-        print("  Processing bolus:", event, "entry:", entry)
+        logger.info("  Processing bolus: %s entry: %s" % (event, entry))
         if not pretend:
             nightscout.upload_entry(entry)
 

tconnectsync/sync/iob.py

@@ -1,4 +1,5 @@
 import arrow
+import logging
 
 from ..parser.nightscout import (
     IOB_ACTIVITYTYPE,
@@ -6,6 +7,8 @@ from ..parser.nightscout import (
 )
 from ..parser.tconnect import TConnectEntry
 
+logger = logging.getLogger(__name__)
+
 """
 Given IOB data input from the therapy timeline CSV, converts it into a digestable format.
 """
@@ -22,19 +25,20 @@ def process_iob_events(iobdata):
 Given processed IOB data, creates a single Nightscout activity definition to store IOB.
 """
 def ns_write_iob_events(nightscout, iobEvents, pretend=False):
+    logger.debug("ns_write_iob_events: querying for last uploaded entry")
     last_upload = nightscout.last_uploaded_activity(IOB_ACTIVITYTYPE)
     last_upload_time = None
     if last_upload:
         last_upload_time = arrow.get(last_upload["created_at"])
-    print("Last Nightscout iob upload:", last_upload_time)
+    logger.info("Last Nightscout iob upload: %s" % last_upload_time)
 
     if not iobEvents or len(iobEvents) == 0:
-        print("No IOB events: skipping")
+        logger.info("No IOB events present from API: skipping")
         return 0
 
     event = iobEvents[-1]
     if last_upload_time and arrow.get(event["time"]) <= last_upload_time:
-        print("  Skipping already uploaded iob event:", event)
+        logger.info("  Skipping already uploaded iob event: %s" % event)
         return 0
 
     entry = NightscoutEntry.iob(
@@ -42,13 +46,13 @@ def ns_write_iob_events(nightscout, iobEvents, pretend=False):
         created_at=event["time"]
     )
 
-    print("  Processing iob:", event, "entry:", entry)
+    logger.info("  Processing iob: %s entry: %s" % (event, entry))
     if not pretend:
         nightscout.upload_entry(entry, entity='activity')
 
     # Delete the previous activity
     if last_upload and '_id' in last_upload:
-        print("  Deleting old iob entry:", last_upload)
+        logger.info("  Deleting old iob entry: %s" % last_upload)
         if not pretend:
             nightscout.delete_entry('activity/{}'.format(last_upload['_id']))
 

tconnectsync/util/__init__.py

@@ -0,0 +1,17 @@
+import arrow
+
+def timeago(timestamp):
+    seconds = (arrow.get() - arrow.get(timestamp)).total_seconds()
+    fmt = '%s ago' if seconds >= 0 else 'in %s'
+    seconds = abs(seconds)
+
+    ret = ''
+    if seconds//86400 > 0:
+        ret += '%d days, ' % (seconds//86400)
+        seconds = seconds % 86400
+    if seconds//3600 > 0:
+        ret += '%d hours, ' % (seconds//3600)
+        seconds = seconds % 3600
+    ret += '%d minutes' % (seconds//60)
+
+    return fmt % ret

tests/api/fake.py

@@ -11,7 +11,7 @@ class ControlIQApi(tconnectsync.api.controliq.ControlIQApi):
     def needs_relogin(self):
         return False
 
-    def get(self, endpoint, query):
+    def _get(self, endpoint, query):
         raise NotImplementedError
 
 class WS2Api(tconnectsync.api.ws2.WS2Api):
@@ -34,7 +34,7 @@ class AndroidApi(tconnectsync.api.android.AndroidApi):
     def needs_relogin(self):
         return False
 
-    def get(self, endpoint, query={}, **kwargs):
+    def _get(self, endpoint, query={}, **kwargs):
         raise NotImplementedError
 
 class TConnectApi(tconnectsync.api.TConnectApi):

tests/api/test_android.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+
+import unittest
+import itertools
+import datetime
+
+from .fake import AndroidApi
+
+from tconnectsync.api.common import ApiException
+
+class TestAndroidApi(unittest.TestCase):
+    def fake_get_with_http_code(self, http_code, expected_endpoint, num_times):
+        tries = 0
+        def fake_get(endpoint, query):
+            nonlocal http_code, expected_endpoint, num_times, tries
+            if endpoint.endswith(expected_endpoint):
+                if tries < num_times:
+                    tries += 1
+                    raise ApiException(http_code, "fake HTTP %d" % http_code)
+
+                return {"faked_json": True}
+
+            raise NotImplementedError
+
+        return fake_get
+
+    def test_last_event_uploaded_works_after_single_http_500(self):
+        android = AndroidApi()
+
+        android._get = self.fake_get_with_http_code(500, "cloud/upload/getlasteventuploaded?sn=1111111", 1)
+
+        self.assertEqual(
+            android.last_event_uploaded(1111111),
+            {
+                "faked_json": True
+            })
+
+    def test_last_event_uploaded_fails_after_two_http_500s(self):
+        android = AndroidApi()
+
+        android._get = self.fake_get_with_http_code(500, "cloud/upload/getlasteventuploaded?sn=1111111", 2)
+
+        self.assertRaises(ApiException, android.last_event_uploaded, 1111111)
+
+    def test_last_event_uploaded_triggers_relogin_after_single_http_401(self):
+        android = AndroidApi()
+        android._email = 'email'
+        android._password = 'password'
+
+        hit_login = []
+        def stub_login(email, password):
+            nonlocal hit_login
+            hit_login.append((email, password))
+
+        android.login = stub_login
+
+        android._get = self.fake_get_with_http_code(401, "cloud/upload/getlasteventuploaded?sn=1111111", 1)
+
+        self.assertEqual(
+            android.last_event_uploaded(1111111),
+            {
+                "faked_json": True
+            })
+
+        self.assertListEqual(hit_login, [
+            ('email', 'password')
+        ])
+
+    def test_last_event_uploaded_fails_after_two_http_401s(self):
+        android = AndroidApi()
+        android._email = 'email'
+        android._password = 'password'
+
+        hit_login = []
+        def stub_login(email, password):
+            nonlocal hit_login
+            hit_login.append((email, password))
+
+        android.login = stub_login
+
+        android._get = self.fake_get_with_http_code(401, "cloud/upload/getlasteventuploaded?sn=1111111", 2)
+
+        self.assertRaises(ApiException, android.last_event_uploaded, 1111111)
+
+        self.assertListEqual(hit_login, [
+            ('email', 'password')
+        ])
+
+if __name__ == '__main__':
+    unittest.main()

tests/api/test_controliq.py

@@ -0,0 +1,244 @@
+#!/usr/bin/env python3
+
+import unittest
+import itertools
+import datetime
+import json
+import requests_mock
+
+from bs4 import BeautifulSoup
+
+from .fake import ControlIQApi
+
+from tconnectsync.api.controliq import ControlIQApi as RealControlIQApi
+from tconnectsync.api.common import ApiException, ApiLoginException, base_headers
+
+class TestControlIQApi(unittest.TestCase):
+    LOGIN_HTML = """
+<html>
+<body>
+    <form method="post" action="./login.aspx?ReturnUrl=%2f" onsubmit="javascript:return WebForm_OnSubmit();" id="form1">
+        <div class="aspNetHidden">
+            <input type="hidden" name="__LASTFOCUS" id="__LASTFOCUS" value="" />
+            <input type="hidden" name="__EVENTTARGET" id="__EVENTTARGET" value="" />
+            <input type="hidden" name="__EVENTARGUMENT" id="__EVENTARGUMENT" value="" />
+            <input type="hidden" name="__VIEWSTATE" id="__VIEWSTATE" value="AAAAA" />
+        </div>
+        <div class="aspNetHidden">
+            <input type="hidden" name="__VIEWSTATEGENERATOR" id="__VIEWSTATEGENERATOR" value="BBBBB" />
+            <input type="hidden" name="__EVENTVALIDATION" id="__EVENTVALIDATION" value="CCCCC" />
+        </div>
+    </form>
+</body>
+</html>
+    """
+
+    LOGIN_POST_DATA = {
+        "__LASTFOCUS": "",
+        "__EVENTTARGET": "ctl00$ContentBody$LoginControl$linkLogin",
+        "__EVENTARGUMENT": "",
+        "__VIEWSTATE": "AAAAA",
+        "__VIEWSTATEGENERATOR": "BBBBB",
+        "__EVENTVALIDATION": "CCCCC",
+        "ctl00$ContentBody$LoginControl$txtLoginEmailAddress": "email@email.com",
+        "txtLoginEmailAddress_ClientState": '{"enabled":true,"emptyMessage":"","validationText":"%s","valueAsString":"%s","lastSetTextBoxValue":"%s"}' % ("email@email.com", "email@email.com", "email@email.com"),
+        "ctl00$ContentBody$LoginControl$txtLoginPassword": "password",
+        "txtLoginPassword_ClientState": '{"enabled":true,"emptyMessage":"","validationText":"%s","valueAsString":"%s","lastSetTextBoxValue":"%s"}' % ("password", "password", "password")
+    }
+
+    def test_build_login_data(self):
+        ciq = ControlIQApi()
+        soup = BeautifulSoup(self.LOGIN_HTML, features='lxml')
+
+        self.assertDictEqual(
+            ciq._build_login_data('email@email.com', 'password', soup),
+            self.LOGIN_POST_DATA)
+
+    def test_login_successful(self):
+        ciq = ControlIQApi()
+        ciq.LOGIN_URL = RealControlIQApi.LOGIN_URL
+        ciq.login = lambda email, password: RealControlIQApi.login(ciq, email, password)
+
+        with requests_mock.Mocker() as m:
+            m.get('https://tconnect.tandemdiabetes.com/login.aspx?ReturnUrl=%2f',
+                request_headers=base_headers(),
+
+                text=self.LOGIN_HTML)
+
+            def post_callback(request, context):
+                context.status_code = 302
+                context.headers['Location'] = '/newlocation'
+                context.cookies['UserGUID'] = 'user_guid'
+                context.cookies['accessToken'] = 'access_tok'
+                context.cookies['accessTokenExpiresAt'] = '2021-05-04T11:18:08.381Z'
+                return ''
+
+            m.post('https://tconnect.tandemdiabetes.com/login.aspx?ReturnUrl=%2f',
+                request_headers={'Referer': ciq.LOGIN_URL, **base_headers()},
+
+                text=post_callback)
+
+            m.post('https://tconnect.tandemdiabetes.com/newlocation',
+                cookies={'cookie': 'value'},
+                headers=base_headers(),
+
+                status_code=200)
+
+            self.assertTrue(ciq.login('email@email.com', 'password'))
+
+            self.assertEqual(ciq.userGuid, 'user_guid')
+            self.assertEqual(ciq.accessToken, 'access_tok')
+            self.assertEqual(ciq.accessTokenExpiresAt, '2021-05-04T11:18:08.381Z')
+
+
+    def test_login_invalid_credentials(self):
+        ciq = ControlIQApi()
+        ciq.LOGIN_URL = RealControlIQApi.LOGIN_URL
+        ciq.login = lambda email, password: RealControlIQApi.login(ciq, email, password)
+
+        with requests_mock.Mocker() as m:
+            m.get('https://tconnect.tandemdiabetes.com/login.aspx?ReturnUrl=%2f',
+                request_headers=base_headers(),
+
+                text=self.LOGIN_HTML)
+
+            def post_callback(request, context):
+                context.status_code = 200
+                return '<html><body>...</body></html>'
+
+            m.post('https://tconnect.tandemdiabetes.com/login.aspx?ReturnUrl=%2f',
+                request_headers={'Referer': ciq.LOGIN_URL, **base_headers()},
+
+                text=post_callback)
+
+            self.assertRaises(ApiLoginException, ciq.login, 'email@email.com', 'password')
+
+            self.assertIsNone(ciq.userGuid)
+            self.assertIsNone(ciq.accessToken)
+            self.assertIsNone(ciq.accessTokenExpiresAt)
+
+    def fake_get_with_http_code(self, http_code, expected_endpoint, num_times):
+        tries = 0
+        def fake_get(endpoint, query):
+            nonlocal http_code, expected_endpoint, num_times, tries
+            if endpoint.endswith(expected_endpoint):
+                if tries < num_times:
+                    tries += 1
+                    raise ApiException(http_code, "fake HTTP %d" % http_code)
+
+                return {"faked_json": True}
+
+            raise NotImplementedError
+
+        return fake_get
+
+    def test_therapy_timeline_works_after_single_http_500(self):
+        ciq = ControlIQApi()
+        ciq.userGuid = 'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee'
+
+        ciq._get = self.fake_get_with_http_code(500, "therapytimeline/users/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", 1)
+
+        self.assertEqual(
+            ciq.therapy_timeline('2021-04-01', '2021-04-02'),
+            {
+                "faked_json": True
+            })
+
+    def test_therapy_timeline_fails_after_two_http_500s(self):
+        ciq = ControlIQApi()
+        ciq.userGuid = 'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee'
+
+        ciq._get = self.fake_get_with_http_code(500, "therapytimeline/users/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", 2)
+
+        self.assertRaises(ApiException, ciq.therapy_timeline, '2021-04-01', '2021-04-02')
+
+    def test_therapy_timeline_triggers_relogin_after_single_http_401(self):
+        ciq = ControlIQApi()
+        ciq.userGuid = 'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee'
+        ciq._email = 'email'
+        ciq._password = 'password'
+
+        hit_login = []
+        def stub_login(email, password):
+            nonlocal hit_login
+            hit_login.append((email, password))
+
+        ciq.login = stub_login
+
+        ciq._get = self.fake_get_with_http_code(401, "therapytimeline/users/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", 1)
+
+        self.assertEqual(
+            ciq.therapy_timeline('2021-04-01', '2021-04-02'),
+            {
+                "faked_json": True
+            })
+
+        self.assertListEqual(hit_login, [
+            ('email', 'password')
+        ])
+
+    def test_therapy_timeline_fails_after_two_http_401s(self):
+        ciq = ControlIQApi()
+        ciq.userGuid = 'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee'
+        ciq._email = 'email'
+        ciq._password = 'password'
+
+        hit_login = []
+        def stub_login(email, password):
+            nonlocal hit_login
+            hit_login.append((email, password))
+
+        ciq.login = stub_login
+
+        ciq._get = self.fake_get_with_http_code(401, "therapytimeline/users/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", 2)
+
+        self.assertRaises(ApiException, ciq.therapy_timeline, '2021-04-01', '2021-04-02')
+
+        self.assertListEqual(hit_login, [
+            ('email', 'password')
+        ])
+
+    def test_therapy_timeline_parses_date(self):
+        ciq = ControlIQApi()
+        ciq.userGuid = 'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee'
+
+        def fake_get(endpoint, query):
+            self.assertTrue(endpoint.endswith("therapytimeline/users/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"))
+            self.assertEqual(query, {
+                "startDate": "04-01-2021",
+                "endDate": "04-02-2021"
+            })
+
+            return {"faked_json": True}
+
+        ciq._get = fake_get
+
+        self.assertEqual(
+            ciq.therapy_timeline(datetime.date(2021, 4, 1), datetime.date(2021, 4, 2)),
+            {
+                "faked_json": True
+            })
+
+    def test_dashboard_summary_parses_date(self):
+        ciq = ControlIQApi()
+        ciq.userGuid = 'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee'
+
+        def fake_get(endpoint, query):
+            self.assertTrue(endpoint.endswith("summary/users/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"))
+            self.assertEqual(query, {
+                "startDate": "04-01-2021",
+                "endDate": "04-02-2021"
+            })
+
+            return {"faked_json": True}
+
+        ciq._get = fake_get
+
+        self.assertEqual(
+            ciq.dashboard_summary(datetime.date(2021, 4, 1), datetime.date(2021, 4, 2)),
+            {
+                "faked_json": True
+            })
+
+if __name__ == '__main__':
+    unittest.main()

tests/api/test_ws2.py

@@ -0,0 +1,129 @@
+#!/usr/bin/env python3
+
+import unittest
+import itertools
+
+from .fake import WS2Api
+
+from tconnectsync.api.common import ApiException
+
+class TestWS2Api(unittest.TestCase):
+    def fake_get_with_http_500(self, num_times):
+        tries = 0
+        def fake_get(endpoint, query):
+            nonlocal tries, num_times
+            if "therapytimeline2csv" in endpoint:
+                if tries < num_times:
+                    tries += 1
+                    raise ApiException(500, "fake HTTP 500")
+
+                return ""
+            raise NotImplementedError
+
+        return fake_get
+
+    def test_therapy_timeline_csv_works_after_two_retries(self):
+        ws2 = WS2Api()
+
+        ws2.get = self.fake_get_with_http_500(2)
+
+        self.assertEqual(
+            ws2.therapy_timeline_csv('2021-04-01', '2021-04-02'),
+            {
+                "readingData": [],
+                "iobData": [],
+                "basalData": [],
+                "bolusData": []
+            })
+
+    def test_therapy_timeline_csv_fails_after_three_retries(self):
+        ws2 = WS2Api()
+
+        ws2.get = self.fake_get_with_http_500(3)
+
+        self.assertRaises(ApiException, ws2.therapy_timeline_csv, '2021-04-01', '2021-04-02')
+
+    RAW_DATA_HEADER = """Tandem Diabetes Care Inc.
+t:connect Therapy Timeline Data Export
+Patient Name, Sample Name
+Patient DOB, 1/1/1990
+Report Generated On, 4/24/2021 7:50:04 PM
+"""
+    RAW_DATA_CGM = """DeviceType,SerialNumber,Description,EventDateTime,Readings (CGM / BGM)
+"t:slim X2 Insulin Pump","11111111","EGV","2021-04-01T00:01:33","235",
+"t:slim X2 Insulin Pump","11111111","EGV","2021-04-01T00:06:33","230",
+"t:slim X2 Insulin Pump","11111111","EGV","2021-04-02T23:31:36","181",
+"""
+    RAW_DATA_IOB = """Type,EventID,EventDateTime,IOB
+"IOB","81","2021-04-01T00:00:19","13.24"
+"IOB","9","2021-04-01T00:03:12","12.80"
+"IOB","81","2021-04-02T23:58:19","4.25"
+"""
+    RAW_DATA_BOLUS = """Type,Description,BG,IOB,BolusRequestID,BolusCompletionID,CompletionDateTime,InsulinDelivered,FoodDelivered,CorrectionDelivered,CompletionStatusID,CompletionStatusDesc,BolusIsComplete,BolexCompletionID,BolexSize,BolexStartDateTime,BolexCompletionDateTime,BolexInsulinDelivered,BolexIOB,BolexCompletionStatusID,BolexCompletionStatusDesc,ExtendedBolusIsComplete,EventDateTime,RequestDateTime,BolusType,BolusRequestOptions,StandardPercent,Duration,CarbSize,UserOverride,TargetBG,CorrectionFactor,FoodBolusSize,CorrectionBolusSize,ActualTotalBolusRequested,IsQuickBolus,EventHistoryReportEventDesc,EventHistoryReportDetails,NoteID,IndexID,Note
+"Bolus","Standard/Correction","141",,"7001.000","7001.000","2021-04-01T12:58:26","13.53","12.50","1.03","3","Completed","1",,,,,,,,,,"2021-04-01T12:53:36","2021-04-01T12:53:36","Carb","Standard/Correction","100.00","0","75","0","110","30.00","12.50","1.03","13.53","0","0","Correction & Food Bolus","CF 1:30 - Carb Ratio 1:6 - Target BG 110","0","1181649","",
+"Bolus","Standard","131","0.71","7003.000","7003.000","2021-04-01T16:03:25","1.50","0.00","0.00","3","Completed","1",,,,,,,,,,"2021-04-01T16:02:04","2021-04-01T16:02:04","Carb","Standard","100.00","0","0","1","110","30.00","0.00","0.00","1.50","0","0","Food Bolus","CF 1:30 - Carb Ratio 1:6 - Target BG 110 | Override: Pump calculated Bolus = 0.0 units","0","1182026","",
+"Bolus","Standard/Correction","168","1.71","7004.000","7004.000","2021-04-01T16:24:08","2.00","0.00","0.00","3","Completed","1",,,,,,,,,,"2021-04-01T16:22:21","2021-04-01T16:22:21","Carb","Standard/Correction","100.00","0","0","1","110","30.00","0.00","0.22","2.00","0","0","Correction & Food Bolus","CF 1:30 - Carb Ratio 1:6 - Target BG 110 | Override: Pump calculated Bolus = 0.2 units","0","1182082","",
+"Bolus","Standard","220","3.98","7032.000","7032.000","2021-04-02T23:16:24","2.50","0.00","0.00","3","Completed","1",,,,,,,,,,"2021-04-02T23:14:33","2021-04-02T23:14:33","Carb","Standard","100.00","0","0","1","110","30.00","0.00","0.00","2.50","0","0","Food Bolus","CF 1:30 - Carb Ratio 1:6 - Target BG 110 | Override: Pump calculated Bolus = 0.0 units","0","1185846","",
+"""
+
+    RAW_DATA_FULL = RAW_DATA_HEADER + "\n" + RAW_DATA_CGM + "\n" + RAW_DATA_IOB + "\n" + RAW_DATA_BOLUS
+
+    PARSED_DATA = {
+        'readingData': [
+            {"DeviceType": "t:slim X2 Insulin Pump", "SerialNumber": "11111111", "Description": "EGV", "EventDateTime": "2021-04-01T00:01:33", "Readings (CGM / BGM)": "235"},
+            {"DeviceType": "t:slim X2 Insulin Pump", "SerialNumber": "11111111", "Description": "EGV", "EventDateTime": "2021-04-01T00:06:33", "Readings (CGM / BGM)": "230"},
+            {"DeviceType": "t:slim X2 Insulin Pump", "SerialNumber": "11111111", "Description": "EGV", "EventDateTime": "2021-04-02T23:31:36", "Readings (CGM / BGM)": "181"}
+        ],
+        'iobData': [
+            {"Type": "IOB", "EventID": "81", "EventDateTime": "2021-04-01T00:00:19", "IOB": "13.24"},
+            {"Type": "IOB", "EventID": "9", "EventDateTime": "2021-04-01T00:03:12", "IOB": "12.80"},
+            {"Type": "IOB", "EventID": "81", "EventDateTime": "2021-04-02T23:58:19", "IOB": "4.25"},
+        ],
+        'basalData': [],
+        'bolusData': [
+            {"Type": "Bolus", "Description": "Standard/Correction", "BG": "141", "IOB": "", "BolusRequestID": "7001.000", "BolusCompletionID": "7001.000", "CompletionDateTime": "2021-04-01T12:58:26", "InsulinDelivered": "13.53", "FoodDelivered": "12.50", "CorrectionDelivered": "1.03", "CompletionStatusID": "3", "CompletionStatusDesc": "Completed", "BolusIsComplete": "1", "BolexCompletionID": "", "BolexSize": "", "BolexStartDateTime": "", "BolexCompletionDateTime": "", "BolexInsulinDelivered": "", "BolexIOB": "", "BolexCompletionStatusID": "", "BolexCompletionStatusDesc": "", "ExtendedBolusIsComplete": "", "EventDateTime": "2021-04-01T12:53:36", "RequestDateTime": "2021-04-01T12:53:36", "BolusType": "Carb", "BolusRequestOptions": "Standard/Correction", "StandardPercent": "100.00", "Duration": "0", "CarbSize": "75", "UserOverride": "0", "TargetBG": "110", "CorrectionFactor": "30.00", "FoodBolusSize": "12.50", "CorrectionBolusSize": "1.03", "ActualTotalBolusRequested": "13.53", "IsQuickBolus": "0", "EventHistoryReportEventDesc": "0", "EventHistoryReportDetails": "Correction & Food Bolus", "NoteID": "CF 1:30 - Carb Ratio 1:6 - Target BG 110", "IndexID": "0", "Note": "1181649"},
+            {"Type": "Bolus", "Description": "Standard", "BG": "131", "IOB": "0.71", "BolusRequestID": "7003.000", "BolusCompletionID": "7003.000", "CompletionDateTime": "2021-04-01T16:03:25", "InsulinDelivered": "1.50", "FoodDelivered": "0.00", "CorrectionDelivered": "0.00", "CompletionStatusID": "3", "CompletionStatusDesc": "Completed", "BolusIsComplete": "1", "BolexCompletionID": "", "BolexSize": "", "BolexStartDateTime": "", "BolexCompletionDateTime": "", "BolexInsulinDelivered": "", "BolexIOB": "", "BolexCompletionStatusID": "", "BolexCompletionStatusDesc": "", "ExtendedBolusIsComplete": "", "EventDateTime": "2021-04-01T16:02:04", "RequestDateTime": "2021-04-01T16:02:04", "BolusType": "Carb", "BolusRequestOptions": "Standard", "StandardPercent": "100.00", "Duration": "0", "CarbSize": "0", "UserOverride": "1", "TargetBG": "110", "CorrectionFactor": "30.00", "FoodBolusSize": "0.00", "CorrectionBolusSize": "0.00", "ActualTotalBolusRequested": "1.50", "IsQuickBolus": "0", "EventHistoryReportEventDesc": "0", "EventHistoryReportDetails": "Food Bolus", "NoteID": "CF 1:30 - Carb Ratio 1:6 - Target BG 110 | Override: Pump calculated Bolus = 0.0 units", "IndexID": "0", "Note": "1182026"},
+            {"Type": "Bolus", "Description": "Standard/Correction", "BG": "168", "IOB": "1.71", "BolusRequestID": "7004.000", "BolusCompletionID": "7004.000", "CompletionDateTime": "2021-04-01T16:24:08", "InsulinDelivered": "2.00", "FoodDelivered": "0.00", "CorrectionDelivered": "0.00", "CompletionStatusID": "3", "CompletionStatusDesc": "Completed", "BolusIsComplete": "1", "BolexCompletionID": "", "BolexSize": "", "BolexStartDateTime": "", "BolexCompletionDateTime": "", "BolexInsulinDelivered": "", "BolexIOB": "", "BolexCompletionStatusID": "", "BolexCompletionStatusDesc": "", "ExtendedBolusIsComplete": "", "EventDateTime": "2021-04-01T16:22:21", "RequestDateTime": "2021-04-01T16:22:21", "BolusType": "Carb", "BolusRequestOptions": "Standard/Correction", "StandardPercent": "100.00", "Duration": "0", "CarbSize": "0", "UserOverride": "1", "TargetBG": "110", "CorrectionFactor": "30.00", "FoodBolusSize": "0.00", "CorrectionBolusSize": "0.22", "ActualTotalBolusRequested": "2.00", "IsQuickBolus": "0", "EventHistoryReportEventDesc": "0", "EventHistoryReportDetails": "Correction & Food Bolus", "NoteID": "CF 1:30 - Carb Ratio 1:6 - Target BG 110 | Override: Pump calculated Bolus = 0.2 units", "IndexID": "0", "Note": "1182082"},
+            {"Type": "Bolus", "Description": "Standard", "BG": "220", "IOB": "3.98", "BolusRequestID": "7032.000", "BolusCompletionID": "7032.000", "CompletionDateTime": "2021-04-02T23:16:24", "InsulinDelivered": "2.50", "FoodDelivered": "0.00", "CorrectionDelivered": "0.00", "CompletionStatusID": "3", "CompletionStatusDesc": "Completed", "BolusIsComplete": "1", "BolexCompletionID": "", "BolexSize": "", "BolexStartDateTime": "", "BolexCompletionDateTime": "", "BolexInsulinDelivered": "", "BolexIOB": "", "BolexCompletionStatusID": "", "BolexCompletionStatusDesc": "", "ExtendedBolusIsComplete": "", "EventDateTime": "2021-04-02T23:14:33", "RequestDateTime": "2021-04-02T23:14:33", "BolusType": "Carb", "BolusRequestOptions": "Standard", "StandardPercent": "100.00", "Duration": "0", "CarbSize": "0", "UserOverride": "1", "TargetBG": "110", "CorrectionFactor": "30.00", "FoodBolusSize": "0.00", "CorrectionBolusSize": "0.00", "ActualTotalBolusRequested": "2.50", "IsQuickBolus": "0", "EventHistoryReportEventDesc": "0", "EventHistoryReportDetails": "Food Bolus", "NoteID": "CF 1:30 - Carb Ratio 1:6 - Target BG 110 | Override: Pump calculated Bolus = 0.0 units", "IndexID": "0", "Note": "1185846"}
+        ]
+    }
+
+    def test_therapy_timeline_csv_parses_full(self):
+        ws2 = WS2Api()
+        ws2.userGuid = 'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee'
+
+        rawData = self.RAW_DATA_FULL
+
+        def fake_get(endpoint, query):
+            nonlocal rawData
+            if endpoint == 'therapytimeline2csv/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/2021-04-01/2021-04-02?format=csv':
+                return rawData
+
+        ws2.get = fake_get
+
+        tt = ws2.therapy_timeline_csv('2021-04-01', '2021-04-02')
+
+        self.assertDictEqual(tt, self.PARSED_DATA)
+
+    def test_therapy_timeline_csv_parses_random_order(self):
+        ws2 = WS2Api()
+        ws2.userGuid = 'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee'
+
+        rawData = ""
+
+        def fake_get(endpoint, query):
+            nonlocal rawData
+            if endpoint == 'therapytimeline2csv/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/2021-04-01/2021-04-02?format=csv':
+                return rawData
+
+        ws2.get = fake_get
+
+        # Randomize the order of all sections
+        for i in itertools.permutations([self.RAW_DATA_HEADER, self.RAW_DATA_CGM, self.RAW_DATA_IOB, self.RAW_DATA_BOLUS], 4):
+            rawData = "\n".join(i)
+
+            tt = ws2.therapy_timeline_csv('2021-04-01', '2021-04-02')
+            self.assertDictEqual(tt, self.PARSED_DATA)
+
+if __name__ == '__main__':
+    unittest.main()

tests/parser/test_nightscout.py

@@ -15,6 +15,7 @@ class TestNightscoutEntry(unittest.TestCase):
                 "reason": "",
                 "duration": 30,
                 "absolute": 1.05,
+                "rate": 1.05,
                 "created_at": "2021-03-16 00:25:21-04:00",
                 "carbs": None,
                 "insulin": None,
@@ -33,6 +34,7 @@ class TestNightscoutEntry(unittest.TestCase):
                 "reason": "Correction",
                 "duration": 5,
                 "absolute": 0.95,
+                "rate": 0.95,
                 "created_at": "2021-03-16 12:25:21-04:00",
                 "carbs": None,
                 "insulin": None,