diff --git a/api.py b/api.py deleted file mode 100644 index 1d78abe..0000000 --- a/api.py +++ /dev/null @@ -1,140 +0,0 @@ -import requests -import json -import urllib -import datetime -import csv -from bs4 import BeautifulSoup - -class TConnectApi: - CONTROLIQ_BASE_URL = 'https://tdcservices.tandemdiabetes.com/tconnect/controliq/api/' - WS2_BASE_URL = 'https://tconnectws2.tandemdiabetes.com/' - LOGIN_URL = 'https://tconnect.tandemdiabetes.com/login.aspx?ReturnUrl=%2f' - - userGuid = None - accessToken = None - accessTokenExpiresAt = None - - def __init__(self, email, password): - if not self.login(email, password): - raise Exception('Unable to authenticate') - - def login(self, email, password): - with requests.Session() as s: - initial = s.get(self.LOGIN_URL) - soup = BeautifulSoup(initial.content, features='lxml') - data = { - "__LASTFOCUS": "", - "__EVENTTARGET": "ctl00$ContentBody$LoginControl$linkLogin", - "__EVENTARGUMENT": "", - "__VIEWSTATE": soup.select_one("#__VIEWSTATE")["value"], - "__VIEWSTATEGENERATOR": soup.select_one("#__VIEWSTATEGENERATOR")["value"], - "__EVENTVALIDATION": soup.select_one("#__EVENTVALIDATION")["value"], - "ctl00$ContentBody$LoginControl$txtLoginEmailAddress": email, - "txtLoginEmailAddress_ClientState": '{"enabled":true,"emptyMessage":"","validationText":"%s","valueAsString":"%s","lastSetTextBoxValue":"%s"}' % (email, email, email), - "ctl00$ContentBody$LoginControl$txtLoginPassword": password, - "txtLoginPassword_ClientState": '{"enabled":true,"emptyMessage":"","validationText":"%s","valueAsString":"%s","lastSetTextBoxValue":"%s"}' % (password, password, password) - } - req = s.post(self.LOGIN_URL, data=data, headers={'Referer': self.LOGIN_URL}, allow_redirects=False) - if req.status_code != 302: - return False - - fwd = s.post(urllib.parse.urljoin(self.LOGIN_URL, req.headers['Location']), cookies=req.cookies) - if fwd.status_code != 200: - return False - - self.userGuid = req.cookies['UserGUID'] - self.accessToken = req.cookies['accessToken'] - self.accessTokenExpiresAt = req.cookies['accessTokenExpiresAt'] - return True - - def api_headers(self): - if not self.accessToken: - raise Exception('No access token provided') - return {'Authorization': 'Bearer %s' % self.accessToken} - - def controliq_api(self, endpoint, query): - r = requests.get(self.CONTROLIQ_BASE_URL + endpoint, query, headers=self.api_headers()) - if r.status_code != 200: - raise ApiException(r.status_code, "ControlIQ API HTTP %s response: %s" % (str(r.status_code), r.text)) - return r.json() - - def ws2_api(self, endpoint, query): - r = requests.get(self.WS2_BASE_URL + endpoint, query, headers=self.api_headers()) - if r.status_code != 200: - raise ApiException(r.status_code, "WS2 API HTTP %s response: %s" % (str(r.status_code), r.text)) - return r.text - - def _parse_date(self, date): - if type(date) == str: - return date - return (date or datetime.datetime.now()).strftime('%m-%d-%Y') - - def therapy_timeline(self, start=None, end=None): - startDate = self._parse_date(start) - endDate = self._parse_date(end) - - return self.controliq_api('therapytimeline/users/%s' % (self.userGuid), { - "startDate": startDate, - "endDate": endDate - }) - - def _split_empty_sections(self, text): - sections = [[]] - sectionIndex = 0 - for line in text.splitlines(): - if len(line.strip()) > 0: - sections[sectionIndex].append(line) - else: - sections.append([]) - sectionIndex += 1 - - return sections + [None] * (4 - len(sections)) - - def _csv_to_dict(self, rawdata): - data = [] - if not rawdata or len(rawdata) == 0: - return data - headers = rawdata[0].split(",") - for row in csv.reader(rawdata[1:]): - data.append({headers[i]: row[i] for i in range(len(row)) if i < len(headers)}) - - return data - - - def therapy_timeline_csv(self, start=None, end=None): - startDate = self._parse_date(start) - endDate = self._parse_date(end) - - req_text = self.ws2_api('therapytimeline2csv/%s/%s/%s' % (self.userGuid, startDate, endDate), {}) - - sections = self._split_empty_sections(req_text) - - readingData = None - iobData = None - basalData = None - bolusData = None - - for s in sections: - if s and len(s) > 2: - firstrow = s[1].replace('"', '').strip() - if firstrow.startswith("t:slim X2 Insulin Pump"): - readingData = s - elif firstrow.startswith("IOB"): - iobData = s - elif firstrow.startswith("Basal"): - basalData = s - elif firstrow.startswith("Bolus"): - bolusData = s - - - return { - "readingData": self._csv_to_dict(readingData), - "iobData": self._csv_to_dict(iobData), - "basalData": self._csv_to_dict(basalData), - "bolusData": self._csv_to_dict(bolusData) - } - -class ApiException(Exception): - def __init__(self, status_code, text, *args, **kwargs): - self.status_code = status_code - super().__init__(text, *args, **kwargs) diff --git a/api/__init__.py b/api/__init__.py new file mode 100644 index 0000000..f48b889 --- /dev/null +++ b/api/__init__.py @@ -0,0 +1,42 @@ +from .android import AndroidApi +from .controliq import ControlIQApi +from .ws2 import WS2Api + +"""A wrapper for the three different t:connect API types.""" +class TConnectApi: + email = None + password = None + + _ciq = None + _ws2 = None + _android = None + + def __init__(self, email, password): + self.email = email + self.password = password + + + @property + def controliq(self): + if self._ciq: + return self._ciq + + self._ciq = ControlIQApi(self.email, self.password) + return self._ciq + + @property + def ws2(self): + if self._ws2: + return self._ws2 + + self._ws2 = WS2Api(self._ciq.userGuid) + return self._ws2 + + @property + def android(self): + if self._android and not self._android.needs_relogin(): + return self._android + + self._android = AndroidApi(self.email, self.password) + return self._android + diff --git a/api/android.py b/api/android.py new file mode 100644 index 0000000..a122d80 --- /dev/null +++ b/api/android.py @@ -0,0 +1,84 @@ +import requests +import json +import urllib +import datetime +import csv +import base64 +import arrow + +from bs4 import BeautifulSoup + +from .common import ApiException + +class AndroidApi: + BASE_URL = 'https://tdcservices.tandemdiabetes.com/' + OAUTH_TOKEN_PATH = 'cloud/oauth2/token' + OAUTH_SCOPES = 'cloud.account cloud.upload cloud.accepttcpp cloud.email cloud.password' + + # These credentials are found in source code + ANDROID_API_USERNAME = base64.b64decode('QzIzMzFDRDYtRDQ1MC00OTVFLTlDMTktNjcyMTUyMzBDODVD').decode() + ANDROID_API_PASSWORD = base64.b64decode('dHo0MzNLVzVRREM5VjdmIXo2QF4ybyZZNlNHR1lo').decode() + + # These tokens are separate from the "standard" tdcservices API + accessToken = None + accessTokenExpiresAt = None + refreshToken = None + refreshTokenExpiresAt = None + userId = None + patientObjectId = None + + def __init__(self, email, password): + self.login(email, password) + + # TODO: auto refresh token + def login(self, email, password): + r = requests.post( + self.BASE_URL + self.OAUTH_TOKEN_PATH, + { + 'username': email, + 'password': password, + 'grant_type': 'password', + 'scope': self.OAUTH_SCOPES + }, + headers={'Content-Type': 'application/x-www-form-urlencoded'}, + auth=requests.auth.HTTPBasicAuth(self.ANDROID_API_USERNAME, self.ANDROID_API_PASSWORD) + ) + + if r.status_code != 200: + raise ApiException(r.status_code, 'Received HTTP %s during login: %s' % (r.status_code, r.text)) + + j = r.json() + self.accessToken = j["accessToken"] + self.accessTokenExpiresAt = j["accessTokenExpiresAt"] + self.refreshToken = j["refreshToken"] + self.refreshTokenExpiresAt = j["refreshTokenExpiresAt"] + self.userId = j["user"]["id"] + self.patientObjectId = j["user"]["patientObjectId"] + + def needs_relogin(self): + diff = (arrow.get(self.refreshTokenExpiresAt) - arrow.get()) + return (diff.seconds <= 5 * 60) + + def api_headers(self): + if not self.accessToken: + raise Exception('No access token') + return {'Authorization': 'Bearer %s' % self.accessToken} + + def get(self, endpoint, query={}, **kwargs): + r = requests.get(self.BASE_URL + endpoint, query, headers=self.api_headers(), **kwargs) + if r.status_code != 200: + raise ApiException(r.status_code, "Internal API HTTP %s response: %s" % (str(r.status_code), r.text)) + return r.json() + + def post(self, endpoint, query={}, **kwargs): + r = requests.post(self.BASE_URL + endpoint, query, headers=self.api_headers(), **kwargs) + if r.status_code != 200: + raise ApiException(r.status_code, "Internal API HTTP %s response: %s" % (str(r.status_code), r.text)) + return r.json() + + """ + Returns the most recent event ID that was uploaded for the given pump. + {'maxPumpEventIndex': , 'processingStatus': 1} + """ + def last_event_uploaded(self, pump_serial_number): + return self.get('cloud/upload/getlasteventuploaded?sn=%d' % pump_serial_number) diff --git a/api/common.py b/api/common.py new file mode 100644 index 0000000..5ef2aa4 --- /dev/null +++ b/api/common.py @@ -0,0 +1,11 @@ +import datetime + +def parse_date(date): + if type(date) == str: + return date + return (date or datetime.datetime.now()).strftime('%m-%d-%Y') + +class ApiException(Exception): + def __init__(self, status_code, text, *args, **kwargs): + self.status_code = status_code + super().__init__(text, *args, **kwargs) diff --git a/api/controliq.py b/api/controliq.py new file mode 100644 index 0000000..1aca367 --- /dev/null +++ b/api/controliq.py @@ -0,0 +1,67 @@ +import requests +import urllib +import datetime +from bs4 import BeautifulSoup + +from .common import parse_date, ApiException + +class ControlIQApi: + BASE_URL = 'https://tdcservices.tandemdiabetes.com/tconnect/controliq/api/' + LOGIN_URL = 'https://tconnect.tandemdiabetes.com/login.aspx?ReturnUrl=%2f' + + userGuid = None + accessToken = None + accessTokenExpiresAt = None + + def __init__(self, email, password): + if not self.login(email, password): + raise Exception('Unable to authenticate') + + def login(self, email, password): + with requests.Session() as s: + initial = s.get(self.LOGIN_URL) + soup = BeautifulSoup(initial.content, features='lxml') + data = { + "__LASTFOCUS": "", + "__EVENTTARGET": "ctl00$ContentBody$LoginControl$linkLogin", + "__EVENTARGUMENT": "", + "__VIEWSTATE": soup.select_one("#__VIEWSTATE")["value"], + "__VIEWSTATEGENERATOR": soup.select_one("#__VIEWSTATEGENERATOR")["value"], + "__EVENTVALIDATION": soup.select_one("#__EVENTVALIDATION")["value"], + "ctl00$ContentBody$LoginControl$txtLoginEmailAddress": email, + "txtLoginEmailAddress_ClientState": '{"enabled":true,"emptyMessage":"","validationText":"%s","valueAsString":"%s","lastSetTextBoxValue":"%s"}' % (email, email, email), + "ctl00$ContentBody$LoginControl$txtLoginPassword": password, + "txtLoginPassword_ClientState": '{"enabled":true,"emptyMessage":"","validationText":"%s","valueAsString":"%s","lastSetTextBoxValue":"%s"}' % (password, password, password) + } + req = s.post(self.LOGIN_URL, data=data, headers={'Referer': self.LOGIN_URL}, allow_redirects=False) + if req.status_code != 302: + return False + + fwd = s.post(urllib.parse.urljoin(self.LOGIN_URL, req.headers['Location']), cookies=req.cookies) + if fwd.status_code != 200: + return False + + self.userGuid = req.cookies['UserGUID'] + self.accessToken = req.cookies['accessToken'] + self.accessTokenExpiresAt = req.cookies['accessTokenExpiresAt'] + return True + + def api_headers(self): + if not self.accessToken: + raise Exception('No access token provided') + return {'Authorization': 'Bearer %s' % self.accessToken} + + def get(self, endpoint, query): + r = requests.get(self.BASE_URL + endpoint, query, headers=self.api_headers()) + if r.status_code != 200: + raise ApiException(r.status_code, "ControlIQ API HTTP %s response: %s" % (str(r.status_code), r.text)) + return r.json() + + def therapy_timeline(self, start=None, end=None): + startDate = parse_date(start) + endDate = parse_date(end) + + return self.get('therapytimeline/users/%s' % (self.userGuid), { + "startDate": startDate, + "endDate": endDate + }) diff --git a/api/ws2.py b/api/ws2.py new file mode 100644 index 0000000..780961e --- /dev/null +++ b/api/ws2.py @@ -0,0 +1,76 @@ +import requests +import datetime +import csv + +from .common import parse_date, ApiException + +class WS2Api: + BASE_URL = 'https://tconnectws2.tandemdiabetes.com/' + + userGuid = None + + def __init__(self, userGuid): + self.userGuid = userGuid + + def get(self, endpoint, query): + r = requests.get(self.BASE_URL + endpoint, query) + if r.status_code != 200: + raise ApiException(r.status_code, "WS2 API HTTP %s response: %s" % (str(r.status_code), r.text)) + return r.text + + + def _split_empty_sections(self, text): + sections = [[]] + sectionIndex = 0 + for line in text.splitlines(): + if len(line.strip()) > 0: + sections[sectionIndex].append(line) + else: + sections.append([]) + sectionIndex += 1 + + return sections + [None] * (4 - len(sections)) + + def _csv_to_dict(self, rawdata): + data = [] + if not rawdata or len(rawdata) == 0: + return data + headers = rawdata[0].split(",") + for row in csv.reader(rawdata[1:]): + data.append({headers[i]: row[i] for i in range(len(row)) if i < len(headers)}) + + return data + + + def therapy_timeline_csv(self, start=None, end=None): + startDate = parse_date(start) + endDate = parse_date(end) + + req_text = self.get('therapytimeline2csv/%s/%s/%s' % (self.userGuid, startDate, endDate), {}) + + sections = self._split_empty_sections(req_text) + + readingData = None + iobData = None + basalData = None + bolusData = None + + for s in sections: + if s and len(s) > 2: + firstrow = s[1].replace('"', '').strip() + if firstrow.startswith("t:slim X2 Insulin Pump"): + readingData = s + elif firstrow.startswith("IOB"): + iobData = s + elif firstrow.startswith("Basal"): + basalData = s + elif firstrow.startswith("Bolus"): + bolusData = s + + + return { + "readingData": self._csv_to_dict(readingData), + "iobData": self._csv_to_dict(iobData), + "basalData": self._csv_to_dict(basalData), + "bolusData": self._csv_to_dict(bolusData) + } diff --git a/main.py b/main.py index e562288..8704591 100644 --- a/main.py +++ b/main.py @@ -7,8 +7,10 @@ import hashlib import requests import arrow import argparse +import time -from api import TConnectApi, ApiException +from api import TConnectApi +from api.common import ApiException from parser import TConnectEntry from nightscout import ( NightscoutEntry, @@ -23,7 +25,12 @@ from nightscout import ( ) try: - from secret import TCONNECT_EMAIL, TCONNECT_PASSWORD + from secret import ( + TCONNECT_EMAIL, + TCONNECT_PASSWORD, + PUMP_SERIAL_NUMBER, + TIMEZONE_NAME + ) except Exception: print('Unable to import secret.py') sys.exit(1) @@ -83,8 +90,9 @@ def ns_write_basal_events(basalEvents, pretend=False): last_upload_time = None if last_upload: last_upload_time = arrow.get(last_upload["created_at"]) - print("Last Nightscout basal upload:", last_upload_time) + print("Last Nightscout basal upload:", arrow.get(last_upload_time, TIMEZONE_NAME)) + add_count = 0 for event in basalEvents: if last_upload_time and arrow.get(event["time"]) < last_upload_time: if pretend: @@ -104,6 +112,8 @@ def ns_write_basal_events(basalEvents, pretend=False): reason=event["delivery_type"] ) + add_count += 1 + print(" Processing basal:", event, "entry:", entry) if recent_needs_update: print("Replacing last uploaded entry:", last_upload) @@ -113,6 +123,8 @@ def ns_write_basal_events(basalEvents, pretend=False): elif not pretend: upload_nightscout(entry) + return add_count + """ Given bolus data input from the therapy timeline CSV, converts it into a digestable format. """ @@ -144,6 +156,7 @@ def ns_write_bolus_events(bolusEvents, pretend=False): last_upload_time = arrow.get(last_upload["created_at"]) print("Last Nightscout bolus upload:", last_upload_time) + add_count = 0 for event in bolusEvents: if last_upload_time and arrow.get(event["completion_time"]) <= last_upload_time: if pretend: @@ -157,10 +170,14 @@ def ns_write_bolus_events(bolusEvents, pretend=False): notes="{}{}{}".format(event["description"], " (Override)" if event["user_override"] == "1" else "", " (Extended)" if event["extended_bolus"] == "1" else "") ) + add_count += 1 + print(" Processing bolus:", event, "entry:", entry) if not pretend: upload_nightscout(entry) + return add_count + """ Given IOB data input from the therapy timeline CSV, converts it into a digestable format. """ @@ -185,12 +202,12 @@ def ns_write_iob_events(iobEvents, pretend=False): if not iobEvents or len(iobEvents) == 0: print("No IOB events: skipping") - return + return 0 event = iobEvents[-1] if last_upload_time and arrow.get(event["time"]) <= last_upload_time: print(" Skipping already uploaded iob event:", event) - return + return 0 entry = NightscoutEntry.iob( iob=event["iob"], @@ -207,10 +224,12 @@ def ns_write_iob_events(iobEvents, pretend=False): if not pretend: delete_nightscout('activity/{}'.format(last_upload['_id'])) + return 1 + def process_time_range(tconnect, time_start, time_end, pretend): print("Downloading t:connect ControlIQ data") try: - ciqBasalData = tconnect.therapy_timeline(time_start, time_end) + ciqBasalData = tconnect.controliq.therapy_timeline(time_start, time_end) except ApiException as e: # The ControlIQ API returns a 404 if the user did not have a ControlIQ enabled # device in the time range which is queried. Since it launched in early 2020, @@ -222,7 +241,7 @@ def process_time_range(tconnect, time_start, time_end, pretend): raise e print("Downloading t:connect CSV data") - csvdata = tconnect.therapy_timeline_csv(time_start, time_end) + csvdata = tconnect.ws2.therapy_timeline_csv(time_start, time_end) readingData = csvdata["readingData"] iobData = csvdata["iobData"] @@ -230,20 +249,24 @@ def process_time_range(tconnect, time_start, time_end, pretend): bolusData = csvdata["bolusData"] if readingData and len(readingData) > 0: - print("Last CGM reading from t:connect:", readingData[-1]) + print("Last CGM reading from t:connect:", readingData[-1]['EventDateTime'] if 'EventDateTime' in readingData[-1] else readingData) + + added = 0 basalEvents = process_ciq_basal_events(ciqBasalData) if csvBasalData: add_csv_basal_events(basalEvents, csvBasalData) - ns_write_basal_events(basalEvents, pretend=pretend) + added += ns_write_basal_events(basalEvents, pretend=pretend) bolusEvents = process_bolus_events(bolusData) - ns_write_bolus_events(bolusEvents, pretend=pretend) + added += ns_write_bolus_events(bolusEvents, pretend=pretend) iobEvents = process_iob_events(iobData) - ns_write_iob_events(iobEvents, pretend=pretend) + added += ns_write_iob_events(iobEvents, pretend=pretend) + + return added def parse_args(): parser = argparse.ArgumentParser(description="Syncs bolus, basal, and IOB data from Tandem Diabetes t:connect to Nightscout.") @@ -251,6 +274,7 @@ def parse_args(): parser.add_argument('--start-date', dest='start_date', type=str, default=None, help='The oldest date to process data from. Must be specified with --end-date.') parser.add_argument('--end-date', dest='end_date', type=str, default=None, help='The newest date to process data until (inclusive). Must be specified with --start-date.') parser.add_argument('--days', dest='days', type=int, default=1, help='The number of days of t:connect data to read in. Cannot be used with --from-date and --until-date.') + parser.add_argument('--auto-update', dest='auto_update', action='store_const', const=True, default=False, help='If set, continuously checks for updates from t:connect and syncs with Nightscout.') return parser.parse_args() @@ -260,6 +284,9 @@ def main(): if args.pretend: print("Pretend mode: will not write to Nightscout") + if args.auto_update and (args.start_date or args.end_date): + raise Exception('Auto-update cannot be used with start/end date') + if args.start_date and args.end_date: time_start = arrow.get(args.start_date) time_end = arrow.get(args.end_date) @@ -270,11 +297,55 @@ def main(): if time_end < time_start: raise Exception('time_start must be before time_end') - print("Processing data between", time_start, "and", time_end) - tconnect = TConnectApi(TCONNECT_EMAIL, TCONNECT_PASSWORD) - process_time_range(tconnect, time_start, time_end, args.pretend) + if args.auto_update: + # Read from android api, find exact interval to cut down on API calls + # Refresh API token. If failure, die, have wrapper script re-run. + + last_event_index = None + last_event_time = None + time_diffs = [] + while True: + last_event = tconnect.android.last_event_uploaded(PUMP_SERIAL_NUMBER) + if not last_event_index or last_event['maxPumpEventIndex'] > last_event_index: + now = time.time() + print('New event index:', last_event['maxPumpEventIndex'], 'last:', last_event_index) + + if args.pretend: + print('Would update now') + else: + added = process_time_range(tconnect, time_start, time_end, args.pretend) + print('Added', added, 'items') + + if last_event_index: + time_diffs.append(now - last_event_time) + print('Time diffs:', time_diffs) + + last_event_index = last_event['maxPumpEventIndex'] + last_event_time = now + else: + print('No event index change:', last_event['maxPumpEventIndex']) + + if len(time_diffs) > 2: + print('Sleeping 60 seconds after unexpected no index change') + time.sleep(60) + continue + + sleep_secs = 60 + if len(time_diffs) > 10: + time_diffs = time_diffs[1:] + + if len(time_diffs) > 2: + sleep_secs = sum(time_diffs) / len(time_diffs) + + # Sleep for a rolling average of time between updates + print('Sleeping for', sleep_secs, 'sec') + time.sleep(sleep_secs) + else: + print("Processing data between", time_start, "and", time_end) + added = process_time_range(tconnect, time_start, time_end, args.pretend) + print("Added", added, "items") if __name__ == '__main__': main() \ No newline at end of file diff --git a/secret.py.example b/secret.py.example index 9c123c2..d3aa495 100644 --- a/secret.py.example +++ b/secret.py.example @@ -1,6 +1,8 @@ TCONNECT_EMAIL = 'email@email.com' TCONNECT_PASSWORD = 'password' +PUMP_SERIAL_NUMBER = 11111111 + NS_URL = 'https://yournightscouturl/' NS_SECRET = 'apisecret'