split up API types, add android api. add auto update option

This commit is contained in:
James Woglom
2020-10-04 00:48:46 -04:00
parent 5cc3f8574b
commit 6fa119720a
8 changed files with 367 additions and 154 deletions
-140
View File
@@ -1,140 +0,0 @@
import requests
import json
import urllib
import datetime
import csv
from bs4 import BeautifulSoup
class TConnectApi:
CONTROLIQ_BASE_URL = 'https://tdcservices.tandemdiabetes.com/tconnect/controliq/api/'
WS2_BASE_URL = 'https://tconnectws2.tandemdiabetes.com/'
LOGIN_URL = 'https://tconnect.tandemdiabetes.com/login.aspx?ReturnUrl=%2f'
userGuid = None
accessToken = None
accessTokenExpiresAt = None
def __init__(self, email, password):
if not self.login(email, password):
raise Exception('Unable to authenticate')
def login(self, email, password):
with requests.Session() as s:
initial = s.get(self.LOGIN_URL)
soup = BeautifulSoup(initial.content, features='lxml')
data = {
"__LASTFOCUS": "",
"__EVENTTARGET": "ctl00$ContentBody$LoginControl$linkLogin",
"__EVENTARGUMENT": "",
"__VIEWSTATE": soup.select_one("#__VIEWSTATE")["value"],
"__VIEWSTATEGENERATOR": soup.select_one("#__VIEWSTATEGENERATOR")["value"],
"__EVENTVALIDATION": soup.select_one("#__EVENTVALIDATION")["value"],
"ctl00$ContentBody$LoginControl$txtLoginEmailAddress": email,
"txtLoginEmailAddress_ClientState": '{"enabled":true,"emptyMessage":"","validationText":"%s","valueAsString":"%s","lastSetTextBoxValue":"%s"}' % (email, email, email),
"ctl00$ContentBody$LoginControl$txtLoginPassword": password,
"txtLoginPassword_ClientState": '{"enabled":true,"emptyMessage":"","validationText":"%s","valueAsString":"%s","lastSetTextBoxValue":"%s"}' % (password, password, password)
}
req = s.post(self.LOGIN_URL, data=data, headers={'Referer': self.LOGIN_URL}, allow_redirects=False)
if req.status_code != 302:
return False
fwd = s.post(urllib.parse.urljoin(self.LOGIN_URL, req.headers['Location']), cookies=req.cookies)
if fwd.status_code != 200:
return False
self.userGuid = req.cookies['UserGUID']
self.accessToken = req.cookies['accessToken']
self.accessTokenExpiresAt = req.cookies['accessTokenExpiresAt']
return True
def api_headers(self):
if not self.accessToken:
raise Exception('No access token provided')
return {'Authorization': 'Bearer %s' % self.accessToken}
def controliq_api(self, endpoint, query):
r = requests.get(self.CONTROLIQ_BASE_URL + endpoint, query, headers=self.api_headers())
if r.status_code != 200:
raise ApiException(r.status_code, "ControlIQ API HTTP %s response: %s" % (str(r.status_code), r.text))
return r.json()
def ws2_api(self, endpoint, query):
r = requests.get(self.WS2_BASE_URL + endpoint, query, headers=self.api_headers())
if r.status_code != 200:
raise ApiException(r.status_code, "WS2 API HTTP %s response: %s" % (str(r.status_code), r.text))
return r.text
def _parse_date(self, date):
if type(date) == str:
return date
return (date or datetime.datetime.now()).strftime('%m-%d-%Y')
def therapy_timeline(self, start=None, end=None):
startDate = self._parse_date(start)
endDate = self._parse_date(end)
return self.controliq_api('therapytimeline/users/%s' % (self.userGuid), {
"startDate": startDate,
"endDate": endDate
})
def _split_empty_sections(self, text):
sections = [[]]
sectionIndex = 0
for line in text.splitlines():
if len(line.strip()) > 0:
sections[sectionIndex].append(line)
else:
sections.append([])
sectionIndex += 1
return sections + [None] * (4 - len(sections))
def _csv_to_dict(self, rawdata):
data = []
if not rawdata or len(rawdata) == 0:
return data
headers = rawdata[0].split(",")
for row in csv.reader(rawdata[1:]):
data.append({headers[i]: row[i] for i in range(len(row)) if i < len(headers)})
return data
def therapy_timeline_csv(self, start=None, end=None):
startDate = self._parse_date(start)
endDate = self._parse_date(end)
req_text = self.ws2_api('therapytimeline2csv/%s/%s/%s' % (self.userGuid, startDate, endDate), {})
sections = self._split_empty_sections(req_text)
readingData = None
iobData = None
basalData = None
bolusData = None
for s in sections:
if s and len(s) > 2:
firstrow = s[1].replace('"', '').strip()
if firstrow.startswith("t:slim X2 Insulin Pump"):
readingData = s
elif firstrow.startswith("IOB"):
iobData = s
elif firstrow.startswith("Basal"):
basalData = s
elif firstrow.startswith("Bolus"):
bolusData = s
return {
"readingData": self._csv_to_dict(readingData),
"iobData": self._csv_to_dict(iobData),
"basalData": self._csv_to_dict(basalData),
"bolusData": self._csv_to_dict(bolusData)
}
class ApiException(Exception):
def __init__(self, status_code, text, *args, **kwargs):
self.status_code = status_code
super().__init__(text, *args, **kwargs)
+42
View File
@@ -0,0 +1,42 @@
from .android import AndroidApi
from .controliq import ControlIQApi
from .ws2 import WS2Api
"""A wrapper for the three different t:connect API types."""
class TConnectApi:
email = None
password = None
_ciq = None
_ws2 = None
_android = None
def __init__(self, email, password):
self.email = email
self.password = password
@property
def controliq(self):
if self._ciq:
return self._ciq
self._ciq = ControlIQApi(self.email, self.password)
return self._ciq
@property
def ws2(self):
if self._ws2:
return self._ws2
self._ws2 = WS2Api(self._ciq.userGuid)
return self._ws2
@property
def android(self):
if self._android and not self._android.needs_relogin():
return self._android
self._android = AndroidApi(self.email, self.password)
return self._android
+84
View File
@@ -0,0 +1,84 @@
import requests
import json
import urllib
import datetime
import csv
import base64
import arrow
from bs4 import BeautifulSoup
from .common import ApiException
class AndroidApi:
BASE_URL = 'https://tdcservices.tandemdiabetes.com/'
OAUTH_TOKEN_PATH = 'cloud/oauth2/token'
OAUTH_SCOPES = 'cloud.account cloud.upload cloud.accepttcpp cloud.email cloud.password'
# These credentials are found in source code
ANDROID_API_USERNAME = base64.b64decode('QzIzMzFDRDYtRDQ1MC00OTVFLTlDMTktNjcyMTUyMzBDODVD').decode()
ANDROID_API_PASSWORD = base64.b64decode('dHo0MzNLVzVRREM5VjdmIXo2QF4ybyZZNlNHR1lo').decode()
# These tokens are separate from the "standard" tdcservices API
accessToken = None
accessTokenExpiresAt = None
refreshToken = None
refreshTokenExpiresAt = None
userId = None
patientObjectId = None
def __init__(self, email, password):
self.login(email, password)
# TODO: auto refresh token
def login(self, email, password):
r = requests.post(
self.BASE_URL + self.OAUTH_TOKEN_PATH,
{
'username': email,
'password': password,
'grant_type': 'password',
'scope': self.OAUTH_SCOPES
},
headers={'Content-Type': 'application/x-www-form-urlencoded'},
auth=requests.auth.HTTPBasicAuth(self.ANDROID_API_USERNAME, self.ANDROID_API_PASSWORD)
)
if r.status_code != 200:
raise ApiException(r.status_code, 'Received HTTP %s during login: %s' % (r.status_code, r.text))
j = r.json()
self.accessToken = j["accessToken"]
self.accessTokenExpiresAt = j["accessTokenExpiresAt"]
self.refreshToken = j["refreshToken"]
self.refreshTokenExpiresAt = j["refreshTokenExpiresAt"]
self.userId = j["user"]["id"]
self.patientObjectId = j["user"]["patientObjectId"]
def needs_relogin(self):
diff = (arrow.get(self.refreshTokenExpiresAt) - arrow.get())
return (diff.seconds <= 5 * 60)
def api_headers(self):
if not self.accessToken:
raise Exception('No access token')
return {'Authorization': 'Bearer %s' % self.accessToken}
def get(self, endpoint, query={}, **kwargs):
r = requests.get(self.BASE_URL + endpoint, query, headers=self.api_headers(), **kwargs)
if r.status_code != 200:
raise ApiException(r.status_code, "Internal API HTTP %s response: %s" % (str(r.status_code), r.text))
return r.json()
def post(self, endpoint, query={}, **kwargs):
r = requests.post(self.BASE_URL + endpoint, query, headers=self.api_headers(), **kwargs)
if r.status_code != 200:
raise ApiException(r.status_code, "Internal API HTTP %s response: %s" % (str(r.status_code), r.text))
return r.json()
"""
Returns the most recent event ID that was uploaded for the given pump.
{'maxPumpEventIndex': <integer>, 'processingStatus': 1}
"""
def last_event_uploaded(self, pump_serial_number):
return self.get('cloud/upload/getlasteventuploaded?sn=%d' % pump_serial_number)
+11
View File
@@ -0,0 +1,11 @@
import datetime
def parse_date(date):
if type(date) == str:
return date
return (date or datetime.datetime.now()).strftime('%m-%d-%Y')
class ApiException(Exception):
def __init__(self, status_code, text, *args, **kwargs):
self.status_code = status_code
super().__init__(text, *args, **kwargs)
+67
View File
@@ -0,0 +1,67 @@
import requests
import urllib
import datetime
from bs4 import BeautifulSoup
from .common import parse_date, ApiException
class ControlIQApi:
BASE_URL = 'https://tdcservices.tandemdiabetes.com/tconnect/controliq/api/'
LOGIN_URL = 'https://tconnect.tandemdiabetes.com/login.aspx?ReturnUrl=%2f'
userGuid = None
accessToken = None
accessTokenExpiresAt = None
def __init__(self, email, password):
if not self.login(email, password):
raise Exception('Unable to authenticate')
def login(self, email, password):
with requests.Session() as s:
initial = s.get(self.LOGIN_URL)
soup = BeautifulSoup(initial.content, features='lxml')
data = {
"__LASTFOCUS": "",
"__EVENTTARGET": "ctl00$ContentBody$LoginControl$linkLogin",
"__EVENTARGUMENT": "",
"__VIEWSTATE": soup.select_one("#__VIEWSTATE")["value"],
"__VIEWSTATEGENERATOR": soup.select_one("#__VIEWSTATEGENERATOR")["value"],
"__EVENTVALIDATION": soup.select_one("#__EVENTVALIDATION")["value"],
"ctl00$ContentBody$LoginControl$txtLoginEmailAddress": email,
"txtLoginEmailAddress_ClientState": '{"enabled":true,"emptyMessage":"","validationText":"%s","valueAsString":"%s","lastSetTextBoxValue":"%s"}' % (email, email, email),
"ctl00$ContentBody$LoginControl$txtLoginPassword": password,
"txtLoginPassword_ClientState": '{"enabled":true,"emptyMessage":"","validationText":"%s","valueAsString":"%s","lastSetTextBoxValue":"%s"}' % (password, password, password)
}
req = s.post(self.LOGIN_URL, data=data, headers={'Referer': self.LOGIN_URL}, allow_redirects=False)
if req.status_code != 302:
return False
fwd = s.post(urllib.parse.urljoin(self.LOGIN_URL, req.headers['Location']), cookies=req.cookies)
if fwd.status_code != 200:
return False
self.userGuid = req.cookies['UserGUID']
self.accessToken = req.cookies['accessToken']
self.accessTokenExpiresAt = req.cookies['accessTokenExpiresAt']
return True
def api_headers(self):
if not self.accessToken:
raise Exception('No access token provided')
return {'Authorization': 'Bearer %s' % self.accessToken}
def get(self, endpoint, query):
r = requests.get(self.BASE_URL + endpoint, query, headers=self.api_headers())
if r.status_code != 200:
raise ApiException(r.status_code, "ControlIQ API HTTP %s response: %s" % (str(r.status_code), r.text))
return r.json()
def therapy_timeline(self, start=None, end=None):
startDate = parse_date(start)
endDate = parse_date(end)
return self.get('therapytimeline/users/%s' % (self.userGuid), {
"startDate": startDate,
"endDate": endDate
})
+76
View File
@@ -0,0 +1,76 @@
import requests
import datetime
import csv
from .common import parse_date, ApiException
class WS2Api:
BASE_URL = 'https://tconnectws2.tandemdiabetes.com/'
userGuid = None
def __init__(self, userGuid):
self.userGuid = userGuid
def get(self, endpoint, query):
r = requests.get(self.BASE_URL + endpoint, query)
if r.status_code != 200:
raise ApiException(r.status_code, "WS2 API HTTP %s response: %s" % (str(r.status_code), r.text))
return r.text
def _split_empty_sections(self, text):
sections = [[]]
sectionIndex = 0
for line in text.splitlines():
if len(line.strip()) > 0:
sections[sectionIndex].append(line)
else:
sections.append([])
sectionIndex += 1
return sections + [None] * (4 - len(sections))
def _csv_to_dict(self, rawdata):
data = []
if not rawdata or len(rawdata) == 0:
return data
headers = rawdata[0].split(",")
for row in csv.reader(rawdata[1:]):
data.append({headers[i]: row[i] for i in range(len(row)) if i < len(headers)})
return data
def therapy_timeline_csv(self, start=None, end=None):
startDate = parse_date(start)
endDate = parse_date(end)
req_text = self.get('therapytimeline2csv/%s/%s/%s' % (self.userGuid, startDate, endDate), {})
sections = self._split_empty_sections(req_text)
readingData = None
iobData = None
basalData = None
bolusData = None
for s in sections:
if s and len(s) > 2:
firstrow = s[1].replace('"', '').strip()
if firstrow.startswith("t:slim X2 Insulin Pump"):
readingData = s
elif firstrow.startswith("IOB"):
iobData = s
elif firstrow.startswith("Basal"):
basalData = s
elif firstrow.startswith("Bolus"):
bolusData = s
return {
"readingData": self._csv_to_dict(readingData),
"iobData": self._csv_to_dict(iobData),
"basalData": self._csv_to_dict(basalData),
"bolusData": self._csv_to_dict(bolusData)
}
+85 -14
View File
@@ -7,8 +7,10 @@ import hashlib
import requests
import arrow
import argparse
import time
from api import TConnectApi, ApiException
from api import TConnectApi
from api.common import ApiException
from parser import TConnectEntry
from nightscout import (
NightscoutEntry,
@@ -23,7 +25,12 @@ from nightscout import (
)
try:
from secret import TCONNECT_EMAIL, TCONNECT_PASSWORD
from secret import (
TCONNECT_EMAIL,
TCONNECT_PASSWORD,
PUMP_SERIAL_NUMBER,
TIMEZONE_NAME
)
except Exception:
print('Unable to import secret.py')
sys.exit(1)
@@ -83,8 +90,9 @@ def ns_write_basal_events(basalEvents, pretend=False):
last_upload_time = None
if last_upload:
last_upload_time = arrow.get(last_upload["created_at"])
print("Last Nightscout basal upload:", last_upload_time)
print("Last Nightscout basal upload:", arrow.get(last_upload_time, TIMEZONE_NAME))
add_count = 0
for event in basalEvents:
if last_upload_time and arrow.get(event["time"]) < last_upload_time:
if pretend:
@@ -104,6 +112,8 @@ def ns_write_basal_events(basalEvents, pretend=False):
reason=event["delivery_type"]
)
add_count += 1
print(" Processing basal:", event, "entry:", entry)
if recent_needs_update:
print("Replacing last uploaded entry:", last_upload)
@@ -113,6 +123,8 @@ def ns_write_basal_events(basalEvents, pretend=False):
elif not pretend:
upload_nightscout(entry)
return add_count
"""
Given bolus data input from the therapy timeline CSV, converts it into a digestable format.
"""
@@ -144,6 +156,7 @@ def ns_write_bolus_events(bolusEvents, pretend=False):
last_upload_time = arrow.get(last_upload["created_at"])
print("Last Nightscout bolus upload:", last_upload_time)
add_count = 0
for event in bolusEvents:
if last_upload_time and arrow.get(event["completion_time"]) <= last_upload_time:
if pretend:
@@ -157,10 +170,14 @@ def ns_write_bolus_events(bolusEvents, pretend=False):
notes="{}{}{}".format(event["description"], " (Override)" if event["user_override"] == "1" else "", " (Extended)" if event["extended_bolus"] == "1" else "")
)
add_count += 1
print(" Processing bolus:", event, "entry:", entry)
if not pretend:
upload_nightscout(entry)
return add_count
"""
Given IOB data input from the therapy timeline CSV, converts it into a digestable format.
"""
@@ -185,12 +202,12 @@ def ns_write_iob_events(iobEvents, pretend=False):
if not iobEvents or len(iobEvents) == 0:
print("No IOB events: skipping")
return
return 0
event = iobEvents[-1]
if last_upload_time and arrow.get(event["time"]) <= last_upload_time:
print(" Skipping already uploaded iob event:", event)
return
return 0
entry = NightscoutEntry.iob(
iob=event["iob"],
@@ -207,10 +224,12 @@ def ns_write_iob_events(iobEvents, pretend=False):
if not pretend:
delete_nightscout('activity/{}'.format(last_upload['_id']))
return 1
def process_time_range(tconnect, time_start, time_end, pretend):
print("Downloading t:connect ControlIQ data")
try:
ciqBasalData = tconnect.therapy_timeline(time_start, time_end)
ciqBasalData = tconnect.controliq.therapy_timeline(time_start, time_end)
except ApiException as e:
# The ControlIQ API returns a 404 if the user did not have a ControlIQ enabled
# device in the time range which is queried. Since it launched in early 2020,
@@ -222,7 +241,7 @@ def process_time_range(tconnect, time_start, time_end, pretend):
raise e
print("Downloading t:connect CSV data")
csvdata = tconnect.therapy_timeline_csv(time_start, time_end)
csvdata = tconnect.ws2.therapy_timeline_csv(time_start, time_end)
readingData = csvdata["readingData"]
iobData = csvdata["iobData"]
@@ -230,20 +249,24 @@ def process_time_range(tconnect, time_start, time_end, pretend):
bolusData = csvdata["bolusData"]
if readingData and len(readingData) > 0:
print("Last CGM reading from t:connect:", readingData[-1])
print("Last CGM reading from t:connect:", readingData[-1]['EventDateTime'] if 'EventDateTime' in readingData[-1] else readingData)
added = 0
basalEvents = process_ciq_basal_events(ciqBasalData)
if csvBasalData:
add_csv_basal_events(basalEvents, csvBasalData)
ns_write_basal_events(basalEvents, pretend=pretend)
added += ns_write_basal_events(basalEvents, pretend=pretend)
bolusEvents = process_bolus_events(bolusData)
ns_write_bolus_events(bolusEvents, pretend=pretend)
added += ns_write_bolus_events(bolusEvents, pretend=pretend)
iobEvents = process_iob_events(iobData)
ns_write_iob_events(iobEvents, pretend=pretend)
added += ns_write_iob_events(iobEvents, pretend=pretend)
return added
def parse_args():
parser = argparse.ArgumentParser(description="Syncs bolus, basal, and IOB data from Tandem Diabetes t:connect to Nightscout.")
@@ -251,6 +274,7 @@ def parse_args():
parser.add_argument('--start-date', dest='start_date', type=str, default=None, help='The oldest date to process data from. Must be specified with --end-date.')
parser.add_argument('--end-date', dest='end_date', type=str, default=None, help='The newest date to process data until (inclusive). Must be specified with --start-date.')
parser.add_argument('--days', dest='days', type=int, default=1, help='The number of days of t:connect data to read in. Cannot be used with --from-date and --until-date.')
parser.add_argument('--auto-update', dest='auto_update', action='store_const', const=True, default=False, help='If set, continuously checks for updates from t:connect and syncs with Nightscout.')
return parser.parse_args()
@@ -260,6 +284,9 @@ def main():
if args.pretend:
print("Pretend mode: will not write to Nightscout")
if args.auto_update and (args.start_date or args.end_date):
raise Exception('Auto-update cannot be used with start/end date')
if args.start_date and args.end_date:
time_start = arrow.get(args.start_date)
time_end = arrow.get(args.end_date)
@@ -270,11 +297,55 @@ def main():
if time_end < time_start:
raise Exception('time_start must be before time_end')
print("Processing data between", time_start, "and", time_end)
tconnect = TConnectApi(TCONNECT_EMAIL, TCONNECT_PASSWORD)
process_time_range(tconnect, time_start, time_end, args.pretend)
if args.auto_update:
# Read from android api, find exact interval to cut down on API calls
# Refresh API token. If failure, die, have wrapper script re-run.
last_event_index = None
last_event_time = None
time_diffs = []
while True:
last_event = tconnect.android.last_event_uploaded(PUMP_SERIAL_NUMBER)
if not last_event_index or last_event['maxPumpEventIndex'] > last_event_index:
now = time.time()
print('New event index:', last_event['maxPumpEventIndex'], 'last:', last_event_index)
if args.pretend:
print('Would update now')
else:
added = process_time_range(tconnect, time_start, time_end, args.pretend)
print('Added', added, 'items')
if last_event_index:
time_diffs.append(now - last_event_time)
print('Time diffs:', time_diffs)
last_event_index = last_event['maxPumpEventIndex']
last_event_time = now
else:
print('No event index change:', last_event['maxPumpEventIndex'])
if len(time_diffs) > 2:
print('Sleeping 60 seconds after unexpected no index change')
time.sleep(60)
continue
sleep_secs = 60
if len(time_diffs) > 10:
time_diffs = time_diffs[1:]
if len(time_diffs) > 2:
sleep_secs = sum(time_diffs) / len(time_diffs)
# Sleep for a rolling average of time between updates
print('Sleeping for', sleep_secs, 'sec')
time.sleep(sleep_secs)
else:
print("Processing data between", time_start, "and", time_end)
added = process_time_range(tconnect, time_start, time_end, args.pretend)
print("Added", added, "items")
if __name__ == '__main__':
main()
+2
View File
@@ -1,6 +1,8 @@
TCONNECT_EMAIL = 'email@email.com'
TCONNECT_PASSWORD = 'password'
PUMP_SERIAL_NUMBER = 11111111
NS_URL = 'https://yournightscouturl/'
NS_SECRET = 'apisecret'