From d00d8e59dd8f456c16c64606c928e4af8364d113 Mon Sep 17 00:00:00 2001 From: James Woglom Date: Thu, 18 Mar 2021 00:47:57 -0400 Subject: [PATCH] refactor: move process and synchronization code outside of main file Splits up basal, bolus, and iob parsing code into separate files in the sync folder. Moves the single- cycle processing code into process.py --- main.py | 248 +--------------------------------- tconnectsync/process.py | 63 +++++++++ tconnectsync/sync/__init__.py | 0 tconnectsync/sync/basal.py | 105 ++++++++++++++ tconnectsync/sync/bolus.py | 63 +++++++++ tconnectsync/sync/iob.py | 58 ++++++++ 6 files changed, 290 insertions(+), 247 deletions(-) create mode 100644 tconnectsync/process.py create mode 100644 tconnectsync/sync/__init__.py create mode 100644 tconnectsync/sync/basal.py create mode 100644 tconnectsync/sync/bolus.py create mode 100644 tconnectsync/sync/iob.py diff --git a/main.py b/main.py index 02f7e1c..a537546 100644 --- a/main.py +++ b/main.py @@ -11,18 +11,7 @@ import time from tconnectsync.api import TConnectApi from tconnectsync.api.common import ApiException -from tconnectsync.parser import TConnectEntry -from tconnectsync.nightscout import ( - NightscoutEntry, - upload_nightscout, - delete_nightscout, - put_nightscout, - last_uploaded_nightscout_entry, - last_uploaded_nightscout_activity, - BASAL_EVENTTYPE, - BOLUS_EVENTTYPE, - IOB_ACTIVITYTYPE -) +from tconnectsync.process import process_time_range try: from secret import ( @@ -35,242 +24,7 @@ except Exception: print('Unable to import secret.py') sys.exit(1) -""" -Merges together input from the therapy timeline API into a digestable format of basal data. -""" -def process_ciq_basal_events(data): - if data is None: - return [] - suspensionEvents = {} - for s in data["suspensionDeliveryEvents"]: - entry = TConnectEntry.parse_suspension_entry(s) - suspensionEvents[entry["time"]] = entry - - basalEvents = [] - for b in data["basal"]["tempDeliveryEvents"]: - basalEvents.append(TConnectEntry.parse_ciq_basal_entry(b, delivery_type="tempDelivery")) - - for b in data["basal"]["algorithmDeliveryEvents"]: - basalEvents.append(TConnectEntry.parse_ciq_basal_entry(b, delivery_type="algorithmDelivery")) - - for b in data["basal"]["profileDeliveryEvents"]: - basalEvents.append(TConnectEntry.parse_ciq_basal_entry(b, delivery_type="profileDelivery")) - - basalEvents.sort(key=lambda x: arrow.get(x["time"])) - - for i in basalEvents: - if i["time"] in suspensionEvents: - i["suspendReason"] = suspensionEvents[i["time"]]["suspendReason"] - - return basalEvents - -""" -Processes basal data input from the therapy timeline CSV (which only exists for pre Control-IQ data) into a digestable format. -""" -def add_csv_basal_events(basalEvents, data): - last_entry = None - for row in data: - entry = TConnectEntry.parse_csv_basal_entry(row) - if last_entry: - diff_mins = (arrow.get(entry["time"]) - arrow.get(last_entry["time"])).seconds // 60 - entry["duration_mins"] = diff_mins - - basalEvents.append(entry) - last_entry = entry - - basalEvents.sort(key=lambda x: arrow.get(x["time"])) - return basalEvents - -""" -Given processed basal data, adds basal events to Nightscout. -""" -def ns_write_basal_events(basalEvents, pretend=False): - last_upload = last_uploaded_nightscout_entry(BASAL_EVENTTYPE) - last_upload_time = None - if last_upload: - last_upload_time = arrow.get(last_upload["created_at"]) - print("Last Nightscout basal upload:", last_upload_time) - - add_count = 0 - for event in basalEvents: - if last_upload_time and arrow.get(event["time"]) < last_upload_time: - if pretend: - print("Skipping basal event before last upload time:", event) - continue - - recent_needs_update = False - if last_upload_time and arrow.get(event["time"]) == last_upload_time: - # If this entry has the same time as the most recent upload, but - # has newer info, then delete and recreate it. - recent_needs_update = (round(last_upload["duration"]) < round(event["duration_mins"])) - - reason = event["delivery_type"] - if "suspendReason" in reason: - reason += " (" + reason["suspendReason"] + ")" - - entry = NightscoutEntry.basal( - value=event["basal_rate"], - duration_mins=event["duration_mins"], - created_at=event["time"], - reason=reason - ) - - add_count += 1 - - print(" Processing basal:", event, "entry:", entry) - if recent_needs_update: - print("Replacing last uploaded entry:", last_upload) - if not pretend: - entry['_id'] = last_upload['_id'] - put_nightscout(entry, entity='treatments') - elif not pretend: - upload_nightscout(entry) - - return add_count - -""" -Given bolus data input from the therapy timeline CSV, converts it into a digestable format. -""" -def process_bolus_events(bolusdata): - bolusEvents = [] - - for b in bolusdata: - parsed = TConnectEntry.parse_bolus_entry(b) - if parsed["completion"] != "Completed": - if parsed["insulin"] and float(parsed["insulin"]) > 0: - # Count non-completed bolus if any insulin was delivered (vs. the amount of insulin requested) - parsed["description"] += " (%s)" % parsed["completion"] - else: - print("Skipping non-completed bolus data:", b, "parsed:", parsed) - continue - bolusEvents.append(parsed) - - bolusEvents.sort(key=lambda event: arrow.get(event["completion_time"] if not event["extended_bolus"] else event["bolex_start_time"])) - - return bolusEvents - -""" -Given processed bolus data, adds bolus events to Nightscout. -""" -def ns_write_bolus_events(bolusEvents, pretend=False): - last_upload = last_uploaded_nightscout_entry(BOLUS_EVENTTYPE) - last_upload_time = None - if last_upload: - last_upload_time = arrow.get(last_upload["created_at"]) - print("Last Nightscout bolus upload:", last_upload_time) - - add_count = 0 - for event in bolusEvents: - if last_upload_time and arrow.get(event["completion_time"]) <= last_upload_time: - if pretend: - print("Skipping basal event before last upload time:", event) - continue - - entry = NightscoutEntry.bolus( - bolus=event["insulin"], - carbs=event["carbs"], - created_at=event["completion_time"] if not event["extended_bolus"] else event["bolex_start_time"], - notes="{}{}{}".format(event["description"], " (Override)" if event["user_override"] == "1" else "", " (Extended)" if event["extended_bolus"] == "1" else "") - ) - - add_count += 1 - - print(" Processing bolus:", event, "entry:", entry) - if not pretend: - upload_nightscout(entry) - - return add_count - -""" -Given IOB data input from the therapy timeline CSV, converts it into a digestable format. -""" -def process_iob_events(iobdata): - iobEvents = [] - for d in iobdata: - iobEvents.append(TConnectEntry.parse_iob_entry(d)) - - iobEvents.sort(key=lambda x: arrow.get(x["time"])) - - return iobEvents - -""" -Given processed IOB data, creates a single Nightscout activity definition to store IOB. -""" -def ns_write_iob_events(iobEvents, pretend=False): - last_upload = last_uploaded_nightscout_activity(IOB_ACTIVITYTYPE) - last_upload_time = None - if last_upload: - last_upload_time = arrow.get(last_upload["created_at"]) - print("Last Nightscout iob upload:", last_upload_time) - - if not iobEvents or len(iobEvents) == 0: - print("No IOB events: skipping") - return 0 - - event = iobEvents[-1] - if last_upload_time and arrow.get(event["time"]) <= last_upload_time: - print(" Skipping already uploaded iob event:", event) - return 0 - - entry = NightscoutEntry.iob( - iob=event["iob"], - created_at=event["time"] - ) - - print(" Processing iob:", event, "entry:", entry) - if not pretend: - upload_nightscout(entry, entity='activity') - - # Delete the previous activity - if last_upload and '_id' in last_upload: - print(" Deleting old iob entry:", last_upload) - if not pretend: - delete_nightscout('activity/{}'.format(last_upload['_id'])) - - return 1 - -def process_time_range(tconnect, time_start, time_end, pretend): - print("Downloading t:connect ControlIQ data") - try: - ciqBasalData = tconnect.controliq.therapy_timeline(time_start, time_end) - except ApiException as e: - # The ControlIQ API returns a 404 if the user did not have a ControlIQ enabled - # device in the time range which is queried. Since it launched in early 2020, - # ignore 404's before February. - if e.status_code == 404 and time_start.date() < datetime.date(2020, 2, 1): - print("Ignoring HTTP 404 for ControlIQ API request before Feb 2020") - ciqBasalData = None - else: - raise e - - print("Downloading t:connect CSV data") - csvdata = tconnect.ws2.therapy_timeline_csv(time_start, time_end) - - readingData = csvdata["readingData"] - iobData = csvdata["iobData"] - csvBasalData = csvdata["basalData"] - bolusData = csvdata["bolusData"] - - if readingData and len(readingData) > 0: - print("Last CGM reading from t:connect:", readingData[-1]['EventDateTime'] if 'EventDateTime' in readingData[-1] else readingData) - - added = 0 - - basalEvents = process_ciq_basal_events(ciqBasalData) - if csvBasalData: - add_csv_basal_events(basalEvents, csvBasalData) - - added += ns_write_basal_events(basalEvents, pretend=pretend) - - - bolusEvents = process_bolus_events(bolusData) - added += ns_write_bolus_events(bolusEvents, pretend=pretend) - - iobEvents = process_iob_events(iobData) - added += ns_write_iob_events(iobEvents, pretend=pretend) - - return added def parse_args(): parser = argparse.ArgumentParser(description="Syncs bolus, basal, and IOB data from Tandem Diabetes t:connect to Nightscout.") diff --git a/tconnectsync/process.py b/tconnectsync/process.py new file mode 100644 index 0000000..f9f0e57 --- /dev/null +++ b/tconnectsync/process.py @@ -0,0 +1,63 @@ +from datetime import datetime + +from .api.common import ApiException +from .sync.basal import ( + process_ciq_basal_events, + add_csv_basal_events, + ns_write_basal_events +) +from .sync.bolus import ( + process_bolus_events, + ns_write_bolus_events +) +from .sync.iob import ( + process_iob_events, + ns_write_iob_events +) + +""" +Given a TConnectApi object and start/end range, performs a single +cycle of synchronizing data within the time range. +If pretend is true, then doesn't actually write data to Nightscout. +""" +def process_time_range(tconnect, time_start, time_end, pretend): + print("Downloading t:connect ControlIQ data") + try: + ciqBasalData = tconnect.controliq.therapy_timeline(time_start, time_end) + except ApiException as e: + # The ControlIQ API returns a 404 if the user did not have a ControlIQ enabled + # device in the time range which is queried. Since it launched in early 2020, + # ignore 404's before February. + if e.status_code == 404 and time_start.date() < datetime.date(2020, 2, 1): + print("Ignoring HTTP 404 for ControlIQ API request before Feb 2020") + ciqBasalData = None + else: + raise e + + print("Downloading t:connect CSV data") + csvdata = tconnect.ws2.therapy_timeline_csv(time_start, time_end) + + readingData = csvdata["readingData"] + iobData = csvdata["iobData"] + csvBasalData = csvdata["basalData"] + bolusData = csvdata["bolusData"] + + if readingData and len(readingData) > 0: + print("Last CGM reading from t:connect:", readingData[-1]['EventDateTime'] if 'EventDateTime' in readingData[-1] else readingData) + + added = 0 + + basalEvents = process_ciq_basal_events(ciqBasalData) + if csvBasalData: + add_csv_basal_events(basalEvents, csvBasalData) + + added += ns_write_basal_events(basalEvents, pretend=pretend) + + + bolusEvents = process_bolus_events(bolusData) + added += ns_write_bolus_events(bolusEvents, pretend=pretend) + + iobEvents = process_iob_events(iobData) + added += ns_write_iob_events(iobEvents, pretend=pretend) + + return added \ No newline at end of file diff --git a/tconnectsync/sync/__init__.py b/tconnectsync/sync/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tconnectsync/sync/basal.py b/tconnectsync/sync/basal.py new file mode 100644 index 0000000..abc5a72 --- /dev/null +++ b/tconnectsync/sync/basal.py @@ -0,0 +1,105 @@ +import arrow + +from ..nightscout import ( + BASAL_EVENTTYPE, + NightscoutEntry, + last_uploaded_nightscout_entry, + put_nightscout, + upload_nightscout +) +from ..parser import TConnectEntry + + +""" +Merges together input from the therapy timeline API into a digestable format of basal data. +""" +def process_ciq_basal_events(data): + if data is None: + return [] + + suspensionEvents = {} + for s in data["suspensionDeliveryEvents"]: + entry = TConnectEntry.parse_suspension_entry(s) + suspensionEvents[entry["time"]] = entry + + basalEvents = [] + for b in data["basal"]["tempDeliveryEvents"]: + basalEvents.append(TConnectEntry.parse_ciq_basal_entry(b, delivery_type="tempDelivery")) + + for b in data["basal"]["algorithmDeliveryEvents"]: + basalEvents.append(TConnectEntry.parse_ciq_basal_entry(b, delivery_type="algorithmDelivery")) + + for b in data["basal"]["profileDeliveryEvents"]: + basalEvents.append(TConnectEntry.parse_ciq_basal_entry(b, delivery_type="profileDelivery")) + + basalEvents.sort(key=lambda x: arrow.get(x["time"])) + + for i in basalEvents: + if i["time"] in suspensionEvents: + i["suspendReason"] = suspensionEvents[i["time"]]["suspendReason"] + + return basalEvents + +""" +Processes basal data input from the therapy timeline CSV (which only exists for pre Control-IQ data) into a digestable format. +""" +def add_csv_basal_events(basalEvents, data): + last_entry = None + for row in data: + entry = TConnectEntry.parse_csv_basal_entry(row) + if last_entry: + diff_mins = (arrow.get(entry["time"]) - arrow.get(last_entry["time"])).seconds // 60 + entry["duration_mins"] = diff_mins + + basalEvents.append(entry) + last_entry = entry + + basalEvents.sort(key=lambda x: arrow.get(x["time"])) + return basalEvents + +""" +Given processed basal data, adds basal events to Nightscout. +""" +def ns_write_basal_events(basalEvents, pretend=False): + last_upload = last_uploaded_nightscout_entry(BASAL_EVENTTYPE) + last_upload_time = None + if last_upload: + last_upload_time = arrow.get(last_upload["created_at"]) + print("Last Nightscout basal upload:", last_upload_time) + + add_count = 0 + for event in basalEvents: + if last_upload_time and arrow.get(event["time"]) < last_upload_time: + if pretend: + print("Skipping basal event before last upload time:", event) + continue + + recent_needs_update = False + if last_upload_time and arrow.get(event["time"]) == last_upload_time: + # If this entry has the same time as the most recent upload, but + # has newer info, then delete and recreate it. + recent_needs_update = (round(last_upload["duration"]) < round(event["duration_mins"])) + + reason = event["delivery_type"] + if "suspendReason" in reason: + reason += " (" + reason["suspendReason"] + ")" + + entry = NightscoutEntry.basal( + value=event["basal_rate"], + duration_mins=event["duration_mins"], + created_at=event["time"], + reason=reason + ) + + add_count += 1 + + print(" Processing basal:", event, "entry:", entry) + if recent_needs_update: + print("Replacing last uploaded entry:", last_upload) + if not pretend: + entry['_id'] = last_upload['_id'] + put_nightscout(entry, entity='treatments') + elif not pretend: + upload_nightscout(entry) + + return add_count diff --git a/tconnectsync/sync/bolus.py b/tconnectsync/sync/bolus.py new file mode 100644 index 0000000..48c76ae --- /dev/null +++ b/tconnectsync/sync/bolus.py @@ -0,0 +1,63 @@ +import arrow + +from ..nightscout import ( + BOLUS_EVENTTYPE, + NightscoutEntry, + last_uploaded_nightscout_entry, + put_nightscout, + upload_nightscout +) +from ..parser import TConnectEntry + +""" +Given bolus data input from the therapy timeline CSV, converts it into a digestable format. +""" +def process_bolus_events(bolusdata): + bolusEvents = [] + + for b in bolusdata: + parsed = TConnectEntry.parse_bolus_entry(b) + if parsed["completion"] != "Completed": + if parsed["insulin"] and float(parsed["insulin"]) > 0: + # Count non-completed bolus if any insulin was delivered (vs. the amount of insulin requested) + parsed["description"] += " (%s)" % parsed["completion"] + else: + print("Skipping non-completed bolus data:", b, "parsed:", parsed) + continue + bolusEvents.append(parsed) + + bolusEvents.sort(key=lambda event: arrow.get(event["completion_time"] if not event["extended_bolus"] else event["bolex_start_time"])) + + return bolusEvents + +""" +Given processed bolus data, adds bolus events to Nightscout. +""" +def ns_write_bolus_events(bolusEvents, pretend=False): + last_upload = last_uploaded_nightscout_entry(BOLUS_EVENTTYPE) + last_upload_time = None + if last_upload: + last_upload_time = arrow.get(last_upload["created_at"]) + print("Last Nightscout bolus upload:", last_upload_time) + + add_count = 0 + for event in bolusEvents: + if last_upload_time and arrow.get(event["completion_time"]) <= last_upload_time: + if pretend: + print("Skipping basal event before last upload time:", event) + continue + + entry = NightscoutEntry.bolus( + bolus=event["insulin"], + carbs=event["carbs"], + created_at=event["completion_time"] if not event["extended_bolus"] else event["bolex_start_time"], + notes="{}{}{}".format(event["description"], " (Override)" if event["user_override"] == "1" else "", " (Extended)" if event["extended_bolus"] == "1" else "") + ) + + add_count += 1 + + print(" Processing bolus:", event, "entry:", entry) + if not pretend: + upload_nightscout(entry) + + return add_count diff --git a/tconnectsync/sync/iob.py b/tconnectsync/sync/iob.py new file mode 100644 index 0000000..65671d1 --- /dev/null +++ b/tconnectsync/sync/iob.py @@ -0,0 +1,58 @@ +import arrow + +from ..nightscout import ( + IOB_ACTIVITYTYPE, + NightscoutEntry, + last_uploaded_nightscout_activity, + delete_nightscout, + upload_nightscout +) +from ..parser import TConnectEntry + +""" +Given IOB data input from the therapy timeline CSV, converts it into a digestable format. +""" +def process_iob_events(iobdata): + iobEvents = [] + for d in iobdata: + iobEvents.append(TConnectEntry.parse_iob_entry(d)) + + iobEvents.sort(key=lambda x: arrow.get(x["time"])) + + return iobEvents + +""" +Given processed IOB data, creates a single Nightscout activity definition to store IOB. +""" +def ns_write_iob_events(iobEvents, pretend=False): + last_upload = last_uploaded_nightscout_activity(IOB_ACTIVITYTYPE) + last_upload_time = None + if last_upload: + last_upload_time = arrow.get(last_upload["created_at"]) + print("Last Nightscout iob upload:", last_upload_time) + + if not iobEvents or len(iobEvents) == 0: + print("No IOB events: skipping") + return 0 + + event = iobEvents[-1] + if last_upload_time and arrow.get(event["time"]) <= last_upload_time: + print(" Skipping already uploaded iob event:", event) + return 0 + + entry = NightscoutEntry.iob( + iob=event["iob"], + created_at=event["time"] + ) + + print(" Processing iob:", event, "entry:", entry) + if not pretend: + upload_nightscout(entry, entity='activity') + + # Delete the previous activity + if last_upload and '_id' in last_upload: + print(" Deleting old iob entry:", last_upload) + if not pretend: + delete_nightscout('activity/{}'.format(last_upload['_id'])) + + return 1 \ No newline at end of file