integration tests for profile upload

This commit is contained in:
James Woglom
2023-01-16 16:58:19 -05:00
parent dd9a9ea72c
commit d9e49e1fc8
8 changed files with 314 additions and 54 deletions
+3 -3
View File
@@ -1,4 +1,4 @@
from typing import List, Tuple
from typing import Dict, List, Tuple
import requests
import urllib
import datetime
@@ -70,7 +70,7 @@ class WebUIScraper:
Returns a mapping between pump/device IDs and information about that device,
including the GUID used for obtaining pump settings.
"""
def my_devices(self):
def my_devices(self) -> Dict[str, DeviceSettings]:
devices = {}
r = self.get('myaccount/my_devices.aspx')
soup = BeautifulSoup(r.content, features='lxml')
@@ -274,7 +274,7 @@ class WebUIScraper:
Wraps a call to my_devices to identify the device GUID from the
given pump serial, and then returns device_settings_from_guid.
"""
def device_settings(self, pump_serial):
def device_settings(self, pump_serial: str) -> Tuple[List[Profile], DeviceSettings]:
devices = self.my_devices()
if str(pump_serial) in devices.keys():
dev = devices[str(pump_serial)]
+3 -1
View File
@@ -7,6 +7,7 @@ IOB = "IOB"
BOLUS_BG = "BOLUS_BG"
CGM = "CGM"
PUMP_EVENTS = "PUMP_EVENTS"
PROFILES = "PROFILES"
DEFAULT_FEATURES = [
BASAL,
@@ -17,7 +18,8 @@ ALL_FEATURES = [
BASAL,
BOLUS,
IOB,
PUMP_EVENTS
PUMP_EVENTS,
PROFILES
]
+17 -28
View File
@@ -43,7 +43,7 @@ class NightscoutApi:
'api-secret': hashlib.sha1(self.secret.encode()).hexdigest()
}, verify=self.verify)
if r.status_code != 200:
raise ApiException(r.status_code, "Nightscout upload response: %s" % r.text)
raise ApiException(r.status_code, "Nightscout upload %s response: %s" % (r.status_code, r.text))
def delete_entry(self, entity):
r = requests.delete(urljoin(self.url, 'api/v1/' + entity + '?api_secret=' + self.secret), json={}, headers={
@@ -52,7 +52,7 @@ class NightscoutApi:
'api-secret': hashlib.sha1(self.secret.encode()).hexdigest()
}, verify=self.verify)
if r.status_code != 200:
raise ApiException(r.status_code, "Nightscout delete response: %s" % r.text)
raise ApiException(r.status_code, "Nightscout delete %s response: %s" % (r.status_code, r.text))
def put_entry(self, ns_format, entity):
r = requests.put(urljoin(self.url, 'api/v1/' + entity + '?api_secret=' + self.secret), json=ns_format, headers={
@@ -61,7 +61,7 @@ class NightscoutApi:
'api-secret': hashlib.sha1(self.secret.encode()).hexdigest()
}, verify=self.verify)
if r.status_code != 200:
raise ApiException(r.status_code, "Nightscout put response: %s" % r.text)
raise ApiException(r.status_code, "Nightscout put %s response: %s" % (r.status_code, r.text))
def last_uploaded_entry(self, eventType, time_start=None, time_end=None):
def internal(t_to_space):
@@ -70,7 +70,7 @@ class NightscoutApi:
'api-secret': hashlib.sha1(self.secret.encode()).hexdigest()
}, verify=self.verify)
if latest.status_code != 200:
raise ApiException(latest.status_code, "Nightscout last_uploaded_entry response: %s" % latest.text)
raise ApiException(latest.status_code, "Nightscout last_uploaded_entry %s response: %s" % (latest.status_code, latest.text))
j = latest.json()
if j and len(j) > 0:
@@ -100,7 +100,7 @@ class NightscoutApi:
'api-secret': hashlib.sha1(self.secret.encode()).hexdigest()
}, verify=self.verify)
if latest.status_code != 200:
raise ApiException(latest.status_code, "Nightscout last_uploaded_bg_entry response: %s" % latest.text)
raise ApiException(latest.status_code, "Nightscout last_uploaded_bg_entry %s response: %s" % (latest.status_code, latest.text))
j = latest.json()
if j and len(j) > 0:
@@ -121,7 +121,7 @@ class NightscoutApi:
'api-secret': hashlib.sha1(self.secret.encode()).hexdigest()
}, verify=self.verify)
if latest.status_code != 200:
raise ApiException(latest.status_code, "Nightscout activity response: %s" % latest.text)
raise ApiException(latest.status_code, "Nightscout activity %s response: %s" % (latest.status_code, latest.text))
j = latest.json()
if j and len(j) > 0:
@@ -147,26 +147,15 @@ class NightscoutApi:
return status.json()
"""
Returns information on configured Nightscout profiles, over the optional time range.
Will only return the most recent profile for the given range.
Returns information on the currently configured Nightscout profile data store
(contains all profiles in Nightscout under one mongo object).
"""
def profiles(self, time_start=None, time_end=None):
def internal(t_to_space):
dateFilter = time_range('created_at', time_start, time_end, t_to_space=t_to_space)
latest = requests.get(urljoin(self.url, 'api/v1/profile.json?' + dateFilter + '&ts=' + str(time.time())), headers={
'api-secret': hashlib.sha1(self.secret.encode()).hexdigest()
}, verify=self.verify)
if latest.status_code != 200:
raise ApiException(latest.status_code, "Nightscout profiles response: %s" % latest.text)
j = latest.json()
if j and len(j) > 0:
return j[0]
return None
ret = internal(False)
if ret is None and (time_start or time_end):
ret = internal(True)
if ret is not None:
logger.warning("profiles with time_start=%s time_end=%s only returned data when timestamps contained a space" % (time_start, time_end))
return ret
def current_profile(self, time_start=None, time_end=None):
r = requests.get(urljoin(self.url, 'api/v1/profile/current?api_secret=' + self.secret), json={}, headers={
'Accept': 'application/json',
'Content-Type': 'application/json',
'api-secret': hashlib.sha1(self.secret.encode()).hexdigest()
}, verify=self.verify)
if r.status_code != 200:
raise ApiException(r.status_code, "Nightscout current_profile %s response: %s" % (r.status_code, r.text))
return r.json()
+21 -13
View File
@@ -29,8 +29,9 @@ from .sync.pump_events import (
process_basalsuspension_events,
ns_write_pump_events
)
from .sync.profile import process_profiles
from .parser.tconnect import TConnectEntry
from .features import BASAL, BOLUS, IOB, BOLUS_BG, CGM, DEFAULT_FEATURES, PUMP_EVENTS
from .features import BASAL, BOLUS, IOB, BOLUS_BG, CGM, DEFAULT_FEATURES, PUMP_EVENTS, PROFILES
from tconnectsync.sync import basal
logger = logging.getLogger(__name__)
@@ -41,18 +42,21 @@ cycle of synchronizing data within the time range.
If pretend is true, then doesn't actually write data to Nightscout.
"""
def process_time_range(tconnect, nightscout, time_start, time_end, pretend, features=DEFAULT_FEATURES):
logger.info("Downloading t:connect ControlIQ data")
try:
ciqTherapyTimelineData = tconnect.controliq.therapy_timeline(time_start, time_end)
except ApiException as e:
# The ControlIQ API returns a 404 if the user did not have a ControlIQ enabled
# device in the time range which is queried. Since it launched in early 2020,
# ignore 404's before February.
if e.status_code == 404 and time_start.date() < datetime.date(2020, 2, 1):
logger.warning("Ignoring HTTP 404 for ControlIQ API request before Feb 2020")
ciqTherapyTimelineData = None
else:
raise e
ciqTherapyTimelineData = None
if BASAL in features or PUMP_EVENTS in features:
logger.info("Downloading t:connect ControlIQ data")
try:
ciqTherapyTimelineData = tconnect.controliq.therapy_timeline(time_start, time_end)
except ApiException as e:
# The ControlIQ API returns a 404 if the user did not have a ControlIQ enabled
# device in the time range which is queried. Since it launched in early 2020,
# ignore 404's before February.
if e.status_code == 404 and time_start.date() < datetime.date(2020, 2, 1):
logger.warning("Ignoring HTTP 404 for ControlIQ API request before Feb 2020")
ciqTherapyTimelineData = None
else:
raise e
csvReadingData = None
csvIobData = None
@@ -182,6 +186,10 @@ def process_time_range(tconnect, nightscout, time_start, time_end, pretend, feat
logger.debug("Writing iob events")
added += ns_write_iob_events(nightscout, iobEvents, pretend=pretend)
logger.debug("Finished writing iob events")
if PROFILES in features:
logger.debug("Running profiles feature")
process_profiles(tconnect, nightscout, pretend=pretend)
logger.info("Wrote %d events to Nightscout this process cycle" % added)
return added
+9
View File
@@ -16,6 +16,13 @@ else:
def get(val, default=None):
return os.environ.get(val, values.get(val, default))
def get_one_of(name, default=None, options=[]):
val = get(name, default)
if val not in options:
print("Error: %s must be one of: %s" % (name, options))
print("Current value: %s" % val)
sys.exit(1)
def get_number(name, default):
val = get(name, default)
try:
@@ -59,6 +66,8 @@ AUTOUPDATE_FAILURE_MINUTES = get_number('AUTOUPDATE_FAILURE_MINUTES', '15') # 15
AUTOUPDATE_RESTART_ON_FAILURE = get_bool('AUTOUPDATE_RESTART_ON_FAILURE', 'true')
AUTOUPDATE_MAX_LOOP_INVOCATIONS = get_number('AUTOUPDATE_MAX_LOOP_INVOCATIONS', '-1')
NIGHTSCOUT_PROFILE_UPLOAD_MODE = get_one_of('NIGHTSCOUT_PROFILE_UPLOAD_MODE', 'add', ['add', 'replace'])
# Default Nightscout profile segment fields which aren't stored by Tandem
NIGHTSCOUT_PROFILE_CARBS_HR_VALUE = get('NIGHTSCOUT_PROFILE_CARBS_HR_VALUE', '20')
NIGHTSCOUT_PROFILE_DELAY_VALUE = get('NIGHTSCOUT_PROFILE_DELAY_VALUE', '20')
+53 -7
View File
@@ -2,22 +2,30 @@ from typing import List, Tuple
import logging
import json
import copy
import arrow
from ..api import TConnectApi
from ..domain.device_settings import Profile, DeviceSettings
from ..parser.nightscout import NightscoutEntry
from ..secret import PUMP_SERIAL_NUMBER
from ..nightscout import NightscoutApi
from ..secret import PUMP_SERIAL_NUMBER, NIGHTSCOUT_PROFILE_UPLOAD_MODE
logger = logging.getLogger(__name__)
def _get_default_serial_number():
return PUMP_SERIAL_NUMBER
def _get_default_upload_mode():
return NIGHTSCOUT_PROFILE_UPLOAD_MODE
def get_pump_profiles(tconnect: TConnectApi, serial_number: int = None) -> Tuple[List[Profile], DeviceSettings]:
all_devices = tconnect.webui.my_devices()
if serial_number is None:
serial_number = PUMP_SERIAL_NUMBER
serial_number = _get_default_serial_number()
if str(serial_number) not in all_devices:
logger.warn("Could not find entry for provided pump serial number in t:connect device list: %s, received: %s", serial_number, all_devices)
return []
return [], None
device = all_devices[str(serial_number)]
@@ -36,7 +44,7 @@ Compare pump device and Nightscout profiles, and return a final dictionary of
Nightscout profile objects, with the pump profile settings overriding what is
currently in Nightscout.
ns_profile_obj is the output from NightscoutApi.profiles() and should be the most
ns_profile_obj is the output from NightscoutApi.current_profile() and should be the most
recent profile object in mongo.
Returns the new Nightscout profile and whether it was changed.
@@ -61,12 +69,12 @@ def compare_profiles(device_profiles: List[Profile], device_settings: DeviceSett
existent_profiles_in_ns = set(device.keys()) & set(ns.keys())
for profile_name in existent_profiles_in_ns:
logger.info("Checking for differences for %s profile between pump and nightscout", profile_name)
logger.debug("Checking for differences for %s profile between pump and nightscout", profile_name)
pump_configured_profile = device[profile_name]
ns_translated_profile = NightscoutEntry.profile_store(pump_configured_profile, device_settings)
ns_configured_profile = ns[profile_name]
logger.info("Comparing %s profile from pump: %s to nightscout: %s", profile_name, ns_translated_profile, ns_configured_profile)
logger.debug("Comparing %s profile from pump: %s to nightscout: %s", profile_name, ns_translated_profile, ns_configured_profile)
if nightscout_profiles_identical(ns_configured_profile, ns_translated_profile):
logger.info("Profile %s identical between pump and nightscout", profile_name)
continue
@@ -95,7 +103,7 @@ def compare_profiles(device_profiles: List[Profile], device_settings: DeviceSett
def nightscout_profiles_identical(configured: dict, translated: dict) -> bool:
if json.dumps(configured, sort_keys=True, indent=None) == json.dumps(translated, sort_keys=True, indent=None):
logger.debug("Initial JSON dump identical")
logger.debug("initial JSON dump identical")
return True
# convert all JSON values into strings
@@ -129,3 +137,41 @@ def nightscout_profiles_identical(configured: dict, translated: dict) -> bool:
logger.debug("profiles not identical")
return False
def setup_new_profile(ns_profile: dict) -> dict:
if '_id' in ns_profile:
del ns_profile['_id']
now = arrow.now().isoformat()
ns_profile['startDate'] = now
ns_profile['created_at'] = now
return ns_profile
def process_profiles(tconnect: TConnectApi, nightscout: NightscoutApi, pretend: bool = False, upload_mode: str = None):
if upload_mode is None:
upload_mode = _get_default_upload_mode()
logger.debug("Checking for differences between pump and nightscout profiles: %s mode", upload_mode)
ns_profile_obj = nightscout.current_profile()
pump_profiles, pump_settings = get_pump_profiles(tconnect)
diff, ns_profile_new = compare_profiles(pump_profiles, pump_settings, ns_profile_obj)
if not diff:
logger.info("Pump and Nightscout profiles up to date")
return
if upload_mode == 'add':
profile_to_upload = setup_new_profile(ns_profile_new)
logger.info("Adding new Nightscout profiles object: %s", profile_to_upload)
if not pretend:
nightscout.upload_entry(profile_to_upload, entity='profile')
elif upload_mode == 'replace':
logger.info("Replacing new Nightscout profiles object: %s", ns_profile_new)
if not pretend:
nightscout.put_entry(ns_profile_new, entity='profile')
else:
raise RuntimeError('invalid upload_mode: %s' % upload_mode)
+7
View File
@@ -44,6 +44,12 @@ class WebUIScraper(tconnectsync.api.webui.WebUIScraper):
def __init__(self, controliq):
self.controliq = controliq
def my_devices(self):
raise NotImplementedError
def device_settings(self, pump_guid):
raise NotImplementedError
class TConnectApi(tconnectsync.api.TConnectApi):
def __init__(self, email=None, password=None):
if email is not None and password is not None:
@@ -54,3 +60,4 @@ class TConnectApi(tconnectsync.api.TConnectApi):
_ciq = ControlIQApi()
_ws2 = WS2Api()
_android = AndroidApi()
_webui = WebUIScraper(_ciq)
+201 -2
View File
@@ -4,17 +4,22 @@ import unittest
import datetime
import pprint
import copy
from unittest.mock import patch
from tconnectsync.process import process_time_range
from tconnectsync.parser.nightscout import EXERCISE_EVENTTYPE, IOB_ACTIVITYTYPE, SLEEP_EVENTTYPE, NightscoutEntry
from tconnectsync.features import BASAL, BOLUS, IOB, PUMP_EVENTS
from tests.domain.test_therapy_event import BOLUS_FULL_EXAMPLES, TestCGMTherapyEvent
from tconnectsync.features import BASAL, BOLUS, IOB, PROFILES, PUMP_EVENTS
from tconnectsync.domain.device_settings import Device
from tests.secrets import build_secrets
from tests.sync.test_profile import DEVICE_PROFILE_A, DEVICE_PROFILE_B, DEVICE_SETTINGS, NS_PROFILE_A, NS_PROFILE_B, build_ns_profile
from .api.fake import TConnectApi
from .nightscout_fake import NightscoutApi
from .sync.test_basal import TestBasalSync
from .sync.test_bolus import TestBolusSync
from .sync.test_iob import TestIOBSync
from .domain.test_therapy_event import BOLUS_FULL_EXAMPLES, TestCGMTherapyEvent
class TestProcessTimeRange(unittest.TestCase):
maxDiff = None
@@ -750,6 +755,200 @@ class TestProcessTimeRange(unittest.TestCase):
self.assertDictEqual(nightscout.put_entries, {})
self.assertListEqual(nightscout.deleted_entries, [])
"""Profile present on pump and not in Nightscout with PROFILES feature enabled, adds profile."""
def test_pump_profile_added(self):
tconnect = TConnectApi()
# datetimes are unused by the API fake
start = datetime.datetime(2021, 5, 1, 0, 0)
end = datetime.datetime(2021, 5, 3, 0, 0)
pump_guid = '00000000-0000-0000-0000-000000000001'
serial_number = '12345'
def fake_my_devices():
return {
serial_number: Device(
name='test',
model_number=serial_number,
status='OK',
guid=pump_guid)
}
tconnect.webui.my_devices = fake_my_devices
def fake_device_settings_from_guid(guid):
if guid != pump_guid:
raise RuntimeError('invalid guid')
return [DEVICE_PROFILE_A.activeProfile()], DEVICE_SETTINGS
tconnect.webui.device_settings_from_guid = fake_device_settings_from_guid
nightscout = NightscoutApi()
def fake_current_profile(time_start=None, time_end=None):
return build_ns_profile({}, '')
nightscout.current_profile = fake_current_profile
with patch("tconnectsync.sync.profile._get_default_upload_mode") as mock_upload_mode, \
patch("tconnectsync.sync.profile._get_default_serial_number") as mock_serial_number:
mock_upload_mode.return_value = 'add'
mock_serial_number.return_value = serial_number
process_time_range(tconnect, nightscout, start, end, pretend=False, features=[PROFILES])
self.assertEqual(len(nightscout.uploaded_entries["profile"]), 1)
self.assertDictEqual(nightscout.uploaded_entries["profile"][0]['store']['A'], NS_PROFILE_A)
self.assertDictEqual(nightscout.put_entries, {})
self.assertListEqual(nightscout.deleted_entries, [])
"""Profile present on pump and in Nightscout with PROFILES feature enabled, does not add profile."""
def test_pump_profile_not_updated(self):
tconnect = TConnectApi()
# datetimes are unused by the API fake
start = datetime.datetime(2021, 5, 1, 0, 0)
end = datetime.datetime(2021, 5, 3, 0, 0)
pump_guid = '00000000-0000-0000-0000-000000000001'
serial_number = '12345'
def fake_my_devices():
return {
serial_number: Device(
name='test',
model_number=serial_number,
status='OK',
guid=pump_guid)
}
tconnect.webui.my_devices = fake_my_devices
def fake_device_settings_from_guid(guid):
if guid != pump_guid:
raise RuntimeError('invalid guid')
return [DEVICE_PROFILE_A.activeProfile()], DEVICE_SETTINGS
tconnect.webui.device_settings_from_guid = fake_device_settings_from_guid
nightscout = NightscoutApi()
def fake_current_profile(time_start=None, time_end=None):
return build_ns_profile({'A': NS_PROFILE_A}, 'A')
nightscout.current_profile = fake_current_profile
with patch("tconnectsync.sync.profile._get_default_upload_mode") as mock_upload_mode, \
patch("tconnectsync.sync.profile._get_default_serial_number") as mock_serial_number:
mock_upload_mode.return_value = 'add'
mock_serial_number.return_value = serial_number
process_time_range(tconnect, nightscout, start, end, pretend=False, features=[PROFILES])
self.assertEqual(len(nightscout.uploaded_entries["profile"]), 0)
self.assertDictEqual(nightscout.put_entries, {})
self.assertListEqual(nightscout.deleted_entries, [])
"""Profile present on pump and in Nightscout with PROFILES feature enabled, with changes on pump, adds new profile object."""
def test_pump_profile_new_entry_added(self):
tconnect = TConnectApi()
# datetimes are unused by the API fake
start = datetime.datetime(2021, 5, 1, 0, 0)
end = datetime.datetime(2021, 5, 3, 0, 0)
pump_guid = '00000000-0000-0000-0000-000000000001'
serial_number = '12345'
def fake_my_devices():
return {
serial_number: Device(
name='test',
model_number=serial_number,
status='OK',
guid=pump_guid)
}
tconnect.webui.my_devices = fake_my_devices
def fake_device_settings_from_guid(guid):
if guid != pump_guid:
raise RuntimeError('invalid guid')
return [DEVICE_PROFILE_A, DEVICE_PROFILE_B.activeProfile()], DEVICE_SETTINGS
tconnect.webui.device_settings_from_guid = fake_device_settings_from_guid
nightscout = NightscoutApi()
def fake_current_profile(time_start=None, time_end=None):
return build_ns_profile({'A': NS_PROFILE_A}, 'A')
nightscout.current_profile = fake_current_profile
with patch("tconnectsync.sync.profile._get_default_upload_mode") as mock_upload_mode, \
patch("tconnectsync.sync.profile._get_default_serial_number") as mock_serial_number:
mock_upload_mode.return_value = 'add'
mock_serial_number.return_value = serial_number
process_time_range(tconnect, nightscout, start, end, pretend=False, features=[PROFILES])
self.assertEqual(len(nightscout.uploaded_entries["profile"]), 1)
self.assertEqual(len(nightscout.uploaded_entries["profile"][0]['store']), 2)
self.assertDictEqual(nightscout.uploaded_entries["profile"][0]['store']['A'], NS_PROFILE_A)
self.assertDictEqual(nightscout.uploaded_entries["profile"][0]['store']['B'], NS_PROFILE_B)
self.assertDictEqual(nightscout.put_entries, {})
self.assertListEqual(nightscout.deleted_entries, [])
"""
Profile present on pump and in Nightscout with PROFILES feature enabled, with changes on pump,
replaces existing profile object with NIGHTSCOUT_PROFILE_UPLOAD_MODE=replace.
"""
def test_pump_profile_new_entry_replaced(self):
tconnect = TConnectApi()
# datetimes are unused by the API fake
start = datetime.datetime(2021, 5, 1, 0, 0)
end = datetime.datetime(2021, 5, 3, 0, 0)
pump_guid = '00000000-0000-0000-0000-000000000001'
serial_number = '12345'
def fake_my_devices():
return {
serial_number: Device(
name='test',
model_number=serial_number,
status='OK',
guid=pump_guid)
}
tconnect.webui.my_devices = fake_my_devices
def fake_device_settings_from_guid(guid):
if guid != pump_guid:
raise RuntimeError('invalid guid')
return [DEVICE_PROFILE_A, DEVICE_PROFILE_B.activeProfile()], DEVICE_SETTINGS
tconnect.webui.device_settings_from_guid = fake_device_settings_from_guid
nightscout = NightscoutApi()
def fake_current_profile(time_start=None, time_end=None):
return build_ns_profile({'A': NS_PROFILE_A}, 'A')
nightscout.current_profile = fake_current_profile
with patch("tconnectsync.sync.profile._get_default_upload_mode") as mock_upload_mode, \
patch("tconnectsync.sync.profile._get_default_serial_number") as mock_serial_number:
mock_upload_mode.return_value = 'replace'
mock_serial_number.return_value = serial_number
process_time_range(tconnect, nightscout, start, end, pretend=False, features=[PROFILES])
self.assertDictEqual(nightscout.uploaded_entries, {})
self.assertEqual(len(nightscout.put_entries["profile"]), 1)
self.assertEqual(len(nightscout.put_entries["profile"][0]['store']), 2)
self.assertDictEqual(nightscout.put_entries["profile"][0]['store']['A'], NS_PROFILE_A)
self.assertDictEqual(nightscout.put_entries["profile"][0]['store']['B'], NS_PROFILE_B)
self.assertListEqual(nightscout.deleted_entries, [])
if __name__ == '__main__':
unittest.main()