more ignore

.github/workflows/python-package.yml

@@ -37,7 +37,16 @@ jobs:
           --ignore 52495 \
           --ignore 52365 \
           --ignore 59956 \
-          --ignore 58755
+          --ignore 58755 \
+          --ignore 67895 \
+          --ignore 61893 \
+          --ignore 61601 \
+          --ignore 62044 \
+          --ignore 67599 \
+          --ignore 72083 \
+          --ignore 71064 \
+          --ignore 71608 \
+          --ignore 72236
     - name: Lint with flake8
       run: |
         # stop the build if there are Python syntax errors or undefined names
@@ -62,4 +71,4 @@ jobs:
     - name: Upload Coverage to Codecov
       uses: codecov/codecov-action@v1
       with:
-        fail_ci_if_error: true
+        fail_ci_if_error: false

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = tconnectsync
-version = 0.9.6
+version = 1.0.0
 author = James Woglom 
 author_email = j@wogloms.net
 description = Syncs Tandem t:connect pump data to Nightscout for the t:slim X2

tconnectsync/__init__.py

@@ -19,7 +19,8 @@ try:
         NS_URL,
         NS_SECRET,
         NS_SKIP_TLS_VERIFY,
-        PUMP_SERIAL_NUMBER
+        PUMP_SERIAL_NUMBER,
+        NS_IGNORE_CONN_ERRORS
     )
     from . import secret
 except Exception as e:
@@ -86,7 +87,7 @@ def main(*args, **kwargs):
 
     tconnect = TConnectApi(TCONNECT_EMAIL, TCONNECT_PASSWORD)
 
-    nightscout = NightscoutApi(NS_URL, NS_SECRET, NS_SKIP_TLS_VERIFY)
+    nightscout = NightscoutApi(NS_URL, NS_SECRET, skip_verify=NS_SKIP_TLS_VERIFY, ignore_conn_errors=NS_IGNORE_CONN_ERRORS)
 
     if args.check_login:
         return check_login(tconnect, time_start, time_end)

tconnectsync/api/android.py

@@ -29,6 +29,10 @@ class AndroidApi:
     ANDROID_API_USERNAME = base64.b64decode('QzIzMzFDRDYtRDQ1MC00OTVFLTlDMTktNjcyMTUyMzBDODVD').decode()
     ANDROID_API_PASSWORD = base64.b64decode('dHo0MzNLVzVRREM5VjdmIXo2QF4ybyZZNlNHR1lo').decode()
 
+    # These credentials are used by tconnect web
+    TCONNECT_WEB_USERNAME = base64.b64decode('M0U2MzU3QkEtRjYyNS00REQyLUI2NUYtNEI1RTgxNDRBQTZG').decode()
+    TCONNECT_WEB_PASSWORD = base64.b64decode('cUMyaXFIc2w3OFFoR0RYdCpMenFwb1pxZTl3eHN6').decode()
+
     ANDROID_USER_AGENT = 'Dalvik/2.1.0 (Linux; U; Android 12; Pixel 4a Build/SP2A.220305.012)'
 
     # These tokens are separate from the "standard" tdcservices API
@@ -65,15 +69,17 @@ class AndroidApi:
             raise ApiLoginException(r.status_code, 'Received HTTP %s during login: %s' % (r.status_code, r.text))
 
         j = r.json()
-        if "user" not in j or not j["user"]:
-            raise ApiException(r.status_code, 'No user details present in AndroidApi oauth response: %s' % r.text)
+        # tconnect web returns a null user
+        # if "user" not in j or not j["user"]:
+        #     raise ApiException(r.status_code, 'No user details present in AndroidApi oauth response: %s' % r.text)
 
         self.accessToken = j["accessToken"]
         self.accessTokenExpiresAt = j["accessTokenExpiresAt"]
         # NOTE: the refresh token is currently unused, instead a new access
         # token is obtained from scratch by re-logging in when it expires.
-        self.refreshToken = j["refreshToken"]
-        self.refreshTokenExpiresAt = j["refreshTokenExpiresAt"]
+        if "refreshToken" in j and "refreshTokenExpiresAt" in j:
+            self.refreshToken = j["refreshToken"]
+            self.refreshTokenExpiresAt = j["refreshTokenExpiresAt"]
         self.userId = j["user"]["id"]
 
         logger.info("Logged in to AndroidApi successfully (expiration: %s, %s)" % (self.accessTokenExpiresAt, timeago(self.accessTokenExpiresAt)))

tconnectsync/api/controliq.py

@@ -14,7 +14,7 @@ class ControlIQApi:
     BASE_URL = 'https://tdcservices.tandemdiabetes.com/'
     LOGIN_URL = 'https://tconnect.tandemdiabetes.com/login.aspx?ReturnUrl=%2f'
 
-    LAST_CONFIRMED_SOFTWARE_VERSION = 't:connect 7.16.0.1'
+    LAST_CONFIRMED_SOFTWARE_VERSION = 't:connect 7.17.2.3'
 
     userGuid = None
     accessToken = None
@@ -51,9 +51,21 @@ class ControlIQApi:
                 raise ApiException(fwd.status_code, 'Error retrieving t:connect login cookies.')
 
             self.userGuid = req.cookies['UserGUID']
-            self.accessToken = req.cookies['accessToken']
-            self.accessTokenExpiresAt = req.cookies['accessTokenExpiresAt']
-            logger.info("Logged in to ControlIQApi successfully (expiration: %s, %s)" % (self.accessTokenExpiresAt, timeago(self.accessTokenExpiresAt)))
+            if 'accessToken' in req.cookies and 'accessTokenExpiresAt' in req.cookies:
+                self.accessToken = req.cookies['accessToken']
+                self.accessTokenExpiresAt = req.cookies['accessTokenExpiresAt']
+                logger.info("Logged in to ControlIQApi successfully via accessToken cookie (expiration: %s, %s)" % (self.accessTokenExpiresAt, timeago(self.accessTokenExpiresAt)))
+            else:
+                logger.info("No accessToken cookie found when logging in to ControlIQApi. Triggering AndroidApi auth")
+
+                from .android import AndroidApi
+                android = AndroidApi(email, password)
+                self.accessToken = android.accessToken
+                self.accessTokenExpiresAt = android.accessTokenExpiresAt
+
+                logger.info("Logged in to AndroidApi successfully via accessToken param (expiration: %s, %s)" % (self.accessTokenExpiresAt, timeago(self.accessTokenExpiresAt)))
+
+
             self.loginSession = s
             return True
 
@@ -74,6 +86,13 @@ class ControlIQApi:
                     contents = "%s[SNIP]%s" % (contents[:500], contents[-500:])
             logger.info("BeautifulSoup parsed contents: %s" % contents)
             pass
+        if not soup.select_one("#__VIEWSTATE"):
+            enc_contents = str(soup.encode_contents())
+            if "Web Page Blocked!" in enc_contents or "Attack ID:" in enc_contents:
+                logger.warn("Being ratelimited/blocked by web application firewall. Sleeping for 30 minutes before retrying.")
+                logger.info("BeautifulSoup parsed contents: %s" % enc_contents)
+                time.sleep(1800)
+                exit(1)
         return {
             "__LASTFOCUS": "",
             "__EVENTTARGET": "ctl00$ContentBody$LoginControl$linkLogin",
@@ -96,6 +115,9 @@ class ControlIQApi:
             return None
 
     def needs_relogin(self):
+        if not self.accessTokenExpiresAt:
+            return False
+
         diff = (arrow.get(self.accessTokenExpiresAt) - arrow.get())
         return (diff.seconds <= 5 * 60)
 

tconnectsync/nightscout.py

@@ -30,10 +30,11 @@ def time_range(field_name, start_time, end_time, t_to_space=False):
 
 logger = logging.getLogger(__name__)
 class NightscoutApi:
-	def __init__(self, url, secret, skip_verify=False):
+	def __init__(self, url, secret, skip_verify=False, ignore_conn_errors=False):
 		self.url = url
 		self.secret = secret
 		self.verify = False if skip_verify else None
+		self.ignore_conn_errors = ignore_conn_errors
 
 
 	def upload_entry(self, ns_format, entity='treatments'):
@@ -76,22 +77,27 @@ class NightscoutApi:
 			if j and len(j) > 0:
 				return j[0]
 			return None
-		
-		ret = None
 		try:
-			ret = internal(False)
-		except ApiException as e:
-			logger.warning("last_uploaded_entry with no t_to_space: %s", e)
 			ret = None
-		if ret is None and (time_start or time_end):
 			try:
-				ret = internal(True)
+				ret = internal(False)
 			except ApiException as e:
-				logger.warning("last_uploaded_entry with t_to_space: %s", e)
+				logger.warning("last_uploaded_entry with no t_to_space: %s", e)
 				ret = None
-			if ret is not None:
-				logger.warning("last_uploaded_entry with eventType=%s time_start=%s time_end=%s only returned data when timestamps contained a space" % (eventType, time_start, time_end))
-		return ret
+			if ret is None and (time_start or time_end):
+				try:
+					ret = internal(True)
+				except ApiException as e:
+					logger.warning("last_uploaded_entry with t_to_space: %s", e)
+					ret = None
+				if ret is not None:
+					logger.warning("last_uploaded_entry with eventType=%s time_start=%s time_end=%s only returned data when timestamps contained a space" % (eventType, time_start, time_end))
+			return ret
+		except requests.exceptions.ConnectionError as e:
+			if self.ignore_conn_errors:
+				logger.warn('Ignoring ConnectionError because ignore_conn_errors=true', e)
+			else:
+				raise e
 	
 	def last_uploaded_bg_entry(self, time_start=None, time_end=None):
 		def internal(t_to_space):
@@ -107,12 +113,19 @@ class NightscoutApi:
 				return j[0]
 			return None
 		
-		ret = internal(False)
-		if ret is None and (time_start or time_end):
-			ret = internal(True)
-			if ret is not None:
-				logger.warning("last_uploaded_bg_entry with time_start=%s time_end=%s only returned data when timestamps contained a space" % (time_start, time_end))
-		return ret
+		try:
+			ret = internal(False)
+			if ret is None and (time_start or time_end):
+				ret = internal(True)
+				if ret is not None:
+					logger.warning("last_uploaded_bg_entry with time_start=%s time_end=%s only returned data when timestamps contained a space" % (time_start, time_end))
+
+			return ret
+		except requests.exceptions.ConnectionError as e:
+			if self.ignore_conn_errors:
+				logger.warn('Ignoring ConnectionError because ignore_conn_errors=true', e)
+			else:
+				raise e
 
 	def last_uploaded_activity(self, activityType, time_start=None, time_end=None):
 		def internal(t_to_space):
@@ -128,12 +141,18 @@ class NightscoutApi:
 				return j[0]
 			return None
 		
-		ret = internal(False)
-		if ret is None and (time_start or time_end):
-			ret = internal(True)
-			if ret is not None:
-				logger.warning("last_uploaded_activity with activityType=%s time_start=%s time_end=%s only returned data when timestamps contained a space" % (activityType, time_start, time_end))
-		return ret
+		try:
+			ret = internal(False)
+			if ret is None and (time_start or time_end):
+				ret = internal(True)
+				if ret is not None:
+					logger.warning("last_uploaded_activity with activityType=%s time_start=%s time_end=%s only returned data when timestamps contained a space" % (activityType, time_start, time_end))
+			return ret
+		except requests.exceptions.ConnectionError as e:
+			if self.ignore_conn_errors:
+				logger.warn('Ignoring ConnectionError because ignore_conn_errors=true', e)
+			else:
+				raise e
 
 	"""
 	Returns general status information about the Nightscout server.

tconnectsync/secret.py

@@ -49,6 +49,7 @@ if not get('NS_SECRET') and get('API_SECRET'):
     NS_SECRET = get('API_SECRET')
 
 NS_SKIP_TLS_VERIFY = get_bool('NS_SKIP_TLS_VERIFY', 'false')
+NS_IGNORE_CONN_ERRORS = get_bool('NS_IGNORE_CONN_ERRORS', 'false')
 
 TIMEZONE_NAME = get('TIMEZONE_NAME', 'America/New_York')
 

tconnectsync/sync/pump_events.py

@@ -192,11 +192,11 @@ def _ns_write_pump_events(nightscout, events, buildNsEventFunc, eventType, prete
 
     add_count = 0
     for event in events:
-        created_at = event["time"]
-        if last_upload_time and arrow.get(created_at) <= last_upload_time:
+        created_at = arrow.get(event["time"])
+        if last_upload_time and created_at <= last_upload_time:
             skip = True
             if "duration_mins" in event.keys() and "duration" in last_upload.keys():
-                if created_at == last_upload["created_at"] and float(event["duration_mins"]) > float(last_upload["duration"]):
+                if created_at == arrow.get(last_upload["created_at"]) and float(event["duration_mins"]) > float(last_upload["duration"]):
                     logger.info("Latest %s event needs updating: duration has increased from %s to %s: %s" % (eventType, last_upload["duration"], event["duration_mins"], event))
                     logger.info("Deleting previous %s: %s" % (eventType, last_upload))
                     nightscout.delete_entry('treatments/%s' % last_upload["_id"])

tests/nightscout_fake.py

@@ -20,7 +20,7 @@ class NightscoutApi(tconnectsync.nightscout.NightscoutApi):
     def put_entry(self, ns_format, entity):
         self.put_entries[entity].append(ns_format)
 
-    def last_uploaded_entry(self, eventType, start_date=None, end_date=None):
+    def last_uploaded_entry(self, eventType, time_start=None, time_end=None):
         raise NotImplementedError
 
     def last_uploaded_activity(self, activityType):

tests/sync/test_pump_events.py

@@ -0,0 +1,149 @@
+#!/usr/bin/env python3
+
+import json
+import unittest
+from typing import Dict
+import copy
+
+from tconnectsync.sync.pump_events import ns_write_pump_events
+
+from ..nightscout_fake import NightscoutApi
+
+
+def _stub_last_uploaded_entry(eventType, time_start=None, time_end=None):
+    return None
+
+class TestNsWritePumpEvents(unittest.TestCase):
+
+    SITE_CHANGE_EVENT = {'time': '2024-02-04 16:27:50-05:00', 'event_type': 'Site/Cartridge Change'}
+    SITE_CHANGE_NS = {'created_at': '2024-02-04 16:27:50-05:00', 'enteredBy': 'Pump (tconnectsync)', 'eventType': 'Site Change', 'notes': 'Site/Cartridge Change','reason': 'Site/Cartridge Change'}
+    def test_write_single_sitechange_event(self):
+        nightscout = NightscoutApi()
+        nightscout.last_uploaded_entry = _stub_last_uploaded_entry
+        ns_write_pump_events(nightscout, [
+            self.SITE_CHANGE_EVENT
+        ])
+
+        self.assertDictEqual({
+            'treatments': [
+                self.SITE_CHANGE_NS
+            ]
+        }, dict(nightscout.uploaded_entries))
+
+    EMPTY_CART_EVENT = {'time': '2024-05-14 09:01:59-04:00', 'event_type': 'Empty Cartridge/Pump Shutdown'}
+    EMPTY_CART_NS = {'created_at': '2024-05-14 09:01:59-04:00', 'enteredBy': 'Pump (tconnectsync)', 'eventType': 'Basal Suspension', 'notes': 'Empty Cartridge/Pump Shutdown', 'reason': 'Empty Cartridge/Pump Shutdown'}
+    def test_write_single_emptycart_event(self):
+        nightscout = NightscoutApi()
+        nightscout.last_uploaded_entry = _stub_last_uploaded_entry
+        ns_write_pump_events(nightscout, [
+            self.EMPTY_CART_EVENT
+        ])
+
+        self.assertDictEqual({
+            'treatments': [
+                self.EMPTY_CART_NS
+            ]
+        }, dict(nightscout.uploaded_entries))
+
+    USER_SUSPENDED_EVENT = {'time': '2024-02-05 09:48:22-05:00', 'event_type': 'User Suspended'}
+    USER_SUSPENDED_NS = {'created_at': '2024-02-05 09:48:22-05:00', 'enteredBy': 'Pump (tconnectsync)', 'eventType': 'Basal Suspension', 'notes': 'User Suspended', 'reason': 'User Suspended'}
+    def test_write_single_usersuspended_event(self):
+        nightscout = NightscoutApi()
+        nightscout.last_uploaded_entry = _stub_last_uploaded_entry
+        ns_write_pump_events(nightscout, [
+            self.USER_SUSPENDED_EVENT
+        ])
+
+        self.assertDictEqual({
+            'treatments': [
+                self.USER_SUSPENDED_NS
+            ]
+        }, dict(nightscout.uploaded_entries))
+    
+    EXERCISE_EVENT = {'time': '2024-03-10 08:53:20-04:00', 'duration_mins': 269.3833333333333, 'event_type': 'Exercise'}
+    EXERCISE_NS = {'created_at': '2024-03-10 08:53:20-04:00', 'duration': 269.3833333333333, 'enteredBy': 'Pump (tconnectsync)', 'eventType': 'Exercise', 'notes': 'Exercise', 'reason': 'Exercise'}
+    EXERCISE_EXTENDED_EVENT = {'time': '2024-03-10 08:53:20-04:00', 'duration_mins': 585.8833333333333, 'event_type': 'Exercise'}
+    EXERCISE_EXTENDED_NS = {'created_at': '2024-03-10 08:53:20-04:00', 'duration': 585.8833333333333, 'enteredBy': 'Pump (tconnectsync)', 'eventType': 'Exercise', 'notes': 'Exercise', 'reason': 'Exercise'}
+    def test_write_exercise_events(self):
+        nightscout = NightscoutApi()
+        nightscout.last_uploaded_entry = _stub_last_uploaded_entry
+        ns_write_pump_events(nightscout, [
+            self.EXERCISE_EVENT
+        ])
+
+        self.assertDictEqual({
+            'treatments': [
+                self.EXERCISE_NS
+            ]
+        }, dict(nightscout.uploaded_entries))
+
+        _id = 'exercise_id'
+        def _last_uploaded_entry(eventType, time_start=None, time_end=None):
+            if eventType == 'Exercise':
+                return {
+                    '_id': _id,
+                    **self.EXERCISE_NS
+                }
+
+        nightscout.last_uploaded_entry = _last_uploaded_entry
+
+        ns_write_pump_events(nightscout, [
+            self.EXERCISE_EXTENDED_EVENT
+        ])
+
+        self.assertListEqual([
+            'treatments/%s' % _id
+        ], nightscout.deleted_entries)
+
+        self.assertDictEqual({
+            'treatments': [
+                self.EXERCISE_NS,
+                self.EXERCISE_EXTENDED_NS
+            ]
+        }, dict(nightscout.uploaded_entries))
+    
+    SLEEP_EVENT = {'time': '2024-02-07 00:00:19-05:00', 'duration_mins': 13.916666666666666, 'event_type': 'Sleep'}
+    SLEEP_NS = {'created_at': '2024-02-07 00:00:19-05:00', 'duration': 13.916666666666666, 'enteredBy': 'Pump (tconnectsync)', 'eventType': 'Sleep', 'notes': 'Sleep', 'reason': 'Sleep'}
+    SLEEP_EXTENDED_EVENT = {'time': '2024-02-07 00:00:19-05:00', 'duration_mins': 74.0, 'event_type': 'Sleep'}
+    SLEEP_EXTENDED_NS = {'created_at': '2024-02-07 00:00:19-05:00', 'duration': 74.0, 'enteredBy': 'Pump (tconnectsync)', 'eventType': 'Sleep', 'notes': 'Sleep', 'reason': 'Sleep'}
+    def test_write_sleep_events(self):
+        nightscout = NightscoutApi()
+        nightscout.last_uploaded_entry = _stub_last_uploaded_entry
+        ns_write_pump_events(nightscout, [
+            self.SLEEP_EVENT
+        ])
+
+        self.assertDictEqual({
+            'treatments': [
+                self.SLEEP_NS
+            ]
+        }, dict(nightscout.uploaded_entries))
+
+        _id = 'sleep_id'
+        def _last_uploaded_entry(eventType, time_start=None, time_end=None):
+            if eventType == 'Sleep':
+                return {
+                    '_id': _id,
+                    **self.SLEEP_NS
+                }
+
+        nightscout.last_uploaded_entry = _last_uploaded_entry
+
+        ns_write_pump_events(nightscout, [
+            self.SLEEP_EXTENDED_EVENT
+        ])
+
+        self.assertListEqual([
+            'treatments/%s' % _id
+        ], nightscout.deleted_entries)
+
+        self.assertDictEqual({
+            'treatments': [
+                self.SLEEP_NS,
+                self.SLEEP_EXTENDED_NS
+            ]
+        }, dict(nightscout.uploaded_entries))
+    
+
+if __name__ == '__main__':
+    unittest.main()