v2.2.3

.codecov.yml

@@ -5,9 +5,9 @@ coverage:
     project:
       default: false
       tconnectsync:
-        paths: 
+        paths:
         - "tconnectsync/"
-        target: '75%'
+        target: '60%'
         threshold: '5%'
       tests:
         paths:

.github/workflows/publish-docker.yml

@@ -38,13 +38,13 @@ jobs:
     steps:
       - name: Check out the repo
         uses: actions/checkout@v2
-      
+
       - name: Log in to Docker Hub
         uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
-      
+
       - name: Extract metadata (tags, labels) for Docker
         id: meta
         uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
@@ -55,7 +55,7 @@ jobs:
             type=ref,event=pr
             type=semver,pattern={{version}}
             type=semver,pattern={{major}}.{{minor}}
-      
+
       - name: Build and push Docker image
         uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
         with:

.github/workflows/python-package.yml

@@ -28,25 +28,25 @@ jobs:
           python -m pip install --upgrade pip
           python -m pip install flake8 pytest pipenv
           pipenv install --system
-      - name: Run pipenv check
-        run: |
-          # DDoS attacks in wheel and setuptools packages, not relevant
-          # root certificate store, not relevant
-            pipenv check \
-              --ignore 51499 \
-              --ignore 52495 \
-              --ignore 52365 \
-              --ignore 59956 \
-              --ignore 58755 \
-              --ignore 67895 \
-              --ignore 61893 \
-              --ignore 61601 \
-              --ignore 62044 \
-              --ignore 67599 \
-              --ignore 72083 \
-              --ignore 71064 \
-              --ignore 71608 \
-              --ignore 72236
+      # - name: Run pipenv check
+      #   run: |
+      #     # DDoS attacks in wheel and setuptools packages, not relevant
+      #     # root certificate store, not relevant
+      #       pipenv check \
+      #         --ignore 51499 \
+      #         --ignore 52495 \
+      #         --ignore 52365 \
+      #         --ignore 59956 \
+      #         --ignore 58755 \
+      #         --ignore 67895 \
+      #         --ignore 61893 \
+      #         --ignore 61601 \
+      #         --ignore 62044 \
+      #         --ignore 67599 \
+      #         --ignore 72083 \
+      #         --ignore 71064 \
+      #         --ignore 71608 \
+      #         --ignore 72236
       - name: Lint with flake8
         run: |
           # stop the build if there are Python syntax errors or undefined names

Pipfile

@@ -27,3 +27,4 @@ typing-extensions = "*"
 tconnectsync = "python3 main.py"
 test = "python3 -m unittest discover -vv"
 build_events = "bash -c 'cd tconnectsync/eventparser && python3 build_events.py > events.py'"
+lint = "bash -c 'flake8 . --count --select=E9,F63,F7,F82 && flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 && echo PASS'"

codecov.yml

@@ -1,6 +0,0 @@
-coverage:
-  status:
-    project:
-      default:
-        target: 50%
-        threshold: null

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = tconnectsync
-version = 2.1.7
+version = 2.2.3
 author = James Woglom
 author_email = j@wogloms.net
 description = Syncs Tandem Source (formerly t:connect) insulin pump data to Nightscout for the t:slim X2

tconnectsync/__init__.py

@@ -120,7 +120,7 @@ def main(*args, **kwargs):
         sys.exit(u.process(tconnect, nightscout, time_start, time_end, args.pretend, features=args.features))
     else:
         tconnectDevice = TandemSourceChooseDevice(secret, tconnect).choose()
-        added, last_event_id = TandemSourceProcessTimeRange(tconnect, nightscout, tconnectDevice, pretend=args.pretend, features=args.features).process(time_start, time_end)
+        added, last_event_id = TandemSourceProcessTimeRange(tconnect, nightscout, tconnectDevice, pretend=args.pretend, secret=secret, features=args.features).process(time_start, time_end)
 
         # return exit code 0 if processed events
         sys.exit(0 if added>0 else 1)

tconnectsync/domain/tandemsource/event_class.py

@@ -24,6 +24,7 @@ class EventClass(set, Enum):
     CGM_START_JOIN_STOP = {*_CGM_START, *_CGM_JOIN, *_CGM_STOP}
     CGM_READING = {events.LidCgmDataGxb, events.LidCgmDataG7, events.LidCgmDataFsl2}
     USER_MODE = {events.LidAaUserModeChange}
+    DEVICE_STATUS = {events.LidDailyBasal}
 
     @staticmethod
     def for_event(evt):

tconnectsync/eventparser/build_events.py

@@ -68,6 +68,15 @@ class {name}(BaseEvent):
     def eventId(self):
         return self.ID
 
+    def todict(self):
+        return dict(
+            id=self.ID,
+            name=self.NAME,
+            seqNum=self.seqNum,
+            eventTimestamp=str(self.eventTimestamp),
+{fields_dict}
+        )
+
 '''
 
 def firstLower(text):
@@ -97,6 +106,14 @@ def build_fields(event_def):
     return '\n'.join([f'{" "*4}{f}' for f in ret])
 
 
+def build_fields_dict(event_def):
+    ret = []
+    for name, field in event_def["data"].items():
+        suffix = 'Raw' if "transform" in field and name[-3:] != 'Raw' else ''
+        f = f'{fieldNameFormat(name)}{suffix}=self.{fieldNameFormat(name)}{suffix},'
+        ret.append(f)
+    return '\n'.join([f'{" "*12}{f}' for f in ret])
+
 
 def build_decode(event_def):
     p1s = []
@@ -134,6 +151,7 @@ def build_event(event_id, event_def):
     return TEMPLATE.format(
         name = eventNameFormat(event_def["name"]),
         fields = build_fields(event_def),
+        fields_dict = build_fields_dict(event_def),
         build_p1 = build_decode(event_def)[0],
         build_p2 = build_decode(event_def)[1],
         transform_funcs = build_transform_funcs(event_def),

tconnectsync/eventparser/raw_event.py

@@ -63,4 +63,11 @@ class RawEvent:
     def eventTimestamp(self):
         return self.timestamp
 
-
+    def todict(self):
+        return dict(
+            id=self.id,
+            name="RawEvent",
+            seqNum=self.seqNum,
+            eventTimestamp=str(self.eventTimestamp),
+            raw=''.join('{:02x}'.format(x) for x in self.raw),
+        )

tconnectsync/eventparser/transforms.py

@@ -116,7 +116,7 @@ def transform_battery_charge_percent(event_def, name, name_fmt, field, tx):
     out += [
         '@property',
         f'def batteryChargePercent(self):',
-        f'    return (256*(self.batterychargepercentmsbRaw%2)+self.batterychargepercentlsbRaw)/512',
+        f'    return (256*(self.batterychargepercentmsbRaw-14)+self.batterychargepercentlsbRaw)/(3*256)',
         ''
     ]
 

tconnectsync/nightscout.py

@@ -71,6 +71,9 @@ class NightscoutApi:
 				'api-secret': hashlib.sha1(self.secret.encode()).hexdigest()
 			}, verify=self.verify)
 			if latest.status_code != 200:
+				if 'as a valid ISO-8601 date' in latest.text:
+					logger.warning("Nightscout last_uploaded_entry %s could not process ISO-8601 date: start=%s end=%s dateFilter=%s" % (eventType, time_start, time_end, dateFilter))
+					return None
 				raise ApiException(latest.status_code, "Nightscout last_uploaded_entry %s response: %s" % (latest.status_code, latest.text))
 
 			j = latest.json()
@@ -106,6 +109,9 @@ class NightscoutApi:
 				'api-secret': hashlib.sha1(self.secret.encode()).hexdigest()
 			}, verify=self.verify)
 			if latest.status_code != 200:
+				if 'as a valid ISO-8601 date' in latest.text:
+					logger.warning("Nightscout last_uploaded_bg_entry could not process ISO-8601 date: start=%s end=%s dateFilter=%s" % (time_start, time_end, dateFilter))
+					return None
 				raise ApiException(latest.status_code, "Nightscout last_uploaded_bg_entry %s response: %s" % (latest.status_code, latest.text))
 
 			j = latest.json()
@@ -134,6 +140,9 @@ class NightscoutApi:
 				'api-secret': hashlib.sha1(self.secret.encode()).hexdigest()
 			}, verify=self.verify)
 			if latest.status_code != 200:
+				if 'as a valid ISO-8601 date' in latest.text:
+					logger.warning("Nightscout activity %s could not process ISO-8601 date: start=%s end=%s dateFilter=%s" % (activityType, time_start, time_end, dateFilter))
+					return None
 				raise ApiException(latest.status_code, "Nightscout activity %s response: %s" % (latest.status_code, latest.text))
 
 			j = latest.json()
@@ -161,6 +170,9 @@ class NightscoutApi:
 				'api-secret': hashlib.sha1(self.secret.encode()).hexdigest()
 			}, verify=self.verify)
 			if latest.status_code != 200:
+				if 'as a valid ISO-8601 date' in latest.text:
+					logger.warning("Nightscout devicestatus could not process ISO-8601 date: start=%s end=%s dateFilter=%s" % (time_start, time_end, dateFilter))
+					return None
 				raise ApiException(latest.status_code, "Nightscout devicestatus %s response: %s" % (latest.status_code, latest.text))
 
 			j = latest.json()
@@ -173,7 +185,7 @@ class NightscoutApi:
 			if ret is None and (time_start or time_end):
 				ret = internal(True)
 				if ret is not None:
-					logger.warning("devicestatus with activityType=%s time_start=%s time_end=%s only returned data when timestamps contained a space" % (activityType, time_start, time_end))
+					logger.warning("devicestatus time_start=%s time_end=%s only returned data when timestamps contained a space" % (time_start, time_end))
 			return ret
 		except requests.exceptions.ConnectionError as e:
 			if self.ignore_conn_errors:

tconnectsync/parser/nightscout.py

@@ -195,14 +195,16 @@ class NightscoutEntry:
         }
 
     @staticmethod
-    def devicestatus(created_at, batteryVoltage, batteryString, pump_event_id=""):
+    def devicestatus(created_at, batteryVoltage, batteryPercent, pump_event_id=""):
         return {
             "device": ENTERED_BY,
             "created_at": created_at,
             "pump": {
+                "clock": created_at,
                 "battery": {
                     "voltage": float(batteryVoltage),
-                    "string": batteryString
+                    "percent": int(batteryPercent) if batteryPercent else None,
+                    "status": "%.0f%s" % (batteryPercent, '%')
                 },
             },
             "pump_event_id": pump_event_id

tconnectsync/secret.py

@@ -70,8 +70,8 @@ AUTOUPDATE_MAX_SLEEP_SECONDS = get_number('AUTOUPDATE_MAX_SLEEP_SECONDS', '1500'
 AUTOUPDATE_UNEXPECTED_NO_INDEX_SLEEP_SECONDS = get_number('AUTOUPDATE_UNEXPECTED_NO_INDEX_SLEEP_SECONDS', '60') # 1 minute
 AUTOUPDATE_USE_FIXED_SLEEP = get_bool('AUTOUPDATE_USE_FIXED_SLEEP', 'false')
 AUTOUPDATE_NO_DATA_FAILURE_MINUTES = get_number('AUTOUPDATE_NO_DATA_FAILURE_MINUTES', '180') # 3 hours
-AUTOUPDATE_FAILURE_MINUTES = get_number('AUTOUPDATE_FAILURE_MINUTES', '15') # 15 minutes
-AUTOUPDATE_RESTART_ON_FAILURE = get_bool('AUTOUPDATE_RESTART_ON_FAILURE', 'true')
+AUTOUPDATE_FAILURE_MINUTES = get_number('AUTOUPDATE_FAILURE_MINUTES', '75') # 75 minutes
+AUTOUPDATE_RESTART_ON_FAILURE = get_bool('AUTOUPDATE_RESTART_ON_FAILURE', 'false')
 AUTOUPDATE_MAX_LOOP_INVOCATIONS = get_number('AUTOUPDATE_MAX_LOOP_INVOCATIONS', '-1')
 
 NIGHTSCOUT_PROFILE_UPLOAD_MODE = get_one_of('NIGHTSCOUT_PROFILE_UPLOAD_MODE', 'add', ['add', 'replace'])

tconnectsync/sync/tandemsource/autoupdate.py

@@ -50,7 +50,7 @@ class TandemSourceAutoupdate:
                 if pretend:
                     logger.info('Would update now if not in pretend mode')
                 else:
-                    added, event_seqnum = ProcessTimeRange(tconnect, nightscout, tconnectDevice, pretend, secret, features=features).process(time_start, time_end)
+                    added, event_seqnum = ProcessTimeRange(tconnect, nightscout, tconnectDevice, pretend, self.secret, features=features).process(time_start, time_end)
                     logger.info('Added %d items from ProcessTimeRange' % added)
                     self.last_successful_process_time_range = now
 
@@ -69,7 +69,12 @@ class TandemSourceAutoupdate:
                 self.last_attempt_time = now
                 self.time_diffs_between_attempts = []
             else:
-                logger.info('No new reported tandemsource data. (cur_max_date: %s)' % cur_max_date_with_events)
+                logger.info('No new reported tandemsource data. cur_max_date: %s (%dm ago) last_event_time: %s (%dm ago)' % (
+                    arrow.get(cur_max_date_with_events) if cur_max_date_with_events else None,
+                    (now - cur_max_date_with_events)//60 if cur_max_date_with_events else None,
+                    arrow.get(self.last_event_time) if self.last_event_time else None,
+                    (now - self.last_event_time)//60 if self.last_event_time else None
+                ))
 
                 # If we haven't seen the pump event index update in AUTOUPDATE_NO_DATA_FAILURE_MINUTES,
                 # then trigger an error and potentially restart.
@@ -93,7 +98,7 @@ class TandemSourceAutoupdate:
                 # above no indexes warning.
                 elif self.last_successful_process_time_range and (now - self.last_successful_process_time_range) >= 60 * self.secret.AUTOUPDATE_FAILURE_MINUTES:
                     logger.error(AutoupdateNoNewDataDetectedError(
-                        "%s: No new data has been detected via the API for %d minutes. " % (datetime.datetime.now(), (now - self.last_successful_process_time_range)//60) +
+                        "%s: No new data has been detected via the API for %d minutes (last: %s). " % (datetime.datetime.now(), (now - self.last_successful_process_time_range)//60, self.last_successful_process_time_range) +
                         "tconnectsync might not be functioning properly."))
 
                     if self.secret.AUTOUPDATE_RESTART_ON_FAILURE:

tconnectsync/sync/tandemsource/process.py

@@ -1,7 +1,7 @@
 import logging
 import collections
 
-from ...features import DEFAULT_FEATURES
+from ...features import DEVICE_STATUS, DEFAULT_FEATURES
 from ...eventparser import events as eventtypes
 from ...domain.tandemsource.event_class import EventClass
 from .process_basal import ProcessBasal
@@ -13,6 +13,7 @@ from .process_cartridge import ProcessCartridge
 from .process_cgm_alert import ProcessCGMAlert
 from .process_cgm_start_join_stop import ProcessCGMStartJoinStop
 from .process_cgm_reading import ProcessCGMReading
+from .process_device_status import ProcessDeviceStatus
 from .process_user_mode import ProcessUserMode
 from .update_profiles import UpdateProfiles
 
@@ -39,6 +40,7 @@ class ProcessTimeRange:
         EventClass.CGM_START_JOIN_STOP.name: ProcessCGMStartJoinStop,
         EventClass.CGM_READING.name: ProcessCGMReading,
         EventClass.USER_MODE.name: ProcessUserMode,
+        EventClass.DEVICE_STATUS.name: ProcessDeviceStatus
     }
 
     updater_classes = [
@@ -46,8 +48,10 @@ class ProcessTimeRange:
     ]
 
     def process(self, time_start, time_end):
-        logger.info(f"ProcessTimeRange time_start={time_start} time_end={time_end} tconnect_device_id={self.tconnect_device_id} features={self.features}")
-        events = self.tconnect.tandemsource.pump_events(self.tconnect_device_id, time_start, time_end, fetch_all_event_types=self.secret.FETCH_ALL_EVENT_TYPES)
+        fetch_all_event_types = self.secret.FETCH_ALL_EVENT_TYPES or DEVICE_STATUS in self.features
+
+        logger.info(f"ProcessTimeRange time_start={time_start} time_end={time_end} tconnect_device_id={self.tconnect_device_id} features={self.features} fetch_all_event_types={fetch_all_event_types}")
+        events = self.tconnect.tandemsource.pump_events(self.tconnect_device_id, time_start, time_end, fetch_all_event_types=fetch_all_event_types)
 
         events_first_time = None
         events_last_time = None

tconnectsync/sync/tandemsource/process_device_status.py

@@ -50,6 +50,7 @@ class ProcessDeviceStatus:
             logger.info("ProcessDeviceStatus: No last_daily_basal_event found for add (time range: %s - %s)" % (time_start, time_end))
             return []
 
+        logger.info("ProcessDeviceStatus: last_daily_basal_event=%s" % (last_daily_basal_event))
 
         ns_entries = []
         ns_entries.append(self.daily_basal_to_nsentry(last_daily_basal_event))
@@ -59,7 +60,7 @@ class ProcessDeviceStatus:
         return NightscoutEntry.devicestatus(
             created_at=event.eventTimestamp.format(),
             batteryVoltage=(float(event.batterylipomillivolts or 0)/1000),
-            batteryString="%.0f%s" % (100*event.batteryChargePercent, '%'),
+            batteryPercent=int(100*event.batteryChargePercent),
             pump_event_id = "%s" % event.seqNum
         )
 

tconnectsync/sync/tandemsource/update_profiles.py

@@ -49,7 +49,7 @@ class UpdateProfiles:
         logger.info("Current pump settings: %s" % pump_settings)
 
         ns_profile_obj = self.nightscout.current_profile()
-        logger.info("Current Nightscout profile: %s" % ns_profile_obj)
+        logger.debug("Current Nightscout profile: %s" % ns_profile_obj)
         if ns_profile_obj is None:
             ns_profile_obj = {}
 
@@ -95,7 +95,7 @@ class UpdateProfiles:
 
         ns = ns_profile_obj.get('store', {})
 
-        logger.info("compare_profiles profile names: device: %s ns: %s", device.keys(), ns.keys())
+        logger.debug("compare_profiles profile names: device: %s ns: %s", device.keys(), ns.keys())
 
         new_ns_profile = copy.deepcopy(ns_profile_obj)
         if not 'store' in new_ns_profile:
@@ -113,12 +113,12 @@ class UpdateProfiles:
 
         existent_profiles_in_ns = set(device.keys()) & set(ns.keys())
         for profile_name in existent_profiles_in_ns:
-            logger.debug("Checking for differences for %s profile between pump and nightscout", profile_name)
+            #logger.debug("Checking for differences for %s profile between pump and nightscout", profile_name)
             pump_configured_profile = device[profile_name]
             ns_translated_profile = NightscoutEntry.tandemsource_profile_store(pump_configured_profile, pump_settings)
             ns_configured_profile = ns[profile_name]
 
-            logger.debug("Comparing %s profile from pump: %s to nightscout: %s", profile_name, ns_translated_profile, ns_configured_profile)
+            #logger.debug("Comparing %s profile from pump: %s to nightscout: %s", profile_name, ns_translated_profile, ns_configured_profile)
             if self.nightscout_profiles_identical(ns_configured_profile, ns_translated_profile):
                 logger.info("Profile %s identical between pump and nightscout", profile_name)
                 continue

tests/sync/tandemsource/test_process_device_status.py

@@ -37,8 +37,10 @@ class TestProcessDeviceStatus(unittest.TestCase):
             'created_at': '2024-12-03 23:40:23-05:00',
             'device': 'Pump (tconnectsync)',
             'pump': {
+                'clock': '2024-12-03 23:40:23-05:00',
                 'battery': {
-                    'string': '48%',
+                    'status': '32%',
+                    'percent': 32,
                     'voltage': 14.08
                 }
             },
@@ -89,8 +91,10 @@ class TestProcessDeviceStatus(unittest.TestCase):
             'created_at': '2024-12-04 02:30:23-05:00',
             'device': 'Pump (tconnectsync)',
             'pump': {
+                'clock': '2024-12-04 02:30:23-05:00',
                 'battery': {
-                    'string': '47%',
+                    'status': '31%',
+                    'percent': 31,
                     'voltage': 13.824
                 }
             },
@@ -131,8 +135,10 @@ class TestProcessDeviceStatus(unittest.TestCase):
             'created_at': '2024-12-04 05:50:23-05:00',
             'device': 'Pump (tconnectsync)',
             'pump': {
+                'clock': '2024-12-04 05:50:23-05:00',
                 'battery': {
-                    'string': '46%',
+                    'status': '30%',
+                    'percent': 30,
                     'voltage': 13.568
                 }
             },