bump to 0.8.10

%s was missing from the format string. This enhances the message to
report the unexpected thearpy type along with the dump of the event that
trigger the exception.

.github/ISSUE_TEMPLATE/setup-help--tconnectsync-heroku.md

@@ -0,0 +1,53 @@
+---
+name: 'Setup help: tconnectsync-heroku'
+about: When experiencing an issue setting up tconnectsync-heroku
+title: ''
+labels: heroku, setup help
+assignees: ''
+
+---
+
+**Describe the problem**
+A clear and concise description of the issue you're experiencing.
+
+
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. ...
+2. ...
+
+
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+
+
+**Have you followed the [Troubleshooting steps in the README?](https://github.com/jwoglom/tconnectsync-heroku#troubleshooting)** (Yes or No)
+
+
+
+**Setup details**
+
+* **On what platform are you using the t:connect mobile app?** (Android or iOS)
+* **What version are you using of the t:connect mobile app?**
+
+
+
+**Heroku log output**
+If applicable, add the full output from the heroku logs in More > View Logs. ([See the instructions in the README.](https://github.com/jwoglom/tconnectsync-heroku#testing))
+
+
+
+**Check Login output**
+If applicable, [please follow the instructions to visit the `check_login` page](https://github.com/jwoglom/tconnectsync-heroku#testing), and copy and paste the output here.
+
+This file may contain sensitive details like your Nightscout URL and pump serial number. Copy the output into a text editor like Textedit or Notepad first and find-and-replace sensitive strings, like your Nightscout URL and pump serial number, if they appear.
+
+The output will also contain pump and CGM related information such as delivery events and blood sugars. If you would prefer to keep the contents private, send an email to tconnectsync<at>wogloms.net with the log output so @jwoglom can investigate.
+
+
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/setup-help--tconnectsync.md

@@ -0,0 +1,49 @@
+---
+name: 'Setup help: tconnectsync'
+about: When experiencing an issue setting up tconnectsync on Windows, Linux, or MacOS.
+title: ''
+labels: setup help
+assignees: ''
+
+---
+
+**Describe the problem**
+A clear and concise description of the issue you're experiencing.
+
+
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. ...
+2. ...
+
+
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+
+
+**Setup details**
+
+* **Operating system:** 
+* **tconnectsync version** (from `--version`): 
+* **On what platform are you using the t:connect mobile app?** (Android or iOS)
+* **What version are you using of the t:connect mobile app?**
+
+
+
+**Terminal output**
+If applicable, add the full output from your terminal from running tconnectsync. Copy the output into a text editor like Textedit or Notepad first and find-and-replace sensitive strings, like your Nightscout URL and pump serial number, if they appear.
+
+
+
+**Check Login output**
+If applicable, please run `--check-login` and upload the tconnectsync-check-output.log file as an attachment on the issue. This file should have sensitive details like your Nightscout URL and pump serial number automatically removed, but double-check before uploading.
+
+This log will contain pump and CGM related information such as delivery events and blood sugars. If you would prefer to keep the contents private, send an email to tconnectsync<at>wogloms.net with the log output so @jwoglom can investigate.
+
+
+
+**Additional context**
+Add any other context about the problem here.

.github/workflows/python-package.yml

@@ -30,7 +30,8 @@ jobs:
         pipenv install --system
     - name: Run pipenv check
       run: |
-        pipenv check
+        pipenv check \
+          --ignore 51499 # DDoS attack in wheel package, which is unsupported in python 3.7
     - name: Lint with flake8
       run: |
         # stop the build if there are Python syntax errors or undefined names

Pipfile.lock

@@ -16,18 +16,18 @@
     "default": {
         "arrow": {
             "hashes": [
-                "sha256:05caf1fd3d9a11a1135b2b6f09887421153b94558e5ef4d090b567b47173ac2b",
-                "sha256:d622c46ca681b5b3e3574fcb60a04e5cc81b9625112d5fb2b44220c36c892177"
+                "sha256:3934b30ca1b9f292376d9db15b19446088d12ec58629bc3f0da28fd55fb633a1",
+                "sha256:5a49ab92e3b7b71d96cd6bfcc4df14efefc9dfa96ea19045815914a6ab6b1fe2"
             ],
             "index": "pypi",
-            "version": "==1.2.2"
+            "version": "==1.2.3"
         },
         "beautifulsoup4": {
             "hashes": [
                 "sha256:58d5c3d29f5a36ffeb94f02f0d786cd53014cf9b3b3951d42e0080d8a9498d30",
                 "sha256:ad9aa55b65ef2808eb405f46cf74df7fcb7044d5cbc26487f96eb2ef2e436693"
             ],
-            "markers": "python_full_version >= '3.6.0'",
+            "markers": "python_version >= '3.6'",
             "version": "==4.11.1"
         },
         "bs4": {
@@ -39,27 +39,27 @@
         },
         "certifi": {
             "hashes": [
-                "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d",
-                "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412"
+                "sha256:0d9c601124e5a6ba9712dbc60d9c53c21e34f5f641fe83002317394311bdce14",
+                "sha256:90c1a32f1d68f940488354e36370f6cca89f0f106db09518524c88d6ed83f382"
             ],
-            "markers": "python_full_version >= '3.6.0'",
-            "version": "==2022.6.15"
+            "markers": "python_version >= '3.6'",
+            "version": "==2022.9.24"
         },
         "charset-normalizer": {
             "hashes": [
-                "sha256:5189b6f22b01957427f35b6a08d9a0bc45b46d3788ef5a92e978433c7a35f8a5",
-                "sha256:575e708016ff3a5e3681541cb9d79312c416835686d054a23accb873b254f413"
+                "sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845",
+                "sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"
             ],
-            "markers": "python_full_version >= '3.6.0'",
-            "version": "==2.1.0"
+            "markers": "python_version >= '3.6'",
+            "version": "==2.1.1"
         },
         "idna": {
             "hashes": [
-                "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff",
-                "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"
+                "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4",
+                "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"
             ],
             "markers": "python_version >= '3.5'",
-            "version": "==3.3"
+            "version": "==3.4"
         },
         "lxml": {
             "hashes": [
@@ -156,11 +156,11 @@
         },
         "python-dotenv": {
             "hashes": [
-                "sha256:b7e3b04a59693c42c36f9ab1cc2acc46fa5df8c78e178fc33a8d4cd05c8d498f",
-                "sha256:d92a187be61fe482e4fd675b6d52200e7be63a12b724abbf931a40ce4fa92938"
+                "sha256:1684eb44636dd462b66c3ee016599815514527ad99965de77f43e0944634a7e5",
+                "sha256:b77d08274639e3d34145dfa6c7008e66df0f04b7be7a75fd0d5292c191d79045"
             ],
             "index": "pypi",
-            "version": "==0.20.0"
+            "version": "==0.21.0"
         },
         "requests": {
             "hashes": [
@@ -172,11 +172,11 @@
         },
         "requests-mock": {
             "hashes": [
-                "sha256:0a2d38a117c08bb78939ec163522976ad59a6b7fdd82b709e23bb98004a44970",
-                "sha256:8d72abe54546c1fc9696fa1516672f1031d72a55a1d66c85184f972a24ba0eba"
+                "sha256:2fdbb637ad17ee15c06f33d31169e71bf9fe2bdb7bc9da26185be0dd8d842699",
+                "sha256:59c9c32419a9fb1ae83ec242d98e889c45bd7d7a65d48375cc243ec08441658b"
             ],
             "index": "pypi",
-            "version": "==1.9.3"
+            "version": "==1.10.0"
         },
         "six": {
             "hashes": [
@@ -191,20 +191,20 @@
                 "sha256:3b2503d3c7084a42b1ebd08116e5f81aadfaea95863628c80a3b774a11b7c759",
                 "sha256:fc53893b3da2c33de295667a0e19f078c14bf86544af307354de5fcf12a3f30d"
             ],
-            "markers": "python_full_version >= '3.6.0'",
+            "markers": "python_version >= '3.6'",
             "version": "==2.3.2.post1"
         },
         "tconnectsync": {
             "path": ".",
-            "version": "==0.7.1"
+            "version": "==0.8.6"
         },
         "urllib3": {
             "hashes": [
-                "sha256:8298d6d56d39be0e3bc13c1c97d133f9b45d797169a0e11cdd0e0489d786f7ec",
-                "sha256:879ba4d1e89654d9769ce13121e0f94310ea32e8d2f8cf587b77c08bbcdb30d6"
+                "sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc",
+                "sha256:c083dd0dce68dbfbe1129d5271cb90f9447dea7d52097c6e0126120c521ddea8"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_version < '4'",
-            "version": "==1.26.10"
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
+            "version": "==1.26.13"
         }
     },
     "develop": {
@@ -217,11 +217,11 @@
         },
         "jedi": {
             "hashes": [
-                "sha256:637c9635fcf47945ceb91cd7f320234a7be540ded6f3e99a50cb6febdfd1ba8d",
-                "sha256:74137626a64a99c8eb6ae5832d99b3bdd7d29a3850fe2aa80a4126b2a7d949ab"
+                "sha256:203c1fd9d969ab8f2119ec0a3342e0b49910045abe6af0a3ae83a5764d54639e",
+                "sha256:bae794c30d07f6d910d32a7048af09b5a39ed740918da923c6b780790ebac612"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==0.18.1"
+            "version": "==0.18.2"
         },
         "parso": {
             "hashes": [
@@ -233,27 +233,27 @@
         },
         "prompt-toolkit": {
             "hashes": [
-                "sha256:859b283c50bde45f5f97829f77a4674d1c1fcd88539364f1b28a37805cfd89c0",
-                "sha256:d8916d3f62a7b67ab353a952ce4ced6a1d2587dfe9ef8ebc30dd7c386751f289"
+                "sha256:535c29c31216c77302877d5120aef6c94ff573748a5b5ca5b1b1f76f5e700c73",
+                "sha256:ced598b222f6f4029c0800cefaa6a17373fb580cd093223003475ce32805c35b"
             ],
             "markers": "python_full_version >= '3.6.2'",
-            "version": "==3.0.30"
+            "version": "==3.0.33"
         },
         "ptpython": {
             "hashes": [
-                "sha256:99636899ab0e4d026e2ecc9368269114f387b4bb5411e57f072b0bde724d9f99",
-                "sha256:eafd4ced27ca5dc370881d4358d1ab5041b32d88d31af8e3c24167fe4af64ed6"
+                "sha256:1ec28ba37959e8e6c4880fcf4748ad5c61341d4c1bfd24a38a4d5b97af1f475a",
+                "sha256:a57b9952eac4a12540a4dfb4ccd4a243403eceb27b0d13245b5e4144c8731d32"
             ],
             "index": "pypi",
-            "version": "==3.0.20"
+            "version": "==3.0.21"
         },
         "pygments": {
             "hashes": [
-                "sha256:5eb116118f9612ff1ee89ac96437bb6b49e8f04d8a13b514ba26f620208e26eb",
-                "sha256:dc9c10fb40944260f6ed4c688ece0cd2048414940f1cea51b8b226318411c519"
+                "sha256:56a8508ae95f98e2b9bdf93a6be5ae3f7d8af858b43e02c5a2ff083726be40c1",
+                "sha256:f643f331ab57ba3c9d89212ee4a2dabc6e94f117cf4eefde99a0574720d14c42"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==2.12.0"
+            "version": "==2.13.0"
         },
         "wcwidth": {
             "hashes": [

README.md

@@ -116,7 +116,10 @@ First, ensure that you have **Python 3** with **Pip** installed:
 
 * **On MacOS:** Open Terminal. Install [Homebrew](https://brew.sh/), and then run `brew install python3`
 * **On Linux:** Follow your distribution's specific instructions.
-  For Debian/Ubuntu based distros, `sudo apt install python3 python3-pip`
+  - For Debian/Ubuntu based distros, `sudo apt install python3 python3-pip`
+  - For CentOS/Rocky Linux 8: 
+    - `sudo dnf install python39-pip`
+    - `sudo alternatives --set python /usr/bin/python3.9` 
 * **On Windows:** Install Ubuntu under the [Windows Subsystem for Linux](https://ubuntu.com/wsl).
   Open the Ubuntu Terminal, then run `sudo apt install python3 python3-pip`.
   Perform the remainder of the steps under the Ubuntu environment.
@@ -126,6 +129,12 @@ Now install the `tconnectsync` package with pip:
 ```
 $ pip3 install tconnectsync
 ```
+To install into a user environment instead of system-wide for a more contained install:
+````
+$ pip3 install --user tconnectsync
+````
+  - This will place the tconnectsync binary file at ``/home/<username>/.local/bin/tconnectsync``
+
 
 If the pip3 command is not found, run `python3 -m pip install tconnectsync` instead.
 
@@ -377,22 +386,27 @@ invoking tconnectsync with no arguments via cron.
 
 If using Pipenv or a virtualenv, make sure that you either prefix the call to main.py with `pipenv run` or source the `bin/activate` file within the virtualenv, so that the proper dependencies are loaded. If not using any kind of virtualenv, you can instead just install the necessary dependencies as specified inside Pipfile globally.
 
-An example configuration in `/etc/crontab` which runs every 15 minutes:
+An system-wide example configuration in `/etc/crontab` which runs every 15 minutes, on the 15 minute mark:
 
 ```bash
 # m         h  dom mon dow user   command
 0,15,30,45  *  *   *   *   root   /path/to/tconnectsync/run.sh
 ```
 
+An example of a user crontab `crontab -e` if not running system-wide, which runs every 15 minutes:
+```
+*/15 * * * * /path/to/tconnectsync/run.sh
+```
+
 You can use one of the same `run.sh` files referenced above, but remove the `--auto-update` flag since you are handling the functionality for running the script periodically yourself.
 
 ## Tandem APIs
 
 This application utilizes three separate Tandem APIs for obtaining t:connect data, referenced here by the identifying part of their URLs:
 
-* [**controliq**](https://github.com/jwoglom/tconnectsync/blob/master/tconnectsync/api/controliq.py) - Contains Control:IQ related data, namely a timeline of all Basal events uploaded by the pump, separated by type (temp basals, algorithmically-updated basals, or profile-updated basals).
+* [**controliq**](https://github.com/jwoglom/tconnectsync/blob/master/tconnectsync/api/controliq.py) - Contains Control:IQ related data, namely a timeline of all Basal events uploaded by the pump, separated by type (temp basals, algorithmically-updated basals, or profile-updated basals). Additionally includes CGM and Bolus data.
 * [**android**](https://github.com/jwoglom/tconnectsync/blob/master/tconnectsync/api/android.py) - Used internally by the t:connect Android app, these API endpoints were discovered by reverse-engineering the Android app. Most of the API endpoints are used for uploading pump data, and tconnectsync uses one endpoint which returns the most recent event ID uploaded by the pump, so we know when more data has been uploaded.
-* [**tconnectws2**](https://github.com/jwoglom/tconnectsync/blob/master/tconnectsync/api/ws2.py) - More legacy than the others, this seems to power the bulk of the main t:connect website. It is used to retrieve a CSV export of non-ControlIQ basal data, as well as bolus and IOB data. (I haven't found any mentions of bolus or IOB data in the Control:IQ-specific API.)
+* [**tconnectws2**](https://github.com/jwoglom/tconnectsync/blob/master/tconnectsync/api/ws2.py) - More legacy than the others, this seems to power the bulk of the main t:connect website. It is used as a last resort due to severe performance issues with this API (see https://github.com/jwoglom/tconnectsync/issues/43). We can use it to retrieve a CSV export of non-ControlIQ basal data, as well as bolus and IOB data. It is only used for bolus data as a fallback, and for pump-reported IOB data if requested. Full tracking of pump events also uses a limited version of this API.
 
 I have only tested tconnectsync with a Tandem pump set in the US Eastern timezone. Tandem's (to us, undocumented) APIs are [a bit loose with timezones](https://github.com/jwoglom/tconnectsync/blob/master/tconnectsync/parser.py#L15), so please let me know if you notice any timezone-related bugs.
 ## Backfilling t:connect Data
@@ -406,3 +420,15 @@ python3 main.py --start-date 2020-01-01 --end-date 2020-03-01
 In order to bulk-import a lot of data, you may need to use shorter intervals, and invoke tconnectsync multiple times. Tandem's API endpoints occasionally return invalid data if you request too large of a data window which causes tconnectsync to error out mid-way through.
 
 One oddity when backfilling data is that the Control:IQ specific API endpoints return errors if they are queried before you updated your pump to utilize Control:IQ. This is [partially worked around in tconnectsync's code](https://github.com/jwoglom/tconnectsync/blob/d841c3811aeff3671d941a7d3ff4b80cce6a219e/main.py#L238), but you might need to update the logic if you did not switch to a Control:IQ enabled pump immediately after launch.
+
+## t:connect API Testing
+
+To test t:connect API endpoints in a Python shell, you can do something like the following:
+
+```python
+import tconnectsync
+tconnectsync.util.cli.enable_logging()
+api = tconnectsync.util.cli.get_api()
+# Make API calls, e.g.
+therapy_timeline = api.controliq.therapy_timeline('2022-08-01', '2022-08-10')
+```

codecov.yml

@@ -0,0 +1,6 @@
+coverage:
+  status:
+    project:
+      default:
+        target: 50%
+        threshold: null

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = tconnectsync
-version = 0.8.1
+version = 0.8.10
 author = James Woglom 
 author_email = j@wogloms.net
 description = Syncs Tandem t:connect pump data to Nightscout for the t:slim X2

tconnectsync/__init__.py

@@ -17,7 +17,8 @@ try:
         TCONNECT_EMAIL,
         TCONNECT_PASSWORD,
         NS_URL,
-        NS_SECRET
+        NS_SECRET,
+        NS_SKIP_TLS_VERIFY
     )
     from . import secret
 except Exception:
@@ -74,7 +75,7 @@ def main(*args, **kwargs):
 
     tconnect = TConnectApi(TCONNECT_EMAIL, TCONNECT_PASSWORD)
 
-    nightscout = NightscoutApi(NS_URL, NS_SECRET)
+    nightscout = NightscoutApi(NS_URL, NS_SECRET, NS_SKIP_TLS_VERIFY)
 
     if args.check_login:
         return check_login(tconnect, time_start, time_end)

tconnectsync/api/controliq.py

@@ -14,9 +14,12 @@ class ControlIQApi:
     BASE_URL = 'https://tdcservices.tandemdiabetes.com/'
     LOGIN_URL = 'https://tconnect.tandemdiabetes.com/login.aspx?ReturnUrl=%2f'
 
+    LAST_CONFIRMED_SOFTWARE_VERSION = 't:connect 7.14.0.1'
+
     userGuid = None
     accessToken = None
     accessTokenExpiresAt = None
+    tconnect_software_ver = None
 
     def __init__(self, email, password):
         self.login(email, password)
@@ -47,6 +50,22 @@ class ControlIQApi:
             return True
 
     def _build_login_data(self, email, password, soup):
+        try:
+            version = soup.select_one("#footer_version").text.strip()
+            self.tconnect_software_ver = version
+            logger.info("Reported tconnect software version: %s" % version)
+            if version != self.LAST_CONFIRMED_SOFTWARE_VERSION:
+                logger.warning("Newer API version than last confirmed working. Saw %s and expected %s" % (version, self.LAST_CONFIRMED_SOFTWARE_VERSION))
+                logger.warning("If you experience any issues, please report them to https://github.com/jwoglom/tconnectsync")
+        except Exception:
+            logger.warning("Unable to find tconnect software version.")
+            contents = "<unknown>"
+            if soup:
+                contents = "%s" % soup.encode_contents()
+                if len(contents) > 1000:
+                    contents = "%s[SNIP]%s" % (contents[:500], contents[-500:])
+            logger.info("BeautifulSoup parsed contents: %s" % contents)
+            pass
         return {
             "__LASTFOCUS": "",
             "__EVENTTARGET": "ctl00$ContentBody$LoginControl$linkLogin",
@@ -126,10 +145,7 @@ class ControlIQApi:
         startDate = parse_date(start)
         endDate = parse_date(end)
 
-        return self.get('tconnect/controliq/api/summary/users/%s' % (self.userGuid), {
-            "startDate": startDate,
-            "endDate": endDate
-        })
+        return self.get('tconnect/controliq/api/summary/users/%s?startDate=%s&endDate=%s' % (self.userGuid, startDate, endDate), {})
     
     """
     Returns active account features, including the date when ControlIQ was enabled.

tconnectsync/api/webui.py

@@ -1,3 +1,4 @@
+from typing import List
 import requests
 import urllib
 import datetime
@@ -7,6 +8,10 @@ import logging
 
 from bs4 import BeautifulSoup
 
+from tconnectsync.domain.device_settings import Device, Profile, ProfileSegment
+from tconnectsync.util import removesuffix, removeprefix
+from tconnectsync.util.constants import MMOLL_TO_MGDL
+
 from .common import base_headers, ApiException
 
 logger = logging.getLogger(__name__)
@@ -92,12 +97,11 @@ class WebUIScraper:
                 settings_guid = settings_a.attrs['href'].split('?guid=')[1]
             
             if serial_number:
-                devices[serial_number] = {
-                    'name': device_name,
-                    'model_number': model_number,
-                    'status': status,
-                    'guid': settings_guid
-                }
+                devices[serial_number] = Device(
+                    name=device_name,
+                    model_number=model_number,
+                    status=status,
+                    guid=settings_guid)
         
         return devices
     
@@ -106,7 +110,7 @@ class WebUIScraper:
     Note that pump_guid is NOT the serial number of the pump, and
     should be obtained from my_devices()[str(serial_number)]['guid']
     """
-    def device_settings_from_guid(self, pump_guid):
+    def device_settings_from_guid(self, pump_guid: str) -> List[Profile]:
         profiles = []
         settings = {}
         r = self.get('myaccount/DeviceSettings.aspx?guid=%s' % pump_guid)
@@ -124,12 +128,33 @@ class WebUIScraper:
         
         return profiles, settings
     
-    def _parse_profile_tbl(self, tbl):
+    def _parse_profile_tbl(self, tbl) -> Profile:
         profile = {}
         profile["title"] = self.strip(tbl.select_one('.setting_title').text)
         profile["active"] = bool(tbl.find(text='Active at the time of upload'))
         profile["segments"] = []
 
+        def parse_basal_rate(rate) -> float:
+            return float(removesuffix(rate, ' u/hr'))
+        
+        def parse_factor(ratio) -> int:
+            return parse_bg_mgdl(removeprefix(ratio, '1u:'))
+
+        def parse_ratio(ratio) -> float:
+            return float(removesuffix(removeprefix(ratio, '1u:'), ' g'))
+        
+        def parse_bg_mgdl(bg) -> int:
+            if bg.endswith(' mg/dL'):
+                return float(removesuffix(bg, ' mg/dL'))
+            elif bg.endswith(' mmol/L'):
+                return float(removesuffix(bg, ' mmol/L')) * MMOLL_TO_MGDL
+            raise ValueError(bg)
+        
+        def hours_to_mins(text) -> int:
+            hrmin = removesuffix(text, " hours")
+            hr, min = hrmin.split(":", 1)
+            return int(min) + int(hr)*60
+
         for tr in tbl.select('tr'):
             # Skip header rows
             if tr.select_one('.setting_bg'):
@@ -149,19 +174,20 @@ class WebUIScraper:
                     t = "12:00 AM"
                 elif display_time == "Noon":
                     t = "12:00 PM"
+
                 segment = {
                     "display_time": display_time,
                     "time": t,
-                    "basal_rate": self.strip(tds[1].text),
-                    "correction_factor": self.strip(tds[2].text),
-                    "carb_ratio": self.strip(tds[3].text),
-                    "target_bg": self.strip(tds[4].text)
+                    "basal_rate": parse_basal_rate(self.strip(tds[1].text)),
+                    "correction_factor": parse_factor(self.strip(tds[2].text)),
+                    "carb_ratio": parse_ratio(self.strip(tds[3].text)),
+                    "target_bg_mgdl": parse_bg_mgdl(self.strip(tds[4].text))
                 }
-                profile["segments"].append(segment)
+                profile["segments"].append(ProfileSegment(**segment))
                 continue
             
             if tr.find(text='Calculated Total Daily Basal'):
-                profile["calculated_total_daily_basal"] = self.strip(tds[1].text)
+                profile["calculated_total_daily_basal"] = float(removesuffix(self.strip(tds[1].text), " units"))
                 continue
             
             # Last row
@@ -175,12 +201,12 @@ class WebUIScraper:
                     key = self.strip(key)
                     val = self.strip(val)
                     if key == 'Duration of Insulin':
-                        profile["insulin_duration"] = val
+                        profile["insulin_duration_min"] = hours_to_mins(val)
                     elif key == 'Carbohydrates':
-                        profile["carbohydrates"] = val
+                        profile["carbs_enabled"] = self.strip(val.lower()) == "on"
 
 
-        return profile
+        return Profile(**profile)
 
     def _parse_settings_tbl(self, tbl):
         outer_tr = tbl.select('tr')[2]

tconnectsync/api/ws2.py

@@ -21,14 +21,14 @@ class WS2Api:
         self.userGuid = userGuid
         self.session = base_session()
 
-    def get(self, endpoint):
-        r = self.session.get(self.BASE_URL + endpoint, headers=base_headers())
+    def get(self, endpoint, **kwargs):
+        r = self.session.get(self.BASE_URL + endpoint, headers=base_headers(), **kwargs)
         if r.status_code != 200:
             raise ApiException(r.status_code, "WS2 API HTTP %s response: %s" % (str(r.status_code), r.text))
         return r.text
 
-    def get_jsonp(self, endpoint):
-        r = self.session.get(self.BASE_URL + endpoint + '?callback=cb', headers=base_headers())
+    def get_jsonp(self, endpoint, **kwargs):
+        r = self.session.get(self.BASE_URL + endpoint + '?callback=cb', headers=base_headers(), **kwargs)
         if r.status_code != 200:
             raise ApiException(r.status_code, "WS2 API HTTP %s response: %s" % (str(r.status_code), r.text))
 
@@ -80,7 +80,7 @@ class WS2Api:
         endDate = parse_date(end)
 
         try:
-            req_text = self.get('therapytimeline2csv/%s/%s/%s?format=csv' % (self.userGuid, startDate, endDate))
+            req_text = self.get('therapytimeline2csv/%s/%s/%s?format=csv' % (self.userGuid, startDate, endDate), timeout=10)
         except ApiException as e:
             # This seems to occur as some kind of soft rate-limit.
             logger.warning("Received ApiException in therapy_timeline_csv: (retry count %d) %s" % (tries, e))
@@ -137,7 +137,7 @@ class WS2Api:
         endDate = parse_date(end)
         arg = "filterbasal/1" if filterbasal else ""
 
-        return self.get_jsonp('basalsuspension/%s/%s/%s/%s' % (self.userGuid, startDate, endDate, arg))
+        return self.get_jsonp('basalsuspension/%s/%s/%s/%s' % (self.userGuid, startDate, endDate, arg), timeout=10)
 
     """
     Returns info on BasalIQ in JSONP format.
@@ -146,4 +146,4 @@ class WS2Api:
         startDate = parse_date(start)
         endDate = parse_date(end)
 
-        return self.get_jsonp('basaliqtech/%s/%s/%s' % (self.userGuid, startDate, endDate))
+        return self.get_jsonp('basaliqtech/%s/%s/%s' % (self.userGuid, startDate, endDate), timeout=10)

tconnectsync/autoupdate.py

@@ -1,5 +1,6 @@
 import time
 import logging
+import datetime
 import sys
 
 from .process import process_time_range
@@ -67,7 +68,7 @@ class Autoupdate:
                             # (we can see the indexes increasing, so we know something's happening!)
                             if (now - last_action_or_start) >= 60 * self.secret.AUTOUPDATE_FAILURE_MINUTES:
                                 logger.error(AutoupdateFailureError(
-                                        "An event index change was recorded, but no new data was found via the API. " +
+                                        ("%s: An event index change was recorded, but no new data was found via the API. " % datetime.datetime.now()) +
                                         "The %s was %d minutes ago. This is a problem with tconnectsync." % 
                                         ("last processed event" if self.last_successful_process_time_range else "start of autoupdate", (now - last_action_or_start)//60)))
 
@@ -75,7 +76,7 @@ class Autoupdate:
                                     logger.error("Exiting with error code due to AUTOUPDATE_RESTART_ON_FAILURE")
                                     return 1
                             else:
-                                logger.warn(AutoupdateFailureWarning("An event index change was recorded, but no new data was found via the API. " +
+                                logger.warning(AutoupdateFailureWarning(("%s: An event index change was recorded, but no new data was found via the API. " % datetime.datetime.now()) +
                                             "The %s was %d minutes ago. Resetting TConnectApi to attempt to solve this problem." % 
                                             ("last processed event" if self.last_successful_process_time_range else "start of autoupdate", (now - last_action_or_start)//60)))
                                 
@@ -107,7 +108,7 @@ class Autoupdate:
                 # The most likely case here is that the pump isn't uploading right now.
                 if self.last_event_time and (now - self.last_event_time) >= 60 * self.secret.AUTOUPDATE_NO_DATA_FAILURE_MINUTES:
                     logger.error(AutoupdateNoEventIndexesDetectedError(
-                        "No new data event indexes have been detected for %d minutes. " % ((now - self.last_event_time)//60) +
+                        "%s: No new data event indexes have been detected for %d minutes. " % (datetime.datetime.now(), (now - self.last_event_time)//60) +
                         "The t:connect app might no longer be functioning."))
 
                     # TODO: restarting doesn't really help anything here.
@@ -124,11 +125,11 @@ class Autoupdate:
                 # above no indexes warning.
                 elif self.last_successful_process_time_range and (now - self.last_successful_process_time_range) >= 60 * self.secret.AUTOUPDATE_FAILURE_MINUTES:
                     logger.error(AutoupdateNoNewDataDetectedError(
-                        "No new data has been detected via the API for %d minutes. " % (now - self.last_successful_process_time_range)//60 +
+                        "%s: No new data has been detected via the API for %d minutes. " % (datetime.datetime.now(), now - self.last_successful_process_time_range)//60 +
                         "tconnectsync might not be functioning properly."))
 
                     if self.secret.AUTOUPDATE_RESTART_ON_FAILURE:
-                        logger.error("Exiting with error code due to AUTOUPDATE_RESTART_ON_FAILURE")
+                        logger.error("%s: Exiting with error code due to AUTOUPDATE_RESTART_ON_FAILURE" % datetime.datetime.now())
                         return 1
 
                 # Track how long we've been retrying
@@ -142,7 +143,7 @@ class Autoupdate:
                 # uploaded, so 
                 if len(self.time_diffs_between_attempts) >= 3:
                     # The pump hasn't sent us data that, based on previous cadence, we were expecting
-                    logger.warn(AutoupdateNoIndexChangeWarning("Sleeping %d seconds after unexpected no index change based on previous cadence. (New data might be delayed.)" % 
+                    logger.warning(AutoupdateNoIndexChangeWarning("Sleeping %d seconds after unexpected no index change based on previous cadence. (New data might be delayed.)" % 
                         int(self.secret.AUTOUPDATE_UNEXPECTED_NO_INDEX_SLEEP_SECONDS)))
                     
                     logger.debug("Last event time: %s, time diffs between attempts: %s" % (self.last_event_time, self.time_diffs_between_attempts))
@@ -203,4 +204,4 @@ class AutoupdateNoNewDataDetectedError(AutoupdateError):
     pass
 
 class AutoupdateNoIndexChangeWarning(AutoupdateWarning):
-    pass
+    pass

tconnectsync/check.py

@@ -2,9 +2,10 @@ import sys
 import time
 import arrow
 import logging
+import traceback
 import pkg_resources
 from datetime import datetime
-from pprint import pformat
+from pprint import pformat as pformat_base
 
 from .nightscout import NightscoutApi
 from .parser.nightscout import BASAL_EVENTTYPE, BOLUS_EVENTTYPE
@@ -21,7 +22,7 @@ Attempts to authenticate with each t:connect API,
 and returns the output of a sample API call from each.
 Also attempts to connect to the Nightscout API.
 """
-def check_login(tconnect, time_start, time_end, verbose=False, sanitize=False):
+def check_login(tconnect, time_start, time_end, verbose=False, sanitize=True):
     errors = 0
 
     loglines = []
@@ -29,6 +30,13 @@ def check_login(tconnect, time_start, time_end, verbose=False, sanitize=False):
         print(*args)
         loglines.append(" ".join([str(i) for i in args]) + "\n")
 
+    def log_err(e):
+        try:
+            out = ''.join(list(traceback.TracebackException.from_exception(e).format()))
+            log(out)
+        except Exception:
+            log("could not log exception traceback: {}".format(e))
+
     def debug(*args):
         if verbose:
             print(*args)
@@ -46,7 +54,7 @@ def check_login(tconnect, time_start, time_end, verbose=False, sanitize=False):
         from .secret import TCONNECT_EMAIL, TCONNECT_PASSWORD, PUMP_SERIAL_NUMBER, NS_URL, NS_SECRET, TIMEZONE_NAME
     except ImportError as e:
         log("Error: Unable to load config file. Please check your .env file or environment variables")
-        log(e)
+        log_err(e)
     
     if not TCONNECT_EMAIL or TCONNECT_EMAIL == 'email@email.com':
         log("Error: You have not specified a TCONNECT_EMAIL")
@@ -76,9 +84,12 @@ def check_login(tconnect, time_start, time_end, verbose=False, sanitize=False):
     try:
         summary = tconnect.controliq.dashboard_summary(time_start, time_end)
         debug("ControlIQ dashboard summary: \n%s" % pformat(summary))
+        log("tconnect_software_ver: %s" % tconnect.controliq.tconnect_software_ver)
     except Exception as e:
         log("Error occurred querying ControlIQ API for dashboard_summary:")
-        log(e)
+        log_err(e)
+        if e and "HTTP 404" in str(e):
+            log("<!> The API returns a 404 error if there is no data for the provided dates (currently %s - %s). Try re-running with an earlier start date using --start-date and --end-date arguments." % (time_start, time_end))
         errors += 1
     
     log("Querying ControlIQ therapy_timeline...")
@@ -96,7 +107,7 @@ def check_login(tconnect, time_start, time_end, verbose=False, sanitize=False):
                 lastBasalDuration = processed_tt[-1]['duration_mins']
     except Exception as e:
         log("Error occurred querying ControlIQ therapy_timeline:")
-        log(e)
+        log_err(e)
         errors += 1
     
     log("Querying ControlIQ therapy_events...")
@@ -105,32 +116,37 @@ def check_login(tconnect, time_start, time_end, verbose=False, sanitize=False):
         debug("controliq therapy_events: \n%s" % pformat(androidevents))
     except Exception as e:
         log("Error occurred querying ControlIQ therapy_events:")
-        log(e)
+        log_err(e)
         errors += 1
     
     log("-----")
 
-    log("Logging in to t:connect WS2 API...")
+    log("Initializing t:connect WS2 API...")
+    ws2_loggedin = False
     try:
         summary = tconnect.ws2.basaliqtech(time_start, time_end)
         debug("WS2 basaliq status: \n%s" % pformat(summary))
+        ws2_loggedin = True
     except Exception as e:
-        log("Error occurred querying WS2 API:")
-        log(e)
+        log("Error occurred querying WS2 API. This is okay so long as you are not using the PUMP_EVENTS or IOB sync features.")
+        log_err(e)
         errors += 1
     
-    log("Querying WS2 therapy_timeline_csv...")
     lastReadingTime = None
-    try:
-        ttcsv = tconnect.ws2.therapy_timeline_csv(time_start, time_end)
-        debug("therapy_timeline_csv: \n%s" % pformat(ttcsv))
-        if ttcsv and "readingData" in ttcsv and len(ttcsv["readingData"]) > 0:
-            log("Last therapy_timeline_csv reading: \n%s" % pformat(ttcsv["readingData"][-1]))
-            lastReadingTime = TConnectEntry._datetime_parse(ttcsv["readingData"][-1]['EventDateTime'])
-    except Exception as e:
-        log("Error occurred querying WS2 therapy_timeline_csv:")
-        log(e)
-        errors += 1
+    if ws2_loggedin:
+        log("Querying WS2 therapy_timeline_csv...")
+        try:
+            ttcsv = tconnect.ws2.therapy_timeline_csv(time_start, time_end)
+            debug("therapy_timeline_csv: \n%s" % pformat(ttcsv))
+            if ttcsv and "readingData" in ttcsv and len(ttcsv["readingData"]) > 0:
+                log("Last therapy_timeline_csv reading: \n%s" % pformat(ttcsv["readingData"][-1]))
+                lastReadingTime = TConnectEntry._datetime_parse(ttcsv["readingData"][-1]['EventDateTime'])
+        except Exception as e:
+            log("Error occurred querying WS2 therapy_timeline_csv. This is okay so long as you are not using the PUMP_EVENTS or IOB sync features.")
+            log_err(e)
+            errors += 1
+    else:
+        log("Not able to log in to WS2 API, so skipping therapy_timeline_csv")
     
     log("-----")
 
@@ -144,7 +160,7 @@ def check_login(tconnect, time_start, time_end, verbose=False, sanitize=False):
         debug("Android last uploaded event: \n%s" % pformat(event))
     except Exception as e:
         log("Error occurred querying Android API:")
-        log(e)
+        log_err(e)
         errors += 1
     
     log("-----")
@@ -162,7 +178,7 @@ def check_login(tconnect, time_start, time_end, verbose=False, sanitize=False):
         debug("Nightscout last uploaded bolus: \n%s" % pformat(last_upload_bolus))
     except Exception as e:
         log("Error occurred querying Nightscout API:")
-        log(e)
+        log_err(e)
         errors += 1
 
     log("-----")
@@ -216,4 +232,8 @@ def run_sanitize(s, sanitizedData):
     for k, v in sanitizedData.items():
         if v and len(str(v)) > 0:
             ret = ret.replace(str(v), '[%s]' % k)
-    return ret
+    return ret
+
+def pformat(*args, **kwargs):
+    kwargs['width'] = 160
+    return pformat_base(*args, **kwargs)

tconnectsync/domain/device_settings.py

@@ -0,0 +1,27 @@
+from dataclasses import dataclass
+from typing import List, Optional
+
+@dataclass
+class Device:
+    name: str
+    model_number: str
+    status: str
+    guid: Optional[str]
+
+@dataclass
+class ProfileSegment:
+    display_time: str # Identical to time except written out as Midnight or Noon
+    time: str
+    basal_rate: float # _ u/hr
+    correction_factor: int # 1u: _ mg/dL
+    carb_ratio: float # 1u: _ g
+    target_bg_mgdl: int
+
+@dataclass
+class Profile:
+    title: str
+    active: bool
+    segments: List[ProfileSegment]
+    calculated_total_daily_basal: float # in units
+    insulin_duration_min: int
+    carbs_enabled: bool

tconnectsync/domain/therapy_event.py

@@ -51,12 +51,52 @@ class CGMTherapyEvent(TherapyEvent):
     },
     """
     @classmethod
-    def parse(self, json):
+    def parse(_, json):
+        self = CGMTherapyEvent()
         TherapyEvent.parse(self, json)
         self.eventID = json['eventID']
         self.egv = json['egv']['estimatedGlucoseValue']
         return self
 
+class BGTherapyEvent(TherapyEvent):
+    eventID = None
+    egv = None
+    """
+    {   
+        'bg': 160, # note in EGV
+        'cgmCalibration': 1, # not in EGV
+        'description': 'BG',
+        'deviceType': 't:slim X2 Insulin Pump',
+        'eventDateTime': '2022-08-20T07:25:24',
+        'eventTypeId': 16,
+        'indexId': 844955,
+        'interactive': 0,
+        'iob': 0.75,
+        'note': {   'active': False,
+                'eventId': 0, # different location than EGV
+                'eventTypeId': 16,
+                'id': 0,
+                'indexId': '',
+                'sourceRecordId': 0},
+        'requestDateTime': '0001-01-01T00:00:00',
+        'serialNumber': 'xxx',
+        'sourceRecId': 793549667,
+        'tempRateActivated': 0,
+        'tempRateCompleted': 0,
+        'tempRateId': 0,
+        'type': 'BG',
+        'uploadId': 748700213}
+    """    
+    @classmethod
+    def parse(_, json):
+        self = BGTherapyEvent()
+        TherapyEvent.parse(self, json)
+        self.eventID = json['note']['eventId']
+        # This is probably not how we want to provide CGM calibrations to Nightscout,
+        # but will just include it as egv data for now to keep the thing from crashing :)
+        self.egv = json['bg']
+        return self
+
 class BolusTherapyEvent(TherapyEvent):
     bolusRequestOptions = None
     REQUEST_AUTOMATIC = "Automatic Bolus/Correction"
@@ -364,4 +404,77 @@ class BolusTherapyEvent(TherapyEvent):
         "tempRateCompleted": 0,
         "tempRateActivated": 0
     }
-    """
+    CGM Calibration (Therapy Event Type BG):
+    {   'bg': 160,
+        'cgmCalibration': 1,
+        'description': 'BG',
+        'deviceType': 't:slim X2 Insulin Pump',
+        'eventDateTime': '2022-08-20T07:25:24',
+        'eventTypeId': 16,
+        'indexId': 844955,
+        'interactive': 0,
+        'iob': 0.75,
+        'note': {   'active': False,
+                    'eventId': 0,
+                    'eventTypeId': 16,
+                    'id': 0,
+                    'indexId': '',
+                    'sourceRecordId': 0},
+        'requestDateTime': '0001-01-01T00:00:00',
+        'serialNumber': 'xxx',
+        'sourceRecId': 793549667,
+        'tempRateActivated': 0,
+        'tempRateCompleted': 0,
+        'tempRateId': 0,
+        'type': 'BG',
+        'uploadId': 0}
+    """
+
+class BasalTherapyEvent(TherapyEvent):
+    """
+    {
+        'basalRate': {
+            'duration': 0, 
+            'percent': 0, 
+            'value': 0.0
+        }, 
+        'displayInHistory': 0, 
+        'eventDateTime': '2022-12-02T00:00:00', 
+        'note': {
+            'id': 0, 
+            'indexId': '16403', 
+            'eventTypeId': 90, 
+            'sourceRecordId': 0, 
+            'eventId': 0, 
+            'active': False
+        }, 
+        'noteDate': {}, 
+        'requestDateTime': '0001-01-01T00:00:00', 
+        'type': 'Basal', 
+        'description': 'NDE', 
+        'sourceRecId': xxx, 
+        'eventTypeId': 0, 
+        'indexId': 0, 
+        'uploadId': 0, 
+        'interactive': 1, 
+        'tempRateId': 0, 
+        'tempRateCompleted': 0, 
+        'tempRateActivated': 0
+    }
+    """
+    basalRateValue = None
+    basalRatePercent = None
+    basalRateDuration = None
+    eventTime = None
+
+
+    @classmethod
+    def parse(_, json):
+        self = CGMTherapyEvent()
+        TherapyEvent.parse(self, json)
+        if 'basalRate' in json:
+            self.basalRateValue = json['basalRate']['value']
+            self.basalRatePercent = json['basalRate']['percent']
+            self.basalRateDuration = json['basalRate']['duration']
+        self.eventTime = json['eventDateTime']
+        return self

tconnectsync/domain/utility.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python3
+
+class Time:
+    def __init__(self, hour: int, min: int):
+        self.hour = hour
+        self.min = min
+    
+    @classmethod
+    def parse(cls, input):
+        if ' ' not in input:
+            raise ValueError('unable to parse time: %s' % input)
+
+        hrmin, ampm = input.split(' ')
+        
+        hr, min = hrmin.split(':')
+        hr = int(hr)
+        min = int(min)
+
+        if ampm.lower() == 'pm':
+            hr += 12
+        elif ampm.lower() != 'am':
+            raise ValueError('unable to parse time: %s' % input)
+        
+        return cls(hr, min)

tconnectsync/nightscout.py

@@ -4,6 +4,7 @@ import hashlib
 import time
 import urllib.parse
 import arrow
+import logging
 
 from urllib.parse import urljoin
 
@@ -11,21 +12,28 @@ from .api.common import ApiException
 from .parser.nightscout import ENTERED_BY
 
 def format_datetime(date):
-	return arrow.get(date).strftime('%Y-%m-%d %H:%M:%S')
+	return arrow.get(date).isoformat()
 
-def time_range(field_name, start_time, end_time):
+def time_range(field_name, start_time, end_time, t_to_space=False):
+	def fmt(date):
+		ret = format_datetime(date)
+		if t_to_space:
+			return ret.replace('T', ' ')
+		return ret
 	arg = ''
 	if start_time:
-		arg += '&find[%s][$gte]=%s' % (field_name, format_datetime(start_time))
+		arg += '&find[%s][$gte]=%s' % (field_name, fmt(start_time))
 	if end_time:
-		arg += '&find[%s][$lte]=%s' % (field_name, format_datetime(end_time))
+		arg += '&find[%s][$lte]=%s' % (field_name, fmt(end_time))
 	return arg
 
 
+logger = logging.getLogger(__name__)
 class NightscoutApi:
-	def __init__(self, url, secret):
+	def __init__(self, url, secret, skip_verify=False):
 		self.url = url
 		self.secret = secret
+		self.verify = False if skip_verify else None
 
 
 	def upload_entry(self, ns_format, entity='treatments'):
@@ -33,7 +41,7 @@ class NightscoutApi:
 			'Accept': 'application/json',
 			'Content-Type': 'application/json',
 			'api-secret': hashlib.sha1(self.secret.encode()).hexdigest()
-		})
+		}, verify=self.verify)
 		if r.status_code != 200:
 			raise ApiException(r.status_code, "Nightscout upload response: %s" % r.text)
 
@@ -42,7 +50,7 @@ class NightscoutApi:
 			'Accept': 'application/json',
 			'Content-Type': 'application/json',
 			'api-secret': hashlib.sha1(self.secret.encode()).hexdigest()
-		})
+		}, verify=self.verify)
 		if r.status_code != 200:
 			raise ApiException(r.status_code, "Nightscout delete response: %s" % r.text)
 
@@ -51,48 +59,81 @@ class NightscoutApi:
 			'Accept': 'application/json',
 			'Content-Type': 'application/json',
 			'api-secret': hashlib.sha1(self.secret.encode()).hexdigest()
-		})
+		}, verify=self.verify)
 		if r.status_code != 200:
 			raise ApiException(r.status_code, "Nightscout put response: %s" % r.text)
 
 	def last_uploaded_entry(self, eventType, time_start=None, time_end=None):
-		dateFilter = time_range('created_at', time_start, time_end)
-		latest = requests.get(urljoin(self.url, 'api/v1/treatments?count=1&find[enteredBy]=' + urllib.parse.quote(ENTERED_BY) + '&find[eventType]=' + urllib.parse.quote(eventType) + dateFilter + '&ts=' + str(time.time())), headers={
-			'api-secret': hashlib.sha1(self.secret.encode()).hexdigest()
-		})
-		if latest.status_code != 200:
-			raise ApiException(latest.status_code, "Nightscout last_uploaded_entry response: %s" % latest.text)
+		def internal(t_to_space):
+			dateFilter = time_range('created_at', time_start, time_end, t_to_space=t_to_space)
+			latest = requests.get(urljoin(self.url, 'api/v1/treatments?count=1&find[enteredBy]=' + urllib.parse.quote(ENTERED_BY) + '&find[eventType]=' + urllib.parse.quote(eventType) + dateFilter + '&ts=' + str(time.time())), headers={
+				'api-secret': hashlib.sha1(self.secret.encode()).hexdigest()
+			}, verify=self.verify)
+			if latest.status_code != 200:
+				raise ApiException(latest.status_code, "Nightscout last_uploaded_entry response: %s" % latest.text)
 
-		j = latest.json()
-		if j and len(j) > 0:
-			return j[0]
-		return None
+			j = latest.json()
+			if j and len(j) > 0:
+				return j[0]
+			return None
+		
+		ret = None
+		try:
+			ret = internal(False)
+		except ApiException as e:
+			logger.warning("last_uploaded_entry with no t_to_space: %s", e)
+			ret = None
+		if ret is None and (time_start or time_end):
+			try:
+				ret = internal(True)
+			except ApiException as e:
+				logger.warning("last_uploaded_entry with t_to_space: %s", e)
+				ret = None
+			if ret is not None:
+				logger.warning("last_uploaded_entry with eventType=%s time_start=%s time_end=%s only returned data when timestamps contained a space" % (eventType, time_start, time_end))
+		return ret
 	
 	def last_uploaded_bg_entry(self, time_start=None, time_end=None):
-		dateFilter = time_range('dateString', time_start, time_end)
-		latest = requests.get(urljoin(self.url, 'api/v1/entries.json?count=1&find[device]=' + urllib.parse.quote(ENTERED_BY) + dateFilter + '&ts=' + str(time.time())), headers={
-			'api-secret': hashlib.sha1(self.secret.encode()).hexdigest()
-		})
-		if latest.status_code != 200:
-			raise ApiException(latest.status_code, "Nightscout last_uploaded_bg_entry response: %s" % latest.text)
+		def internal(t_to_space):
+			dateFilter = time_range('dateString', time_start, time_end, t_to_space=t_to_space)
+			latest = requests.get(urljoin(self.url, 'api/v1/entries.json?count=1&find[device]=' + urllib.parse.quote(ENTERED_BY) + dateFilter + '&ts=' + str(time.time())), headers={
+				'api-secret': hashlib.sha1(self.secret.encode()).hexdigest()
+			}, verify=self.verify)
+			if latest.status_code != 200:
+				raise ApiException(latest.status_code, "Nightscout last_uploaded_bg_entry response: %s" % latest.text)
 
-		j = latest.json()
-		if j and len(j) > 0:
-			return j[0]
-		return None
+			j = latest.json()
+			if j and len(j) > 0:
+				return j[0]
+			return None
+		
+		ret = internal(False)
+		if ret is None and (time_start or time_end):
+			ret = internal(True)
+			if ret is not None:
+				logger.warning("last_uploaded_bg_entry with time_start=%s time_end=%s only returned data when timestamps contained a space" % (time_start, time_end))
+		return ret
 
 	def last_uploaded_activity(self, activityType, time_start=None, time_end=None):
-		dateFilter = time_range('created_at', time_start, time_end)
-		latest = requests.get(urljoin(self.url, 'api/v1/activity?find[enteredBy]=' + urllib.parse.quote(ENTERED_BY) + '&find[activityType]=' + urllib.parse.quote(activityType) + dateFilter + '&ts=' + str(time.time())), headers={
-			'api-secret': hashlib.sha1(self.secret.encode()).hexdigest()
-		})
-		if latest.status_code != 200:
-			raise ApiException(latest.status_code, "Nightscout activity response: %s" % latest.text)
+		def internal(t_to_space):
+			dateFilter = time_range('created_at', time_start, time_end, t_to_space=t_to_space)
+			latest = requests.get(urljoin(self.url, 'api/v1/activity?find[enteredBy]=' + urllib.parse.quote(ENTERED_BY) + '&find[activityType]=' + urllib.parse.quote(activityType) + dateFilter + '&ts=' + str(time.time())), headers={
+				'api-secret': hashlib.sha1(self.secret.encode()).hexdigest()
+			}, verify=self.verify)
+			if latest.status_code != 200:
+				raise ApiException(latest.status_code, "Nightscout activity response: %s" % latest.text)
 
-		j = latest.json()
-		if j and len(j) > 0:
-			return j[0]
-		return None
+			j = latest.json()
+			if j and len(j) > 0:
+				return j[0]
+			return None
+		
+		ret = internal(False)
+		if ret is None and (time_start or time_end):
+			ret = internal(True)
+			if ret is not None:
+				logger.warning("last_uploaded_activity with activityType=%s time_start=%s time_end=%s only returned data when timestamps contained a space" % (activityType, time_start, time_end))
+		return ret
 
 	"""
 	Returns general status information about the Nightscout server.
@@ -100,7 +141,7 @@ class NightscoutApi:
 	def api_status(self):
 		status = requests.get(urljoin(self.url, 'api/v1/status.json'), headers={
 			'api-secret': hashlib.sha1(self.secret.encode()).hexdigest()
-		})
+		}, verify=self.verify)
 		if status.status_code != 200:
 			raise Exception('HTTP error status code (%d) from Nightscout: %s' % (status.status_code, status.text))
 		return status.json()

tconnectsync/parser/ciq_therapy_events.py

@@ -1,4 +1,4 @@
-from tconnectsync.domain.therapy_event import BolusTherapyEvent, CGMTherapyEvent
+from tconnectsync.domain.therapy_event import BolusTherapyEvent, CGMTherapyEvent, BGTherapyEvent, BasalTherapyEvent
 from tconnectsync.parser.tconnect import TConnectEntry
 
 import logging
@@ -8,12 +8,20 @@ logger = logging.getLogger(__name__)
 def split_therapy_events(ciqTherapyEvents):
     bolusEvents = []
     cgmEvents = []
+    bgEvents = []
+    basalEvents = []
     for e in ciqTherapyEvents['event']:
         event = TConnectEntry.parse_therapy_event(e)
-        if type(event) == BolusTherapyEvent:
+        if isinstance(event, BolusTherapyEvent):
             bolusEvents.append(event)
-        elif type(event) == CGMTherapyEvent:
+        elif isinstance(event, CGMTherapyEvent): 
             cgmEvents.append(event)
+        elif isinstance(event, BGTherapyEvent):
+            bgEvents.append(event)
+        elif isinstance(event, BasalTherapyEvent):
+            basalEvents.append(event)
+
         
-    logger.debug("split_therapy_events: %d bolus, %d CGM" % (len(bolusEvents), len(cgmEvents)))
+    logger.debug("split_therapy_events: %d bolus, %d CGM, %d BG, %d basal" % (len(bolusEvents), len(cgmEvents), len(bgEvents), len(basalEvents)))
+    # TODO: BG events (CGM Calibration) values are not currently returned from ciq_therapy_events.py
     return bolusEvents, cgmEvents

tconnectsync/parser/tconnect.py

@@ -3,7 +3,7 @@ import sys
 import arrow
 from tconnectsync.domain.bolus import Bolus
 
-from tconnectsync.domain.therapy_event import BolusTherapyEvent, CGMTherapyEvent
+from tconnectsync.domain.therapy_event import BolusTherapyEvent, CGMTherapyEvent, BGTherapyEvent, BasalTherapyEvent
 
 try:
     from ..secret import TIMEZONE_NAME
@@ -200,6 +200,10 @@ class TConnectEntry:
             return BolusTherapyEvent.parse(data)
         elif data["type"] == "CGM":
             return CGMTherapyEvent.parse(data)
+        elif data["type"] == "BG":
+            return BGTherapyEvent.parse(data)
+        elif data["type"] == "Basal":
+            return BasalTherapyEvent.parse(data)
         
         raise UnknownTherapyEventException(data)
 
@@ -213,4 +217,5 @@ class UnknownBasalSuspensionEventException(Exception):
 
 class UnknownTherapyEventException(Exception):
     def __init__(self, data):
-        super().__init__("Unknown therapy event type: " % data)
+        typ = data["type"]
+        super().__init__(f"Unknown therapy event type: {typ} in {data}")

tconnectsync/process.py

@@ -86,19 +86,19 @@ def process_time_range(tconnect, nightscout, time_start, time_end, pretend, feat
        ciqFallingBack or \
        BOLUS_BG in features or \
        IOB in features:
-        logger.warn("Downloading t:connect CSV data")
+        logger.warning("Downloading t:connect CSV data")
         if bolusFallingBack:
-            logger.warn("Falling back on WS2 CSV data source because BOLUS is an enabled feature and CIQ bolus data was empty!!")
+            logger.warning("Falling back on WS2 CSV data source because BOLUS is an enabled feature and CIQ bolus data was empty!!")
         if ciqFallingBack:
-            logger.warn("Falling back on WS2 CSV data source because CGM is an enabled feature and CIQ cgm data was empty!!")
+            logger.warning("Falling back on WS2 CSV data source because CGM is an enabled feature and CIQ cgm data was empty!!")
         if BOLUS_BG in features:
-            logger.warn("Falling back on WS2 CSV data source because BOLUS_BG is an enabled feature. " +
+            logger.warning("Falling back on WS2 CSV data source because BOLUS_BG is an enabled feature. " +
                         "Please consider disabling this feature to improve synchronization reliability.")
         if IOB in features:
-            logger.warn("Falling back on WS2 CSV data source because IOB is an enabled feature. " +
+            logger.warning("Falling back on WS2 CSV data source because IOB is an enabled feature. " +
                         "Please consider disabling this feature to improve synchronization reliability.")
         
-        logger.warn("<!!> The WS2 data source is unreliable and may prevent timely synchronization")
+        logger.warning("<!!> The WS2 data source is unreliable and may prevent timely synchronization")
         csvdata = tconnect.ws2.therapy_timeline_csv(time_start, time_end)
 
         csvReadingData = csvdata["readingData"]
@@ -147,8 +147,8 @@ def process_time_range(tconnect, nightscout, time_start, time_end, pretend, feat
         pumpEvents = process_ciq_activity_events(ciqTherapyTimelineData)
         logger.debug("CIQ activity events: %s" % pumpEvents)
 
-        logger.warn("Using WS2 data source for basalsuspension because PUMP_EVENTS is an enabled feature")
-        logger.warn("<!!> The WS2 data source is unreliable and may prevent timely synchronization")
+        logger.warning("Using WS2 data source for basalsuspension because PUMP_EVENTS is an enabled feature")
+        logger.warning("<!!> The WS2 data source is unreliable and may prevent timely synchronization")
         ws2BasalSuspension = tconnect.ws2.basalsuspension(time_start, time_end)
 
         bsPumpEvents = process_basalsuspension_events(ws2BasalSuspension)
@@ -167,7 +167,7 @@ def process_time_range(tconnect, nightscout, time_start, time_end, pretend, feat
             bolusEvents = process_bolus_events(ciqBolusData, source="ciq")
 
         if csvBolusData and not bolusEvents:
-            logger.warn("Falling back on non-CIQ csvBolusData")
+            logger.warning("Falling back on non-CIQ csvBolusData")
             bolusEvents = process_bolus_events(csvBolusData, source="csv")
             logger.debug("ciq bolusEvents: %s" % bolusEvents)
         
@@ -184,4 +184,4 @@ def process_time_range(tconnect, nightscout, time_start, time_end, pretend, feat
             logger.debug("Finished writing iob events")
 
     logger.info("Wrote %d events to Nightscout this process cycle" % added)
-    return added
+    return added

tconnectsync/secret.py

@@ -31,7 +31,7 @@ def get_bool(name, default):
 TCONNECT_EMAIL = get('TCONNECT_EMAIL', 'email@email.com')
 TCONNECT_PASSWORD = get('TCONNECT_PASSWORD', 'password')
 
-PUMP_SERIAL_NUMBER = get_number('PUMP_SERIAL_NUMBER', '11111111')
+PUMP_SERIAL_NUMBER = int(get_number('PUMP_SERIAL_NUMBER', '11111111'))
 
 NS_URL = get('NS_URL', 'https://yournightscouturl/')
 NS_SECRET = get('NS_SECRET', 'apisecret')
@@ -40,6 +40,8 @@ if not get('NS_SECRET') and get('API_SECRET'):
     print('API_SECRET environment variable is set, overriding NS_SECRET')
     NS_SECRET = get('API_SECRET')
 
+NS_SKIP_TLS_VERIFY = get_bool('NS_SKIP_TLS_VERIFY', 'false')
+
 TIMEZONE_NAME = get('TIMEZONE_NAME', 'America/New_York')
 
 if not get('TIMEZONE_NAME') and get('TZ'):

tconnectsync/sync/bolus.py

@@ -34,6 +34,12 @@ def process_bolus_events(bolusdata, cgmEvents=None, source=""):
             else:
                 logger.warning("Skipping non-completed %s bolus data (was a bolus in progress?): %s parsed: %s" % (source, b, parsed))
                 continue
+            if parsed.is_extended_bolus:
+                if not parsed.bolex_start_time and not parsed.start_time:
+                    logger.warning("Skipping non-completed %s extended bolus data with no start time: %s parsed: %s" % (source, b, parsed))
+                elif not parsed.bolex_start_time and parsed.start_time:
+                    logger.warning("Setting bolex_start_time to start_time for non-completed %s extended bolus: %s parsed: %s" % (source, b, parsed))
+                    parsed.bolex_start_time = parsed.start_time
 
         if parsed.bg and cgmEvents:
             requested_at = parsed.request_time if not parsed.extended_bolus else parsed.bolex_start_time

tconnectsync/util/__init__.py

@@ -1,5 +1,8 @@
 import arrow
 
+from . import cli
+from . import constants
+
 def timeago(timestamp):
     seconds = (arrow.get() - arrow.get(timestamp)).total_seconds()
     fmt = '%s ago' if seconds >= 0 else 'in %s'
@@ -15,3 +18,14 @@ def timeago(timestamp):
     ret += '%d minutes' % (seconds//60)
 
     return fmt % ret
+
+# String methods only available in python 3.9+
+def removesuffix(input_string, suffix):
+    if suffix and input_string.endswith(suffix):
+        return input_string[:-len(suffix)]
+    return input_string
+
+def removeprefix(input_string, prefix):
+    if prefix and input_string.startswith(prefix):
+        return input_string[len(prefix):]
+    return input_string

tconnectsync/util/cli.py

@@ -0,0 +1,20 @@
+import logging
+
+"""
+Enables logging at the specified level for all loggers
+inside the tconnectsync package, and sets up a basicConfig
+to print those log messages to stderr.
+"""
+def enable_logging(level=logging.DEBUG):
+    logging.basicConfig()
+    for logger in logging.root.manager.loggerDict:
+        if logger.startswith('tconnectsync'):
+            logging.getLogger(logger).setLevel(level)
+
+"""
+Returns a TConnectApi object with default secret parameters.
+"""
+def get_api():
+    from ..api import TConnectApi
+    from ..secret import TCONNECT_EMAIL, TCONNECT_PASSWORD
+    return TConnectApi(TCONNECT_EMAIL, TCONNECT_PASSWORD)

tconnectsync/util/constants.py

@@ -0,0 +1,4 @@
+
+# http://www.soc-bdr.org/rds/authors/unit_tables_conversions_and_genetic_dictionaries/conversion_glucose_mg_dl_to_mmol_l/index_en.html
+MMOLL_TO_MGDL = 18.0182
+MGDL_TO_MMOLL = 0.0555

tests/api/test_controliq.py

@@ -221,12 +221,10 @@ class TestControlIQApi(unittest.TestCase):
         ciq = ControlIQApi()
         ciq.userGuid = 'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee'
 
-        def fake_get(endpoint, query):
+        def fake_get(raw_endpoint, ignored_query):
+            endpoint, query = raw_endpoint.split("?")
             self.assertTrue(endpoint.endswith("summary/users/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"))
-            self.assertEqual(query, {
-                "startDate": "04-01-2021",
-                "endDate": "04-02-2021"
-            })
+            self.assertEqual(query, "startDate=04-01-2021&endDate=04-02-2021")
 
             return {"faked_json": True}
 

tests/api/test_webui.py

@@ -11,6 +11,7 @@ import requests_mock
 from bs4 import BeautifulSoup
 
 from tconnectsync.api.webui import WebUIScraper
+from tconnectsync.domain.device_settings import Device, Profile, ProfileSegment
 
 from .fake import ControlIQApi
 
@@ -293,30 +294,30 @@ class TestWebUIScraper(unittest.TestCase):
 
             devices = webui.my_devices()
             self.assertDictEqual(devices, {
-                '100001': {
+                '100001': Device(**{
                     'name': 't:slim X2™ Insulin Pump', 
                     'model_number': '001002717', 
                     'status': 'Activated —  Dec 30 2021', 
                     'guid': '00000000-0000-0000-0000-000000000001'
-                },
-                '10000002': {
+                }),
+                '10000002': Device(**{
                     'name': 't:slim X2™ Insulin Pump', 
                     'model_number': '001000354', 
                     'status': 'Activated —  Oct 26 2021', 
                     'guid': '00000000-0000-0000-0000-000000000002'
-                }, 
-                '100003': {
+                }), 
+                '100003': Device(**{
                     'name': 't:slim X2™ Insulin Pump', 
                     'model_number': '001000096', 
                     'status': 'Activated —  Nov 20 2017', 
                     'guid': '00000000-0000-0000-0000-000000000003'
-                }, 
-                'ABCDEFGH': {
+                }), 
+                'ABCDEFGH': Device(**{
                     'name': 'OneTouch Verio IQ', 
                     'model_number': 'VERIO IQ', 
                     'status': 'Activated —  Jan 17 2018', 
                     'guid': None
-                }})
+                })})
     
     PUMP_SETTINGS_HTML = """
 <!DOCTYPE html
@@ -956,42 +957,42 @@ class TestWebUIScraper(unittest.TestCase):
                 text=self.PUMP_SETTINGS_HTML)
 
             profiles, settings = webui.device_settings_from_guid('00000000-0000-0000-0000-000000000001')
-            self.assertListEqual(profiles, [{
+            self.assertListEqual(profiles, [Profile(**{
                 'title': 'A', 
                 'active': True, 
-                'segments': [{
+                'segments': [ProfileSegment(**{
                     'display_time': 'Midnight', 
                     'time': '12:00 AM', 
-                    'basal_rate': '0.800 u/hr', 
-                    'correction_factor': '1u:30 mg/dL', 
-                    'carb_ratio': '1u:6.0 g', 
-                    'target_bg': '110 mg/dL'
-                }, {
+                    'basal_rate': 0.800,
+                    'correction_factor': 30.0,
+                    'carb_ratio': 6.0,
+                    'target_bg_mgdl': 110
+                }), ProfileSegment(**{
                     'display_time': '6:00 AM', 
                     'time': '6:00 AM', 
-                    'basal_rate': '1.250 u/hr', 
-                    'correction_factor': '1u:30 mg/dL', 
-                    'carb_ratio': '1u:6.0 g', 
-                    'target_bg': '110 mg/dL'
-                }, {
+                    'basal_rate': 1.250,
+                    'correction_factor': 30,
+                    'carb_ratio': 6.0,
+                    'target_bg_mgdl': 110
+                }), ProfileSegment(**{
                     'display_time': '11:00 AM', 
                     'time': '11:00 AM', 
-                    'basal_rate': '1.000 u/hr', 
-                    'correction_factor': '1u:30 mg/dL', 
-                    'carb_ratio': '1u:6.0 g', 
-                    'target_bg': '110 mg/dL'
-                }, {
+                    'basal_rate': 1.000,
+                    'correction_factor': 30,
+                    'carb_ratio': 6.0,
+                    'target_bg_mgdl': 110
+                }), ProfileSegment(**{
                     'display_time': 'Noon', 
                     'time': '12:00 PM', 
-                    'basal_rate': '0.800 u/hr', 
-                    'correction_factor': '1u:30 mg/dL', 
-                    'carb_ratio': '1u:6.0 g', 
-                    'target_bg': '110 mg/dL'
-                }], 
-                'calculated_total_daily_basal': '21.65 units', 
-                'insulin_duration': '5:00 hours', 
-                'carbohydrates': 'On'
-            }])
+                    'basal_rate': 0.800,
+                    'correction_factor': 30,
+                    'carb_ratio': 6.0,
+                    'target_bg_mgdl': 110
+                })], 
+                'calculated_total_daily_basal': 21.65, 
+                'insulin_duration_min': 5*60,
+                'carbs_enabled': True
+            })])
 
             self.assertDictEqual(settings, {
                 'Alerts': {

tests/api/test_ws2.py

@@ -10,7 +10,7 @@ from tconnectsync.api.common import ApiException
 class TestWS2Api(unittest.TestCase):
     def fake_get_with_http_500(self, num_times):
         tries = 0
-        def fake_get(endpoint):
+        def fake_get(endpoint, **kwargs):
             nonlocal tries, num_times
             if "therapytimeline2csv" in endpoint:
                 if tries < num_times:
@@ -94,7 +94,7 @@ Report Generated On, 4/24/2021 7:50:04 PM
 
         rawData = self.RAW_DATA_FULL
 
-        def fake_get(endpoint):
+        def fake_get(endpoint, **kwargs):
             nonlocal rawData
             if endpoint == 'therapytimeline2csv/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/2021-04-01/2021-04-02?format=csv':
                 return rawData
@@ -111,7 +111,7 @@ Report Generated On, 4/24/2021 7:50:04 PM
 
         rawData = ""
 
-        def fake_get(endpoint):
+        def fake_get(endpoint, **kwargs):
             nonlocal rawData
             if endpoint == 'therapytimeline2csv/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/2021-04-01/2021-04-02?format=csv':
                 return rawData

tests/test_autoupdate.py

@@ -24,9 +24,9 @@ def build_mock_logger():
         logger.error(*args, **kwargs)
     
     def fake_warn(*args, **kwargs):
-        logger.warn(*args, **kwargs)
+        logger.warning(*args, **kwargs)
         
-    with patch("tconnectsync.autoupdate.logger.error") as mock_error, patch("tconnectsync.autoupdate.logger.warn") as mock_warn:
+    with patch("tconnectsync.autoupdate.logger.error") as mock_error, patch("tconnectsync.autoupdate.logger.warning") as mock_warn:
         mock_error.side_effect = fake_error
         mock_warn.side_effect = fake_warn
         yield (mock_error, mock_warn)