Normalize Treatment, Entry and Device Status object dates to be all in UTC Strings (#4658)

* Normalize Treatment object dates to be all in UTC Strings, as expected by the codebase

* Also normalize the device status data

* Sort the results so results with mixed dates are returned in correct order

* Use MomentJS to correctly parse the offsets & output the offset to objects at all times

* Removing a debug console log message causing excessive logging

* Use UTC date values for queries even if client asks for zoned dates

* Remove extra sorting based on date. Code clarifications

* * Add missing parseZone() call to treatments
* Check isValid() date on entries

* Use isValid() to check date validity for device status data

lib/plugins/index.js

@@ -106,8 +106,6 @@ function init(ctx) {
       }
     });
 
-    console.log("Plugins registered", enabledPlugins);
-
   };
 
   plugins.isPluginEnabled = function isPluginEnabled(pluginName) {

lib/server/devicestatus.js

@@ -1,14 +1,18 @@
 'use strict';
 
+var moment = require('moment');
 var find_options = require('./query');
 
 function storage (collection, ctx) {
   var ObjectID = require('mongodb').ObjectID;
 
   function create(obj, fn) {
-    if (! obj.hasOwnProperty('created_at')){
-      obj.created_at = (new Date()).toISOString();
-    }
+    
+    // Normalize all dates to UTC
+    const d = moment(obj.created_at).isValid() ? moment.parseZone(obj.created_at) : moment();
+    obj.created_at = d.toISOString();
+    obj.utcOffset = d.utcOffset();
+    
     api().insert(obj, function (err, doc) {
       if (err != null && err.message) {
         console.log('Error inserting the device status object', err.message);

lib/server/entries.js

@@ -3,6 +3,7 @@
 var es = require('event-stream');
 var find_options = require('./query');
 var ObjectID = require('mongodb').ObjectID;
+var moment = require('moment');
 
 /**********\
  * Entries
@@ -87,6 +88,16 @@ function storage(env, ctx) {
         totalCreated = 0;
 
     docs.forEach(function(doc) {
+
+      // Normalize dates to be in UTC, store offset in utcOffset
+
+      var _sysTime = moment(doc.dateString).isValid() ? moment.parseZone(doc.dateString) : moment(doc.date);
+      _sysTime = _sysTime.isValid() ? _sysTime : moment();
+
+      doc.utcOffset = _sysTime.utcOffset();
+      doc.sysTime = _sysTime.toISOString();
+      if (doc.dateString) doc.dateString = doc.sysTime;
+
       var query  = (doc.sysTime && doc.type) ? {sysTime: doc.sysTime, type: doc.type} : doc;
       api( ).update(query, doc, {upsert: true}, function (err) {
         firstErr = firstErr || err;

lib/server/query.js

@@ -56,6 +56,17 @@ function default_options (opts) {
 function enforceDateFilter (query, opts) {
   var dateValue = query[opts.dateField];
 
+  // rewrite dates to ISO UTC strings so queries work as expected
+  if (dateValue) {
+    Object.keys(dateValue).forEach(function(key) {
+      let dateString = dateValue[key];
+      if (isNaN(dateString)) {
+        dateString = dateString.replace(' ', '+'); // some clients don't excape the plus
+        dateValue[key] = new Date(dateString).toISOString();
+      }
+    });
+  }
+
   if (!dateValue && !query.dateString && true !== opts.noDateFilter) {
     var minDate = Date.now( ) - opts.deltaAgo;
     query[opts.dateField] = {

lib/server/treatments.js

@@ -2,6 +2,7 @@
 
 var _ = require('lodash');
 var async = require('async');
+var moment = require('moment');
 
 var find_options = require('./query');
 
@@ -143,12 +144,20 @@ function storage (env, ctx) {
 
 function prepareData(obj) {
 
+  // Convert all dates to UTC dates
+
+  const d = moment(obj.created_at).isValid() ? moment.parseZone(obj.created_at) : moment();
+  obj.created_at = d.toISOString();
+
   var results = {
-    //TODO: validate format of created_at
-    created_at: obj.created_at || new Date().toISOString()
+    created_at: obj.created_at
     , preBolusCarbs: ''
   };
 
+  const offset = d.utcOffset();
+  obj.utcOffset = offset;
+  results.offset = offset;
+
   obj.glucose = Number(obj.glucose);
   obj.targetTop = Number(obj.targetTop);
   obj.targetBottom = Number(obj.targetBottom);

tests/api.devicestatus.test.js

@@ -60,6 +60,7 @@ describe('Devicestatus API', function ( ) {
             .expect(200)
             .expect(function (response) {
               response.body[0].xdripjs.state.should.equal(6);
+              response.body[0].utcOffset.should.equal(0);
             })
             .end(function (err) {
               if (err) {

tests/api.entries.test.js

@@ -271,7 +271,9 @@ describe('Entries REST api', function ( ) {
             .set('api-secret', self.env.api_secret || '')
             .expect(200)
             .expect(function (response) {
-              response.body[0].sgv.should.equal('199');
+              var entry = response.body[0];
+              entry.sgv.should.equal('199');
+              entry.utcOffset.should.equal(-420);
             })
             .end(function (err) {
               if (err) {

tests/api.treatments.test.js

@@ -4,6 +4,7 @@ var _ = require('lodash');
 var request = require('supertest');
 var should = require('should');
 var language = require('../lib/language')();
+var _moment = require('moment');
 
 describe('Treatment API', function ( ) {
   this.timeout(10000);
@@ -60,6 +61,44 @@ describe('Treatment API', function ( ) {
     });
   });
 
+
+  it('post single treatments in zoned time format', function (done) {
+   
+    var current_time = Date.now();
+    console.log('Testing date with local format: ', _moment(current_time).format("YYYY-MM-DDTHH:mm:ss.SSSZZ"));
+      
+    self.ctx.treatments().remove({ }, function ( ) {
+      request(self.app)
+        .post('/api/treatments/')
+        .set('api-secret', self.env.api_secret || '')
+        .send({eventType: 'Meal Bolus', created_at: _moment(current_time).format("YYYY-MM-DDTHH:mm:ss.SSSZZ"), carbs: '30', insulin: '2.00', glucose: 100, glucoseType: 'Finger', units: 'mg/dl'})
+        .expect(200)
+        .end(function (err) {
+          if (err) {
+            done(err);
+          } else {
+            self.ctx.treatments.list({}, function (err, list) {
+              var sorted = _.sortBy(list, function (treatment) {
+                return treatment.created_at;
+              });
+              console.log(sorted);
+              sorted.length.should.equal(1);
+              sorted[0].glucose.should.equal(100);
+              should.not.exist(sorted[0].eventTime);
+              sorted[0].insulin.should.equal(2);
+              sorted[0].carbs.should.equal(30);
+              var zonedTime = _moment(current_time).utc().format("YYYY-MM-DDTHH:mm:ss.SSS") + "Z";
+              sorted[0].created_at.should.equal(zonedTime);
+              sorted[0].utcOffset.should.equal(-1* new Date().getTimezoneOffset());
+              done();
+            });
+          }
+        });
+
+    });
+  });
+
+
   it('post a treatment array', function (done) {
     self.ctx.treatments().remove({ }, function ( ) {
       request(self.app)
@@ -101,7 +140,7 @@ describe('Treatment API', function ( ) {
           , {eventType: 'BG Check', glucose: 100, units: 'mg/dl', created_at: now}
           , {eventType: 'BG Check', glucose: 100, units: 'mg/dl', created_at: now}
           , {eventType: 'BG Check', glucose: 100, units: 'mg/dl', created_at: now}
-          , {eventType: 'Meal Bolus', carbs: '30', insulin: '2.00', preBolus: '15', glucose: 100, glucoseType: 'Finger', units: 'mg/dl'}
+          , {eventType: 'Meal Bolus', created_at: now, carbs: '30', insulin: '2.00', preBolus: '15', glucose: 100, glucoseType: 'Finger', units: 'mg/dl'}
         ])
         .expect(200)
         .end(function (err) {
@@ -125,6 +164,7 @@ describe('Treatment API', function ( ) {
         });
     });
   });
+
   it('post a treatment, query, delete, verify gone', function (done) {
     // insert a treatment - needs to be unique from example data
     console.log('Inserting treatment entry');