Merge pull request #8474 from nightscout/fix/mongo-save-hardening

Fix/mongo save hardening

lib/api/activity/index.js

@@ -6,30 +6,7 @@ var _isArray = require('lodash/isArray');
 
 var consts = require('../../constants');
 var moment = require('moment');
-
+var objectIdValidation = require('../shared/objectid-validation');
-/**
- * Validate MongoDB ObjectId format.
- * Accepts: undefined, null, or 24-character hex string.
- * Rejects: anything else (UUIDs, short strings, numbers, objects).
- */
-function isValidObjectId(id) {
-    if (id === undefined || id === null) return true;
-    if (typeof id !== 'string') return false;
-    return /^[a-fA-F0-9]{24}$/.test(id);
-}
-
-/**
- * Validate _id field for each document in an array.
- * @returns {Object|null} - null if all valid, or {index, id} of first invalid
- */
-function findInvalidId(docs) {
-    for (var i = 0; i < docs.length; i++) {
-        if (!isValidObjectId(docs[i]._id)) {
-            return { index: i, id: docs[i]._id };
-        }
-    }
-    return null;
-}
 
 function configure(app, wares, ctx) {
     var express = require('express')
@@ -98,7 +75,7 @@ function configure(app, wares, ctx) {
             }
 
             // Validate _id fields before storage (return 400 on invalid)
-            var invalid = findInvalidId(activity);
+            var invalid = objectIdValidation.findInvalidId(activity);
             if (invalid) {
                 return res.sendJSONStatus(res, consts.HTTP_BAD_REQUEST,
                     'Invalid _id format', 'Must be 24-character hex string or omit for auto-generation. Got: ' + String(invalid.id));
@@ -119,7 +96,7 @@ function configure(app, wares, ctx) {
 
         api.delete('/activity/:_id', ctx.authorization.isPermitted('api:activity:delete'), function(req, res) {
             // Validate _id parameter
-            if (!isValidObjectId(req.params._id)) {
+            if (!objectIdValidation.isValidObjectId(req.params._id)) {
                 return res.sendJSONStatus(res, consts.HTTP_BAD_REQUEST,
                     'Invalid _id format', 'Must be 24-character hex string. Got: ' + String(req.params._id));
             }
@@ -133,7 +110,7 @@ function configure(app, wares, ctx) {
             var data = req.body;
 
             // Validate _id if provided
-            if (!isValidObjectId(data._id)) {
+            if (!objectIdValidation.isValidObjectId(data._id)) {
                 return res.sendJSONStatus(res, consts.HTTP_BAD_REQUEST,
                     'Invalid _id format', 'Must be 24-character hex string. Got: ' + String(data._id));
             }
@@ -159,4 +136,3 @@ function configure(app, wares, ctx) {
 }
 
 module.exports = configure;
-

lib/api/devicestatus/index.js

@@ -5,33 +5,7 @@ const moment = require('moment');
 const { query } = require('express');
 const _take = require('lodash/take');
 const _ = require('lodash');
-
+const objectIdValidation = require('../shared/objectid-validation');
-/**
- * Validate MongoDB ObjectId format.
- * Accepts: undefined, null, or 24-character hex string.
- * Rejects: anything else (UUIDs, short strings, numbers, objects).
- * @param {*} id - The _id value to validate
- * @returns {boolean} - true if valid or empty, false if invalid format
- */
-function isValidObjectId(id) {
-    if (id === undefined || id === null) return true;  // Will auto-generate
-    if (typeof id !== 'string') return false;
-    return /^[a-fA-F0-9]{24}$/.test(id);
-}
-
-/**
- * Validate _id field for each document in an array.
- * @param {Array} docs - Array of documents to validate
- * @returns {Object|null} - null if all valid, or {index, id} of first invalid
- */
-function findInvalidId(docs) {
-    for (var i = 0; i < docs.length; i++) {
-        if (!isValidObjectId(docs[i]._id)) {
-            return { index: i, id: docs[i]._id };
-        }
-    }
-    return null;
-}
 
 function configure (app, wares, ctx, env) {
   var express = require('express')
@@ -102,7 +76,7 @@ function configure (app, wares, ctx, env) {
       }
 
       // Validate _id fields before storage (return 400 on invalid)
-      var invalid = findInvalidId(statuses);
+      var invalid = objectIdValidation.findInvalidId(statuses);
       if (invalid) {
         return res.sendJSONStatus(res, consts.HTTP_BAD_REQUEST,
           'Invalid _id format', 'Must be 24-character hex string or omit for auto-generation. Got: ' + String(invalid.id));
@@ -154,7 +128,7 @@ function configure (app, wares, ctx, env) {
 
     api.delete('/devicestatus/:id', ctx.authorization.isPermitted('api:devicestatus:delete'), function(req, res, next) {
       // Validate _id parameter (unless wildcard)
-      if (req.params.id !== '*' && !isValidObjectId(req.params.id)) {
+      if (req.params.id !== '*' && !objectIdValidation.isValidObjectId(req.params.id)) {
         return res.sendJSONStatus(res, consts.HTTP_BAD_REQUEST,
           'Invalid _id format', 'Must be 24-character hex string. Got: ' + String(req.params.id));
       }

lib/api/food/index.js

@@ -2,30 +2,7 @@
 
 var _isArray = require('lodash/isArray');
 var consts = require('../../constants');
-
+var objectIdValidation = require('../shared/objectid-validation');
-/**
- * Validate MongoDB ObjectId format.
- * Accepts: undefined, null, or 24-character hex string.
- * Rejects: anything else (UUIDs, short strings, numbers, objects).
- */
-function isValidObjectId(id) {
-    if (id === undefined || id === null) return true;
-    if (typeof id !== 'string') return false;
-    return /^[a-fA-F0-9]{24}$/.test(id);
-}
-
-/**
- * Validate _id field for each document in an array.
- * @returns {Object|null} - null if all valid, or {index, id} of first invalid
- */
-function findInvalidId(docs) {
-    for (var i = 0; i < docs.length; i++) {
-        if (!isValidObjectId(docs[i]._id)) {
-            return { index: i, id: docs[i]._id };
-        }
-    }
-    return null;
-}
 
 function configure (app, wares, ctx) {
     var express = require('express'),
@@ -75,7 +52,7 @@ function configure (app, wares, ctx) {
         }
 
         // Validate _id fields before storage (return 400 on invalid)
-        var invalid = findInvalidId(data);
+        var invalid = objectIdValidation.findInvalidId(data);
         if (invalid) {
             return res.sendJSONStatus(res, consts.HTTP_BAD_REQUEST,
                 'Invalid _id format', 'Must be 24-character hex string or omit for auto-generation. Got: ' + String(invalid.id));
@@ -103,7 +80,7 @@ function configure (app, wares, ctx) {
         }
 
         // Validate _id fields before storage (return 400 on invalid)
-        var invalid = findInvalidId(data);
+        var invalid = objectIdValidation.findInvalidId(data);
         if (invalid) {
             return res.sendJSONStatus(res, consts.HTTP_BAD_REQUEST,
                 'Invalid _id format', 'Must be 24-character hex string. Got: ' + String(invalid.id));
@@ -123,7 +100,7 @@ function configure (app, wares, ctx) {
       // delete record
       api.delete('/food/:_id', ctx.authorization.isPermitted('api:food:delete'), function(req, res) {
         // Validate _id parameter
-        if (!isValidObjectId(req.params._id)) {
+        if (!objectIdValidation.isValidObjectId(req.params._id)) {
             return res.sendJSONStatus(res, consts.HTTP_BAD_REQUEST,
                 'Invalid _id format', 'Must be 24-character hex string. Got: ' + String(req.params._id));
         }
@@ -141,4 +118,3 @@ function configure (app, wares, ctx) {
 }
 
 module.exports = configure;
-

lib/api/profile/index.js

@@ -1,33 +1,7 @@
 'use strict';
 
 var consts = require('../../constants');
-
+var objectIdValidation = require('../shared/objectid-validation');
-/**
- * Validate MongoDB ObjectId format.
- * Accepts: undefined, null, or 24-character hex string.
- * Rejects: anything else (UUIDs, short strings, numbers, objects).
- * @param {*} id - The _id value to validate
- * @returns {boolean} - true if valid or empty, false if invalid format
- */
-function isValidObjectId(id) {
-    if (id === undefined || id === null) return true;  // Will auto-generate
-    if (typeof id !== 'string') return false;
-    return /^[a-fA-F0-9]{24}$/.test(id);
-}
-
-/**
- * Validate _id field for each document in an array.
- * @param {Array} docs - Array of documents to validate
- * @returns {Object|null} - null if all valid, or {index, id} of first invalid
- */
-function findInvalidId(docs) {
-    for (var i = 0; i < docs.length; i++) {
-        if (!isValidObjectId(docs[i]._id)) {
-            return { index: i, id: docs[i]._id };
-        }
-    }
-    return null;
-}
 
 function configure (app, wares, ctx) {
     var express = require('express'),
@@ -97,7 +71,7 @@ function configure (app, wares, ctx) {
             }
 
             // Validate _id fields before storage (return 400 on invalid)
-            var invalid = findInvalidId(data);
+            var invalid = objectIdValidation.findInvalidId(data);
             if (invalid) {
                 return res.sendJSONStatus(res, consts.HTTP_BAD_REQUEST,
                     'Invalid _id format', 'Must be 24-character hex string or omit for auto-generation. Got: ' + String(invalid.id));
@@ -125,7 +99,7 @@ function configure (app, wares, ctx) {
             var data = req.body;
 
             // Validate _id if provided (required for PUT, must be valid format)
-            if (!isValidObjectId(data._id)) {
+            if (!objectIdValidation.isValidObjectId(data._id)) {
                 return res.sendJSONStatus(res, consts.HTTP_BAD_REQUEST,
                     'Invalid _id format', 'Must be 24-character hex string. Got: ' + String(data._id));
             }
@@ -145,7 +119,7 @@ function configure (app, wares, ctx) {
 
         api.delete('/profile/:_id', ctx.authorization.isPermitted('api:profile:delete'), function(req, res) {
             // Validate _id parameter
-            if (!isValidObjectId(req.params._id)) {
+            if (!objectIdValidation.isValidObjectId(req.params._id)) {
                 return res.sendJSONStatus(res, consts.HTTP_BAD_REQUEST,
                     'Invalid _id format', 'Must be 24-character hex string. Got: ' + String(req.params._id));
             }
@@ -164,4 +138,3 @@ function configure (app, wares, ctx) {
 }
 
 module.exports = configure;
-

lib/api/shared/objectid-validation.js

@@ -0,0 +1,26 @@
+'use strict';
+
+var OBJECT_ID_PATTERN = /^[a-fA-F0-9]{24}$/;
+
+function isValidObjectId(id) {
+  if (id === undefined || id === null) {
+    return true;
+  }
+
+  return typeof id === 'string' && OBJECT_ID_PATTERN.test(id);
+}
+
+function findInvalidId(docs) {
+  for (var i = 0; i < docs.length; i++) {
+    if (!isValidObjectId(docs[i]._id)) {
+      return { index: i, id: docs[i]._id };
+    }
+  }
+
+  return null;
+}
+
+module.exports = {
+  findInvalidId: findInvalidId,
+  isValidObjectId: isValidObjectId
+};

lib/api3/storage/mongoCollection/find.js

@@ -16,6 +16,14 @@ function toSafeInt (value, defaultValue) {
   return Number.isFinite(parsed) ? parsed : defaultValue;
 }
 
+function normalizeDocs (docs, options) {
+  if (!options || options.normalize !== false) {
+    _.each(docs, utils.normalizeDoc);
+  }
+
+  return docs;
+}
+
 
 /**
  * Find single document by identifier
@@ -24,27 +32,15 @@ function toSafeInt (value, defaultValue) {
  * @param {Object} projection
  * @param {Object} options
  */
-function findOne (col, identifier, projection, options) {
+async function findOne (col, identifier, projection, options) {
 
-  return new Promise(function (resolve, reject) {
+  const filter = utils.filterForOne(identifier);
+  const result = await col.find(filter)
+    .project(projection)
+    .sort({ identifier: -1 }) // document with identifier first (not the fallback one)
+    .toArray();
 
-    const filter = utils.filterForOne(identifier);
+  return normalizeDocs(result, options);
-
-    col.find(filter)
-      .project(projection)
-      .sort({ identifier: -1 }) // document with identifier first (not the fallback one)
-      .toArray(function mongoDone (err, result) {
-
-        if (err) {
-          reject(err);
-        } else {
-          if (!options || options.normalize !== false) {
-            _.each(result, utils.normalizeDoc);
-          }
-          resolve(result);
-        }
-      });
-  });
 }
 
 
@@ -55,56 +51,33 @@ function findOne (col, identifier, projection, options) {
  * @param {Object} projection
  * @param {Object} options
  */
-function findOneFilter (col, filter, projection, options) {
+async function findOneFilter (col, filter, projection, options) {
 
-  return new Promise(function (resolve, reject) {
+  const result = await col.find(filter)
+    .project(projection)
+    .sort({ identifier: -1 }) // document with identifier first (not the fallback one)
+    .toArray();
 
-    col.find(filter)
+  return normalizeDocs(result, options);
-      .project(projection)
-      .sort({ identifier: -1 }) // document with identifier first (not the fallback one)
-      .toArray(function mongoDone (err, result) {
-
-        if (err) {
-          reject(err);
-        } else {
-          if (!options || options.normalize !== false) {
-            _.each(result, utils.normalizeDoc);
-          }
-          resolve(result);
-        }
-      });
-  });
 }
 
 
 /**
  * Find many documents matching the filtering criteria
  */
-function findMany (col, args) {
+async function findMany (col, args) {
   const logicalOperator = args.logicalOperator || 'and';
-  return new Promise(function (resolve, reject) {
+  const filter = utils.parseFilter(args.filter, logicalOperator, args.onlyValid);
+  const safeLimit = toSafeInt(args.limit, 1000);
+  const safeSkip = toSafeInt(args.skip, 0);
+  const result = await col.find(filter)
+    .sort(args.sort)
+    .limit(safeLimit)
+    .skip(safeSkip)
+    .project(args.projection)
+    .toArray();
 
-    const filter = utils.parseFilter(args.filter, logicalOperator, args.onlyValid);
+  return normalizeDocs(result, args.options);
-    const safeLimit = toSafeInt(args.limit, 1000);
-    const safeSkip = toSafeInt(args.skip, 0);
-
-    col.find(filter)
-      .sort(args.sort)
-      .limit(safeLimit)
-      .skip(safeSkip)
-      .project(args.projection)
-      .toArray(function mongoDone (err, result) {
-
-        if (err) {
-          reject(err);
-        } else {
-          if (!args.options || args.options.normalize !== false) {
-            _.each(result, utils.normalizeDoc);
-          }
-          resolve(result);
-        }
-      });
-  });
 }
 
 

lib/api3/storage/mongoCollection/index.js

@@ -46,44 +46,29 @@ function MongoCollection (ctx, env, colName) {
   /**
    * Get server version
    */
-  self.version = function version () {
+  self.version = async function version () {
 
-    return new Promise(function (resolve, reject) {
+    const result = await ctx.store.db.admin().buildInfo();
 
-      ctx.store.db.admin().buildInfo({}, function mongoDone (err, result) {
+    return {
-
+      storage: 'mongodb',
-        err
+      version: result.version
-          ? reject(err)
+    };
-          : resolve({
-            storage: 'mongodb',
-            version: result.version
-          });
-      });
-    });
   };
 
 
   /**
    * Get timestamp (e.g. srvModified) of the last modified document
    */
-  self.getLastModified = function getLastModified (fieldName) {
+  self.getLastModified = async function getLastModified (fieldName) {
 
-    return new Promise(function (resolve, reject) {
+    const [ result ] = await self.col.find()
+      .sort({ [fieldName]: -1 })
+      .limit(1)
+      .project({ [fieldName]: 1 })
+      .toArray();
 
-      self.col.find()
+    return result;
-
-        .sort({ [fieldName]: -1 })
-
-        .limit(1)
-
-        .project({ [fieldName]: 1 })
-
-        .toArray(function mongoDone (err, [ result ]) {
-          err
-            ? reject(err)
-            : resolve(result);
-        });
-    });
   }
 }
 

lib/api3/storage/mongoCollection/modify.js

@@ -9,24 +9,16 @@ const utils = require('./utils')
  * @param {Object} doc
  * @param {Object} options
  */
-function insertOne (col, doc, options) {
+async function insertOne (col, doc, options) {
 
-  return new Promise(function (resolve, reject) {
+  const result = await col.insertOne(doc);
+  const identifier = doc.identifier || result.insertedId.toString();
 
-    col.insertOne(doc, function mongoDone(err, result) {
+  if (!options || options.normalize !== false) {
+    delete doc._id;
+  }
 
-      if (err) {
+  return identifier;
-        reject(err);
-      } else {
-        const identifier = doc.identifier || result.insertedId.toString();
-
-        if (!options || options.normalize !== false) {
-          delete doc._id;
-        }
-        resolve(identifier);
-      }
-    });
-  });
 }
 
 
@@ -36,20 +28,12 @@ function insertOne (col, doc, options) {
  * @param {string} identifier
  * @param {Object} doc
  */
-function replaceOne (col, identifier, doc) {
+async function replaceOne (col, identifier, doc) {
 
-  return new Promise(function (resolve, reject) {
+  const filter = utils.filterForOne(identifier);
+  const result = await col.replaceOne(filter, doc, { upsert: true });
 
-    const filter = utils.filterForOne(identifier);
+  return result.matchedCount;
-
-    col.replaceOne(filter, doc, { upsert: true }, function mongoDone(err, result) {
-      if (err) {
-        reject(err);
-      } else {
-        resolve(result.matchedCount);
-      }
-    });
-  });
 }
 
 
@@ -59,20 +43,12 @@ function replaceOne (col, identifier, doc) {
  * @param {string} identifier
  * @param {object} setFields
  */
-function updateOne (col, identifier, setFields) {
+async function updateOne (col, identifier, setFields) {
 
-  return new Promise(function (resolve, reject) {
+  const filter = utils.filterForOne(identifier);
+  const result = await col.updateOne(filter, { $set: setFields });
 
-    const filter = utils.filterForOne(identifier);
+  return { updated: result.modifiedCount };
-
-    col.updateOne(filter, { $set: setFields }, function mongoDone(err, result) {
-      if (err) {
-        reject(err);
-      } else {
-        resolve({ updated: result.modifiedCount });
-      }
-    });
-  });
 }
 
 
@@ -81,40 +57,24 @@ function updateOne (col, identifier, setFields) {
  * @param {Object} col
  * @param {string} identifier
  */
-function deleteOne (col, identifier) {
+async function deleteOne (col, identifier) {
 
-  return new Promise(function (resolve, reject) {
+  const filter = utils.filterForOne(identifier);
+  const result = await col.deleteOne(filter);
 
-    const filter = utils.filterForOne(identifier);
+  return { deleted: result.deletedCount };
-
-    col.deleteOne(filter, function mongoDone(err, result) {
-      if (err) {
-        reject(err);
-      } else {
-        resolve({ deleted: result.deletedCount });
-      }
-    });
-  });
 }
 
 
 /**
  * Permanently remove many documents matching any of filtering criteria
  */
-function deleteManyOr (col, filterDef) {
+async function deleteManyOr (col, filterDef) {
 
-  return new Promise(function (resolve, reject) {
+  const filter = utils.parseFilter(filterDef, 'or');
+  const result = await col.deleteMany(filter);
 
-    const filter = utils.parseFilter(filterDef, 'or');
+  return { deleted: result.deletedCount };
-
-    col.deleteMany(filter, function mongoDone(err, result) {
-      if (err) {
-        reject(err);
-      } else {
-        resolve({ deleted: result.deletedCount });
-      }
-    });
-  });
 }
 
 

lib/api3/storage/mongoCollection/utils.js

@@ -2,7 +2,7 @@
 
 const _ = require('lodash')
   , checkForHexRegExp = new RegExp("^[0-9a-fA-F]{24}$")
-  , ObjectID = require('mongodb-legacy').ObjectId
+  , ObjectID = require('mongodb').ObjectId
 ;
 
 

lib/authorization/storage.js

@@ -3,7 +3,8 @@
 var _ = require('lodash');
 var crypto = require('crypto');
 var shiroTrie = require('shiro-trie');
-var ObjectID = require('mongodb-legacy').ObjectId;
+var ObjectID = require('mongodb').ObjectId;
+var runWithCallback = require('../storage/run-with-callback');
 
 var find_options = require('../server/query');
 
@@ -22,27 +23,44 @@ function init (env, ctx) {
     return find_options(opts, storage.queryOpts);
   }
 
+  function normalizeRequiredObjectId(id) {
+    if (id === undefined || id === null || id === '') {
+      return { error: 'Missing _id for update' };
+    }
+
+    try {
+      return { value: new ObjectID(id) };
+    } catch (err) {
+      return { error: 'Invalid _id format: ' + String(id) };
+    }
+  }
+
   function create (collection) {
     function doCreate(obj, fn) {
       if (!Object.prototype.hasOwnProperty.call(obj, 'created_at')) {
         obj.created_at = (new Date()).toISOString();
       }
-      collection.insertOne(obj, function (err, doc) {
+
-        if (err != null && err.message) {
+      return runWithCallback(async function () {
-          console.log('Data insertion error', err.message);
+        try {
-          fn(err.message, null);
+          await collection.insertOne(obj);
-          return;
+        } catch (err) {
+          if (err != null && err.message) {
+            console.log('Data insertion error', err.message);
+            throw err.message;
+          }
+          throw err;
         }
-        storage.reload(function loaded() {
+
-          fn(null, obj);
+        await storageReload();
-        });
+        return obj;
-      });
+      }, fn);
     }
     return doCreate;
   }
 
   function list (collection) {
-     function doList(opts, fn) {
+    function doList(opts, fn) {
       // these functions, find, sort, and limit, are used to
       // dynamically configure the request, based on the options we've
       // been given
@@ -60,18 +78,14 @@ function init (env, ctx) {
         return this;
       }
 
-      // handle all the results
-      function toArray(err, entries) {
-        fn(err, entries);
-      }
-
       console.log('Loading',opts);
 
-      // now just stitch them all together
+      return runWithCallback(function () {
-      limit.call(collection
+        return limit.call(collection
           .find(query_for(opts))
           .sort(sort())
-      ).toArray(toArray);
+        ).toArray();
+      }, fn);
     }
 
     return doList;
@@ -79,27 +93,33 @@ function init (env, ctx) {
 
   function remove (collection) {
     function doRemove (_id, callback) {
-      collection.deleteOne({ '_id': new ObjectID(_id) }, function (err) {
+      return runWithCallback(async function () {
-        storage.reload(function loaded() {
+        await collection.deleteOne({ '_id': new ObjectID(_id) });
-          callback(err, null);
+        await storageReload();
-        });
+        return null;
-      });
+      }, callback);
     }
     return doRemove;
   }
 
   function save (collection) {
     function doSave (obj, callback) {
-      obj._id = new ObjectID(obj._id);
+      var idResult = normalizeRequiredObjectId(obj && obj._id);
+      if (idResult.error) {
+        callback(idResult.error, null);
+        return;
+      }
+
+      obj._id = idResult.value;
       if (!obj.created_at) {
         obj.created_at = (new Date()).toISOString();
       }
-      collection.insertOne(obj, function (err) {
+
-        //id should be added for new docs
+      return runWithCallback(async function () {
-        storage.reload(function loaded() {
+        await collection.replaceOne({ _id: obj._id }, obj, { upsert: true });
-          callback(err, obj);
+        await storageReload();
-        });
+        return obj;
-      });
+      }, callback);
     }
     return doSave;
   }
@@ -183,6 +203,20 @@ function init (env, ctx) {
 
   };
 
+  function storageReload () {
+    return runWithCallback(function () {
+      return new Promise(function (resolve, reject) {
+        storage.reload(function loaded(err) {
+          if (err) {
+            reject(err);
+            return;
+          }
+          resolve();
+        });
+      });
+    });
+  }
+
   storage.findRole = function findRole (roleName) {
     return _.find(storage.roles, {name: roleName});
   };

lib/data/dataloader.js

@@ -493,19 +493,27 @@ function loadDeviceStatus(ddata, env, ctx, callback) {
 }
 
 function loadDatabaseStats(ddata, ctx, callback) {
-    ctx.store.db.stats(function mongoDone (err, result) {
+    Promise.resolve()
+      .then(function () {
+        return ctx.store.db.stats();
+      })
+      .then(function (result) {
+        if (result) {
+          ddata.dbstats = {
+            dataSize: result.dataSize,
+            indexSize: result.indexSize
+          };
+        }
+      })
+      .catch(function (err) {
+        console.log("Problem loading database stats");
         if (err) {
-            console.log("Problem loading database stats");
+          console.error(err);
-        }
-        if (!err && result) {
-            ddata.dbstats = { 
-                  dataSize:  result.dataSize
-                , indexSize: result.indexSize
-            };
         }
+      })
+      .finally(function () {
         callback();
-    });
+      });
 }
 
 module.exports = init;
-

lib/server/activity.js

@@ -1,10 +1,19 @@
 'use strict';
 
 var find_options = require('./query');
+var runWithCallback = require('../storage/run-with-callback');
 
 
 function storage (env, ctx) {
-   var ObjectID = require('mongodb-legacy').ObjectId;
+   var ObjectID = require('mongodb').ObjectId;
+
+  function normalizeObjectId(id) {
+    try {
+      return new ObjectID(id);
+    } catch (err) {
+      return new ObjectID();
+    }
+  }
 
   function create (docs, fn) {
     if (docs.length === 0) {
@@ -26,10 +35,14 @@ function storage (env, ctx) {
       };
     });
 
-    api().bulkWrite(bulkOps, { ordered: true }, function(err, bulkResult) {
+    return runWithCallback(async function () {
-      if (err) {
+      var bulkResult;
+
+      try {
+        bulkResult = await api().bulkWrite(bulkOps, { ordered: true });
+      } catch (err) {
         console.error('Problem upserting activity batch', err);
-        return fn(err, []);
+        throw err;
       }
 
       // Assign _ids from upserted results
@@ -39,20 +52,26 @@ function storage (env, ctx) {
         });
       }
 
-      fn(null, docs);
+      return docs;
+    }, function (err, result) {
+      if (err) {
+        fn(err, []);
+        return;
+      }
+      fn(null, result);
     });
   }
 
 
   function save (obj, fn) {
-    obj._id = new ObjectID(obj._id);
+    obj._id = normalizeObjectId(obj._id);
     if (!Object.prototype.hasOwnProperty.call(obj, 'created_at')) {
       obj.created_at = (new Date( )).toISOString( );
     }
-    api().insertOne(obj, function (err) {
+    return runWithCallback(async function () {
-      //id should be added for new docs
+      await api().replaceOne({ _id: obj._id }, obj, { upsert: true });
-      fn(err, obj);
+      return obj;
-    });
+    }, fn);
   }
 
   function query_for (opts) {
@@ -77,21 +96,19 @@ function storage (env, ctx) {
       return this;
     }
 
-    // handle all the results
+    return runWithCallback(function () {
-    function toArray (err, entries) {
+      return limit.call(api( )
-      fn(err, entries);
-    }
-
-    // now just stitch them all together
-    limit.call(api( )
         .find(query_for(opts))
         .sort(sort( ))
-    ).toArray(toArray);
+      ).toArray();
+    }, fn);
   }
   
   function remove (_id, fn) {
     var objId = new ObjectID(_id);
-    return api( ).deleteOne({ '_id': objId }, fn);
+    return runWithCallback(function () {
+      return api().deleteOne({ '_id': objId });
+    }, fn);
   }
 
   function api ( ) {
@@ -112,4 +129,3 @@ storage.queryOpts = {
 };
 
 module.exports = storage;
-

lib/server/aggregate.js

@@ -1,4 +1,5 @@
 var find_options = require('./query');
+var runWithCallback = require('../storage/run-with-callback');
 
 function create (conf, api) {
 
@@ -21,7 +22,9 @@ function create (conf, api) {
     var groupBy = [ {$match: query } ].concat(pipeline).concat(template( ));
     console.log('$match query', query);
     console.log('AGGREGATE', groupBy);
-    api( ).aggregate(groupBy, done);
+    return runWithCallback(function () {
+      return api().aggregate(groupBy).toArray();
+    }, done);
   }
 
   return aggregate;
@@ -29,4 +32,3 @@ function create (conf, api) {
 }
 
 module.exports = create;
-

lib/server/devicestatus.js

@@ -2,6 +2,7 @@
 
 var moment = require('moment');
 var find_options = require('./query');
+var runWithCallback = require('../storage/run-with-callback');
 
 function truncatePredictions (obj, maxSize) {
   if (!maxSize || maxSize <= 0) return obj;
@@ -52,12 +53,15 @@ function storage (env, ctx) {
       truncatePredictions(obj, predictionsMaxSize);
     });
 
-    // Use insertMany for batch insert
+    return runWithCallback(async function () {
-    api().insertMany(statuses, { ordered: true }, function(err, insertResult) {
+      var insertResult;
-      if (err) {
+
+      try {
+        // Use insertMany for batch insert
+        insertResult = await api().insertMany(statuses, { ordered: true });
+      } catch (err) {
         console.log('Error inserting device status objects', err.message);
-        fn(err.message || err, null);
+        throw err.message || err;
-        return;
       }
 
       // Assign _ids from insertMany result
@@ -75,8 +79,8 @@ function storage (env, ctx) {
       });
 
       ctx.bus.emit('data-received');
-      fn(null, statuses);
+      return statuses;
-    });
+    }, fn);
   }
 
   function last (fn) {
@@ -111,24 +115,19 @@ function storage (env, ctx) {
       return this;
     }
 
-    // handle all the results
+    return runWithCallback(function () {
-    function toArray (err, entries) {
+      return limit.call(api()
-      fn(err, entries);
+        .find(query_for(opts))
-    }
+        .sort(sort())
-
+      ).toArray();
-    // now just stitch them all together
+    }, fn);
-    limit.call(api()
-      .find(query_for(opts))
-      .sort(sort())
-    ).toArray(toArray);
   }
 
   function remove (opts, fn) {
 
-    function removed (err, stat) {
+    return runWithCallback(async function () {
-
+      var stat = await api().deleteMany(query_for(opts));
-      console.log('removed', err, stat);
+      console.log('removed', null, stat);
-
       ctx.bus.emit('data-update', {
         type: 'devicestatus'
         , op: 'remove'
@@ -136,11 +135,8 @@ function storage (env, ctx) {
         , changes: opts.find._id
       });
 
-      fn(err, stat);
+      return stat;
-    }
+    }, fn);
-
-    return api().deleteMany(
-      query_for(opts), removed);
   }
 
   function api () {

lib/server/entries.js

@@ -2,8 +2,9 @@
 
 var es = require('event-stream');
 var find_options = require('./query');
-var ObjectId = require('mongodb-legacy').ObjectId;
+var ObjectId = require('mongodb').ObjectId;
 var moment = require('moment');
+var runWithCallback = require('../storage/run-with-callback');
 
 // REQ-SYNC-072: Pattern to match valid MongoDB ObjectId hex strings
 var OBJECT_ID_HEX_RE = /^[0-9a-fA-F]{24}$/;
@@ -37,20 +38,17 @@ function storage (env, ctx) {
     }
 
     // handle all the results
-    function toArray (err, entries) {
+    return runWithCallback(function () {
-      fn(err, entries);
+      return limit.call(api()
-    }
+        .find(query_for(opts))
-
+        .sort(sort())
-    // now just stitch them all together
+      ).toArray();
-    limit.call(api()
+    }, fn);
-      .find(query_for(opts))
-      .sort(sort())
-    ).toArray(toArray);
   }
 
   function remove (opts, fn) {
-    api().deleteMany(query_for(opts), function(err, stat) {
+    return runWithCallback(async function () {
-
+      var stat = await api().deleteMany(query_for(opts));
       ctx.bus.emit('data-update', {
         type: 'entries'
         , op: 'remove'
@@ -60,8 +58,8 @@ function storage (env, ctx) {
 
       //TODO: this is triggering a read from Mongo, we can do better
       ctx.bus.emit('data-received');
-      fn(err, stat);
+      return stat;
-    });
+    }, fn);
   }
 
   // return writable stream to lint each sgv record passing through it
@@ -126,12 +124,15 @@ function storage (env, ctx) {
       };
     });
 
-    // Use bulkWrite for batch upsert
+    return runWithCallback(async function () {
-    api().bulkWrite(bulkOps, { ordered: true }, function(err, bulkResult) {
+      var bulkResult;
-      if (err) {
+
+      try {
+        // Use bulkWrite for batch upsert
+        bulkResult = await api().bulkWrite(bulkOps, { ordered: true });
+      } catch (err) {
         console.error('Problem upserting entries batch', err);
-        fn(err, docs);
+        throw err;
-        return;
       }
 
       // Assign _ids from upserted results
@@ -148,18 +149,20 @@ function storage (env, ctx) {
       });
 
       ctx.bus.emit('data-received');
-      fn(null, docs);
+      return docs;
+    }, function (err, result) {
+      if (err) {
+        fn(err, docs);
+        return;
+      }
+      fn(null, result);
     });
   }
 
   function getEntry (id, fn) {
-    api().findOne({ "_id": new ObjectId(id) }, function(err, entry) {
+    return runWithCallback(function () {
-      if (err) {
+      return api().findOne({ "_id": new ObjectId(id) });
-        fn(err);
+    }, fn);
-      } else {
-        fn(null, entry);
-      }
-    });
   }
 
   function query_for (opts) {

lib/server/food.js

@@ -1,7 +1,16 @@
 'use strict';
 
 function storage (env, ctx) {
-   var ObjectID = require('mongodb-legacy').ObjectId;
+   var ObjectID = require('mongodb').ObjectId;
+  var runWithCallback = require('../storage/run-with-callback');
+
+  function normalizeObjectId(id) {
+    try {
+      return new ObjectID(id);
+    } catch (err) {
+      return new ObjectID();
+    }
+  }
 
   function create (docs, fn) {
     // Normalize to array for consistent handling (allows direct storage calls with single objects)
@@ -26,10 +35,14 @@ function storage (env, ctx) {
       };
     });
 
-    api().bulkWrite(bulkOps, { ordered: true }, function(err, bulkResult) {
+    return runWithCallback(async function () {
-      if (err) {
+      var bulkResult;
+
+      try {
+        bulkResult = await api().bulkWrite(bulkOps, { ordered: true });
+      } catch (err) {
         console.error('Problem upserting food batch', err);
-        return fn(err, []);
+        throw err;
       }
 
       // Assign _ids from upserted results
@@ -39,7 +52,13 @@ function storage (env, ctx) {
         });
       }
 
-      fn(null, docs);
+      return docs;
+    }, function (err, result) {
+      if (err) {
+        fn(err, []);
+        return;
+      }
+      fn(null, result);
     });
   }
 
@@ -55,30 +74,28 @@ function storage (env, ctx) {
 
     // Build bulkWrite operations for batch upsert
     var bulkOps = docs.map(function(doc) {
-      try {
+      doc._id = normalizeObjectId(doc._id);
-        doc._id = new ObjectID(doc._id);
-      } catch (err){
-        console.error(err);
-        doc._id = new ObjectID();
-      }
       if (!doc.created_at) {
         doc.created_at = (new Date()).toISOString();
       }
 
-      var query = (doc.created_at && doc._id) ? { _id: doc._id, created_at: doc.created_at } : doc;
       return {
         replaceOne: {
-          filter: query,
+          filter: { _id: doc._id },
           replacement: doc,
           upsert: true
         }
       };
     });
 
-    api().bulkWrite(bulkOps, { ordered: true }, function(err, bulkResult) {
+    return runWithCallback(async function () {
-      if (err) {
+      var bulkResult;
+
+      try {
+        bulkResult = await api().bulkWrite(bulkOps, { ordered: true });
+      } catch (err) {
         console.error('Problem saving food batch', err);
-        return fn(err, []);
+        throw err;
       }
 
       // Assign _ids from upserted results
@@ -88,25 +105,39 @@ function storage (env, ctx) {
         });
       }
 
-      fn(null, docs);
+      return docs;
+    }, function (err, result) {
+      if (err) {
+        fn(err, []);
+        return;
+      }
+      fn(null, result);
     });
   }
 
   function list (fn) {
-    return api( ).find({ }).toArray(fn);
+    return runWithCallback(function () {
+      return api().find({ }).toArray();
+    }, fn);
   }
   
   function listquickpicks (fn) {
-    return api( ).find({ $and: [ { 'type': 'quickpick'} , { 'hidden' : 'false' } ] }).sort({'position': 1}).toArray(fn);
+    return runWithCallback(function () {
+      return api().find({ $and: [ { 'type': 'quickpick'} , { 'hidden' : 'false' } ] }).sort({'position': 1}).toArray();
+    }, fn);
   }
   
   function listregular (fn) {
-    return api( ).find( { 'type': 'food'} ).toArray(fn);
+    return runWithCallback(function () {
+      return api().find( { 'type': 'food'} ).toArray();
+    }, fn);
   }
   
   function remove (_id, fn) {
     var objId = new ObjectID(_id);
-    return api( ).deleteOne({ '_id': objId }, fn);
+    return runWithCallback(function () {
+      return api().deleteOne({ '_id': objId });
+    }, fn);
   }
 
 

lib/server/profile.js

@@ -2,9 +2,10 @@
 
 var find_options = require('./query');
 var consts = require('../constants');
+var runWithCallback = require('../storage/run-with-callback');
 
 function storage (collection, ctx) {
-   var ObjectID = require('mongodb-legacy').ObjectId;
+   var ObjectID = require('mongodb').ObjectId;
 
   function create (objOrArray, fn) {
     // Normalize to array (supports both single object and array inputs)
@@ -13,7 +14,7 @@ function storage (collection, ctx) {
     if (docs.length === 0) {
       fn(null, []);
       ctx.bus.emit('data-received');
-      return;
+      return Promise.resolve([]);
     }
 
     // Add created_at to each document
@@ -23,16 +24,27 @@ function storage (collection, ctx) {
       }
     });
 
-    api().insertMany(docs, function (err, result) {
+    const promise = runWithCallback(async function () {
+      const result = await api().insertMany(docs);
+      if (result && result.insertedIds) {
+        Object.keys(result.insertedIds).forEach(function (index) {
+          if (!docs[index]._id) {
+            docs[index]._id = result.insertedIds[index];
+          }
+        });
+      }
+      return docs;
+    }, function (err, result) {
       if (err) {
-        console.log("Error saving profile data", docs, err);
+        console.log('Error saving profile data', docs, err);
         fn(err);
         return;
       }
-      // Return the inserted documents with _id (NightscoutKit expects array)
+      fn(null, result);
-      fn(null, docs);
     });
+
     ctx.bus.emit('data-received');
+    return promise;
   }
 
   function save (obj, fn) {
@@ -45,16 +57,20 @@ function storage (collection, ctx) {
       obj.created_at = (new Date( )).toISOString( );
     }
     // Match existing profiles by _id only. The profile editor rewrites created_at on save.
-    api().replaceOne({ _id: obj._id }, obj, { upsert: true }, function (err) {
+    const promise = runWithCallback(async function () {
-      //id should be added for new docs
+      await api().replaceOne({ _id: obj._id }, obj, { upsert: true });
-      fn(err, obj);
+      return obj;
-    });
+    }, fn);
+
     ctx.bus.emit('data-received');
+    return promise;
   }
 
   function list (fn, count) {
     const limit = count !== null ? count : Number(consts.PROFILES_DEFAULT_COUNT);
-    return api( ).find({ }).limit(limit).sort({startDate: -1}).toArray(fn);
+    return runWithCallback(function () {
+      return api().find({ }).limit(limit).sort({startDate: -1}).toArray();
+    }, fn);
   }
 
   function list_query (opts, fn) {
@@ -71,10 +87,12 @@ function storage (collection, ctx) {
         return this;
     }
 
-    return limit.call(api()
+    return runWithCallback(function () {
-      .find(query_for(opts))
+      return limit.call(api()
-      .sort(opts && opts.sort && query_sort(opts) || { startDate: -1 }), opts)
+        .find(query_for(opts))
-      .toArray(fn);
+        .sort(opts && opts.sort && query_sort(opts) || { startDate: -1 }), opts)
+        .toArray();
+    }, fn);
   }
 
   function query_for (opts) {
@@ -100,14 +118,19 @@ function storage (collection, ctx) {
 
 
   function last (fn) {
-    return api().find().sort({startDate: -1}).limit(1).toArray(fn);
+    return runWithCallback(function () {
+      return api().find().sort({startDate: -1}).limit(1).toArray();
+    }, fn);
   }
 
   function remove (_id, fn) {
     var objId = new ObjectID(_id);
-    api( ).deleteOne({ '_id': objId }, fn);
+    const promise = runWithCallback(function () {
+      return api().deleteOne({ '_id': objId });
+    }, fn);
 
     ctx.bus.emit('data-received');
+    return promise;
   }
 
   function api () {

lib/server/query.js

@@ -1,7 +1,7 @@
 'use strict';
 
 const traverse = require('traverse');
-const ObjectID = require('mongodb-legacy').ObjectId;
+const ObjectID = require('mongodb').ObjectId;
 const moment = require('moment');
 const OBJECT_ID_HEX_RE = /^[0-9a-fA-F]{24}$/;
 const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;

lib/server/treatments.js

@@ -4,9 +4,10 @@ var _ = require('lodash');
 var async = require('async');
 var moment = require('moment');
 var find_options = require('./query');
+var runWithCallback = require('../storage/run-with-callback');
 
 function storage (env, ctx) {
-  var ObjectID = require('mongodb-legacy').ObjectId;
+  var ObjectID = require('mongodb').ObjectId;
   var OBJECT_ID_HEX_RE = /^[0-9a-fA-F]{24}$/;
 
   function create (objOrArray, fn) {
@@ -61,10 +62,14 @@ function storage (env, ctx) {
         };
       });
 
-      api().bulkWrite(bulkOps, { ordered: true }, function(err, bulkResult) {
+      return runWithCallback(async function () {
-        if (err) {
+        var bulkResult;
+
+        try {
+          bulkResult = await api().bulkWrite(bulkOps, { ordered: true });
+        } catch (err) {
           console.error('Problem upserting treatments batch', err);
-          return done(err, []);
+          throw err;
         }
 
         // Assign _ids from upserted results
@@ -82,9 +87,10 @@ function storage (env, ctx) {
 
         if (docsNeedingId.length > 0) {
           var identifiers = docsNeedingId.map(function(obj) { return obj.identifier; });
-          
+
-          api().find({ identifier: { $in: identifiers } }).toArray(function(findErr, existing) {
+          try {
-            if (!findErr && existing) {
+            var existing = await api().find({ identifier: { $in: identifiers } }).toArray();
+            if (existing) {
               var idMap = {};
               existing.forEach(function(doc) {
                 if (doc.identifier) idMap[doc.identifier] = doc._id;
@@ -95,16 +101,9 @@ function storage (env, ctx) {
                 }
               });
             }
-
+          } catch (findErr) {
-            ctx.bus.emit('data-update', {
+            // Preserve existing behavior: still report success even if the id lookup fails.
-              type: 'treatments',
+          }
-              op: 'update',
-              changes: ctx.ddata.processRawDataForRuntime(objOrArray)
-            });
-
-            done(null, objOrArray);
-          });
-          return;
         }
 
         ctx.bus.emit('data-update', {
@@ -113,7 +112,13 @@ function storage (env, ctx) {
           changes: ctx.ddata.processRawDataForRuntime(objOrArray)
         });
 
-        done(null, objOrArray);
+        return objOrArray;
+      }, function (err, result) {
+        if (err) {
+          done(err, []);
+          return;
+        }
+        done(null, result);
       });
     } else {
       upsert(objOrArray, function upserted (err, docs) {
@@ -130,29 +135,34 @@ function storage (env, ctx) {
     var results = prepareData(obj);
     var query = upsertQueryFor(obj, results);
 
-    api( ).replaceOne(query, obj, {upsert: true}, function complete (err, updateResults) {
+    (async function () {
+      try {
+        var updateResults = await api().replaceOne(query, obj, {upsert: true});
 
-      if (err) console.error('Problem upserting treatment', err);
+        if (updateResults) {
-
+          if (updateResults.upsertedCount == 1) {
-      if (updateResults) {
+            obj._id = updateResults.upsertedId;
-        if (updateResults.upsertedCount == 1) {
+          } else if (updateResults.matchedCount >= 1 && obj.identifier && !obj._id) {
-          obj._id = updateResults.upsertedId;
+            // REQ-SYNC-072: On update by identifier, fetch the existing _id
-        } else if (updateResults.matchedCount >= 1 && obj.identifier && !obj._id) {
+            try {
-          // REQ-SYNC-072: On update by identifier, fetch the existing _id
+              var existing = await api().findOne(query);
-          api().findOne(query, function(findErr, existing) {
+              if (existing) {
-            if (!findErr && existing) {
+                obj._id = existing._id;
-              obj._id = existing._id;
+              }
+            } catch (findErr) {
+              // Preserve existing behavior: update success does not fail if the lookup fails.
             }
-            finishUpsert(err, obj, results);
+          }
-          });
-          return;
         }
+
+        await finishUpsert(null, obj, results);
+      } catch (err) {
+        console.error('Problem upserting treatment', err);
+        await finishUpsert(err, obj, results);
       }
+    })();
 
-      finishUpsert(err, obj, results);
+    async function finishUpsert(err, obj, results) {
-    });
-
-    function finishUpsert(err, obj, results) {
       // TODO document this feature
       if (!err && obj.preBolus) {
         //create a new object to insert copying only the needed fields
@@ -170,24 +180,28 @@ function storage (env, ctx) {
           created_at: pbTreat.created_at,
           eventType: pbTreat.eventType
         };
-        api( ).replaceOne(pbQuery, pbTreat, {upsert: true}, function pbComplete (err, updateResults) {
+        var updateResults;
+        try {
+          updateResults = await api().replaceOne(pbQuery, pbTreat, {upsert: true});
+        } catch (pbErr) {
+          err = pbErr;
+        }
 
-          if (updateResults) {
+        if (updateResults) {
-            if (updateResults.upsertedCount == 1) {
+          if (updateResults.upsertedCount == 1) {
-              pbTreat._id = updateResults.upsertedId
+            pbTreat._id = updateResults.upsertedId;
-            }
           }
+        }
 
-          var treatments = _.compact([obj, pbTreat]);
+        var treatments = _.compact([obj, pbTreat]);
 
-          ctx.bus.emit('data-update', {
+        ctx.bus.emit('data-update', {
-            type: 'treatments',
+          type: 'treatments',
-            op: 'update',
+          op: 'update',
-            changes: ctx.ddata.processRawDataForRuntime(treatments)
+          changes: ctx.ddata.processRawDataForRuntime(treatments)
-          });
-
-          fn(err, treatments);
         });
+
+        fn(err, treatments);
       } else {
 
         ctx.bus.emit('data-update', {
@@ -210,10 +224,12 @@ function storage (env, ctx) {
       return this;
     }
 
-    return limit.call(api()
+    return runWithCallback(function () {
-      .find(query_for(opts))
+      return limit.call(api()
-      .sort(opts && opts.sort || {created_at: -1}), opts)
+        .find(query_for(opts))
-      .toArray(fn);
+        .sort(opts && opts.sort || {created_at: -1}), opts)
+        .toArray();
+    }, fn);
   }
 
   function query_for (opts) {
@@ -234,20 +250,21 @@ function storage (env, ctx) {
   }
 
   function remove (opts, fn) {
-    return api( ).deleteMany(query_for(opts), {}, function (err, stat) {
+    return runWithCallback(async function () {
-        //TODO: this is triggering a read from Mongo, we can do better
+      var stat = await api().deleteMany(query_for(opts), {});
-        //console.log('Treatment removed', opts); // , stat);
+      //TODO: this is triggering a read from Mongo, we can do better
+      //console.log('Treatment removed', opts); // , stat);
 
-        ctx.bus.emit('data-update', {
+      ctx.bus.emit('data-update', {
-          type: 'treatments',
+        type: 'treatments',
-          op: 'remove',
+        op: 'remove',
-          count: stat.deletedCount,
+        count: stat.deletedCount,
-          changes: opts.find._id
+        changes: opts.find._id
-        });
-
-        ctx.bus.emit('data-received');
-        fn(err, stat);
       });
+
+      ctx.bus.emit('data-received');
+      return stat;
+    }, fn);
   }
 
   function save (obj, fn) {
@@ -256,52 +273,43 @@ function storage (env, ctx) {
 
     var query = upsertQueryFor(obj, { created_at: obj.created_at });
 
-    function saved (err, updateResults) {
+    const promise = runWithCallback(async function () {
-      if (!err) {
+      var updateResults = await api().replaceOne(query, obj, {upsert: true});
-        if (updateResults && updateResults.upsertedCount == 1) {
+
-          obj._id = updateResults.upsertedId;
+      if (updateResults && updateResults.upsertedCount == 1) {
-        } else if (updateResults && updateResults.matchedCount >= 1 && obj.identifier && !obj._id) {
+        obj._id = updateResults.upsertedId;
-          // REQ-SYNC-072: On update by identifier, fetch the existing _id
+      } else if (updateResults && updateResults.matchedCount >= 1 && obj.identifier && !obj._id) {
-          api().findOne(query, function(findErr, existing) {
+        // REQ-SYNC-072: On update by identifier, fetch the existing _id
-            if (!findErr && existing) {
+        try {
-              obj._id = existing._id;
+          var existing = await api().findOne(query);
-            }
+          if (existing) {
-            finishSave(err, obj);
+            obj._id = existing._id;
-          });
+          }
-          return;
+        } catch (findErr) {
+          // Preserve existing behavior: update success does not fail if the lookup fails.
         }
-        // console.log('Treatment updated', created);
-
-        ctx.ddata.processRawDataForRuntime(obj);
-
-        ctx.bus.emit('data-update', {
-          type: 'treatments',
-          op: 'update',
-          changes: ctx.ddata.processRawDataForRuntime([obj])
-        });
-
       }
-      if (err) console.error('Problem saving treating', err);
 
-      fn(err, obj);
+      ctx.ddata.processRawDataForRuntime(obj);
-    }
+      ctx.bus.emit('data-update', {
+        type: 'treatments',
+        op: 'update',
+        changes: ctx.ddata.processRawDataForRuntime([obj])
+      });
 
-    function finishSave(err, obj) {
+      return obj;
-      if (!err) {
+    }, function (err, result) {
-        ctx.ddata.processRawDataForRuntime(obj);
+      if (err) {
-        ctx.bus.emit('data-update', {
+        console.error('Problem saving treating', err);
-          type: 'treatments',
+        fn(err, obj);
-          op: 'update',
+        return;
-          changes: ctx.ddata.processRawDataForRuntime([obj])
-        });
       }
-      if (err) console.error('Problem saving treating', err);
-      fn(err, obj);
-    }
 
-    api().replaceOne(query, obj, {upsert: true}, saved);
+      fn(null, result);
+    });
 
     ctx.bus.emit('data-received');
+    return promise;
   }
 
   function api ( ) {

lib/server/websocket.js

@@ -2,7 +2,7 @@
 
 var times = require('../times');
 var calcData = require('../data/calcdelta');
-var ObjectID = require('mongodb-legacy').ObjectId;
+var ObjectID = require('mongodb').ObjectId;
 const forwarded = require('forwarded-for');
 
 function getRemoteIP (req) {
@@ -230,25 +230,23 @@ function init (env, ctx, server) {
         }
         var id = safeObjectID(data._id);
 
-        ctx.store.collection(collection).updateOne({ '_id': id }
+        (async function () {
-          , { $set: data.data }
+          try {
-          , function(err, results) {
+            var mongoCollection = ctx.store.collection(collection);
-
+            await mongoCollection.updateOne({ '_id': id }, { $set: data.data });
-            if (!err) {
+            var results = await mongoCollection.findOne({ '_id': id });
-              ctx.store.collection(collection).findOne({ '_id': id }
+            console.log('Got results', results);
-                , function(err, results) {
+            if (results !== null) {
-                  console.log('Got results', results);
+              ctx.bus.emit('data-update', {
-                  if (!err && results !== null) {
+                type: data.collection
-                    ctx.bus.emit('data-update', {
+                , op: 'update'
-                      type: data.collection
+                , changes: ctx.ddata.processRawDataForRuntime([results])
-                      , op: 'update'
+              });
-                      , changes: ctx.ddata.processRawDataForRuntime([results])
-                    });
-                  }
-                });
             }
+          } catch (err) {
+            console.error(err);
           }
-        );
+        })();
 
         if (callback) {
           callback({ result: 'success' });
@@ -278,23 +276,23 @@ function init (env, ctx, server) {
         }
 
         var objId = safeObjectID(data._id);
-        ctx.store.collection(collection).updateOne({ '_id': objId }, { $unset: data.data }
+        (async function () {
-          , function(err, results) {
+          try {
-
+            var mongoCollection = ctx.store.collection(collection);
-            if (!err) {
+            await mongoCollection.updateOne({ '_id': objId }, { $unset: data.data });
-              ctx.store.collection(collection).findOne({ '_id': objId }
+            var results = await mongoCollection.findOne({ '_id': objId });
-                , function(err, results) {
+            console.log('Got results', results);
-                  console.log('Got results', results);
+            if (results !== null) {
-                  if (!err && results !== null) {
+              ctx.bus.emit('data-update', {
-                    ctx.bus.emit('data-update', {
+                type: data.collection
-                      type: data.collection
+                , op: 'update'
-                      , op: 'update'
+                , changes: ctx.ddata.processRawDataForRuntime([results])
-                      , changes: ctx.ddata.processRawDataForRuntime([results])
+              });
-                    });
-                  }
-                });
             }
-          });
+          } catch (err) {
+            console.error(err);
+          }
+        })();
 
         if (callback) {
           callback({ result: 'success' });
@@ -328,40 +326,50 @@ function init (env, ctx, server) {
         // Handle array input: process each item sequentially
         if (Array.isArray(data.data)) {
           console.log(LOG_WS + 'dbAdd received array with ' + data.data.length + ' items');
-          var results = [];
+          (async function () {
-          var processIndex = 0;
+            var results = [];
 
-          function processNextItem() {
+            for (var processIndex = 0; processIndex < data.data.length; processIndex += 1) {
-            if (processIndex >= data.data.length) {
+              var itemData = {
-              if (callback) {
+                collection: data.collection,
-                callback(results);
+                data: data.data[processIndex]
-              }
+              };
-              return;
+              var itemResult = await processSingleDbAdd(itemData, collection, maxtimediff);
-            }
-
-            var itemData = {
-              collection: data.collection,
-              data: data.data[processIndex]
-            };
-
-            processIndex++;
-            processSingleDbAdd(itemData, collection, maxtimediff, function(itemResult) {
               if (itemResult && itemResult.length > 0) {
                 results = results.concat(itemResult);
               }
-              processNextItem();
+            }
-            });
-          }
 
-          processNextItem();
+            if (callback) {
+              callback(results);
+            }
+          })().catch(function (err) {
+            console.error(err);
+            if (callback) {
+              callback([]);
+            }
+          });
           return;
         }
 
         // Single object processing
-        processSingleDbAdd(data, collection, maxtimediff, callback);
+        processSingleDbAdd(data, collection, maxtimediff)
+          .then(function (result) {
+            if (callback) {
+              callback(result);
+            }
+          })
+          .catch(function (err) {
+            console.error(err);
+            if (callback) {
+              callback([]);
+            }
+          });
       });
 
-      function processSingleDbAdd(data, collection, maxtimediff, callback) {
+      async function processSingleDbAdd(data, collection, maxtimediff) {
+        var mongoCollection = ctx.store.collection(collection);
+
         if (data.collection === 'treatments' && !('eventType' in data.data)) {
           data.data.eventType = '<none>';
         }
@@ -382,97 +390,86 @@ function init (env, ctx, server) {
           }
 
           // try to find exact match
-          ctx.store.collection(collection).find(query).toArray(function findResult (err, array) {
+          try {
-            if (err) {
+            var array = await mongoCollection.find(query).toArray();
-              console.error(err);
-              callback([]);
-              return;
-            }
-
             if (array.length > 0) {
               console.log(LOG_DEDUP + 'Exact match');
-              if (callback) {
+              return [array[0]];
-                callback([array[0]]);
-              }
-              return;
             }
+          } catch (err) {
+            console.error(err);
+            return [];
+          }
 
-            var selected = false;
+          var selected = false;
-            var query_similiar = {
+          var query_similiar = {
-              created_at: { $gte: new Date(new Date(data.data.created_at).getTime() - maxtimediff).toISOString(), $lte: new Date(new Date(data.data.created_at).getTime() + maxtimediff).toISOString() }
+            created_at: { $gte: new Date(new Date(data.data.created_at).getTime() - maxtimediff).toISOString(), $lte: new Date(new Date(data.data.created_at).getTime() + maxtimediff).toISOString() }
-            };
+          };
-            if (data.data.insulin) {
+          if (data.data.insulin) {
-              query_similiar.insulin = data.data.insulin;
+            query_similiar.insulin = data.data.insulin;
-              selected = true;
+            selected = true;
+          }
+          if (data.data.carbs) {
+            query_similiar.carbs = data.data.carbs;
+            selected = true;
+          }
+          if (data.data.percent) {
+            query_similiar.percent = data.data.percent;
+            selected = true;
+          }
+          if (data.data.absolute) {
+            query_similiar.absolute = data.data.absolute;
+            selected = true;
+          }
+          if (data.data.duration) {
+            query_similiar.duration = data.data.duration;
+            selected = true;
+          }
+          if (data.data.NSCLIENT_ID) {
+            query_similiar.NSCLIENT_ID = data.data.NSCLIENT_ID;
+            selected = true;
+          }
+          // if none assigned add at least eventType
+          if (!selected) {
+            query_similiar.eventType = data.data.eventType;
+          }
+          // try to find similiar
+          try {
+            var similar = await mongoCollection.find(query_similiar).toArray();
+            // if found similiar just update date. next time it will match exactly
+            if (similar.length > 0) {
+              console.log(LOG_DEDUP + 'Found similiar', similar[0]);
+              similar[0].created_at = data.data.created_at;
+              var objId = safeObjectID(similar[0]._id);
+              await mongoCollection.updateOne({ '_id': objId }, { $set: { created_at: data.data.created_at } });
+              ctx.bus.emit('data-received');
+              return [similar[0]];
             }
-            if (data.data.carbs) {
+          } catch (err) {
-              query_similiar.carbs = data.data.carbs;
+            console.error(err);
-              selected = true;
+            return [];
-            }
+          }
-            if (data.data.percent) {
-              query_similiar.percent = data.data.percent;
-              selected = true;
-            }
-            if (data.data.absolute) {
-              query_similiar.absolute = data.data.absolute;
-              selected = true;
-            }
-            if (data.data.duration) {
-              query_similiar.duration = data.data.duration;
-              selected = true;
-            }
-            if (data.data.NSCLIENT_ID) {
-              query_similiar.NSCLIENT_ID = data.data.NSCLIENT_ID;
-              selected = true;
-            }
-            // if none assigned add at least eventType
-            if (!selected) {
-              query_similiar.eventType = data.data.eventType;
-            }
-            // try to find similiar
-            ctx.store.collection(collection).find(query_similiar).toArray(function findSimiliarResult (err, array) {
-              // if found similiar just update date. next time it will match exactly
 
-              if (err) {
+          // if not found create new record
-                console.error(err);
+          console.log(LOG_DEDUP + 'Adding new record');
-                callback([]);
+          try {
-                return;
+            var insertResult = await mongoCollection.insertOne(data.data);
-              }
+            var doc = data.data;
-
+            doc._id = insertResult.insertedId;
-              if (array.length > 0) {
+            ctx.bus.emit('data-update', {
-                console.log(LOG_DEDUP + 'Found similiar', array[0]);
+              type: data.collection
-                array[0].created_at = data.data.created_at;
+              , op: 'update'
-                var objId = safeObjectID(array[0]._id);
+              , changes: ctx.ddata.processRawDataForRuntime([doc])
-                ctx.store.collection(collection).updateOne({ '_id': objId }, { $set: { created_at: data.data.created_at } });
-                if (callback) {
-                  callback([array[0]]);
-                }
-                ctx.bus.emit('data-received');
-                return;
-              }
-              // if not found create new record
-              console.log(LOG_DEDUP + 'Adding new record');
-              ctx.store.collection(collection).insertOne(data.data, function insertResult (err, ops) {
-                if (err != null && err.message) {
-                  console.log('treatments data insertion error: ', err.message);
-                  return;
-                }
-
-                var doc = data.data;
-                doc._id = ops.insertedId;
-                ctx.bus.emit('data-update', {
-                  type: data.collection
-                  , op: 'update'
-                  , changes: ctx.ddata.processRawDataForRuntime([doc])
-                });
-
-                if (callback) {
-                  callback([doc]);
-                }
-                ctx.bus.emit('data-received');
-              });
             });
-          });
+            ctx.bus.emit('data-received');
+            return [doc];
+          } catch (err) {
+            if (err != null && err.message) {
+              console.log('treatments data insertion error: ', err.message);
+              return [];
+            }
+            throw err;
+          }
           // devicestatus deduping
         } else if (data.collection === 'devicestatus') {
           var queryDev;
@@ -485,62 +482,54 @@ function init (env, ctx, server) {
           }
 
           // try to find exact match
-          ctx.store.collection(collection).find(queryDev).toArray(function findResult (err, array) {
+          try {
-            if (err) {
+            var existingStatus = await mongoCollection.find(queryDev).toArray();
-              console.error(err);
+            if (existingStatus.length > 0) {
-              callback([]);
-              return;
-            }
-
-            if (array.length > 0) {
               console.log(LOG_DEDUP + 'Devicestatus exact match');
-              if (callback) {
+              return [existingStatus[0]];
-                callback([array[0]]);
-              }
-              return;
             }
+          } catch (err) {
+            console.error(err);
+            return [];
+          }
 
-          });
+          try {
-
+            var devicestatusInsertResult = await mongoCollection.insertOne(data.data);
-          ctx.store.collection(collection).insertOne(data.data, function insertResult (err, ops) {
+            var devicestatusDoc = data.data;
-            if (err != null && err.message) {
+            devicestatusDoc._id = devicestatusInsertResult.insertedId;
-              console.log('devicestatus insertion error: ', err.message);
-              return;
-            }
-
-            var doc = data.data;
-            doc._id = ops.insertedId;
             ctx.bus.emit('data-update', {
               type: 'devicestatus'
               , op: 'update'
-              , changes: ctx.ddata.processRawDataForRuntime([doc])
+              , changes: ctx.ddata.processRawDataForRuntime([devicestatusDoc])
             });
-
-            if (callback) {
-              callback([doc]);
-            }
             ctx.bus.emit('data-received');
-          });
+            return [devicestatusDoc];
-        } else {
+          } catch (err) {
-          ctx.store.collection(collection).insertOne(data.data, function insertResult (err, ops) {
             if (err != null && err.message) {
-              console.log(data.collection + ' insertion error: ', err.message);
+              console.log('devicestatus insertion error: ', err.message);
-              return;
+              return [];
             }
-
+            throw err;
-            var doc = data.data;
+          }
-            doc._id = ops.insertedId;
+        } else {
+          try {
+            var genericInsertResult = await mongoCollection.insertOne(data.data);
+            var genericDoc = data.data;
+            genericDoc._id = genericInsertResult.insertedId;
             ctx.bus.emit('data-update', {
               type: data.collection
               , op: 'update'
-              , changes: ctx.ddata.processRawDataForRuntime([doc])
+              , changes: ctx.ddata.processRawDataForRuntime([genericDoc])
             });
-
-            if (callback) {
-              callback([doc]);
-            }
             ctx.bus.emit('data-received');
-          });
+            return [genericDoc];
+          } catch (err) {
+            if (err != null && err.message) {
+              console.log(data.collection + ' insertion error: ', err.message);
+              return [];
+            }
+            throw err;
+          }
         }
       }
 
@@ -562,19 +551,19 @@ function init (env, ctx, server) {
         }
 
         var objId = safeObjectID(data._id);
-        ctx.store.collection(collection).deleteOne({ '_id': objId }
+        (async function () {
-          , function(err, stat) {
+          try {
-
+            var stat = await ctx.store.collection(collection).deleteOne({ '_id': objId });
-            if (!err) {
+            ctx.bus.emit('data-update', {
-              ctx.bus.emit('data-update', {
+              type: data.collection
-                type: data.collection
+              , op: 'remove'
-                , op: 'remove'
+              , count: stat.deletedCount
-                , count: stat.deletedCount
+              , changes: data._id
-                , changes: data._id
+            });
-              });
+          } catch (err) {
-
+            console.error(err);
-            }
+          }
-          });
+        })();
 
         if (callback) {
           callback({ result: 'success' });

lib/storage/mongo-storage.js

@@ -1,6 +1,6 @@
 'use strict';
 
-const MongoClient = require('mongodb-legacy').MongoClient;
+const MongoClient = require('mongodb').MongoClient;
 
 const mongo = {
   client: null,
@@ -10,6 +10,44 @@ const mongo = {
 const DEFAULT_POOL_SIZE = 5;
 const LEGACY_POOL_SIZE = 100;
 
+function getRetryDelay(attempt) {
+  return attempt > 15 ? 60000 : attempt * 3000;
+}
+
+function wait(ms) {
+  return new Promise(function (resolve) {
+    setTimeout(resolve, ms);
+  });
+}
+
+async function closeClient(client) {
+  if (!client || typeof client.close !== 'function') {
+    return;
+  }
+
+  try {
+    await client.close();
+  } catch (err) {
+    console.log('Error closing failed MongoDB client: %j', err);
+  }
+}
+
+function wrapConnectionError(err) {
+  if (err && err.name === 'MongoReadOnlyConnectionError') {
+    return err;
+  }
+
+  if (err && err.message && err.message.includes('AuthenticationFailed')) {
+    return new Error('MongoDB authentication failed! Double check the URL has the right username and password in MONGODB_URI.');
+  }
+
+  return new Error('MONGODB_URI seems invalid: ' + err.message);
+}
+
+function isRetryableConnectionError(err) {
+  return !!(err && err.name === 'MongoServerSelectionError' && !(err.message && err.message.includes('AuthenticationFailed')));
+}
+
 function getPoolOptions(env) {
   const poolSize = env.mongo_pool_size 
     ? parseInt(env.mongo_pool_size, 10) 
@@ -75,6 +113,8 @@ function init(env, cb, forceNewConnection) {
       if (cb && cb.call) {
         cb(null, mongo);
       }
+
+      return Promise.resolve(mongo);
     } else {
       if (!env.storageURI) {
         throw new Error('MongoDB connection string is missing. Please set MONGODB_URI environment variable');
@@ -84,56 +124,82 @@ function init(env, cb, forceNewConnection) {
       console.log('Setting up new connection to MongoDB with pool options:', poolOptions);
       
       const options = {
-        useNewUrlParser: true,
-        useUnifiedTopology: true,
         ...poolOptions,
       };
 
-      const connect_with_retry = async function (i) {
+      const connectWithRetry = async function () {
+        let attempt = 1;
 
-        mongo.client = new MongoClient(env.storageURI, options);
+        if (forceNewConnection) {
-        setupPoolMonitoring(mongo.client, env);
+          const previousClient = mongo.client;
-        
+          mongo.client = null;
-        try {
+          mongo.db = null;
-          await mongo.client.connect();
+          await closeClient(previousClient);
+        }
 
-          console.log('Successfully established connection to MongoDB');
+        while (true) {
+          let client = null;
 
-          const dbName = mongo.client.s.options.dbName;
+          try {
-          mongo.db = mongo.client.db(dbName);
+            client = new MongoClient(env.storageURI, options);
+            mongo.client = client;
+            setupPoolMonitoring(client, env);
 
-          const result = await mongo.db.command({ connectionStatus: 1 });
+            await client.connect();
-          const roles = result.authInfo.authenticatedUserRoles;
-          if (roles && roles.length > 0 && roles[0].role == 'readAnyDatabase') {
-            console.error('Mongo user is read only');
-            cb(new Error('MongoDB connection is in read only mode! Go back to MongoDB configuration and check your database user has read and write access.'), null);
-          }
 
-          console.log('Mongo user role seems ok:', roles);
+            console.log('Successfully established connection to MongoDB');
 
-          // If there is a valid callback, then invoke the function to perform the callback
+            mongo.db = client.db();
-          if (cb && cb.call) {
-            cb(null, mongo);
-          }
-        } catch (err) {
-          if (err.message && err.message.includes('AuthenticationFailed')) {
-            console.log('Authentication to Mongo failed');
-            cb(new Error('MongoDB authentication failed! Double check the URL has the right username and password in MONGODB_URI.'), null);
-            return;
-          }
 
-          if (err.name && err.name === "MongoServerSelectionError") {
+            const result = await mongo.db.command({ connectionStatus: 1 });
-            const timeout = (i > 15) ? 60000 : i * 3000;
+            const roles = result.authInfo.authenticatedUserRoles;
+            if (roles && roles.length > 0 && roles[0].role == 'readAnyDatabase') {
+              console.error('Mongo user is read only');
+              const readOnlyError = new Error('MongoDB connection is in read only mode! Go back to MongoDB configuration and check your database user has read and write access.');
+              readOnlyError.name = 'MongoReadOnlyConnectionError';
+              throw readOnlyError;
+            }
+
+            console.log('Mongo user role seems ok:', roles);
+            return mongo;
+          } catch (err) {
+            const retryable = isRetryableConnectionError(err);
+            const wrappedError = wrapConnectionError(err);
+
+            if (err && err.message && err.message.includes('AuthenticationFailed')) {
+              console.log('Authentication to Mongo failed');
+            }
+
+            mongo.db = null;
+            if (mongo.client === client) {
+              mongo.client = null;
+            }
+
+            await closeClient(client);
+
+            if (!retryable) {
+              throw wrappedError;
+            }
+
+            const timeout = getRetryDelay(attempt);
             console.log('Error connecting to MongoDB: %j - retrying in ' + timeout / 1000 + ' sec', err);
-            setTimeout(connect_with_retry, timeout, i + 1);
+            await wait(timeout);
-            if (i == 1) cb(new Error('MongoDB connection failed! Double check the MONGODB_URI setting in Heroku.'), null);
+            attempt += 1;
-          } else {
-            cb(new Error('MONGODB_URI seems invalid: ' + err.message));
           }
         }
       };
 
-      return connect_with_retry(1);
+      const promise = connectWithRetry();
+
+      if (cb && cb.call) {
+        promise.then(function (store) {
+          cb(null, store);
+        }, function (err) {
+          cb(err, null);
+        });
+      }
+
+      return promise;
 
     }
   }
@@ -146,10 +212,8 @@ function init(env, cb, forceNewConnection) {
     fields.forEach(function (field) {
       const name = collection.collectionName + "." + field;
       console.info('ensuring index for: ' + name);
-      collection.createIndex(field, { 'background': true }, function (err) {
+      collection.createIndex(field).catch(function (err) {
-        if (err) {
+        console.error('unable to ensureIndex for: ' + name + ' - ' + err);
-          console.error('unable to ensureIndex for: ' + name + ' - ' + err);
-        }
       });
     });
   };
@@ -161,3 +225,4 @@ module.exports = init;
 module.exports.DEFAULT_POOL_SIZE = DEFAULT_POOL_SIZE;
 module.exports.LEGACY_POOL_SIZE = LEGACY_POOL_SIZE;
 module.exports.getPoolOptions = getPoolOptions;
+module.exports.getRetryDelay = getRetryDelay;

lib/storage/run-with-callback.js

@@ -0,0 +1,20 @@
+'use strict';
+
+function runWithCallback (work, callback) {
+  const promise = Promise.resolve().then(work);
+
+  if (callback && callback.call) {
+    promise.then(
+      function onSuccess(result) {
+        callback(null, result);
+      },
+      function onError(err) {
+        callback(err, null);
+      }
+    );
+  }
+
+  return promise;
+}
+
+module.exports = runWithCallback;

package-lock.json

@@ -56,7 +56,7 @@
         "moment-timezone": "^0.5.31",
         "moment-timezone-data-webpack-plugin": "^1.5.0",
         "mongo-url-parser": "^1.0.2",
-        "mongodb-legacy": "^5.0.0",
+        "mongodb": "^5.9.2",
         "mongomock": "^0.1.2",
         "nightscout-connect": "^0.0.12",
         "node-cache": "^4.2.1",
@@ -6947,16 +6947,6 @@
         "node": ">=12"
       }
     },
-    "node_modules/mongodb-legacy": {
-      "version": "5.0.0",
-      "license": "Apache-2.0",
-      "dependencies": {
-        "mongodb": "^5.0.0"
-      },
-      "engines": {
-        "node": ">=14.20.1"
-      }
-    },
     "node_modules/mongomock": {
       "version": "0.1.2",
       "dependencies": {

package.json

@@ -90,9 +90,9 @@
     "npm": ">=10.x"
   },
   "dependencies": {
-    "@mongodb-js/saslprep": "^1.4.5",
     "@babel/core": "^7.18.10",
     "@babel/preset-env": "^7.18.10",
+    "@mongodb-js/saslprep": "^1.4.5",
     "@parse/node-apn": "^5.1.3",
     "acorn": "^8.0.5",
     "acorn-jsx": "^5.3.1",
@@ -136,7 +136,7 @@
     "moment-timezone": "^0.5.31",
     "moment-timezone-data-webpack-plugin": "^1.5.0",
     "mongo-url-parser": "^1.0.2",
-    "mongodb-legacy": "^5.0.0",
+    "mongodb": "^5.9.2",
     "mongomock": "^0.1.2",
     "nightscout-connect": "^0.0.12",
     "node-cache": "^4.2.1",

tests/XX_clean.test.js

@@ -26,16 +26,12 @@ describe('Clean MONGO after tests', function ( ) {
     });
   });
 
-  it('wipe treatment data', function (done) {
+  it('wipe treatment data', async function () {
-    self.ctx.treatments().deleteMany({ }, function ( ) {
+    await self.ctx.treatments().deleteMany({ });
-        done();
-    });
   });
 
-  it('wipe entries data', function (done) {
+  it('wipe entries data', async function () {
-    self.ctx.entries().deleteMany({ }, function ( ) {
+    await self.ctx.entries().deleteMany({ });
-        done();
-    });
   });
   
 });

tests/api.deduplication.test.js

@@ -55,12 +55,14 @@ describe('v1 API Deduplication Behavior', function() {
       find: { created_at: { '$gte': '1999-01-01T00:00:00.000Z' } } 
     }, function() {
       // Use deleteMany for faster cleanup of entries
-      self.ctx.entries().deleteMany({}, function() {
+      self.ctx.entries().deleteMany({})
-        // Also clear devicestatus to reduce database load
+        .then(function() {
-        self.ctx.devicestatus.remove({
+          // Also clear devicestatus to reduce database load
-          find: { created_at: { '$gte': '1999-01-01T00:00:00.000Z' } }
+          self.ctx.devicestatus.remove({
-        }, done);
+            find: { created_at: { '$gte': '1999-01-01T00:00:00.000Z' } }
-      });
+          }, done);
+        })
+        .catch(done);
     });
   });
 

tests/api.entries.test.js

@@ -62,12 +62,12 @@ describe('Entries REST api', function ( ) {
 
   });
 
-  afterEach(function (done) {
+  afterEach(async function () {
-    self.archive( ).deleteMany({ }, done);
+    await self.archive( ).deleteMany({ });
   });
 
-  after(function (done) {
+  after(async function () {
-    self.archive( ).deleteMany({ }, done);
+    await self.archive( ).deleteMany({ });
   });
 
   // keep this test pinned at or near the top in order to validate all

tests/api.entries.uuid.test.js

@@ -43,12 +43,12 @@ describe('Entry sysTime+type dedup (Baseline)', function() {
     });
   });
   
-  afterEach(function(done) {
+  afterEach(async function() {
-    self.archive().deleteMany({}, done);
+    await self.archive().deleteMany({});
   });
   
-  after(function(done) {
+  after(async function() {
-    self.archive().deleteMany({}, done);
+    await self.archive().deleteMany({});
   });
   
   /**
@@ -98,14 +98,14 @@ describe('Entry sysTime+type dedup (Baseline)', function() {
             if (err2) return done(err2);
             
             // Verify: only 1 entry exists with updated sgv 125
-            self.archive().find({ date: timestamp }).toArray(function(err3, docs) {
+            self.archive().find({ date: timestamp }).toArray()
-              if (err3) return done(err3);
+              .then(function(docs) {
-              
+                docs.should.have.lengthOf(1);
-              docs.should.have.lengthOf(1);
+                docs[0].sgv.should.equal(125);
-              docs[0].sgv.should.equal(125);
+                docs[0].direction.should.equal('FortyFiveUp');
-              docs[0].direction.should.equal('FortyFiveUp');
+                done();
-              done();
+              })
-            });
+              .catch(done);
           });
       });
   });
@@ -155,14 +155,14 @@ describe('Entry sysTime+type dedup (Baseline)', function() {
             if (err2) return done(err2);
             
             // Verify: 2 entries exist (different types)
-            self.archive().find({ date: timestamp }).toArray(function(err3, docs) {
+            self.archive().find({ date: timestamp }).toArray()
-              if (err3) return done(err3);
+              .then(function(docs) {
-              
+                docs.should.have.lengthOf(2);
-              docs.should.have.lengthOf(2);
+                var types = docs.map(d => d.type).sort();
-              var types = docs.map(d => d.type).sort();
+                types.should.eql(['mbg', 'sgv']);
-              types.should.eql(['mbg', 'sgv']);
+                done();
-              done();
+              })
-            });
+              .catch(done);
           });
       });
   });
@@ -213,12 +213,12 @@ describe('Entry sysTime+type dedup (Baseline)', function() {
             if (err2) return done(err2);
             
             // Verify: 2 entries exist
-            self.archive().find({ type: 'sgv', date: { $in: [timestamp1, timestamp2] } }).toArray(function(err3, docs) {
+            self.archive().find({ type: 'sgv', date: { $in: [timestamp1, timestamp2] } }).toArray()
-              if (err3) return done(err3);
+              .then(function(docs) {
-              
+                docs.should.have.lengthOf(2);
-              docs.should.have.lengthOf(2);
+                done();
-              done();
+              })
-            });
+              .catch(done);
           });
       });
   });
@@ -249,12 +249,12 @@ describe('Entry UUID _id handling (GAP-SYNC-045)', function() {
     });
   });
   
-  afterEach(function(done) {
+  afterEach(async function() {
-    self.archive().deleteMany({}, done);
+    await self.archive().deleteMany({});
   });
   
-  after(function(done) {
+  after(async function() {
-    self.archive().deleteMany({}, done);
+    await self.archive().deleteMany({});
   });
   
   /**
@@ -286,14 +286,14 @@ describe('Entry UUID _id handling (GAP-SYNC-045)', function() {
         if (err) return done(err);
         
         // Entry should be created (may have ObjectId _id, UUID in identifier)
-        self.archive().find({ date: timestamp }).toArray(function(err2, docs) {
+        self.archive().find({ date: timestamp }).toArray()
-          if (err2) return done(err2);
+          .then(function(docs) {
-          
+            docs.should.have.lengthOf(1);
-          docs.should.have.lengthOf(1);
+            docs[0].sgv.should.equal(120);
-          docs[0].sgv.should.equal(120);
+            // Note: After fix, expect docs[0].identifier === uuid
-          // Note: After fix, expect docs[0].identifier === uuid
+            done();
-          done();
+          })
-        });
+          .catch(done);
       });
   });
   
@@ -347,13 +347,13 @@ describe('Entry UUID _id handling (GAP-SYNC-045)', function() {
             if (err2) return done(err2);
             
             // Verify: single entry, updated value
-            self.archive().find({ date: timestamp }).toArray(function(err3, docs) {
+            self.archive().find({ date: timestamp }).toArray()
-              if (err3) return done(err3);
+              .then(function(docs) {
-              
+                docs.should.have.lengthOf(1);
-              docs.should.have.lengthOf(1);
+                docs[0].sgv.should.equal(125);
-              docs[0].sgv.should.equal(125);
+                done();
-              done();
+              })
-            });
+              .catch(done);
           });
       });
   });
@@ -412,13 +412,13 @@ describe('Entry UUID _id handling (GAP-SYNC-045)', function() {
             if (err2) return done(err2);
             
             // Verify: single entry (dedup by sysTime+type, not UUID)
-            self.archive().find({ date: timestamp }).toArray(function(err3, docs) {
+            self.archive().find({ date: timestamp }).toArray()
-              if (err3) return done(err3);
+              .then(function(docs) {
-              
+                docs.should.have.lengthOf(1);
-              docs.should.have.lengthOf(1);
+                docs[0].sgv.should.equal(125);
-              docs[0].sgv.should.equal(125);
+                done();
-              done();
+              })
-            });
+              .catch(done);
           });
       });
   });
@@ -469,14 +469,14 @@ describe('Entry UUID _id handling (GAP-SYNC-045)', function() {
         if (err) return done(err);
         
         // Verify: all 3 entries created
-        self.archive().find({ date: { $in: [timestamp1, timestamp2, timestamp3] } }).toArray(function(err2, docs) {
+        self.archive().find({ date: { $in: [timestamp1, timestamp2, timestamp3] } }).toArray()
-          if (err2) return done(err2);
+          .then(function(docs) {
-          
+            docs.should.have.lengthOf(3);
-          docs.should.have.lengthOf(3);
+            var sgvValues = docs.map(d => d.sgv).sort();
-          var sgvValues = docs.map(d => d.sgv).sort();
+            sgvValues.should.eql([120, 125, 130]);
-          sgvValues.should.eql([120, 125, 130]);
+            done();
-          done();
+          })
-        });
+          .catch(done);
       });
   });
   
@@ -500,9 +500,7 @@ describe('Entry UUID _id handling (GAP-SYNC-045)', function() {
       dateString: sysTime,
       sysTime: sysTime,
       device: 'Trio'
-    }, function(err) {
+    }).then(function() {
-      if (err) return done(err);
-      
       // POST via API with same timestamp
       var entry = {
         _id: uuid,
@@ -513,7 +511,7 @@ describe('Entry UUID _id handling (GAP-SYNC-045)', function() {
         dateString: sysTime,
         device: 'Trio'
       };
-      
+
       request(self.app)
         .post('/entries/')
         .set('api-secret', self.known)
@@ -521,17 +519,17 @@ describe('Entry UUID _id handling (GAP-SYNC-045)', function() {
         .expect(200)
         .end(function(err2, res) {
           if (err2) return done(err2);
-          
+
           // Verify: single entry, updated value
-          self.archive().find({ date: timestamp }).toArray(function(err3, docs) {
+          self.archive().find({ date: timestamp }).toArray()
-            if (err3) return done(err3);
+            .then(function(docs) {
-            
+              docs.should.have.lengthOf(1);
-            docs.should.have.lengthOf(1);
+              docs[0].sgv.should.equal(125);
-            docs[0].sgv.should.equal(125);
+              done();
-            done();
+            })
-          });
+            .catch(done);
         });
-    });
+    }).catch(done);
   });
   
   /**
@@ -563,15 +561,15 @@ describe('Entry UUID _id handling (GAP-SYNC-045)', function() {
         if (err) return done(err);
         
         // Verify: entry has identifier field with UUID
-        self.archive().find({ date: timestamp }).toArray(function(err2, docs) {
+        self.archive().find({ date: timestamp }).toArray()
-          if (err2) return done(err2);
+          .then(function(docs) {
-          
+            docs.should.have.lengthOf(1);
-          docs.should.have.lengthOf(1);
+            docs[0].should.have.property('identifier', uuid);
-          docs[0].should.have.property('identifier', uuid);
+            // _id should be ObjectId, not UUID
-          // _id should be ObjectId, not UUID
+            docs[0]._id.should.not.equal(uuid);
-          docs[0]._id.should.not.equal(uuid);
+            done();
-          done();
+          })
-        });
+          .catch(done);
       });
   });
 });

tests/api.objectid-validation.test.js

@@ -0,0 +1,34 @@
+'use strict';
+
+var should = require('should');
+
+var objectIdValidation = require('../lib/api/shared/objectid-validation');
+
+describe('API ObjectId validation helper', function () {
+  it('accepts missing ids for create-tolerant routes', function () {
+    should(objectIdValidation.isValidObjectId(undefined)).be.true();
+    should(objectIdValidation.isValidObjectId(null)).be.true();
+  });
+
+  it('accepts 24-character hex ids and rejects non-hex values', function () {
+    should(objectIdValidation.isValidObjectId('507f1f77bcf86cd799439011')).be.true();
+    should(objectIdValidation.isValidObjectId('550e8400-e29b-41d4-a716-446655440000')).be.false();
+    should(objectIdValidation.isValidObjectId(123)).be.false();
+  });
+
+  it('finds the first invalid id in a batch', function () {
+    objectIdValidation.findInvalidId([
+      { _id: '507f1f77bcf86cd799439011' },
+      { _id: 'invalid-id' },
+      { _id: '507f191e810c19729de860ea' }
+    ]).should.eql({ index: 1, id: 'invalid-id' });
+  });
+
+  it('returns null when all ids are valid or omitted', function () {
+    should.not.exist(objectIdValidation.findInvalidId([
+      { _id: '507f1f77bcf86cd799439011' },
+      {},
+      { _id: null }
+    ]));
+  });
+});

tests/api.security.test.js

@@ -15,6 +15,22 @@ describe('Security of REST API V1', function() {
 
   var known = 'b723e97aa97846eb92d5264f084b2823f57c4aa1';
 
+  function rolesCollection() {
+    return self.ctx.store.collection(self.env.authentication_collections_prefix + 'roles');
+  }
+
+  function subjectsCollection() {
+    return self.ctx.store.collection(self.env.authentication_collections_prefix + 'subjects');
+  }
+
+  async function getBearerToken(accessToken) {
+    const res = await request(self.app)
+      .get('/api/v2/authorization/request/' + accessToken)
+      .expect(200);
+
+    return res.body.token;
+  }
+
   before(function(done) {
     var api = require('../lib/api/');
     delete process.env.API_SECRET;
@@ -26,6 +42,7 @@ describe('Security of REST API V1', function() {
     self.app = require('express')();
     self.app.enable('api');
     require('../lib/server/bootevent')(self.env, language).boot(async function booted (ctx) {
+      self.ctx = ctx;
       self.app.use('/api/v1', api(self.env, ctx));
       self.app.use('/api/v2/authorization', ctx.authorization.endpoints);
       let authResult = await authSubject(ctx.authorization.storage);
@@ -159,4 +176,140 @@ describe('Security of REST API V1', function() {
     });
   });
 
+  describe('Authorization admin save endpoints', function () {
+    beforeEach(async function () {
+      await rolesCollection().deleteMany({ name: /^api-security-role/ });
+      await subjectsCollection().deleteMany({ name: /^api-security-subject/ });
+    });
+
+    afterEach(async function () {
+      await rolesCollection().deleteMany({ name: /^api-security-role/ });
+      await subjectsCollection().deleteMany({ name: /^api-security-subject/ });
+    });
+
+    it('PUT /api/v2/authorization/subjects updates an existing subject by _id', async function () {
+      const insertResult = await subjectsCollection().insertOne({
+        name: 'api-security-subject-update',
+        roles: ['readable'],
+        notes: 'original',
+        created_at: '2024-10-26T20:32:49.173Z'
+      });
+      const token = await getBearerToken(self.token.adminAll);
+
+      await request(self.app)
+        .put('/api/v2/authorization/subjects')
+        .set('Authorization', 'Bearer ' + token)
+        .send({
+          _id: insertResult.insertedId.toString(),
+          name: 'api-security-subject-update',
+          roles: ['admin'],
+          notes: 'updated',
+          created_at: '2024-10-26T21:32:49.173Z'
+        })
+        .expect(200);
+
+      const docs = await subjectsCollection().find({ name: 'api-security-subject-update' }).toArray();
+      docs.length.should.equal(1);
+      docs[0].roles.should.deepEqual(['admin']);
+      docs[0].notes.should.equal('updated');
+    });
+
+    it('PUT /api/v2/authorization/roles updates an existing role by _id', async function () {
+      const insertResult = await rolesCollection().insertOne({
+        name: 'api-security-role-update',
+        permissions: ['api:entries:read'],
+        notes: 'original',
+        created_at: '2024-10-26T20:32:49.173Z'
+      });
+      const token = await getBearerToken(self.token.adminAll);
+
+      await request(self.app)
+        .put('/api/v2/authorization/roles')
+        .set('Authorization', 'Bearer ' + token)
+        .send({
+          _id: insertResult.insertedId.toString(),
+          name: 'api-security-role-update',
+          permissions: ['api:entries:update'],
+          notes: 'updated',
+          created_at: '2024-10-26T21:32:49.173Z'
+        })
+        .expect(200);
+
+      const docs = await rolesCollection().find({ name: 'api-security-role-update' }).toArray();
+      docs.length.should.equal(1);
+      docs[0].permissions.should.deepEqual(['api:entries:update']);
+      docs[0].notes.should.equal('updated');
+    });
+
+    it('PUT /api/v2/authorization/subjects fails when _id is missing', async function () {
+      const token = await getBearerToken(self.token.adminAll);
+
+      await request(self.app)
+        .put('/api/v2/authorization/subjects')
+        .set('Authorization', 'Bearer ' + token)
+        .send({
+          name: 'api-security-subject-missing-id',
+          roles: ['readable'],
+          notes: 'should fail'
+        })
+        .expect(500);
+
+      const docs = await subjectsCollection().find({ name: 'api-security-subject-missing-id' }).toArray();
+      docs.length.should.equal(0);
+    });
+
+    it('PUT /api/v2/authorization/subjects fails when _id is invalid', async function () {
+      const token = await getBearerToken(self.token.adminAll);
+
+      await request(self.app)
+        .put('/api/v2/authorization/subjects')
+        .set('Authorization', 'Bearer ' + token)
+        .send({
+          _id: 'not-a-valid-objectid',
+          name: 'api-security-subject-invalid-id',
+          roles: ['readable'],
+          notes: 'should fail'
+        })
+        .expect(500);
+
+      const docs = await subjectsCollection().find({ name: 'api-security-subject-invalid-id' }).toArray();
+      docs.length.should.equal(0);
+    });
+
+    it('PUT /api/v2/authorization/roles fails when _id is missing', async function () {
+      const token = await getBearerToken(self.token.adminAll);
+
+      await request(self.app)
+        .put('/api/v2/authorization/roles')
+        .set('Authorization', 'Bearer ' + token)
+        .send({
+          name: 'api-security-role-missing-id',
+          permissions: ['api:entries:read'],
+          notes: 'should fail'
+        })
+        .expect(500);
+
+      const docs = await rolesCollection().find({ name: 'api-security-role-missing-id' }).toArray();
+      docs.length.should.equal(0);
+    });
+
+    it('PUT /api/v2/authorization/roles fails when _id is invalid', async function () {
+      const token = await getBearerToken(self.token.adminAll);
+
+      await request(self.app)
+        .put('/api/v2/authorization/roles')
+        .set('Authorization', 'Bearer ' + token)
+        .send({
+          _id: 'not-a-valid-objectid',
+          name: 'api-security-role-invalid-id',
+          permissions: ['api:entries:read'],
+          notes: 'should fail'
+        })
+        .expect(500);
+
+      const docs = await rolesCollection().find({ name: 'api-security-role-invalid-id' }).toArray();
+      docs.length.should.equal(0);
+    });
+  });
+
 });

tests/api.shape-handling.test.js

@@ -352,16 +352,12 @@ describe('API Shape Handling - Single Object vs Array Input', function () {
 
   describe('Entries API - /api/entries/', function () {
     
-    beforeEach(function (done) {
+    beforeEach(async function () {
-      self.ctx.entries().deleteMany({}, function () {
+      await self.ctx.entries().deleteMany({});
-        done();
-      });
     });
 
-    afterEach(function (done) {
+    afterEach(async function () {
-      self.ctx.entries().deleteMany({}, function () {
+      await self.ctx.entries().deleteMany({});
-        done();
-      });
     });
 
     it('POST accepts single SGV entry object', function (done) {
@@ -838,7 +834,95 @@ describe('API Shape Handling - Single Object vs Array Input', function () {
               p.should.have.property('_id');
             });
             done();
-          });
+        });
+      });
+    });
+
+    describe('Legacy PUT update semantics', function () {
+      beforeEach(async function () {
+        await self.ctx.food().deleteMany({});
+        await self.ctx.activity().deleteMany({});
+      });
+
+      afterEach(async function () {
+        await self.ctx.food().deleteMany({});
+        await self.ctx.activity().deleteMany({});
+      });
+
+      it('PUT /api/food/ updates an existing record by _id when created_at changes', function (done) {
+        self.ctx.food.create({
+          name: 'API shape food',
+          category: 'Test',
+          carbs: 20,
+          protein: 10,
+          fat: 5
+        }, function (createErr, docs) {
+          if (createErr) return done(createErr);
+
+          var created = docs[0];
+          request(self.app)
+            .put('/api/food/')
+            .set('api-secret', known)
+            .send({
+              _id: created._id.toString(),
+              name: 'API shape food',
+              category: 'Test',
+              carbs: 25,
+              protein: 10,
+              fat: 5,
+              created_at: '2024-10-26T21:32:49.173Z'
+            })
+            .expect(200)
+            .end(function (err) {
+              if (err) return done(err);
+
+              self.ctx.food().find({ _id: created._id }).toArray()
+                .then(function (updatedDocs) {
+                  updatedDocs.length.should.equal(1);
+                  updatedDocs[0].carbs.should.equal(25);
+                  updatedDocs[0].created_at.should.equal('2024-10-26T21:32:49.173Z');
+                  done();
+                })
+                .catch(done);
+            });
+        });
+      });
+
+      it('PUT /api/activity/ updates an existing record by _id', function (done) {
+        self.ctx.activity.create([{
+          created_at: '2024-10-26T20:32:49.173Z',
+          heartrate: 80,
+          steps: 100,
+          activitylevel: 'walking'
+        }], function (createErr, docs) {
+          if (createErr) return done(createErr);
+
+          var created = docs[0];
+          request(self.app)
+            .put('/api/activity/')
+            .set('api-secret', known)
+            .send({
+              _id: created._id.toString(),
+              created_at: '2024-10-26T21:32:49.173Z',
+              heartrate: 95,
+              steps: 250,
+              activitylevel: 'running'
+            })
+            .expect(200)
+            .end(function (err) {
+              if (err) return done(err);
+
+              self.ctx.activity().find({ _id: created._id }).toArray()
+                .then(function (updatedDocs) {
+                  updatedDocs.length.should.equal(1);
+                  updatedDocs[0].heartrate.should.equal(95);
+                  updatedDocs[0].steps.should.equal(250);
+                  updatedDocs[0].activitylevel.should.equal('running');
+                  done();
+                })
+                .catch(done);
+            });
+        });
       });
     });
   });

tests/api.unauthorized.test.js

@@ -38,12 +38,12 @@ describe('authed REST api', function ( ) {
     this.archive.create(creating, done);
   });
 
-  afterEach(function (done) {
+  afterEach(async function () {
-    this.archive( ).deleteMany({ }, done);
+    await this.archive( ).deleteMany({ });
   });
 
-  after(function (done) {
+  after(async function () {
-    this.archive( ).deleteMany({ }, done);
+    await this.archive( ).deleteMany({ });
   });
 
   it('disallow unauthorized POST', function (done) {

tests/api3.storage.find.test.js

@@ -3,13 +3,17 @@
 require('should');
 
 const find = require('../lib/api3/storage/mongoCollection/find');
+const { ObjectId } = require('mongodb');
 
-describe('API3 mongoCollection findMany', function () {
+describe('API3 mongoCollection find helpers', function () {
-  function createStubCollection (observed) {
+  function createStubCursor (observed, docs) {
     return {
       find: function () {
         return this;
       },
+      project: function () {
+        return this;
+      },
       sort: function () {
         return this;
       },
@@ -21,11 +25,17 @@ describe('API3 mongoCollection findMany', function () {
         observed.skip = value;
         return this;
       },
-      project: function () {
+      toArray: function () {
-        return this;
+        observed.toArrayCalls = (observed.toArrayCalls || 0) + 1;
-      },
+        return Promise.resolve(docs || []);
-      toArray: function (callback) {
+      }
-        callback(null, []);
+    };
+  }
+
+  function createStubCollection (observed, docs) {
+    return {
+      find: function () {
+        return createStubCursor(observed, docs);
       }
     };
   }
@@ -44,6 +54,7 @@ describe('API3 mongoCollection findMany', function () {
 
     observed.limit.should.equal(5);
     observed.skip.should.equal(2);
+    observed.toArrayCalls.should.equal(1);
     result.should.eql([]);
   });
 
@@ -63,4 +74,30 @@ describe('API3 mongoCollection findMany', function () {
     observed.limit.should.equal(5);
     observed.skip.should.equal(2);
   });
+
+  it('normalizes findOne results when using promise-based cursors', async function () {
+    const observed = {};
+    const docId = new ObjectId();
+    const col = createStubCollection(observed, [{ _id: docId, type: 'sgv' }]);
+
+    const result = await find.findOne(col, docId.toString(), {});
+
+    observed.toArrayCalls.should.equal(1);
+    result.should.have.length(1);
+    result[0].should.have.property('identifier', docId.toString());
+    result[0].should.not.have.property('_id');
+  });
+
+  it('supports promise-based findOneFilter without normalization when requested', async function () {
+    const observed = {};
+    const docId = new ObjectId();
+    const col = createStubCollection(observed, [{ _id: docId, type: 'mbg' }]);
+
+    const result = await find.findOneFilter(col, { type: 'mbg' }, {}, { normalize: false });
+
+    observed.toArrayCalls.should.equal(1);
+    result.should.have.length(1);
+    result[0].should.have.property('_id');
+    result[0]._id.toString().should.equal(docId.toString());
+  });
 });

tests/api3.storage.modify.test.js

@@ -0,0 +1,161 @@
+'use strict';
+
+require('should');
+
+const { ObjectId } = require('mongodb');
+const modify = require('../lib/api3/storage/mongoCollection/modify');
+const MongoCollection = require('../lib/api3/storage/mongoCollection');
+
+describe('API3 mongoCollection promise-based helpers', function () {
+  describe('modify helpers', function () {
+    it('insertOne uses promise-based collection methods and keeps existing behavior', async function () {
+      const doc = { _id: new ObjectId(), type: 'sgv' };
+      const insertedId = doc._id.toString();
+      const col = {
+        insertOne: function (receivedDoc) {
+          arguments.length.should.equal(1);
+          receivedDoc.should.equal(doc);
+          return Promise.resolve({ insertedId: receivedDoc._id });
+        }
+      };
+
+      const identifier = await modify.insertOne(col, doc);
+
+      identifier.should.equal(insertedId);
+      doc.should.not.have.property('_id');
+    });
+
+    it('replaceOne uses promise-based upsert without a callback', async function () {
+      const doc = { value: 42 };
+      const col = {
+        replaceOne: function (filter, receivedDoc, options) {
+          arguments.length.should.equal(3);
+          filter.should.eql({ $or: [{ identifier: 'record-1' }] });
+          receivedDoc.should.equal(doc);
+          options.should.eql({ upsert: true });
+          return Promise.resolve({ matchedCount: 1 });
+        }
+      };
+
+      const matchedCount = await modify.replaceOne(col, 'record-1', doc);
+
+      matchedCount.should.equal(1);
+    });
+
+    it('update and delete helpers use promise-based collection methods', async function () {
+      const col = {
+        updateOne: function (filter, update) {
+          arguments.length.should.equal(2);
+          filter.should.eql({ $or: [{ identifier: 'record-2' }] });
+          update.should.eql({ $set: { value: 84 } });
+          return Promise.resolve({ modifiedCount: 1 });
+        },
+        deleteOne: function (filter) {
+          arguments.length.should.equal(1);
+          filter.should.eql({ $or: [{ identifier: 'record-2' }] });
+          return Promise.resolve({ deletedCount: 1 });
+        },
+        deleteMany: function (filter) {
+          arguments.length.should.equal(1);
+          filter.should.eql({ $or: [{ kind: 'sgv' }, { kind: 'mbg' }] });
+          return Promise.resolve({ deletedCount: 2 });
+        }
+      };
+
+      const updated = await modify.updateOne(col, 'record-2', { value: 84 });
+      const deletedOne = await modify.deleteOne(col, 'record-2');
+      const deletedMany = await modify.deleteManyOr(col, [
+        { field: 'kind', operator: 'eq', value: 'sgv' },
+        { field: 'kind', operator: 'eq', value: 'mbg' }
+      ]);
+
+      updated.should.eql({ updated: 1 });
+      deletedOne.should.eql({ deleted: 1 });
+      deletedMany.should.eql({ deleted: 2 });
+    });
+  });
+
+  describe('collection metadata helpers', function () {
+    function createCursor (observed, docs) {
+      return {
+        sort: function (spec) {
+          observed.sort = spec;
+          return this;
+        },
+        limit: function (value) {
+          observed.limit = value;
+          return this;
+        },
+        project: function (projection) {
+          observed.projection = projection;
+          return this;
+        },
+        toArray: function () {
+          observed.toArrayCalls = (observed.toArrayCalls || 0) + 1;
+          return Promise.resolve(docs);
+        }
+      };
+    }
+
+    function createStorage (observed, docs) {
+      const col = {
+        find: function () {
+          arguments.length.should.equal(0);
+          observed.findCalls = (observed.findCalls || 0) + 1;
+          return createCursor(observed, docs);
+        }
+      };
+
+      const ctx = {
+        store: {
+          collection: function (name) {
+            observed.collectionName = name;
+            return col;
+          },
+          ensureIndexes: function (receivedCol, fields) {
+            receivedCol.should.equal(col);
+            observed.indexFields = fields;
+          },
+          db: {
+            admin: function () {
+              return {
+                buildInfo: function () {
+                  arguments.length.should.equal(0);
+                  observed.buildInfoCalls = (observed.buildInfoCalls || 0) + 1;
+                  return Promise.resolve({ version: '5.9.2' });
+                }
+              };
+            }
+          }
+        }
+      };
+
+      return new MongoCollection(ctx, {}, 'entries');
+    }
+
+    it('version uses promise-based buildInfo', async function () {
+      const observed = {};
+      const storage = createStorage(observed, []);
+
+      const version = await storage.version();
+
+      observed.collectionName.should.equal('entries');
+      observed.buildInfoCalls.should.equal(1);
+      version.should.eql({ storage: 'mongodb', version: '5.9.2' });
+    });
+
+    it('getLastModified uses promise-based cursor methods', async function () {
+      const observed = {};
+      const storage = createStorage(observed, [{ srvModified: 1234 }]);
+
+      const lastModified = await storage.getLastModified('srvModified');
+
+      observed.findCalls.should.equal(1);
+      observed.sort.should.eql({ srvModified: -1 });
+      observed.limit.should.equal(1);
+      observed.projection.should.eql({ srvModified: 1 });
+      observed.toArrayCalls.should.equal(1);
+      lastModified.should.eql({ srvModified: 1234 });
+    });
+  });
+});

tests/cache-objectid-compat.test.js

@@ -12,7 +12,7 @@
  * On mongodb driver 3.x (v15.0.6), Object.keys(new ObjectID()) returned
  * ['_bsontype','id'], making _.isEmpty return false (correct).
  *
- * On mongodb driver 5.x / mongodb-legacy (dev), Object.keys(new ObjectId())
+ * On mongodb driver 5.x, Object.keys(new ObjectId())
  * returns [], making _.isEmpty return true (incorrect – treats valid
  * ObjectId _id as empty).
  *
@@ -21,7 +21,7 @@
  */
 
 const _ = require('lodash');
-const { ObjectId } = require('mongodb-legacy');
+const { ObjectId } = require('mongodb');
 const should = require('should');
 
 describe('Cache ObjectId compatibility', function () {

tests/concurrent-writes.test.js

@@ -306,16 +306,12 @@ describe('Concurrent Write Tests - MongoDB 5.x Compatibility', function () {
 
   describe('Simultaneous POST requests to entries', function () {
 
-    beforeEach(function (done) {
+    beforeEach(async function () {
-      self.ctx.entries().deleteMany({}, function () {
+      await self.ctx.entries().deleteMany({});
-        done();
-      });
     });
 
-    afterEach(function (done) {
+    afterEach(async function () {
-      self.ctx.entries().deleteMany({}, function () {
+      await self.ctx.entries().deleteMany({});
-        done();
-      });
     });
 
     it('handles 5 simultaneous single entry POSTs', function (done) {
@@ -422,9 +418,11 @@ describe('Concurrent Write Tests - MongoDB 5.x Compatibility', function () {
     beforeEach(function (done) {
       self.ctx.treatments.remove({ find: { created_at: { '$gte': '1999-01-01T00:00:00.000Z' } } }, function () {
         self.ctx.devicestatus.remove({ find: { created_at: { '$gte': '1999-01-01T00:00:00.000Z' } } }, function () {
-          self.ctx.entries().deleteMany({}, function () {
+          self.ctx.entries().deleteMany({})
-            done();
+            .then(function () {
-          });
+              done();
+            })
+            .catch(done);
         });
       });
     });
@@ -432,9 +430,11 @@ describe('Concurrent Write Tests - MongoDB 5.x Compatibility', function () {
     afterEach(function (done) {
       self.ctx.treatments.remove({ find: { created_at: { '$gte': '1999-01-01T00:00:00.000Z' } } }, function () {
         self.ctx.devicestatus.remove({ find: { created_at: { '$gte': '1999-01-01T00:00:00.000Z' } } }, function () {
-          self.ctx.entries().deleteMany({}, function () {
+          self.ctx.entries().deleteMany({})
-            done();
+            .then(function () {
-          });
+              done();
+            })
+            .catch(done);
         });
       });
     });

tests/dataloader.test.js

@@ -0,0 +1,66 @@
+'use strict';
+
+require('should');
+
+const dataloaderInit = require('../lib/data/dataloader');
+const createDData = require('../lib/data/ddata');
+
+describe('dataloader', function () {
+  it('completes update when db.stats is promise-based', function (done) {
+    const ddata = createDData();
+    ddata.processTreatments = function () {};
+    const ctx = {
+      settings: {},
+      language: {
+        translate: function (value) { return value; }
+      },
+      cache: {
+        isEmpty: function () { return true; },
+        insertData: function (key, results) { return results; }
+      },
+      ddata: ddata,
+      entries: {
+        list: function (query, callback) { callback(null, []); }
+      },
+      treatments: {
+        list: function (query, callback) { callback(null, []); }
+      },
+      profile: {
+        last: function (callback) { callback(null, []); }
+      },
+      food: {
+        list: function (callback) { callback(null, []); }
+      },
+      devicestatus: {
+        list: function (query, callback) { callback(null, []); }
+      },
+      activity: {
+        list: function (query, callback) { callback(null, []); }
+      },
+      store: {
+        db: {
+          stats: function () {
+            return Promise.resolve({ dataSize: 123, indexSize: 456 });
+          }
+        }
+      }
+    };
+    const env = {
+      settings: {
+        isEnabled: function () { return false; },
+        units: 'mg/dl'
+      },
+      extendedSettings: {}
+    };
+    const loader = dataloaderInit(env, ctx);
+
+    loader.update(ddata, function (err) {
+      should.not.exist(err);
+      ddata.dbstats.should.eql({
+        dataSize: 123,
+        indexSize: 456
+      });
+      done();
+    });
+  });
+});

tests/fixtures/test-guard.js

@@ -75,7 +75,14 @@ function guardedDeleteMany(collection, filter, callback) {
   }
   
   // Safe to proceed
-  return collection.deleteMany(filter, callback);
+  const promise = collection.deleteMany(filter);
+  if (callback) {
+    promise.then(
+      function onSuccess(result) { callback(null, result); },
+      function onError(err) { callback(err); }
+    );
+  }
+  return promise;
 }
 
 /**
@@ -95,7 +102,14 @@ function guardedDrop(collection, callback) {
     throw err;
   }
   
-  return collection.drop(callback);
+  const promise = collection.drop();
+  if (callback) {
+    promise.then(
+      function onSuccess(result) { callback(null, result); },
+      function onError(err) { callback(err); }
+    );
+  }
+  return promise;
 }
 
 /**

tests/issue-6923-legacy-uuid.test.js

@@ -63,10 +63,8 @@ describe('Issue #6923: Legacy UUID override edit/delete', function () {
     return self.ctx.store.collection(self.env.treatments_collection);
   }
 
-  beforeEach(function (done) {
+  beforeEach(async function () {
-    rawCollection().deleteMany({}, function () {
+    await rawCollection().deleteMany({});
-      done();
-    });
   });
 
   /**
@@ -75,16 +73,19 @@ describe('Issue #6923: Legacy UUID override edit/delete', function () {
    */
   function insertLegacyDoc (callback) {
     var doc = Object.assign({}, LEGACY_OVERRIDE);
-    rawCollection().insertOne(doc, function (err) {
+    rawCollection().insertOne(doc)
-      should.not.exist(err);
+      .then(function () {
-      rawCollection().findOne({ _id: LEGACY_UUID }, function (err, stored) {
+        return rawCollection().findOne({ _id: LEGACY_UUID });
-        should.not.exist(err);
+      })
+      .then(function (stored) {
         should.exist(stored, 'Legacy doc should exist after direct insert');
         stored._id.should.equal(LEGACY_UUID);
         should.not.exist(stored.identifier, 'Legacy doc must NOT have identifier field');
         callback(stored);
+      })
+      .catch(function (err) {
+        should.not.exist(err);
       });
-    });
   }
 
   describe('DELETE legacy UUID override via API', function () {
@@ -130,10 +131,8 @@ describe('Issue #6923: Legacy UUID override edit/delete', function () {
 
         // Check the database after the server has had time to process the upsert
         setTimeout(function () {
-          rawCollection().find({ eventType: 'Temporary Override' }).toArray(function (err, docs) {
+          rawCollection().find({ eventType: 'Temporary Override' }).toArray()
-            try {
+            .then(function (docs) {
-              should.not.exist(err);
-
               docs.length.should.equal(1,
                 'PUT should update the existing legacy override, not create a duplicate. '
                 + 'Found ' + docs.length + ' documents. '
@@ -143,10 +142,8 @@ describe('Issue #6923: Legacy UUID override edit/delete', function () {
               );
 
               done();
-            } catch (e) {
+            })
-              done(e);
+            .catch(done);
-            }
-          });
         }, 5000);
       });
     });

tests/mongo-storage.retry.test.js

@@ -0,0 +1,134 @@
+'use strict';
+
+var should = require('should');
+var mongodb = require('mongodb');
+
+describe('mongo storage retry lifecycle', function () {
+  var originalMongoClient = mongodb.MongoClient;
+  var originalSetTimeout = global.setTimeout;
+
+  function setMongoClient(fakeMongoClient) {
+    Object.defineProperty(mongodb, 'MongoClient', {
+      configurable: true,
+      enumerable: true,
+      value: fakeMongoClient,
+      writable: true
+    });
+  }
+
+  afterEach(function () {
+    setMongoClient(originalMongoClient);
+    global.setTimeout = originalSetTimeout;
+    delete require.cache[require.resolve('../lib/storage/mongo-storage')];
+  });
+
+  it('closes failed retry clients and calls back only after a successful retry', function (done) {
+    var createdClients = [];
+    var connectAttempts = 0;
+    var retryDelays = [];
+
+    function FakeMongoClient() {
+      this.closed = 0;
+      createdClients.push(this);
+    }
+
+    FakeMongoClient.prototype.on = function () {};
+    FakeMongoClient.prototype.connect = function () {
+      connectAttempts += 1;
+
+      if (connectAttempts === 1) {
+        var err = new Error('server selection failed');
+        err.name = 'MongoServerSelectionError';
+        return Promise.reject(err);
+      }
+
+      return Promise.resolve();
+    };
+    FakeMongoClient.prototype.db = function () {
+      return {
+        databaseName: 'testdb',
+        command: function () {
+          return Promise.resolve({ authInfo: { authenticatedUserRoles: [] } });
+        },
+        collection: function (name) {
+          return { collectionName: name };
+        }
+      };
+    };
+    FakeMongoClient.prototype.close = function () {
+      this.closed += 1;
+      return Promise.resolve();
+    };
+
+    setMongoClient(FakeMongoClient);
+    global.setTimeout = function (fn, ms) {
+      retryDelays.push(ms);
+      Promise.resolve().then(fn);
+      return 1;
+    };
+
+    delete require.cache[require.resolve('../lib/storage/mongo-storage')];
+    var store = require('../lib/storage/mongo-storage');
+    var callbackCount = 0;
+
+    store({ storageURI: 'mongodb://example/testdb' }, function (err, db) {
+      callbackCount += 1;
+
+      should.not.exist(err);
+      should.exist(db);
+      callbackCount.should.equal(1);
+      connectAttempts.should.equal(2);
+      createdClients.length.should.equal(2);
+      createdClients[0].closed.should.equal(1);
+      createdClients[1].closed.should.equal(0);
+      retryDelays.should.eql([3000]);
+      db.db.databaseName.should.equal('testdb');
+
+      done();
+    }, true);
+  });
+
+  it('closes the client and reports authentication failure once', function (done) {
+    var createdClients = [];
+    var retryDelays = [];
+
+    function FakeMongoClient() {
+      this.closed = 0;
+      createdClients.push(this);
+    }
+
+    FakeMongoClient.prototype.on = function () {};
+    FakeMongoClient.prototype.connect = function () {
+      return Promise.reject(new Error('AuthenticationFailed: bad auth'));
+    };
+    FakeMongoClient.prototype.close = function () {
+      this.closed += 1;
+      return Promise.resolve();
+    };
+
+    setMongoClient(FakeMongoClient);
+    global.setTimeout = function (fn, ms) {
+      retryDelays.push(ms);
+      Promise.resolve().then(fn);
+      return 1;
+    };
+
+    delete require.cache[require.resolve('../lib/storage/mongo-storage')];
+    var store = require('../lib/storage/mongo-storage');
+    var callbackCount = 0;
+
+    store({ storageURI: 'mongodb://example/testdb' }, function (err, db) {
+      callbackCount += 1;
+
+      should.exist(err);
+      should.not.exist(db);
+      err.message.should.equal('MongoDB authentication failed! Double check the URL has the right username and password in MONGODB_URI.');
+      callbackCount.should.equal(1);
+      createdClients.length.should.equal(1);
+      createdClients[0].closed.should.equal(1);
+      retryDelays.should.eql([]);
+
+      done();
+    }, true);
+  });
+});

tests/mongo-storage.test.js

@@ -30,6 +30,42 @@ describe('mongo storage', function () {
     });
   });
 
+  it('Uses the default database from the connection string via the public client API', function (done) {
+    var store = require('../lib/storage/mongo-storage');
+    store(env, function (err, db) {
+      should.not.exist(err);
+      should.exist(db.db);
+      should.exist(db.client);
+
+      db.client.db().databaseName.should.equal(db.db.databaseName);
+      db.db.databaseName.should.equal('testdb');
+      done();
+    });
+  });
+
+  it('ensureIndexes uses createIndex without the legacy background option', function (done) {
+    var store = require('../lib/storage/mongo-storage');
+    var calls = [];
+
+    store(env, function (err, db) {
+      should.not.exist(err);
+
+      db.ensureIndexes({
+        collectionName: 'entries',
+        createIndex: function (field, options) {
+          calls.push({ field: field, options: options });
+          return Promise.resolve();
+        }
+      }, ['date']);
+
+      calls.length.should.equal(1);
+      calls[0].field.should.equal('date');
+      should.not.exist(calls[0].options);
+
+      done();
+    });
+  });
+
   it('When no connection-string is given the storage-class should throw an error.', function (done) {
     delete env.storageURI;
     should.not.exist(env.storageURI);
@@ -58,4 +94,3 @@ describe('mongo storage', function () {
   });
 
 });
-

tests/sgv-devicestatus.test.js

@@ -37,8 +37,8 @@ describe('Loop SGV Entry Upload Tests', function() {
     });
   });
 
-  beforeEach(function(done) {
+  beforeEach(async function() {
-    self.ctx.entries().deleteMany({}, done);
+    await self.ctx.entries().deleteMany({});
   });
 
   describe('TEST-SGV-001: Single SGV entry', function() {

tests/storage.shape-handling.test.js

@@ -173,16 +173,12 @@ describe('Storage Layer Shape Handling - Direct Storage Tests', function () {
 
   describe('Entries Storage - lib/server/entries.js', function () {
     
-    beforeEach(function (done) {
+    beforeEach(async function () {
-      self.ctx.entries().deleteMany({}, function () {
+      await self.ctx.entries().deleteMany({});
-        done();
-      });
     });
 
-    afterEach(function (done) {
+    afterEach(async function () {
-      self.ctx.entries().deleteMany({}, function () {
+      await self.ctx.entries().deleteMany({});
-        done();
-      });
     });
 
     it('create() accepts single entry in array', function (done) {
@@ -218,16 +214,12 @@ describe('Storage Layer Shape Handling - Direct Storage Tests', function () {
 
   describe('Profile Storage - lib/server/profile.js', function () {
     
-    beforeEach(function (done) {
+    beforeEach(async function () {
-      self.ctx.profile().deleteMany({}, function () {
+      await self.ctx.profile().deleteMany({});
-        done();
-      });
     });
 
-    afterEach(function (done) {
+    afterEach(async function () {
-      self.ctx.profile().deleteMany({}, function () {
+      await self.ctx.profile().deleteMany({});
-        done();
-      });
     });
 
     it('create() accepts single profile object', function (done) {
@@ -347,13 +339,14 @@ describe('Storage Layer Shape Handling - Direct Storage Tests', function () {
         self.ctx.profile.save(updated, function (saveErr) {
           should.not.exist(saveErr);
 
-          self.ctx.profile().find({ _id: savedId }).toArray(function (findErr, docs) {
+          self.ctx.profile().find({ _id: savedId }).toArray()
-            should.not.exist(findErr);
+            .then(function (docs) {
-            docs.length.should.equal(1);
+              docs.length.should.equal(1);
-            docs[0].store.Default.dia.should.equal(4);
+              docs[0].store.Default.dia.should.equal(4);
-            docs[0].created_at.should.equal('2024-10-26T21:32:49.173Z');
+              docs[0].created_at.should.equal('2024-10-26T21:32:49.173Z');
-            done();
+              done();
-          });
+            })
+            .catch(done);
         });
       });
     });
@@ -438,28 +431,25 @@ describe('Storage Layer Shape Handling - Direct Storage Tests', function () {
         should.not.exist(err);
         saved.created_at.should.equal('2020-01-01T00:00:00.000Z');
 
-        self.ctx.profile().find({ _id: saved._id }).toArray(function (findErr, docs) {
+        self.ctx.profile().find({ _id: saved._id }).toArray()
-          should.not.exist(findErr);
+          .then(function (docs) {
-          docs.length.should.equal(1);
+            docs.length.should.equal(1);
-          docs[0].created_at.should.equal('2020-01-01T00:00:00.000Z');
+            docs[0].created_at.should.equal('2020-01-01T00:00:00.000Z');
-          done();
+            done();
-        });
+          })
+          .catch(done);
       });
     });
   });
 
   describe('Food Storage - lib/server/food.js', function () {
     
-    beforeEach(function (done) {
+    beforeEach(async function () {
-      self.ctx.food().deleteMany({}, function () {
+      await self.ctx.food().deleteMany({});
-        done();
-      });
     });
 
-    afterEach(function (done) {
+    afterEach(async function () {
-      self.ctx.food().deleteMany({}, function () {
+      await self.ctx.food().deleteMany({});
-        done();
-      });
     });
 
     it('create() accepts single food object', function (done) {
@@ -480,20 +470,59 @@ describe('Storage Layer Shape Handling - Direct Storage Tests', function () {
         done();
       });
     });
+
+    it('save() updates an existing food by _id when created_at changes', function (done) {
+      self.ctx.food.create({
+        name: 'Test Food',
+        category: 'Test',
+        carbs: 20,
+        protein: 10,
+        fat: 5
+      }, function (err, createdDocs) {
+        should.not.exist(err);
+        should.exist(createdDocs);
+        createdDocs.should.be.an.Array();
+        createdDocs.length.should.equal(1);
+
+        var savedId = createdDocs[0]._id;
+        var updated = {
+          _id: savedId.toString(),
+          name: 'Updated Food',
+          category: 'Test',
+          carbs: 25,
+          protein: 10,
+          fat: 5,
+          created_at: '2024-10-26T21:32:49.173Z'
+        };
+
+        self.ctx.food.save(updated, function (saveErr, savedDocs) {
+          should.not.exist(saveErr);
+          should.exist(savedDocs);
+          savedDocs.should.be.an.Array();
+          savedDocs.length.should.equal(1);
+
+          self.ctx.food().find({ _id: savedId }).toArray()
+            .then(function (docs) {
+              docs.length.should.equal(1);
+              docs[0].name.should.equal('Updated Food');
+              docs[0].carbs.should.equal(25);
+              docs[0].created_at.should.equal('2024-10-26T21:32:49.173Z');
+              done();
+            })
+            .catch(done);
+        });
+      });
+    });
   });
 
   describe('Activity Storage - lib/server/activity.js', function () {
     
-    beforeEach(function (done) {
+    beforeEach(async function () {
-      self.ctx.activity().deleteMany({}, function () {
+      await self.ctx.activity().deleteMany({});
-        done();
-      });
     });
 
-    afterEach(function (done) {
+    afterEach(async function () {
-      self.ctx.activity().deleteMany({}, function () {
+      await self.ctx.activity().deleteMany({});
-        done();
-      });
     });
 
     it('create() accepts array of activity objects (single object not supported)', function (done) {
@@ -511,6 +540,124 @@ describe('Storage Layer Shape Handling - Direct Storage Tests', function () {
         done();
       });
     });
+
+    it('save() updates an existing activity by _id', function (done) {
+      self.ctx.activity.create([{
+        created_at: '2024-10-26T20:32:49.173Z',
+        heartrate: 80,
+        steps: 100,
+        activitylevel: 'walking'
+      }], function (err, createdDocs) {
+        should.not.exist(err);
+        should.exist(createdDocs);
+        createdDocs.length.should.equal(1);
+        should.exist(createdDocs[0]._id);
+
+        var savedId = createdDocs[0]._id;
+        var updated = {
+          _id: savedId.toString(),
+          created_at: '2024-10-26T21:32:49.173Z',
+          heartrate: 95,
+          steps: 250,
+          activitylevel: 'running'
+        };
+
+        self.ctx.activity.save(updated, function (saveErr, savedDoc) {
+          should.not.exist(saveErr);
+          should.exist(savedDoc);
+          savedDoc._id.toString().should.equal(savedId.toString());
+
+          self.ctx.activity().find({ _id: savedId }).toArray()
+            .then(function (docs) {
+              docs.length.should.equal(1);
+              docs[0].heartrate.should.equal(95);
+              docs[0].steps.should.equal(250);
+              docs[0].activitylevel.should.equal('running');
+              docs[0].created_at.should.equal('2024-10-26T21:32:49.173Z');
+              done();
+            })
+            .catch(done);
+        });
+      });
+    });
+  });
+
+  describe('Authorization Storage - lib/authorization/storage.js', function () {
+    function rolesCollection() {
+      return self.ctx.store.collection(self.env.authentication_collections_prefix + 'roles');
+    }
+
+    function subjectsCollection() {
+      return self.ctx.store.collection(self.env.authentication_collections_prefix + 'subjects');
+    }
+
+    beforeEach(async function () {
+      await rolesCollection().deleteMany({ name: 'mongo-save-role' });
+      await subjectsCollection().deleteMany({ name: 'mongo-save-subject' });
+    });
+
+    afterEach(async function () {
+      await rolesCollection().deleteMany({ name: 'mongo-save-role' });
+      await subjectsCollection().deleteMany({ name: 'mongo-save-subject' });
+    });
+
+    it('saveRole() updates an existing role without duplicating it', function (done) {
+      rolesCollection().insertOne({
+        name: 'mongo-save-role',
+        permissions: ['api:entries:read'],
+        notes: 'original',
+        created_at: '2024-10-26T20:32:49.173Z'
+      }).then(function (result) {
+        self.ctx.authorization.storage.saveRole({
+          _id: result.insertedId.toString(),
+          name: 'mongo-save-role',
+          permissions: ['api:entries:update'],
+          notes: 'updated',
+          created_at: '2024-10-26T21:32:49.173Z'
+        }, function (saveErr) {
+          should.not.exist(saveErr);
+
+          rolesCollection().find({ name: 'mongo-save-role' }).toArray()
+            .then(function (docs) {
+              docs.length.should.equal(1);
+              docs[0].permissions.should.deepEqual(['api:entries:update']);
+              docs[0].notes.should.equal('updated');
+              docs[0].created_at.should.equal('2024-10-26T21:32:49.173Z');
+              done();
+            })
+            .catch(done);
+        });
+      }).catch(done);
+    });
+
+    it('saveSubject() updates an existing subject without duplicating it', function (done) {
+      subjectsCollection().insertOne({
+        name: 'mongo-save-subject',
+        roles: ['readable'],
+        notes: 'original',
+        created_at: '2024-10-26T20:32:49.173Z'
+      }).then(function (result) {
+        self.ctx.authorization.storage.saveSubject({
+          _id: result.insertedId.toString(),
+          name: 'mongo-save-subject',
+          roles: ['admin'],
+          notes: 'updated',
+          created_at: '2024-10-26T21:32:49.173Z'
+        }, function (saveErr) {
+          should.not.exist(saveErr);
+
+          subjectsCollection().find({ name: 'mongo-save-subject' }).toArray()
+            .then(function (docs) {
+              docs.length.should.equal(1);
+              docs[0].roles.should.deepEqual(['admin']);
+              docs[0].notes.should.equal('updated');
+              docs[0].created_at.should.equal('2024-10-26T21:32:49.173Z');
+              done();
+            })
+            .catch(done);
+        });
+      }).catch(done);
+    });
   });
 });
 
@@ -533,71 +680,61 @@ describe('MongoDB insertOne vs insertMany Behavior', function () {
 
   describe('Direct MongoDB operations - testing insertOne with array data', function () {
     
-    it('insertOne with object inserts correctly', function (done) {
+    it('insertOne with object inserts correctly', async function () {
       var testCollection = self.ctx.store.collection('test_shape_handling');
-      
+
-      testCollection.deleteMany({}, function () {
+      await testCollection.deleteMany({});
-        testCollection.insertOne({ type: 'test', value: 42 }, function (err, result) {
+      var result = await testCollection.insertOne({ type: 'test', value: 42 });
-          should.not.exist(err);
+      should.exist(result);
-          should.exist(result);
+      result.insertedId.should.be.ok();
-          result.insertedId.should.be.ok();
+
-          
+      var docs = await testCollection.find({}).toArray();
-          testCollection.find({}).toArray(function (err, docs) {
+      docs.length.should.equal(1);
-            docs.length.should.equal(1);
+      docs[0].value.should.equal(42);
-            docs[0].value.should.equal(42);
+      await testCollection.deleteMany({});
-            testCollection.deleteMany({}, done);
-          });
-        });
-      });
     });
 
-    it('insertOne with array creates single document containing array (NOT multiple docs)', function (done) {
+    it('insertOne with array creates single document containing array (NOT multiple docs)', async function () {
       var testCollection = self.ctx.store.collection('test_shape_handling');
-      
+
-      testCollection.deleteMany({}, function () {
+      await testCollection.deleteMany({});
-        var arrayData = [
+      var arrayData = [
-          { type: 'test', value: 1 },
+        { type: 'test', value: 1 },
-          { type: 'test', value: 2 },
+        { type: 'test', value: 2 },
-          { type: 'test', value: 3 }
+        { type: 'test', value: 3 }
-        ];
+      ];
-        
+
-        testCollection.insertOne(arrayData, function (err, result) {
+      try {
-          if (err) {
+        await testCollection.insertOne(arrayData);
-            console.log('insertOne with array error:', err.message);
+      } catch (err) {
-            done();
+        console.log('insertOne with array error:', err.message);
-          } else {
+        await testCollection.deleteMany({});
-            testCollection.find({}).toArray(function (err, docs) {
+        return;
-              console.log('Documents after insertOne with array:', JSON.stringify(docs, null, 2));
+      }
-              console.log('Number of documents:', docs.length);
+
-              
+      var docs = await testCollection.find({}).toArray();
-              testCollection.deleteMany({}, done);
+      console.log('Documents after insertOne with array:', JSON.stringify(docs, null, 2));
-            });
+      console.log('Number of documents:', docs.length);
-          }
+      await testCollection.deleteMany({});
-        });
-      });
     });
 
-    it('insertMany with array creates multiple documents', function (done) {
+    it('insertMany with array creates multiple documents', async function () {
       var testCollection = self.ctx.store.collection('test_shape_handling');
-      
+
-      testCollection.deleteMany({}, function () {
+      await testCollection.deleteMany({});
-        var arrayData = [
+      var arrayData = [
-          { type: 'test', value: 1 },
+        { type: 'test', value: 1 },
-          { type: 'test', value: 2 },
+        { type: 'test', value: 2 },
-          { type: 'test', value: 3 }
+        { type: 'test', value: 3 }
-        ];
+      ];
-        
+
-        testCollection.insertMany(arrayData, function (err, result) {
+      var result = await testCollection.insertMany(arrayData);
-          should.not.exist(err);
+      should.exist(result);
-          should.exist(result);
+      result.insertedCount.should.equal(3);
-          result.insertedCount.should.equal(3);
+
-          
+      var docs = await testCollection.find({}).toArray();
-          testCollection.find({}).toArray(function (err, docs) {
+      docs.length.should.equal(3);
-            docs.length.should.equal(3);
+      await testCollection.deleteMany({});
-            testCollection.deleteMany({}, done);
-          });
-        });
-      });
     });
   });
 });

tests/uuid-handling.test.js

@@ -12,6 +12,7 @@
 
 var request = require('supertest');
 var should = require('should');
+var ObjectID = require('mongodb').ObjectId;
 var language = require('../lib/language')();
 var api = require('../lib/api/');
 
@@ -243,7 +244,6 @@ describe('UUID_HANDLING=true', function() {
   });
   
   it('UUID-ON-003: ObjectId still works normally', function(done) {
-    var ObjectID = require('mongodb').ObjectId;
     var testId = new ObjectID();
     
     // Insert treatment with ObjectId
@@ -491,7 +491,6 @@ describe('UUID Edge Cases', function() {
   });
   
   it('UUID-EDGE-007: Valid ObjectId still works normally', function(done) {
-    var ObjectID = require('mongodb').ObjectId;
     var testId = new ObjectID();
     
     self.ctx.treatments.create([{

tests/websocket.shape-handling.test.js

@@ -79,6 +79,10 @@ describe('WebSocket Shape Handling - dbAdd Single vs Array Input', function () {
     return self.ctx.store.collection(self.env.treatments_collection);
   }
 
+  function foodCollection() {
+    return self.ctx.food();
+  }
+
   describe('dbAdd with treatments collection', function () {
     
     beforeEach(function (done) {
@@ -278,8 +282,7 @@ describe('WebSocket Shape Handling - dbAdd Single vs Array Input', function () {
           eventType: 'Note',
           created_at: createdAt,
           notes: 'legacy original'
-        }, function (insertErr) {
+        }).then(function () {
-          if (insertErr) return done(insertErr);
 
           socket.emit('dbUpdate', {
             collection: 'treatments',
@@ -293,7 +296,9 @@ describe('WebSocket Shape Handling - dbAdd Single vs Array Input', function () {
 
             waitForConditionWithWarning({
               condition: function (cb) {
-                treatmentsCollection().findOne({ _id: legacyId }, cb);
+                treatmentsCollection().findOne({ _id: legacyId })
+                  .then(function (doc) { cb(null, doc); })
+                  .catch(cb);
               },
               assertion: function (doc) {
                 should.exist(doc);
@@ -303,7 +308,7 @@ describe('WebSocket Shape Handling - dbAdd Single vs Array Input', function () {
               operationName: 'verify websocket dbUpdate with custom string _id'
             });
           });
-        });
+        }).catch(done);
       });
     });
   });
@@ -328,8 +333,7 @@ describe('WebSocket Shape Handling - dbAdd Single vs Array Input', function () {
           eventType: 'Note',
           created_at: createdAt,
           notes: 'remove me'
-        }, function (insertErr) {
+        }).then(function () {
-          if (insertErr) return done(insertErr);
 
           socket.emit('dbUpdateUnset', {
             collection: 'treatments',
@@ -343,7 +347,9 @@ describe('WebSocket Shape Handling - dbAdd Single vs Array Input', function () {
 
             waitForConditionWithWarning({
               condition: function (cb) {
-                treatmentsCollection().findOne({ _id: legacyId }, cb);
+                treatmentsCollection().findOne({ _id: legacyId })
+                  .then(function (doc) { cb(null, doc); })
+                  .catch(cb);
               },
               assertion: function (doc) {
                 should.exist(doc);
@@ -353,7 +359,7 @@ describe('WebSocket Shape Handling - dbAdd Single vs Array Input', function () {
               operationName: 'verify websocket dbUpdateUnset with custom string _id'
             });
           });
-        });
+        }).catch(done);
       });
     });
   });
@@ -408,8 +414,7 @@ describe('WebSocket Shape Handling - dbAdd Single vs Array Input', function () {
           eventType: 'Note',
           created_at: createdAt,
           notes: 'delete me'
-        }, function (insertErr) {
+        }).then(function () {
-          if (insertErr) return done(insertErr);
 
           socket.emit('dbRemove', {
             collection: 'treatments',
@@ -420,7 +425,9 @@ describe('WebSocket Shape Handling - dbAdd Single vs Array Input', function () {
 
             waitForConditionWithWarning({
               condition: function (cb) {
-                treatmentsCollection().findOne({ _id: legacyId }, cb);
+                treatmentsCollection().findOne({ _id: legacyId })
+                  .then(function (doc) { cb(null, doc); })
+                  .catch(cb);
               },
               assertion: function (doc) {
                 should.not.exist(doc);
@@ -429,7 +436,7 @@ describe('WebSocket Shape Handling - dbAdd Single vs Array Input', function () {
               operationName: 'verify websocket dbRemove with custom string _id'
             });
           });
-        });
+        }).catch(done);
       });
     });
   });
@@ -455,8 +462,7 @@ describe('WebSocket Shape Handling - dbAdd Single vs Array Input', function () {
           eventType: 'Note',
           created_at: originalCreatedAt,
           notes: 'existing legacy note'
-        }, function (insertErr) {
+        }).then(function () {
-          if (insertErr) return done(insertErr);
 
           socket.emit('dbAdd', {
             collection: 'treatments',
@@ -473,7 +479,9 @@ describe('WebSocket Shape Handling - dbAdd Single vs Array Input', function () {
 
             waitForConditionWithWarning({
               condition: function (cb) {
-                treatmentsCollection().findOne({ _id: legacyId }, cb);
+                treatmentsCollection().findOne({ _id: legacyId })
+                  .then(function (doc) { cb(null, doc); })
+                  .catch(cb);
               },
               assertion: function (doc) {
                 should.exist(doc);
@@ -483,8 +491,132 @@ describe('WebSocket Shape Handling - dbAdd Single vs Array Input', function () {
               operationName: 'verify websocket dbAdd dedupe with custom string _id'
             });
           });
+        }).catch(done);
+      });
+    });
+  });
+
+  describe('generic collection raw write watchpoints', function () {
+
+    beforeEach(async function () {
+      await foodCollection().deleteMany({});
+    });
+
+    it('dbAdd preserves custom string _id values for generic collections', function (done) {
+      connectAndAuthorize(function (err, socket, authResult) {
+        if (err) return done(err);
+
+        authResult.write.should.equal(true);
+
+        var legacyId = 'legacy-food-id-add';
+        socket.emit('dbAdd', {
+          collection: 'food',
+          data: {
+            _id: legacyId,
+            name: 'ws food',
+            carbs: 15
+          }
+        }, function (result) {
+          should.exist(result);
+          result.should.be.instanceof(Array);
+          result.length.should.equal(1);
+          result[0]._id.should.equal(legacyId);
+
+          waitForConditionWithWarning({
+            condition: function (cb) {
+              foodCollection().findOne({ _id: legacyId })
+                .then(function (doc) { cb(null, doc); })
+                .catch(cb);
+            },
+            assertion: function (doc) {
+              should.exist(doc);
+              doc.name.should.equal('ws food');
+              doc.carbs.should.equal(15);
+            },
+            done: done,
+            operationName: 'verify websocket dbAdd generic collection custom string _id'
+          });
         });
       });
     });
+
+    it('dbUpdate supports custom string _id values for generic collections', function (done) {
+      connectAndAuthorize(function (err, socket, authResult) {
+        if (err) return done(err);
+
+        authResult.write.should.equal(true);
+
+        var legacyId = 'legacy-food-id-update';
+        foodCollection().insertOne({
+          _id: legacyId,
+          name: 'original food',
+          carbs: 10,
+          protein: 2
+        }).then(function () {
+          socket.emit('dbUpdate', {
+            collection: 'food',
+            _id: legacyId,
+            data: {
+              carbs: 18,
+              protein: 4
+            }
+          }, function (updateResult) {
+            should.exist(updateResult);
+            updateResult.result.should.equal('success');
+
+            waitForConditionWithWarning({
+              condition: function (cb) {
+                foodCollection().findOne({ _id: legacyId })
+                  .then(function (doc) { cb(null, doc); })
+                  .catch(cb);
+              },
+              assertion: function (doc) {
+                should.exist(doc);
+                doc.carbs.should.equal(18);
+                doc.protein.should.equal(4);
+              },
+              done: done,
+              operationName: 'verify websocket dbUpdate generic collection custom string _id'
+            });
+          });
+        }).catch(done);
+      });
+    });
+
+    it('dbRemove supports custom string _id values for generic collections', function (done) {
+      connectAndAuthorize(function (err, socket, authResult) {
+        if (err) return done(err);
+
+        authResult.write.should.equal(true);
+
+        var legacyId = 'legacy-food-id-remove';
+        foodCollection().insertOne({
+          _id: legacyId,
+          name: 'remove food',
+          carbs: 9
+        }).then(function () {
+          socket.emit('dbRemove', {
+            collection: 'food',
+            _id: legacyId
+          }, function (removeResult) {
+            should.exist(removeResult);
+            removeResult.result.should.equal('success');
+
+            waitForConditionWithWarning({
+              condition: function (cb) {
+                foodCollection().findOne({ _id: legacyId })
+                  .then(function (doc) { cb(null, doc); })
+                  .catch(cb);
+              },
+              assertion: function (doc) {
+                should.not.exist(doc);
+              },
+              done: done,
+              operationName: 'verify websocket dbRemove generic collection custom string _id'
+            });
+          });
+        }).catch(done);
+      });
+    });
   });
 });