bckelley/cgm-remote-monitor
1
0
Fork 0
mirror of https://github.com/bckelley/cgm-remote-monitor.git synced 2026-08-24 03:14:12 -05:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #8474 from nightscout/fix/mongo-save-hardening

Browse Source 
Fix/mongo save hardening
This commit is contained in:
Ben West
2026-04-19 17:14:40 -07:00
committed by GitHub
parent e807e1a6b6 69b620dd2f
commit b8d5c02d52
45 changed files with 2058 additions and 1051 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/api/activity/index.js
+4 -28
View File
@@ -6,30 +6,7 @@ var _isArray = require('lodash/isArray');
var consts = require('../../constants');
var moment = require('moment');
/**
* Validate MongoDB ObjectId format.
* Accepts: undefined, null, or 24-character hex string.
* Rejects: anything else (UUIDs, short strings, numbers, objects).
*/
function isValidObjectId(id) {
if (id === undefined || id === null) return true;
if (typeof id !== 'string') return false;
return /^[a-fA-F0-9]{24}$/.test(id);
}
/**
* Validate _id field for each document in an array.
* @returns {Object|null} - null if all valid, or {index, id} of first invalid
*/
function findInvalidId(docs) {
for (var i = 0; i < docs.length; i++) {
if (!isValidObjectId(docs[i]._id)) {
return { index: i, id: docs[i]._id };
}
}
return null;
}
var objectIdValidation = require('../shared/objectid-validation');
function configure(app, wares, ctx) {
var express = require('express')
@@ -98,7 +75,7 @@ function configure(app, wares, ctx) {
}
// Validate _id fields before storage (return 400 on invalid)
var invalid = findInvalidId(activity);
var invalid = objectIdValidation.findInvalidId(activity);
if (invalid) {
return res.sendJSONStatus(res, consts.HTTP_BAD_REQUEST,
'Invalid _id format', 'Must be 24-character hex string or omit for auto-generation. Got: ' + String(invalid.id));
@@ -119,7 +96,7 @@ function configure(app, wares, ctx) {
api.delete('/activity/:_id', ctx.authorization.isPermitted('api:activity:delete'), function(req, res) {
// Validate _id parameter
if (!isValidObjectId(req.params._id)) {
if (!objectIdValidation.isValidObjectId(req.params._id)) {
return res.sendJSONStatus(res, consts.HTTP_BAD_REQUEST,
'Invalid _id format', 'Must be 24-character hex string. Got: ' + String(req.params._id));
}
@@ -133,7 +110,7 @@ function configure(app, wares, ctx) {
var data = req.body;
// Validate _id if provided
if (!isValidObjectId(data._id)) {
if (!objectIdValidation.isValidObjectId(data._id)) {
return res.sendJSONStatus(res, consts.HTTP_BAD_REQUEST,
'Invalid _id format', 'Must be 24-character hex string. Got: ' + String(data._id));
}
@@ -159,4 +136,3 @@ function configure(app, wares, ctx) {
}
module.exports = configure;
lib/api/devicestatus/index.js
+3 -29
View File
@@ -5,33 +5,7 @@ const moment = require('moment');
const { query } = require('express');
const _take = require('lodash/take');
const _ = require('lodash');
/**
* Validate MongoDB ObjectId format.
* Accepts: undefined, null, or 24-character hex string.
* Rejects: anything else (UUIDs, short strings, numbers, objects).
* @param {*} id - The _id value to validate
* @returns {boolean} - true if valid or empty, false if invalid format
*/
function isValidObjectId(id) {
if (id === undefined || id === null) return true; // Will auto-generate
if (typeof id !== 'string') return false;
return /^[a-fA-F0-9]{24}$/.test(id);
}
/**
* Validate _id field for each document in an array.
* @param {Array} docs - Array of documents to validate
* @returns {Object|null} - null if all valid, or {index, id} of first invalid
*/
function findInvalidId(docs) {
for (var i = 0; i < docs.length; i++) {
if (!isValidObjectId(docs[i]._id)) {
return { index: i, id: docs[i]._id };
}
}
return null;
}
const objectIdValidation = require('../shared/objectid-validation');
function configure (app, wares, ctx, env) {
var express = require('express')
@@ -102,7 +76,7 @@ function configure (app, wares, ctx, env) {
}
// Validate _id fields before storage (return 400 on invalid)
var invalid = findInvalidId(statuses);
var invalid = objectIdValidation.findInvalidId(statuses);
if (invalid) {
return res.sendJSONStatus(res, consts.HTTP_BAD_REQUEST,
'Invalid _id format', 'Must be 24-character hex string or omit for auto-generation. Got: ' + String(invalid.id));
@@ -154,7 +128,7 @@ function configure (app, wares, ctx, env) {
api.delete('/devicestatus/:id', ctx.authorization.isPermitted('api:devicestatus:delete'), function(req, res, next) {
// Validate _id parameter (unless wildcard)
if (req.params.id !== '*' && !isValidObjectId(req.params.id)) {
if (req.params.id !== '*' && !objectIdValidation.isValidObjectId(req.params.id)) {
return res.sendJSONStatus(res, consts.HTTP_BAD_REQUEST,
'Invalid _id format', 'Must be 24-character hex string. Got: ' + String(req.params.id));
}
lib/api/food/index.js
+4 -28
View File
@@ -2,30 +2,7 @@
var _isArray = require('lodash/isArray');
var consts = require('../../constants');
/**
* Validate MongoDB ObjectId format.
* Accepts: undefined, null, or 24-character hex string.
* Rejects: anything else (UUIDs, short strings, numbers, objects).
*/
function isValidObjectId(id) {
if (id === undefined || id === null) return true;
if (typeof id !== 'string') return false;
return /^[a-fA-F0-9]{24}$/.test(id);
}
/**
* Validate _id field for each document in an array.
* @returns {Object|null} - null if all valid, or {index, id} of first invalid
*/
function findInvalidId(docs) {
for (var i = 0; i < docs.length; i++) {
if (!isValidObjectId(docs[i]._id)) {
return { index: i, id: docs[i]._id };
}
}
return null;
}
var objectIdValidation = require('../shared/objectid-validation');
function configure (app, wares, ctx) {
var express = require('express'),
@@ -75,7 +52,7 @@ function configure (app, wares, ctx) {
}
// Validate _id fields before storage (return 400 on invalid)
var invalid = findInvalidId(data);
var invalid = objectIdValidation.findInvalidId(data);
if (invalid) {
return res.sendJSONStatus(res, consts.HTTP_BAD_REQUEST,
'Invalid _id format', 'Must be 24-character hex string or omit for auto-generation. Got: ' + String(invalid.id));
@@ -103,7 +80,7 @@ function configure (app, wares, ctx) {
}
// Validate _id fields before storage (return 400 on invalid)
var invalid = findInvalidId(data);
var invalid = objectIdValidation.findInvalidId(data);
if (invalid) {
return res.sendJSONStatus(res, consts.HTTP_BAD_REQUEST,
'Invalid _id format', 'Must be 24-character hex string. Got: ' + String(invalid.id));
@@ -123,7 +100,7 @@ function configure (app, wares, ctx) {
// delete record
api.delete('/food/:_id', ctx.authorization.isPermitted('api:food:delete'), function(req, res) {
// Validate _id parameter
if (!isValidObjectId(req.params._id)) {
if (!objectIdValidation.isValidObjectId(req.params._id)) {
return res.sendJSONStatus(res, consts.HTTP_BAD_REQUEST,
'Invalid _id format', 'Must be 24-character hex string. Got: ' + String(req.params._id));
}
@@ -141,4 +118,3 @@ function configure (app, wares, ctx) {
}
module.exports = configure;
lib/api/profile/index.js
+4 -31
View File
@@ -1,33 +1,7 @@
'use strict';
var consts = require('../../constants');
/**
* Validate MongoDB ObjectId format.
* Accepts: undefined, null, or 24-character hex string.
* Rejects: anything else (UUIDs, short strings, numbers, objects).
* @param {*} id - The _id value to validate
* @returns {boolean} - true if valid or empty, false if invalid format
*/
function isValidObjectId(id) {
if (id === undefined || id === null) return true; // Will auto-generate
if (typeof id !== 'string') return false;
return /^[a-fA-F0-9]{24}$/.test(id);
}
/**
* Validate _id field for each document in an array.
* @param {Array} docs - Array of documents to validate
* @returns {Object|null} - null if all valid, or {index, id} of first invalid
*/
function findInvalidId(docs) {
for (var i = 0; i < docs.length; i++) {
if (!isValidObjectId(docs[i]._id)) {
return { index: i, id: docs[i]._id };
}
}
return null;
}
var objectIdValidation = require('../shared/objectid-validation');
function configure (app, wares, ctx) {
var express = require('express'),
@@ -97,7 +71,7 @@ function configure (app, wares, ctx) {
}
// Validate _id fields before storage (return 400 on invalid)
var invalid = findInvalidId(data);
var invalid = objectIdValidation.findInvalidId(data);
if (invalid) {
return res.sendJSONStatus(res, consts.HTTP_BAD_REQUEST,
'Invalid _id format', 'Must be 24-character hex string or omit for auto-generation. Got: ' + String(invalid.id));
@@ -125,7 +99,7 @@ function configure (app, wares, ctx) {
var data = req.body;
// Validate _id if provided (required for PUT, must be valid format)
if (!isValidObjectId(data._id)) {
if (!objectIdValidation.isValidObjectId(data._id)) {
return res.sendJSONStatus(res, consts.HTTP_BAD_REQUEST,
'Invalid _id format', 'Must be 24-character hex string. Got: ' + String(data._id));
}
@@ -145,7 +119,7 @@ function configure (app, wares, ctx) {
api.delete('/profile/:_id', ctx.authorization.isPermitted('api:profile:delete'), function(req, res) {
// Validate _id parameter
if (!isValidObjectId(req.params._id)) {
if (!objectIdValidation.isValidObjectId(req.params._id)) {
return res.sendJSONStatus(res, consts.HTTP_BAD_REQUEST,
'Invalid _id format', 'Must be 24-character hex string. Got: ' + String(req.params._id));
}
@@ -164,4 +138,3 @@ function configure (app, wares, ctx) {
}
module.exports = configure;
lib/api/shared/objectid-validation.js
+26
View File
@@ -0,0 +1,26 @@
'use strict';
var OBJECT_ID_PATTERN = /^[a-fA-F0-9]{24}$/;
function isValidObjectId(id) {
if (id === undefined || id === null) {
return true;
}
return typeof id === 'string' && OBJECT_ID_PATTERN.test(id);
}
function findInvalidId(docs) {
for (var i = 0; i < docs.length; i++) {
if (!isValidObjectId(docs[i]._id)) {
return { index: i, id: docs[i]._id };
}
}
return null;
}
module.exports = {
findInvalidId: findInvalidId,
isValidObjectId: isValidObjectId
};
lib/api3/storage/mongoCollection/find.js
+32 -59
View File
@@ -16,6 +16,14 @@ function toSafeInt (value, defaultValue) {
return Number.isFinite(parsed) ? parsed : defaultValue;
}
function normalizeDocs (docs, options) {
if (!options || options.normalize !== false) {
_.each(docs, utils.normalizeDoc);
}
return docs;
}
/**
* Find single document by identifier
@@ -24,27 +32,15 @@ function toSafeInt (value, defaultValue) {
* @param {Object} projection
* @param {Object} options
*/
function findOne (col, identifier, projection, options) {
async function findOne (col, identifier, projection, options) {
return new Promise(function (resolve, reject) {
const filter = utils.filterForOne(identifier);
const result = await col.find(filter)
.project(projection)
.sort({ identifier: -1 }) // document with identifier first (not the fallback one)
.toArray();
const filter = utils.filterForOne(identifier);
col.find(filter)
.project(projection)
.sort({ identifier: -1 }) // document with identifier first (not the fallback one)
.toArray(function mongoDone (err, result) {
if (err) {
reject(err);
} else {
if (!options || options.normalize !== false) {
_.each(result, utils.normalizeDoc);
}
resolve(result);
}
});
});
return normalizeDocs(result, options);
}
@@ -55,56 +51,33 @@ function findOne (col, identifier, projection, options) {
* @param {Object} projection
* @param {Object} options
*/
function findOneFilter (col, filter, projection, options) {
async function findOneFilter (col, filter, projection, options) {
return new Promise(function (resolve, reject) {
const result = await col.find(filter)
.project(projection)
.sort({ identifier: -1 }) // document with identifier first (not the fallback one)
.toArray();
col.find(filter)
.project(projection)
.sort({ identifier: -1 }) // document with identifier first (not the fallback one)
.toArray(function mongoDone (err, result) {
if (err) {
reject(err);
} else {
if (!options || options.normalize !== false) {
_.each(result, utils.normalizeDoc);
}
resolve(result);
}
});
});
return normalizeDocs(result, options);
}
/**
* Find many documents matching the filtering criteria
*/
function findMany (col, args) {
async function findMany (col, args) {
const logicalOperator = args.logicalOperator || 'and';
return new Promise(function (resolve, reject) {
const filter = utils.parseFilter(args.filter, logicalOperator, args.onlyValid);
const safeLimit = toSafeInt(args.limit, 1000);
const safeSkip = toSafeInt(args.skip, 0);
const result = await col.find(filter)
.sort(args.sort)
.limit(safeLimit)
.skip(safeSkip)
.project(args.projection)
.toArray();
const filter = utils.parseFilter(args.filter, logicalOperator, args.onlyValid);
const safeLimit = toSafeInt(args.limit, 1000);
const safeSkip = toSafeInt(args.skip, 0);
col.find(filter)
.sort(args.sort)
.limit(safeLimit)
.skip(safeSkip)
.project(args.projection)
.toArray(function mongoDone (err, result) {
if (err) {
reject(err);
} else {
if (!args.options || args.options.normalize !== false) {
_.each(result, utils.normalizeDoc);
}
resolve(result);
}
});
});
return normalizeDocs(result, args.options);
}
lib/api3/storage/mongoCollection/index.js
+13 -28
View File
@@ -46,44 +46,29 @@ function MongoCollection (ctx, env, colName) {
/**
* Get server version
*/
self.version = function version () {
self.version = async function version () {
return new Promise(function (resolve, reject) {
const result = await ctx.store.db.admin().buildInfo();
ctx.store.db.admin().buildInfo({}, function mongoDone (err, result) {
err
? reject(err)
: resolve({
storage: 'mongodb',
version: result.version
});
});
});
return {
storage: 'mongodb',
version: result.version
};
};
/**
* Get timestamp (e.g. srvModified) of the last modified document
*/
self.getLastModified = function getLastModified (fieldName) {
self.getLastModified = async function getLastModified (fieldName) {
return new Promise(function (resolve, reject) {
const [ result ] = await self.col.find()
.sort({ [fieldName]: -1 })
.limit(1)
.project({ [fieldName]: 1 })
.toArray();
self.col.find()
.sort({ [fieldName]: -1 })
.limit(1)
.project({ [fieldName]: 1 })
.toArray(function mongoDone (err, [ result ]) {
err
? reject(err)
: resolve(result);
});
});
return result;
}
}
lib/api3/storage/mongoCollection/modify.js
+23 -63
View File
@@ -9,24 +9,16 @@ const utils = require('./utils')
* @param {Object} doc
* @param {Object} options
*/
function insertOne (col, doc, options) {
async function insertOne (col, doc, options) {
return new Promise(function (resolve, reject) {
const result = await col.insertOne(doc);
const identifier = doc.identifier || result.insertedId.toString();
col.insertOne(doc, function mongoDone(err, result) {
if (!options || options.normalize !== false) {
delete doc._id;
}
if (err) {
reject(err);
} else {
const identifier = doc.identifier || result.insertedId.toString();
if (!options || options.normalize !== false) {
delete doc._id;
}
resolve(identifier);
}
});
});
return identifier;
}
@@ -36,20 +28,12 @@ function insertOne (col, doc, options) {
* @param {string} identifier
* @param {Object} doc
*/
function replaceOne (col, identifier, doc) {
async function replaceOne (col, identifier, doc) {
return new Promise(function (resolve, reject) {
const filter = utils.filterForOne(identifier);
const result = await col.replaceOne(filter, doc, { upsert: true });
const filter = utils.filterForOne(identifier);
col.replaceOne(filter, doc, { upsert: true }, function mongoDone(err, result) {
if (err) {
reject(err);
} else {
resolve(result.matchedCount);
}
});
});
return result.matchedCount;
}
@@ -59,20 +43,12 @@ function replaceOne (col, identifier, doc) {
* @param {string} identifier
* @param {object} setFields
*/
function updateOne (col, identifier, setFields) {
async function updateOne (col, identifier, setFields) {
return new Promise(function (resolve, reject) {
const filter = utils.filterForOne(identifier);
const result = await col.updateOne(filter, { $set: setFields });
const filter = utils.filterForOne(identifier);
col.updateOne(filter, { $set: setFields }, function mongoDone(err, result) {
if (err) {
reject(err);
} else {
resolve({ updated: result.modifiedCount });
}
});
});
return { updated: result.modifiedCount };
}
@@ -81,40 +57,24 @@ function updateOne (col, identifier, setFields) {
* @param {Object} col
* @param {string} identifier
*/
function deleteOne (col, identifier) {
async function deleteOne (col, identifier) {
return new Promise(function (resolve, reject) {
const filter = utils.filterForOne(identifier);
const result = await col.deleteOne(filter);
const filter = utils.filterForOne(identifier);
col.deleteOne(filter, function mongoDone(err, result) {
if (err) {
reject(err);
} else {
resolve({ deleted: result.deletedCount });
}
});
});
return { deleted: result.deletedCount };
}
/**
* Permanently remove many documents matching any of filtering criteria
*/
function deleteManyOr (col, filterDef) {
async function deleteManyOr (col, filterDef) {
return new Promise(function (resolve, reject) {
const filter = utils.parseFilter(filterDef, 'or');
const result = await col.deleteMany(filter);
const filter = utils.parseFilter(filterDef, 'or');
col.deleteMany(filter, function mongoDone(err, result) {
if (err) {
reject(err);
} else {
resolve({ deleted: result.deletedCount });
}
});
});
return { deleted: result.deletedCount };
}
lib/api3/storage/mongoCollection/utils.js
+1 -1
View File
@@ -2,7 +2,7 @@
const _ = require('lodash')
, checkForHexRegExp = new RegExp("^[0-9a-fA-F]{24}$")
, ObjectID = require('mongodb-legacy').ObjectId
, ObjectID = require('mongodb').ObjectId
;
lib/authorization/storage.js
+65 -31
View File
@@ -3,7 +3,8 @@
var _ = require('lodash');
var crypto = require('crypto');
var shiroTrie = require('shiro-trie');
var ObjectID = require('mongodb-legacy').ObjectId;
var ObjectID = require('mongodb').ObjectId;
var runWithCallback = require('../storage/run-with-callback');
var find_options = require('../server/query');
@@ -22,27 +23,44 @@ function init (env, ctx) {
return find_options(opts, storage.queryOpts);
}
function normalizeRequiredObjectId(id) {
if (id === undefined || id === null || id === '') {
return { error: 'Missing _id for update' };
}
try {
return { value: new ObjectID(id) };
} catch (err) {
return { error: 'Invalid _id format: ' + String(id) };
}
}
function create (collection) {
function doCreate(obj, fn) {
if (!Object.prototype.hasOwnProperty.call(obj, 'created_at')) {
obj.created_at = (new Date()).toISOString();
}
collection.insertOne(obj, function (err, doc) {
if (err != null && err.message) {
console.log('Data insertion error', err.message);
fn(err.message, null);
return;
return runWithCallback(async function () {
try {
await collection.insertOne(obj);
} catch (err) {
if (err != null && err.message) {
console.log('Data insertion error', err.message);
throw err.message;
}
throw err;
}
storage.reload(function loaded() {
fn(null, obj);
});
});
await storageReload();
return obj;
}, fn);
}
return doCreate;
}
function list (collection) {
function doList(opts, fn) {
function doList(opts, fn) {
// these functions, find, sort, and limit, are used to
// dynamically configure the request, based on the options we've
// been given
@@ -60,18 +78,14 @@ function init (env, ctx) {
return this;
}
// handle all the results
function toArray(err, entries) {
fn(err, entries);
}
console.log('Loading',opts);
// now just stitch them all together
limit.call(collection
return runWithCallback(function () {
return limit.call(collection
.find(query_for(opts))
.sort(sort())
).toArray(toArray);
).toArray();
}, fn);
}
return doList;
@@ -79,27 +93,33 @@ function init (env, ctx) {
function remove (collection) {
function doRemove (_id, callback) {
collection.deleteOne({ '_id': new ObjectID(_id) }, function (err) {
storage.reload(function loaded() {
callback(err, null);
});
});
return runWithCallback(async function () {
await collection.deleteOne({ '_id': new ObjectID(_id) });
await storageReload();
return null;
}, callback);
}
return doRemove;
}
function save (collection) {
function doSave (obj, callback) {
obj._id = new ObjectID(obj._id);
var idResult = normalizeRequiredObjectId(obj && obj._id);
if (idResult.error) {
callback(idResult.error, null);
return;
}
obj._id = idResult.value;
if (!obj.created_at) {
obj.created_at = (new Date()).toISOString();
}
collection.insertOne(obj, function (err) {
//id should be added for new docs
storage.reload(function loaded() {
callback(err, obj);
});
});
return runWithCallback(async function () {
await collection.replaceOne({ _id: obj._id }, obj, { upsert: true });
await storageReload();
return obj;
}, callback);
}
return doSave;
}
@@ -183,6 +203,20 @@ function init (env, ctx) {
};
function storageReload () {
return runWithCallback(function () {
return new Promise(function (resolve, reject) {
storage.reload(function loaded(err) {
if (err) {
reject(err);
return;
}
resolve();
});
});
});
}
storage.findRole = function findRole (roleName) {
return _.find(storage.roles, {name: roleName});
};
lib/data/dataloader.js
+18 -10
View File
@@ -493,19 +493,27 @@ function loadDeviceStatus(ddata, env, ctx, callback) {
}
function loadDatabaseStats(ddata, ctx, callback) {
ctx.store.db.stats(function mongoDone (err, result) {
Promise.resolve()
.then(function () {
return ctx.store.db.stats();
})
.then(function (result) {
if (result) {
ddata.dbstats = {
dataSize: result.dataSize,
indexSize: result.indexSize
};
}
})
.catch(function (err) {
console.log("Problem loading database stats");
if (err) {
console.log("Problem loading database stats");
}
if (!err && result) {
ddata.dbstats = {
dataSize: result.dataSize
, indexSize: result.indexSize
};
console.error(err);
}
})
.finally(function () {
callback();
});
});
}
module.exports = init;
lib/server/activity.js
+36 -20
View File
@@ -1,10 +1,19 @@
'use strict';
var find_options = require('./query');
var runWithCallback = require('../storage/run-with-callback');
function storage (env, ctx) {
var ObjectID = require('mongodb-legacy').ObjectId;
var ObjectID = require('mongodb').ObjectId;
function normalizeObjectId(id) {
try {
return new ObjectID(id);
} catch (err) {
return new ObjectID();
}
}
function create (docs, fn) {
if (docs.length === 0) {
@@ -26,10 +35,14 @@ function storage (env, ctx) {
};
});
api().bulkWrite(bulkOps, { ordered: true }, function(err, bulkResult) {
if (err) {
return runWithCallback(async function () {
var bulkResult;
try {
bulkResult = await api().bulkWrite(bulkOps, { ordered: true });
} catch (err) {
console.error('Problem upserting activity batch', err);
return fn(err, []);
throw err;
}
// Assign _ids from upserted results
@@ -39,20 +52,26 @@ function storage (env, ctx) {
});
}
fn(null, docs);
return docs;
}, function (err, result) {
if (err) {
fn(err, []);
return;
}
fn(null, result);
});
}
function save (obj, fn) {
obj._id = new ObjectID(obj._id);
obj._id = normalizeObjectId(obj._id);
if (!Object.prototype.hasOwnProperty.call(obj, 'created_at')) {
obj.created_at = (new Date( )).toISOString( );
}
api().insertOne(obj, function (err) {
//id should be added for new docs
fn(err, obj);
});
return runWithCallback(async function () {
await api().replaceOne({ _id: obj._id }, obj, { upsert: true });
return obj;
}, fn);
}
function query_for (opts) {
@@ -77,21 +96,19 @@ function storage (env, ctx) {
return this;
}
// handle all the results
function toArray (err, entries) {
fn(err, entries);
}
// now just stitch them all together
limit.call(api( )
return runWithCallback(function () {
return limit.call(api( )
.find(query_for(opts))
.sort(sort( ))
).toArray(toArray);
).toArray();
}, fn);
}
function remove (_id, fn) {
var objId = new ObjectID(_id);
return api( ).deleteOne({ '_id': objId }, fn);
return runWithCallback(function () {
return api().deleteOne({ '_id': objId });
}, fn);
}
function api ( ) {
@@ -112,4 +129,3 @@ storage.queryOpts = {
};
module.exports = storage;
lib/server/aggregate.js
+4 -2
View File
@@ -1,4 +1,5 @@
var find_options = require('./query');
var runWithCallback = require('../storage/run-with-callback');
function create (conf, api) {
@@ -21,7 +22,9 @@ function create (conf, api) {
var groupBy = [ {$match: query } ].concat(pipeline).concat(template( ));
console.log('$match query', query);
console.log('AGGREGATE', groupBy);
api( ).aggregate(groupBy, done);
return runWithCallback(function () {
return api().aggregate(groupBy).toArray();
}, done);
}
return aggregate;
@@ -29,4 +32,3 @@ function create (conf, api) {
}
module.exports = create;
lib/server/devicestatus.js
+22 -26
View File
@@ -2,6 +2,7 @@
var moment = require('moment');
var find_options = require('./query');
var runWithCallback = require('../storage/run-with-callback');
function truncatePredictions (obj, maxSize) {
if (!maxSize || maxSize <= 0) return obj;
@@ -52,12 +53,15 @@ function storage (env, ctx) {
truncatePredictions(obj, predictionsMaxSize);
});
// Use insertMany for batch insert
api().insertMany(statuses, { ordered: true }, function(err, insertResult) {
if (err) {
return runWithCallback(async function () {
var insertResult;
try {
// Use insertMany for batch insert
insertResult = await api().insertMany(statuses, { ordered: true });
} catch (err) {
console.log('Error inserting device status objects', err.message);
fn(err.message || err, null);
return;
throw err.message || err;
}
// Assign _ids from insertMany result
@@ -75,8 +79,8 @@ function storage (env, ctx) {
});
ctx.bus.emit('data-received');
fn(null, statuses);
});
return statuses;
}, fn);
}
function last (fn) {
@@ -111,24 +115,19 @@ function storage (env, ctx) {
return this;
}
// handle all the results
function toArray (err, entries) {
fn(err, entries);
}
// now just stitch them all together
limit.call(api()
.find(query_for(opts))
.sort(sort())
).toArray(toArray);
return runWithCallback(function () {
return limit.call(api()
.find(query_for(opts))
.sort(sort())
).toArray();
}, fn);
}
function remove (opts, fn) {
function removed (err, stat) {
console.log('removed', err, stat);
return runWithCallback(async function () {
var stat = await api().deleteMany(query_for(opts));
console.log('removed', null, stat);
ctx.bus.emit('data-update', {
type: 'devicestatus'
, op: 'remove'
@@ -136,11 +135,8 @@ function storage (env, ctx) {
, changes: opts.find._id
});
fn(err, stat);
}
return api().deleteMany(
query_for(opts), removed);
return stat;
}, fn);
}
function api () {
lib/server/entries.js
+30 -27
View File
@@ -2,8 +2,9 @@
var es = require('event-stream');
var find_options = require('./query');
var ObjectId = require('mongodb-legacy').ObjectId;
var ObjectId = require('mongodb').ObjectId;
var moment = require('moment');
var runWithCallback = require('../storage/run-with-callback');
// REQ-SYNC-072: Pattern to match valid MongoDB ObjectId hex strings
var OBJECT_ID_HEX_RE = /^[0-9a-fA-F]{24}$/;
@@ -37,20 +38,17 @@ function storage (env, ctx) {
}
// handle all the results
function toArray (err, entries) {
fn(err, entries);
}
// now just stitch them all together
limit.call(api()
.find(query_for(opts))
.sort(sort())
).toArray(toArray);
return runWithCallback(function () {
return limit.call(api()
.find(query_for(opts))
.sort(sort())
).toArray();
}, fn);
}
function remove (opts, fn) {
api().deleteMany(query_for(opts), function(err, stat) {
return runWithCallback(async function () {
var stat = await api().deleteMany(query_for(opts));
ctx.bus.emit('data-update', {
type: 'entries'
, op: 'remove'
@@ -60,8 +58,8 @@ function storage (env, ctx) {
//TODO: this is triggering a read from Mongo, we can do better
ctx.bus.emit('data-received');
fn(err, stat);
});
return stat;
}, fn);
}
// return writable stream to lint each sgv record passing through it
@@ -126,12 +124,15 @@ function storage (env, ctx) {
};
});
// Use bulkWrite for batch upsert
api().bulkWrite(bulkOps, { ordered: true }, function(err, bulkResult) {
if (err) {
return runWithCallback(async function () {
var bulkResult;
try {
// Use bulkWrite for batch upsert
bulkResult = await api().bulkWrite(bulkOps, { ordered: true });
} catch (err) {
console.error('Problem upserting entries batch', err);
fn(err, docs);
return;
throw err;
}
// Assign _ids from upserted results
@@ -148,18 +149,20 @@ function storage (env, ctx) {
});
ctx.bus.emit('data-received');
fn(null, docs);
return docs;
}, function (err, result) {
if (err) {
fn(err, docs);
return;
}
fn(null, result);
});
}
function getEntry (id, fn) {
api().findOne({ "_id": new ObjectId(id) }, function(err, entry) {
if (err) {
fn(err);
} else {
fn(null, entry);
}
});
return runWithCallback(function () {
return api().findOne({ "_id": new ObjectId(id) });
}, fn);
}
function query_for (opts) {
lib/server/food.js
+52 -21
View File
@@ -1,7 +1,16 @@
'use strict';
function storage (env, ctx) {
var ObjectID = require('mongodb-legacy').ObjectId;
var ObjectID = require('mongodb').ObjectId;
var runWithCallback = require('../storage/run-with-callback');
function normalizeObjectId(id) {
try {
return new ObjectID(id);
} catch (err) {
return new ObjectID();
}
}
function create (docs, fn) {
// Normalize to array for consistent handling (allows direct storage calls with single objects)
@@ -26,10 +35,14 @@ function storage (env, ctx) {
};
});
api().bulkWrite(bulkOps, { ordered: true }, function(err, bulkResult) {
if (err) {
return runWithCallback(async function () {
var bulkResult;
try {
bulkResult = await api().bulkWrite(bulkOps, { ordered: true });
} catch (err) {
console.error('Problem upserting food batch', err);
return fn(err, []);
throw err;
}
// Assign _ids from upserted results
@@ -39,7 +52,13 @@ function storage (env, ctx) {
});
}
fn(null, docs);
return docs;
}, function (err, result) {
if (err) {
fn(err, []);
return;
}
fn(null, result);
});
}
@@ -55,30 +74,28 @@ function storage (env, ctx) {
// Build bulkWrite operations for batch upsert
var bulkOps = docs.map(function(doc) {
try {
doc._id = new ObjectID(doc._id);
} catch (err){
console.error(err);
doc._id = new ObjectID();
}
doc._id = normalizeObjectId(doc._id);
if (!doc.created_at) {
doc.created_at = (new Date()).toISOString();
}
var query = (doc.created_at && doc._id) ? { _id: doc._id, created_at: doc.created_at } : doc;
return {
replaceOne: {
filter: query,
filter: { _id: doc._id },
replacement: doc,
upsert: true
}
};
});
api().bulkWrite(bulkOps, { ordered: true }, function(err, bulkResult) {
if (err) {
return runWithCallback(async function () {
var bulkResult;
try {
bulkResult = await api().bulkWrite(bulkOps, { ordered: true });
} catch (err) {
console.error('Problem saving food batch', err);
return fn(err, []);
throw err;
}
// Assign _ids from upserted results
@@ -88,25 +105,39 @@ function storage (env, ctx) {
});
}
fn(null, docs);
return docs;
}, function (err, result) {
if (err) {
fn(err, []);
return;
}
fn(null, result);
});
}
function list (fn) {
return api( ).find({ }).toArray(fn);
return runWithCallback(function () {
return api().find({ }).toArray();
}, fn);
}
function listquickpicks (fn) {
return api( ).find({ $and: [ { 'type': 'quickpick'} , { 'hidden' : 'false' } ] }).sort({'position': 1}).toArray(fn);
return runWithCallback(function () {
return api().find({ $and: [ { 'type': 'quickpick'} , { 'hidden' : 'false' } ] }).sort({'position': 1}).toArray();
}, fn);
}
function listregular (fn) {
return api( ).find( { 'type': 'food'} ).toArray(fn);
return runWithCallback(function () {
return api().find( { 'type': 'food'} ).toArray();
}, fn);
}
function remove (_id, fn) {
var objId = new ObjectID(_id);
return api( ).deleteOne({ '_id': objId }, fn);
return runWithCallback(function () {
return api().deleteOne({ '_id': objId });
}, fn);
}
lib/server/profile.js
+40 -17
View File
@@ -2,9 +2,10 @@
var find_options = require('./query');
var consts = require('../constants');
var runWithCallback = require('../storage/run-with-callback');
function storage (collection, ctx) {
var ObjectID = require('mongodb-legacy').ObjectId;
var ObjectID = require('mongodb').ObjectId;
function create (objOrArray, fn) {
// Normalize to array (supports both single object and array inputs)
@@ -13,7 +14,7 @@ function storage (collection, ctx) {
if (docs.length === 0) {
fn(null, []);
ctx.bus.emit('data-received');
return;
return Promise.resolve([]);
}
// Add created_at to each document
@@ -23,16 +24,27 @@ function storage (collection, ctx) {
}
});
api().insertMany(docs, function (err, result) {
const promise = runWithCallback(async function () {
const result = await api().insertMany(docs);
if (result && result.insertedIds) {
Object.keys(result.insertedIds).forEach(function (index) {
if (!docs[index]._id) {
docs[index]._id = result.insertedIds[index];
}
});
}
return docs;
}, function (err, result) {
if (err) {
console.log("Error saving profile data", docs, err);
console.log('Error saving profile data', docs, err);
fn(err);
return;
}
// Return the inserted documents with _id (NightscoutKit expects array)
fn(null, docs);
fn(null, result);
});
ctx.bus.emit('data-received');
return promise;
}
function save (obj, fn) {
@@ -45,16 +57,20 @@ function storage (collection, ctx) {
obj.created_at = (new Date( )).toISOString( );
}
// Match existing profiles by _id only. The profile editor rewrites created_at on save.
api().replaceOne({ _id: obj._id }, obj, { upsert: true }, function (err) {
//id should be added for new docs
fn(err, obj);
});
const promise = runWithCallback(async function () {
await api().replaceOne({ _id: obj._id }, obj, { upsert: true });
return obj;
}, fn);
ctx.bus.emit('data-received');
return promise;
}
function list (fn, count) {
const limit = count !== null ? count : Number(consts.PROFILES_DEFAULT_COUNT);
return api( ).find({ }).limit(limit).sort({startDate: -1}).toArray(fn);
return runWithCallback(function () {
return api().find({ }).limit(limit).sort({startDate: -1}).toArray();
}, fn);
}
function list_query (opts, fn) {
@@ -71,10 +87,12 @@ function storage (collection, ctx) {
return this;
}
return limit.call(api()
.find(query_for(opts))
.sort(opts && opts.sort && query_sort(opts) || { startDate: -1 }), opts)
.toArray(fn);
return runWithCallback(function () {
return limit.call(api()
.find(query_for(opts))
.sort(opts && opts.sort && query_sort(opts) || { startDate: -1 }), opts)
.toArray();
}, fn);
}
function query_for (opts) {
@@ -100,14 +118,19 @@ function storage (collection, ctx) {
function last (fn) {
return api().find().sort({startDate: -1}).limit(1).toArray(fn);
return runWithCallback(function () {
return api().find().sort({startDate: -1}).limit(1).toArray();
}, fn);
}
function remove (_id, fn) {
var objId = new ObjectID(_id);
api( ).deleteOne({ '_id': objId }, fn);
const promise = runWithCallback(function () {
return api().deleteOne({ '_id': objId });
}, fn);
ctx.bus.emit('data-received');
return promise;
}
function api () {
lib/server/query.js
+1 -1
View File
@@ -1,7 +1,7 @@
'use strict';
const traverse = require('traverse');
const ObjectID = require('mongodb-legacy').ObjectId;
const ObjectID = require('mongodb').ObjectId;
const moment = require('moment');
const OBJECT_ID_HEX_RE = /^[0-9a-fA-F]{24}$/;
const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
lib/server/treatments.js
+110 -102
View File
@@ -4,9 +4,10 @@ var _ = require('lodash');
var async = require('async');
var moment = require('moment');
var find_options = require('./query');
var runWithCallback = require('../storage/run-with-callback');
function storage (env, ctx) {
var ObjectID = require('mongodb-legacy').ObjectId;
var ObjectID = require('mongodb').ObjectId;
var OBJECT_ID_HEX_RE = /^[0-9a-fA-F]{24}$/;
function create (objOrArray, fn) {
@@ -61,10 +62,14 @@ function storage (env, ctx) {
};
});
api().bulkWrite(bulkOps, { ordered: true }, function(err, bulkResult) {
if (err) {
return runWithCallback(async function () {
var bulkResult;
try {
bulkResult = await api().bulkWrite(bulkOps, { ordered: true });
} catch (err) {
console.error('Problem upserting treatments batch', err);
return done(err, []);
throw err;
}
// Assign _ids from upserted results
@@ -83,8 +88,9 @@ function storage (env, ctx) {
if (docsNeedingId.length > 0) {
var identifiers = docsNeedingId.map(function(obj) { return obj.identifier; });
api().find({ identifier: { $in: identifiers } }).toArray(function(findErr, existing) {
if (!findErr && existing) {
try {
var existing = await api().find({ identifier: { $in: identifiers } }).toArray();
if (existing) {
var idMap = {};
existing.forEach(function(doc) {
if (doc.identifier) idMap[doc.identifier] = doc._id;
@@ -95,16 +101,9 @@ function storage (env, ctx) {
}
});
}
ctx.bus.emit('data-update', {
type: 'treatments',
op: 'update',
changes: ctx.ddata.processRawDataForRuntime(objOrArray)
});
done(null, objOrArray);
});
return;
} catch (findErr) {
// Preserve existing behavior: still report success even if the id lookup fails.
}
}
ctx.bus.emit('data-update', {
@@ -113,7 +112,13 @@ function storage (env, ctx) {
changes: ctx.ddata.processRawDataForRuntime(objOrArray)
});
done(null, objOrArray);
return objOrArray;
}, function (err, result) {
if (err) {
done(err, []);
return;
}
done(null, result);
});
} else {
upsert(objOrArray, function upserted (err, docs) {
@@ -130,29 +135,34 @@ function storage (env, ctx) {
var results = prepareData(obj);
var query = upsertQueryFor(obj, results);
api( ).replaceOne(query, obj, {upsert: true}, function complete (err, updateResults) {
(async function () {
try {
var updateResults = await api().replaceOne(query, obj, {upsert: true});
if (err) console.error('Problem upserting treatment', err);
if (updateResults) {
if (updateResults.upsertedCount == 1) {
obj._id = updateResults.upsertedId;
} else if (updateResults.matchedCount >= 1 && obj.identifier && !obj._id) {
// REQ-SYNC-072: On update by identifier, fetch the existing _id
api().findOne(query, function(findErr, existing) {
if (!findErr && existing) {
obj._id = existing._id;
if (updateResults) {
if (updateResults.upsertedCount == 1) {
obj._id = updateResults.upsertedId;
} else if (updateResults.matchedCount >= 1 && obj.identifier && !obj._id) {
// REQ-SYNC-072: On update by identifier, fetch the existing _id
try {
var existing = await api().findOne(query);
if (existing) {
obj._id = existing._id;
}
} catch (findErr) {
// Preserve existing behavior: update success does not fail if the lookup fails.
}
finishUpsert(err, obj, results);
});
return;
}
}
await finishUpsert(null, obj, results);
} catch (err) {
console.error('Problem upserting treatment', err);
await finishUpsert(err, obj, results);
}
})();
finishUpsert(err, obj, results);
});
function finishUpsert(err, obj, results) {
async function finishUpsert(err, obj, results) {
// TODO document this feature
if (!err && obj.preBolus) {
//create a new object to insert copying only the needed fields
@@ -170,24 +180,28 @@ function storage (env, ctx) {
created_at: pbTreat.created_at,
eventType: pbTreat.eventType
};
api( ).replaceOne(pbQuery, pbTreat, {upsert: true}, function pbComplete (err, updateResults) {
var updateResults;
try {
updateResults = await api().replaceOne(pbQuery, pbTreat, {upsert: true});
} catch (pbErr) {
err = pbErr;
}
if (updateResults) {
if (updateResults.upsertedCount == 1) {
pbTreat._id = updateResults.upsertedId
}
if (updateResults) {
if (updateResults.upsertedCount == 1) {
pbTreat._id = updateResults.upsertedId;
}
}
var treatments = _.compact([obj, pbTreat]);
var treatments = _.compact([obj, pbTreat]);
ctx.bus.emit('data-update', {
type: 'treatments',
op: 'update',
changes: ctx.ddata.processRawDataForRuntime(treatments)
});
fn(err, treatments);
ctx.bus.emit('data-update', {
type: 'treatments',
op: 'update',
changes: ctx.ddata.processRawDataForRuntime(treatments)
});
fn(err, treatments);
} else {
ctx.bus.emit('data-update', {
@@ -210,10 +224,12 @@ function storage (env, ctx) {
return this;
}
return limit.call(api()
.find(query_for(opts))
.sort(opts && opts.sort || {created_at: -1}), opts)
.toArray(fn);
return runWithCallback(function () {
return limit.call(api()
.find(query_for(opts))
.sort(opts && opts.sort || {created_at: -1}), opts)
.toArray();
}, fn);
}
function query_for (opts) {
@@ -234,20 +250,21 @@ function storage (env, ctx) {
}
function remove (opts, fn) {
return api( ).deleteMany(query_for(opts), {}, function (err, stat) {
//TODO: this is triggering a read from Mongo, we can do better
//console.log('Treatment removed', opts); // , stat);
return runWithCallback(async function () {
var stat = await api().deleteMany(query_for(opts), {});
//TODO: this is triggering a read from Mongo, we can do better
//console.log('Treatment removed', opts); // , stat);
ctx.bus.emit('data-update', {
type: 'treatments',
op: 'remove',
count: stat.deletedCount,
changes: opts.find._id
});
ctx.bus.emit('data-received');
fn(err, stat);
ctx.bus.emit('data-update', {
type: 'treatments',
op: 'remove',
count: stat.deletedCount,
changes: opts.find._id
});
ctx.bus.emit('data-received');
return stat;
}, fn);
}
function save (obj, fn) {
@@ -256,52 +273,43 @@ function storage (env, ctx) {
var query = upsertQueryFor(obj, { created_at: obj.created_at });
function saved (err, updateResults) {
if (!err) {
if (updateResults && updateResults.upsertedCount == 1) {
obj._id = updateResults.upsertedId;
} else if (updateResults && updateResults.matchedCount >= 1 && obj.identifier && !obj._id) {
// REQ-SYNC-072: On update by identifier, fetch the existing _id
api().findOne(query, function(findErr, existing) {
if (!findErr && existing) {
obj._id = existing._id;
}
finishSave(err, obj);
});
return;
const promise = runWithCallback(async function () {
var updateResults = await api().replaceOne(query, obj, {upsert: true});
if (updateResults && updateResults.upsertedCount == 1) {
obj._id = updateResults.upsertedId;
} else if (updateResults && updateResults.matchedCount >= 1 && obj.identifier && !obj._id) {
// REQ-SYNC-072: On update by identifier, fetch the existing _id
try {
var existing = await api().findOne(query);
if (existing) {
obj._id = existing._id;
}
} catch (findErr) {
// Preserve existing behavior: update success does not fail if the lookup fails.
}
// console.log('Treatment updated', created);
ctx.ddata.processRawDataForRuntime(obj);
ctx.bus.emit('data-update', {
type: 'treatments',
op: 'update',
changes: ctx.ddata.processRawDataForRuntime([obj])
});
}
if (err) console.error('Problem saving treating', err);
fn(err, obj);
}
ctx.ddata.processRawDataForRuntime(obj);
ctx.bus.emit('data-update', {
type: 'treatments',
op: 'update',
changes: ctx.ddata.processRawDataForRuntime([obj])
});
function finishSave(err, obj) {
if (!err) {
ctx.ddata.processRawDataForRuntime(obj);
ctx.bus.emit('data-update', {
type: 'treatments',
op: 'update',
changes: ctx.ddata.processRawDataForRuntime([obj])
});
return obj;
}, function (err, result) {
if (err) {
console.error('Problem saving treating', err);
fn(err, obj);
return;
}
if (err) console.error('Problem saving treating', err);
fn(err, obj);
}
api().replaceOne(query, obj, {upsert: true}, saved);
fn(null, result);
});
ctx.bus.emit('data-received');
return promise;
}
function api ( ) {
lib/server/websocket.js
+184 -195
View File
@@ -2,7 +2,7 @@
var times = require('../times');
var calcData = require('../data/calcdelta');
var ObjectID = require('mongodb-legacy').ObjectId;
var ObjectID = require('mongodb').ObjectId;
const forwarded = require('forwarded-for');
function getRemoteIP (req) {
@@ -230,25 +230,23 @@ function init (env, ctx, server) {
}
var id = safeObjectID(data._id);
ctx.store.collection(collection).updateOne({ '_id': id }
, { $set: data.data }
, function(err, results) {
if (!err) {
ctx.store.collection(collection).findOne({ '_id': id }
, function(err, results) {
console.log('Got results', results);
if (!err && results !== null) {
ctx.bus.emit('data-update', {
type: data.collection
, op: 'update'
, changes: ctx.ddata.processRawDataForRuntime([results])
});
}
});
(async function () {
try {
var mongoCollection = ctx.store.collection(collection);
await mongoCollection.updateOne({ '_id': id }, { $set: data.data });
var results = await mongoCollection.findOne({ '_id': id });
console.log('Got results', results);
if (results !== null) {
ctx.bus.emit('data-update', {
type: data.collection
, op: 'update'
, changes: ctx.ddata.processRawDataForRuntime([results])
});
}
} catch (err) {
console.error(err);
}
);
})();
if (callback) {
callback({ result: 'success' });
@@ -278,23 +276,23 @@ function init (env, ctx, server) {
}
var objId = safeObjectID(data._id);
ctx.store.collection(collection).updateOne({ '_id': objId }, { $unset: data.data }
, function(err, results) {
if (!err) {
ctx.store.collection(collection).findOne({ '_id': objId }
, function(err, results) {
console.log('Got results', results);
if (!err && results !== null) {
ctx.bus.emit('data-update', {
type: data.collection
, op: 'update'
, changes: ctx.ddata.processRawDataForRuntime([results])
});
}
});
(async function () {
try {
var mongoCollection = ctx.store.collection(collection);
await mongoCollection.updateOne({ '_id': objId }, { $unset: data.data });
var results = await mongoCollection.findOne({ '_id': objId });
console.log('Got results', results);
if (results !== null) {
ctx.bus.emit('data-update', {
type: data.collection
, op: 'update'
, changes: ctx.ddata.processRawDataForRuntime([results])
});
}
});
} catch (err) {
console.error(err);
}
})();
if (callback) {
callback({ result: 'success' });
@@ -328,40 +326,50 @@ function init (env, ctx, server) {
// Handle array input: process each item sequentially
if (Array.isArray(data.data)) {
console.log(LOG_WS + 'dbAdd received array with ' + data.data.length + ' items');
var results = [];
var processIndex = 0;
(async function () {
var results = [];
function processNextItem() {
if (processIndex >= data.data.length) {
if (callback) {
callback(results);
}
return;
}
var itemData = {
collection: data.collection,
data: data.data[processIndex]
};
processIndex++;
processSingleDbAdd(itemData, collection, maxtimediff, function(itemResult) {
for (var processIndex = 0; processIndex < data.data.length; processIndex += 1) {
var itemData = {
collection: data.collection,
data: data.data[processIndex]
};
var itemResult = await processSingleDbAdd(itemData, collection, maxtimediff);
if (itemResult && itemResult.length > 0) {
results = results.concat(itemResult);
}
processNextItem();
});
}
}
processNextItem();
if (callback) {
callback(results);
}
})().catch(function (err) {
console.error(err);
if (callback) {
callback([]);
}
});
return;
}
// Single object processing
processSingleDbAdd(data, collection, maxtimediff, callback);
processSingleDbAdd(data, collection, maxtimediff)
.then(function (result) {
if (callback) {
callback(result);
}
})
.catch(function (err) {
console.error(err);
if (callback) {
callback([]);
}
});
});
function processSingleDbAdd(data, collection, maxtimediff, callback) {
async function processSingleDbAdd(data, collection, maxtimediff) {
var mongoCollection = ctx.store.collection(collection);
if (data.collection === 'treatments' && !('eventType' in data.data)) {
data.data.eventType = '<none>';
}
@@ -382,97 +390,86 @@ function init (env, ctx, server) {
}
// try to find exact match
ctx.store.collection(collection).find(query).toArray(function findResult (err, array) {
if (err) {
console.error(err);
callback([]);
return;
}
try {
var array = await mongoCollection.find(query).toArray();
if (array.length > 0) {
console.log(LOG_DEDUP + 'Exact match');
if (callback) {
callback([array[0]]);
}
return;
return [array[0]];
}
} catch (err) {
console.error(err);
return [];
}
var selected = false;
var query_similiar = {
created_at: { $gte: new Date(new Date(data.data.created_at).getTime() - maxtimediff).toISOString(), $lte: new Date(new Date(data.data.created_at).getTime() + maxtimediff).toISOString() }
};
if (data.data.insulin) {
query_similiar.insulin = data.data.insulin;
selected = true;
var selected = false;
var query_similiar = {
created_at: { $gte: new Date(new Date(data.data.created_at).getTime() - maxtimediff).toISOString(), $lte: new Date(new Date(data.data.created_at).getTime() + maxtimediff).toISOString() }
};
if (data.data.insulin) {
query_similiar.insulin = data.data.insulin;
selected = true;
}
if (data.data.carbs) {
query_similiar.carbs = data.data.carbs;
selected = true;
}
if (data.data.percent) {
query_similiar.percent = data.data.percent;
selected = true;
}
if (data.data.absolute) {
query_similiar.absolute = data.data.absolute;
selected = true;
}
if (data.data.duration) {
query_similiar.duration = data.data.duration;
selected = true;
}
if (data.data.NSCLIENT_ID) {
query_similiar.NSCLIENT_ID = data.data.NSCLIENT_ID;
selected = true;
}
// if none assigned add at least eventType
if (!selected) {
query_similiar.eventType = data.data.eventType;
}
// try to find similiar
try {
var similar = await mongoCollection.find(query_similiar).toArray();
// if found similiar just update date. next time it will match exactly
if (similar.length > 0) {
console.log(LOG_DEDUP + 'Found similiar', similar[0]);
similar[0].created_at = data.data.created_at;
var objId = safeObjectID(similar[0]._id);
await mongoCollection.updateOne({ '_id': objId }, { $set: { created_at: data.data.created_at } });
ctx.bus.emit('data-received');
return [similar[0]];
}
if (data.data.carbs) {
query_similiar.carbs = data.data.carbs;
selected = true;
}
if (data.data.percent) {
query_similiar.percent = data.data.percent;
selected = true;
}
if (data.data.absolute) {
query_similiar.absolute = data.data.absolute;
selected = true;
}
if (data.data.duration) {
query_similiar.duration = data.data.duration;
selected = true;
}
if (data.data.NSCLIENT_ID) {
query_similiar.NSCLIENT_ID = data.data.NSCLIENT_ID;
selected = true;
}
// if none assigned add at least eventType
if (!selected) {
query_similiar.eventType = data.data.eventType;
}
// try to find similiar
ctx.store.collection(collection).find(query_similiar).toArray(function findSimiliarResult (err, array) {
// if found similiar just update date. next time it will match exactly
} catch (err) {
console.error(err);
return [];
}
if (err) {
console.error(err);
callback([]);
return;
}
if (array.length > 0) {
console.log(LOG_DEDUP + 'Found similiar', array[0]);
array[0].created_at = data.data.created_at;
var objId = safeObjectID(array[0]._id);
ctx.store.collection(collection).updateOne({ '_id': objId }, { $set: { created_at: data.data.created_at } });
if (callback) {
callback([array[0]]);
}
ctx.bus.emit('data-received');
return;
}
// if not found create new record
console.log(LOG_DEDUP + 'Adding new record');
ctx.store.collection(collection).insertOne(data.data, function insertResult (err, ops) {
if (err != null && err.message) {
console.log('treatments data insertion error: ', err.message);
return;
}
var doc = data.data;
doc._id = ops.insertedId;
ctx.bus.emit('data-update', {
type: data.collection
, op: 'update'
, changes: ctx.ddata.processRawDataForRuntime([doc])
});
if (callback) {
callback([doc]);
}
ctx.bus.emit('data-received');
});
// if not found create new record
console.log(LOG_DEDUP + 'Adding new record');
try {
var insertResult = await mongoCollection.insertOne(data.data);
var doc = data.data;
doc._id = insertResult.insertedId;
ctx.bus.emit('data-update', {
type: data.collection
, op: 'update'
, changes: ctx.ddata.processRawDataForRuntime([doc])
});
});
ctx.bus.emit('data-received');
return [doc];
} catch (err) {
if (err != null && err.message) {
console.log('treatments data insertion error: ', err.message);
return [];
}
throw err;
}
// devicestatus deduping
} else if (data.collection === 'devicestatus') {
var queryDev;
@@ -485,62 +482,54 @@ function init (env, ctx, server) {
}
// try to find exact match
ctx.store.collection(collection).find(queryDev).toArray(function findResult (err, array) {
if (err) {
console.error(err);
callback([]);
return;
}
if (array.length > 0) {
try {
var existingStatus = await mongoCollection.find(queryDev).toArray();
if (existingStatus.length > 0) {
console.log(LOG_DEDUP + 'Devicestatus exact match');
if (callback) {
callback([array[0]]);
}
return;
return [existingStatus[0]];
}
} catch (err) {
console.error(err);
return [];
}
});
ctx.store.collection(collection).insertOne(data.data, function insertResult (err, ops) {
if (err != null && err.message) {
console.log('devicestatus insertion error: ', err.message);
return;
}
var doc = data.data;
doc._id = ops.insertedId;
try {
var devicestatusInsertResult = await mongoCollection.insertOne(data.data);
var devicestatusDoc = data.data;
devicestatusDoc._id = devicestatusInsertResult.insertedId;
ctx.bus.emit('data-update', {
type: 'devicestatus'
, op: 'update'
, changes: ctx.ddata.processRawDataForRuntime([doc])
, changes: ctx.ddata.processRawDataForRuntime([devicestatusDoc])
});
if (callback) {
callback([doc]);
}
ctx.bus.emit('data-received');
});
} else {
ctx.store.collection(collection).insertOne(data.data, function insertResult (err, ops) {
return [devicestatusDoc];
} catch (err) {
if (err != null && err.message) {
console.log(data.collection + ' insertion error: ', err.message);
return;
console.log('devicestatus insertion error: ', err.message);
return [];
}
var doc = data.data;
doc._id = ops.insertedId;
throw err;
}
} else {
try {
var genericInsertResult = await mongoCollection.insertOne(data.data);
var genericDoc = data.data;
genericDoc._id = genericInsertResult.insertedId;
ctx.bus.emit('data-update', {
type: data.collection
, op: 'update'
, changes: ctx.ddata.processRawDataForRuntime([doc])
, changes: ctx.ddata.processRawDataForRuntime([genericDoc])
});
if (callback) {
callback([doc]);
}
ctx.bus.emit('data-received');
});
return [genericDoc];
} catch (err) {
if (err != null && err.message) {
console.log(data.collection + ' insertion error: ', err.message);
return [];
}
throw err;
}
}
}
@@ -562,19 +551,19 @@ function init (env, ctx, server) {
}
var objId = safeObjectID(data._id);
ctx.store.collection(collection).deleteOne({ '_id': objId }
, function(err, stat) {
if (!err) {
ctx.bus.emit('data-update', {
type: data.collection
, op: 'remove'
, count: stat.deletedCount
, changes: data._id
});
}
});
(async function () {
try {
var stat = await ctx.store.collection(collection).deleteOne({ '_id': objId });
ctx.bus.emit('data-update', {
type: data.collection
, op: 'remove'
, count: stat.deletedCount
, changes: data._id
});
} catch (err) {
console.error(err);
}
})();
if (callback) {
callback({ result: 'success' });
lib/storage/mongo-storage.js
+104 -39
View File
@@ -1,6 +1,6 @@
'use strict';
const MongoClient = require('mongodb-legacy').MongoClient;
const MongoClient = require('mongodb').MongoClient;
const mongo = {
client: null,
@@ -10,6 +10,44 @@ const mongo = {
const DEFAULT_POOL_SIZE = 5;
const LEGACY_POOL_SIZE = 100;
function getRetryDelay(attempt) {
return attempt > 15 ? 60000 : attempt * 3000;
}
function wait(ms) {
return new Promise(function (resolve) {
setTimeout(resolve, ms);
});
}
async function closeClient(client) {
if (!client || typeof client.close !== 'function') {
return;
}
try {
await client.close();
} catch (err) {
console.log('Error closing failed MongoDB client: %j', err);
}
}
function wrapConnectionError(err) {
if (err && err.name === 'MongoReadOnlyConnectionError') {
return err;
}
if (err && err.message && err.message.includes('AuthenticationFailed')) {
return new Error('MongoDB authentication failed! Double check the URL has the right username and password in MONGODB_URI.');
}
return new Error('MONGODB_URI seems invalid: ' + err.message);
}
function isRetryableConnectionError(err) {
return !!(err && err.name === 'MongoServerSelectionError' && !(err.message && err.message.includes('AuthenticationFailed')));
}
function getPoolOptions(env) {
const poolSize = env.mongo_pool_size
? parseInt(env.mongo_pool_size, 10)
@@ -75,6 +113,8 @@ function init(env, cb, forceNewConnection) {
if (cb && cb.call) {
cb(null, mongo);
}
return Promise.resolve(mongo);
} else {
if (!env.storageURI) {
throw new Error('MongoDB connection string is missing. Please set MONGODB_URI environment variable');
@@ -84,56 +124,82 @@ function init(env, cb, forceNewConnection) {
console.log('Setting up new connection to MongoDB with pool options:', poolOptions);
const options = {
useNewUrlParser: true,
useUnifiedTopology: true,
...poolOptions,
};
const connect_with_retry = async function (i) {
const connectWithRetry = async function () {
let attempt = 1;
mongo.client = new MongoClient(env.storageURI, options);
setupPoolMonitoring(mongo.client, env);
if (forceNewConnection) {
const previousClient = mongo.client;
mongo.client = null;
mongo.db = null;
await closeClient(previousClient);
}
try {
await mongo.client.connect();
while (true) {
let client = null;
console.log('Successfully established connection to MongoDB');
try {
client = new MongoClient(env.storageURI, options);
mongo.client = client;
setupPoolMonitoring(client, env);
const dbName = mongo.client.s.options.dbName;
mongo.db = mongo.client.db(dbName);
await client.connect();
const result = await mongo.db.command({ connectionStatus: 1 });
const roles = result.authInfo.authenticatedUserRoles;
if (roles && roles.length > 0 && roles[0].role == 'readAnyDatabase') {
console.error('Mongo user is read only');
cb(new Error('MongoDB connection is in read only mode! Go back to MongoDB configuration and check your database user has read and write access.'), null);
}
console.log('Successfully established connection to MongoDB');
console.log('Mongo user role seems ok:', roles);
mongo.db = client.db();
// If there is a valid callback, then invoke the function to perform the callback
if (cb && cb.call) {
cb(null, mongo);
}
} catch (err) {
if (err.message && err.message.includes('AuthenticationFailed')) {
console.log('Authentication to Mongo failed');
cb(new Error('MongoDB authentication failed! Double check the URL has the right username and password in MONGODB_URI.'), null);
return;
}
const result = await mongo.db.command({ connectionStatus: 1 });
const roles = result.authInfo.authenticatedUserRoles;
if (roles && roles.length > 0 && roles[0].role == 'readAnyDatabase') {
console.error('Mongo user is read only');
const readOnlyError = new Error('MongoDB connection is in read only mode! Go back to MongoDB configuration and check your database user has read and write access.');
readOnlyError.name = 'MongoReadOnlyConnectionError';
throw readOnlyError;
}
if (err.name && err.name === "MongoServerSelectionError") {
const timeout = (i > 15) ? 60000 : i * 3000;
console.log('Mongo user role seems ok:', roles);
return mongo;
} catch (err) {
const retryable = isRetryableConnectionError(err);
const wrappedError = wrapConnectionError(err);
if (err && err.message && err.message.includes('AuthenticationFailed')) {
console.log('Authentication to Mongo failed');
}
mongo.db = null;
if (mongo.client === client) {
mongo.client = null;
}
await closeClient(client);
if (!retryable) {
throw wrappedError;
}
const timeout = getRetryDelay(attempt);
console.log('Error connecting to MongoDB: %j - retrying in ' + timeout / 1000 + ' sec', err);
setTimeout(connect_with_retry, timeout, i + 1);
if (i == 1) cb(new Error('MongoDB connection failed! Double check the MONGODB_URI setting in Heroku.'), null);
} else {
cb(new Error('MONGODB_URI seems invalid: ' + err.message));
await wait(timeout);
attempt += 1;
}
}
};
return connect_with_retry(1);
const promise = connectWithRetry();
if (cb && cb.call) {
promise.then(function (store) {
cb(null, store);
}, function (err) {
cb(err, null);
});
}
return promise;
}
}
@@ -146,10 +212,8 @@ function init(env, cb, forceNewConnection) {
fields.forEach(function (field) {
const name = collection.collectionName + "." + field;
console.info('ensuring index for: ' + name);
collection.createIndex(field, { 'background': true }, function (err) {
if (err) {
console.error('unable to ensureIndex for: ' + name + ' - ' + err);
}
collection.createIndex(field).catch(function (err) {
console.error('unable to ensureIndex for: ' + name + ' - ' + err);
});
});
};
@@ -161,3 +225,4 @@ module.exports = init;
module.exports.DEFAULT_POOL_SIZE = DEFAULT_POOL_SIZE;
module.exports.LEGACY_POOL_SIZE = LEGACY_POOL_SIZE;
module.exports.getPoolOptions = getPoolOptions;
module.exports.getRetryDelay = getRetryDelay;
lib/storage/run-with-callback.js
+20
View File
@@ -0,0 +1,20 @@
'use strict';
function runWithCallback (work, callback) {
const promise = Promise.resolve().then(work);
if (callback && callback.call) {
promise.then(
function onSuccess(result) {
callback(null, result);
},
function onError(err) {
callback(err, null);
}
);
}
return promise;
}
module.exports = runWithCallback;
package-lock.json
Generated
+1 -11
View File
@@ -56,7 +56,7 @@
"moment-timezone": "^0.5.31",
"moment-timezone-data-webpack-plugin": "^1.5.0",
"mongo-url-parser": "^1.0.2",
"mongodb-legacy": "^5.0.0",
"mongodb": "^5.9.2",
"mongomock": "^0.1.2",
"nightscout-connect": "^0.0.12",
"node-cache": "^4.2.1",
@@ -6947,16 +6947,6 @@
"node": ">=12"
}
},
"node_modules/mongodb-legacy": {
"version": "5.0.0",
"license": "Apache-2.0",
"dependencies": {
"mongodb": "^5.0.0"
},
"engines": {
"node": ">=14.20.1"
}
},
"node_modules/mongomock": {
"version": "0.1.2",
"dependencies": {
package.json
+2 -2
View File
@@ -90,9 +90,9 @@
"npm": ">=10.x"
},
"dependencies": {
"@mongodb-js/saslprep": "^1.4.5",
"@babel/core": "^7.18.10",
"@babel/preset-env": "^7.18.10",
"@mongodb-js/saslprep": "^1.4.5",
"@parse/node-apn": "^5.1.3",
"acorn": "^8.0.5",
"acorn-jsx": "^5.3.1",
@@ -136,7 +136,7 @@
"moment-timezone": "^0.5.31",
"moment-timezone-data-webpack-plugin": "^1.5.0",
"mongo-url-parser": "^1.0.2",
"mongodb-legacy": "^5.0.0",
"mongodb": "^5.9.2",
"mongomock": "^0.1.2",
"nightscout-connect": "^0.0.12",
"node-cache": "^4.2.1",
tests/XX_clean.test.js
+4 -8
View File
@@ -26,16 +26,12 @@ describe('Clean MONGO after tests', function ( ) {
});
});
it('wipe treatment data', function (done) {
self.ctx.treatments().deleteMany({ }, function ( ) {
done();
});
it('wipe treatment data', async function () {
await self.ctx.treatments().deleteMany({ });
});
it('wipe entries data', function (done) {
self.ctx.entries().deleteMany({ }, function ( ) {
done();
});
it('wipe entries data', async function () {
await self.ctx.entries().deleteMany({ });
});
});
tests/api.deduplication.test.js
+8 -6
View File
@@ -55,12 +55,14 @@ describe('v1 API Deduplication Behavior', function() {
find: { created_at: { '$gte': '1999-01-01T00:00:00.000Z' } }
}, function() {
// Use deleteMany for faster cleanup of entries
self.ctx.entries().deleteMany({}, function() {
// Also clear devicestatus to reduce database load
self.ctx.devicestatus.remove({
find: { created_at: { '$gte': '1999-01-01T00:00:00.000Z' } }
}, done);
});
self.ctx.entries().deleteMany({})
.then(function() {
// Also clear devicestatus to reduce database load
self.ctx.devicestatus.remove({
find: { created_at: { '$gte': '1999-01-01T00:00:00.000Z' } }
}, done);
})
.catch(done);
});
});
tests/api.entries.test.js
+4 -4
View File
@@ -62,12 +62,12 @@ describe('Entries REST api', function ( ) {
});
afterEach(function (done) {
self.archive( ).deleteMany({ }, done);
afterEach(async function () {
await self.archive( ).deleteMany({ });
});
after(function (done) {
self.archive( ).deleteMany({ }, done);
after(async function () {
await self.archive( ).deleteMany({ });
});
// keep this test pinned at or near the top in order to validate all
tests/api.entries.uuid.test.js
+78 -80
View File
@@ -43,12 +43,12 @@ describe('Entry sysTime+type dedup (Baseline)', function() {
});
});
afterEach(function(done) {
self.archive().deleteMany({}, done);
afterEach(async function() {
await self.archive().deleteMany({});
});
after(function(done) {
self.archive().deleteMany({}, done);
after(async function() {
await self.archive().deleteMany({});
});
/**
@@ -98,14 +98,14 @@ describe('Entry sysTime+type dedup (Baseline)', function() {
if (err2) return done(err2);
// Verify: only 1 entry exists with updated sgv 125
self.archive().find({ date: timestamp }).toArray(function(err3, docs) {
if (err3) return done(err3);
docs.should.have.lengthOf(1);
docs[0].sgv.should.equal(125);
docs[0].direction.should.equal('FortyFiveUp');
done();
});
self.archive().find({ date: timestamp }).toArray()
.then(function(docs) {
docs.should.have.lengthOf(1);
docs[0].sgv.should.equal(125);
docs[0].direction.should.equal('FortyFiveUp');
done();
})
.catch(done);
});
});
});
@@ -155,14 +155,14 @@ describe('Entry sysTime+type dedup (Baseline)', function() {
if (err2) return done(err2);
// Verify: 2 entries exist (different types)
self.archive().find({ date: timestamp }).toArray(function(err3, docs) {
if (err3) return done(err3);
docs.should.have.lengthOf(2);
var types = docs.map(d => d.type).sort();
types.should.eql(['mbg', 'sgv']);
done();
});
self.archive().find({ date: timestamp }).toArray()
.then(function(docs) {
docs.should.have.lengthOf(2);
var types = docs.map(d => d.type).sort();
types.should.eql(['mbg', 'sgv']);
done();
})
.catch(done);
});
});
});
@@ -213,12 +213,12 @@ describe('Entry sysTime+type dedup (Baseline)', function() {
if (err2) return done(err2);
// Verify: 2 entries exist
self.archive().find({ type: 'sgv', date: { $in: [timestamp1, timestamp2] } }).toArray(function(err3, docs) {
if (err3) return done(err3);
docs.should.have.lengthOf(2);
done();
});
self.archive().find({ type: 'sgv', date: { $in: [timestamp1, timestamp2] } }).toArray()
.then(function(docs) {
docs.should.have.lengthOf(2);
done();
})
.catch(done);
});
});
});
@@ -249,12 +249,12 @@ describe('Entry UUID _id handling (GAP-SYNC-045)', function() {
});
});
afterEach(function(done) {
self.archive().deleteMany({}, done);
afterEach(async function() {
await self.archive().deleteMany({});
});
after(function(done) {
self.archive().deleteMany({}, done);
after(async function() {
await self.archive().deleteMany({});
});
/**
@@ -286,14 +286,14 @@ describe('Entry UUID _id handling (GAP-SYNC-045)', function() {
if (err) return done(err);
// Entry should be created (may have ObjectId _id, UUID in identifier)
self.archive().find({ date: timestamp }).toArray(function(err2, docs) {
if (err2) return done(err2);
docs.should.have.lengthOf(1);
docs[0].sgv.should.equal(120);
// Note: After fix, expect docs[0].identifier === uuid
done();
});
self.archive().find({ date: timestamp }).toArray()
.then(function(docs) {
docs.should.have.lengthOf(1);
docs[0].sgv.should.equal(120);
// Note: After fix, expect docs[0].identifier === uuid
done();
})
.catch(done);
});
});
@@ -347,13 +347,13 @@ describe('Entry UUID _id handling (GAP-SYNC-045)', function() {
if (err2) return done(err2);
// Verify: single entry, updated value
self.archive().find({ date: timestamp }).toArray(function(err3, docs) {
if (err3) return done(err3);
docs.should.have.lengthOf(1);
docs[0].sgv.should.equal(125);
done();
});
self.archive().find({ date: timestamp }).toArray()
.then(function(docs) {
docs.should.have.lengthOf(1);
docs[0].sgv.should.equal(125);
done();
})
.catch(done);
});
});
});
@@ -412,13 +412,13 @@ describe('Entry UUID _id handling (GAP-SYNC-045)', function() {
if (err2) return done(err2);
// Verify: single entry (dedup by sysTime+type, not UUID)
self.archive().find({ date: timestamp }).toArray(function(err3, docs) {
if (err3) return done(err3);
docs.should.have.lengthOf(1);
docs[0].sgv.should.equal(125);
done();
});
self.archive().find({ date: timestamp }).toArray()
.then(function(docs) {
docs.should.have.lengthOf(1);
docs[0].sgv.should.equal(125);
done();
})
.catch(done);
});
});
});
@@ -469,14 +469,14 @@ describe('Entry UUID _id handling (GAP-SYNC-045)', function() {
if (err) return done(err);
// Verify: all 3 entries created
self.archive().find({ date: { $in: [timestamp1, timestamp2, timestamp3] } }).toArray(function(err2, docs) {
if (err2) return done(err2);
docs.should.have.lengthOf(3);
var sgvValues = docs.map(d => d.sgv).sort();
sgvValues.should.eql([120, 125, 130]);
done();
});
self.archive().find({ date: { $in: [timestamp1, timestamp2, timestamp3] } }).toArray()
.then(function(docs) {
docs.should.have.lengthOf(3);
var sgvValues = docs.map(d => d.sgv).sort();
sgvValues.should.eql([120, 125, 130]);
done();
})
.catch(done);
});
});
@@ -500,9 +500,7 @@ describe('Entry UUID _id handling (GAP-SYNC-045)', function() {
dateString: sysTime,
sysTime: sysTime,
device: 'Trio'
}, function(err) {
if (err) return done(err);
}).then(function() {
// POST via API with same timestamp
var entry = {
_id: uuid,
@@ -523,15 +521,15 @@ describe('Entry UUID _id handling (GAP-SYNC-045)', function() {
if (err2) return done(err2);
// Verify: single entry, updated value
self.archive().find({ date: timestamp }).toArray(function(err3, docs) {
if (err3) return done(err3);
docs.should.have.lengthOf(1);
docs[0].sgv.should.equal(125);
done();
});
self.archive().find({ date: timestamp }).toArray()
.then(function(docs) {
docs.should.have.lengthOf(1);
docs[0].sgv.should.equal(125);
done();
})
.catch(done);
});
});
}).catch(done);
});
/**
@@ -563,15 +561,15 @@ describe('Entry UUID _id handling (GAP-SYNC-045)', function() {
if (err) return done(err);
// Verify: entry has identifier field with UUID
self.archive().find({ date: timestamp }).toArray(function(err2, docs) {
if (err2) return done(err2);
docs.should.have.lengthOf(1);
docs[0].should.have.property('identifier', uuid);
// _id should be ObjectId, not UUID
docs[0]._id.should.not.equal(uuid);
done();
});
self.archive().find({ date: timestamp }).toArray()
.then(function(docs) {
docs.should.have.lengthOf(1);
docs[0].should.have.property('identifier', uuid);
// _id should be ObjectId, not UUID
docs[0]._id.should.not.equal(uuid);
done();
})
.catch(done);
});
});
});
tests/api.objectid-validation.test.js
+34
View File
@@ -0,0 +1,34 @@
'use strict';
var should = require('should');
var objectIdValidation = require('../lib/api/shared/objectid-validation');
describe('API ObjectId validation helper', function () {
it('accepts missing ids for create-tolerant routes', function () {
should(objectIdValidation.isValidObjectId(undefined)).be.true();
should(objectIdValidation.isValidObjectId(null)).be.true();
});
it('accepts 24-character hex ids and rejects non-hex values', function () {
should(objectIdValidation.isValidObjectId('507f1f77bcf86cd799439011')).be.true();
should(objectIdValidation.isValidObjectId('550e8400-e29b-41d4-a716-446655440000')).be.false();
should(objectIdValidation.isValidObjectId(123)).be.false();
});
it('finds the first invalid id in a batch', function () {
objectIdValidation.findInvalidId([
{ _id: '507f1f77bcf86cd799439011' },
{ _id: 'invalid-id' },
{ _id: '507f191e810c19729de860ea' }
]).should.eql({ index: 1, id: 'invalid-id' });
});
it('returns null when all ids are valid or omitted', function () {
should.not.exist(objectIdValidation.findInvalidId([
{ _id: '507f1f77bcf86cd799439011' },
{},
{ _id: null }
]));
});
});
tests/api.security.test.js
+153
View File
@@ -15,6 +15,22 @@ describe('Security of REST API V1', function() {
var known = 'b723e97aa97846eb92d5264f084b2823f57c4aa1';
function rolesCollection() {
return self.ctx.store.collection(self.env.authentication_collections_prefix + 'roles');
}
function subjectsCollection() {
return self.ctx.store.collection(self.env.authentication_collections_prefix + 'subjects');
}
async function getBearerToken(accessToken) {
const res = await request(self.app)
.get('/api/v2/authorization/request/' + accessToken)
.expect(200);
return res.body.token;
}
before(function(done) {
var api = require('../lib/api/');
delete process.env.API_SECRET;
@@ -26,6 +42,7 @@ describe('Security of REST API V1', function() {
self.app = require('express')();
self.app.enable('api');
require('../lib/server/bootevent')(self.env, language).boot(async function booted (ctx) {
self.ctx = ctx;
self.app.use('/api/v1', api(self.env, ctx));
self.app.use('/api/v2/authorization', ctx.authorization.endpoints);
let authResult = await authSubject(ctx.authorization.storage);
@@ -159,4 +176,140 @@ describe('Security of REST API V1', function() {
});
});
describe('Authorization admin save endpoints', function () {
beforeEach(async function () {
await rolesCollection().deleteMany({ name: /^api-security-role/ });
await subjectsCollection().deleteMany({ name: /^api-security-subject/ });
});
afterEach(async function () {
await rolesCollection().deleteMany({ name: /^api-security-role/ });
await subjectsCollection().deleteMany({ name: /^api-security-subject/ });
});
it('PUT /api/v2/authorization/subjects updates an existing subject by _id', async function () {
const insertResult = await subjectsCollection().insertOne({
name: 'api-security-subject-update',
roles: ['readable'],
notes: 'original',
created_at: '2024-10-26T20:32:49.173Z'
});
const token = await getBearerToken(self.token.adminAll);
await request(self.app)
.put('/api/v2/authorization/subjects')
.set('Authorization', 'Bearer ' + token)
.send({
_id: insertResult.insertedId.toString(),
name: 'api-security-subject-update',
roles: ['admin'],
notes: 'updated',
created_at: '2024-10-26T21:32:49.173Z'
})
.expect(200);
const docs = await subjectsCollection().find({ name: 'api-security-subject-update' }).toArray();
docs.length.should.equal(1);
docs[0].roles.should.deepEqual(['admin']);
docs[0].notes.should.equal('updated');
});
it('PUT /api/v2/authorization/roles updates an existing role by _id', async function () {
const insertResult = await rolesCollection().insertOne({
name: 'api-security-role-update',
permissions: ['api:entries:read'],
notes: 'original',
created_at: '2024-10-26T20:32:49.173Z'
});
const token = await getBearerToken(self.token.adminAll);
await request(self.app)
.put('/api/v2/authorization/roles')
.set('Authorization', 'Bearer ' + token)
.send({
_id: insertResult.insertedId.toString(),
name: 'api-security-role-update',
permissions: ['api:entries:update'],
notes: 'updated',
created_at: '2024-10-26T21:32:49.173Z'
})
.expect(200);
const docs = await rolesCollection().find({ name: 'api-security-role-update' }).toArray();
docs.length.should.equal(1);
docs[0].permissions.should.deepEqual(['api:entries:update']);
docs[0].notes.should.equal('updated');
});
it('PUT /api/v2/authorization/subjects fails when _id is missing', async function () {
const token = await getBearerToken(self.token.adminAll);
await request(self.app)
.put('/api/v2/authorization/subjects')
.set('Authorization', 'Bearer ' + token)
.send({
name: 'api-security-subject-missing-id',
roles: ['readable'],
notes: 'should fail'
})
.expect(500);
const docs = await subjectsCollection().find({ name: 'api-security-subject-missing-id' }).toArray();
docs.length.should.equal(0);
});
it('PUT /api/v2/authorization/subjects fails when _id is invalid', async function () {
const token = await getBearerToken(self.token.adminAll);
await request(self.app)
.put('/api/v2/authorization/subjects')
.set('Authorization', 'Bearer ' + token)
.send({
_id: 'not-a-valid-objectid',
name: 'api-security-subject-invalid-id',
roles: ['readable'],
notes: 'should fail'
})
.expect(500);
const docs = await subjectsCollection().find({ name: 'api-security-subject-invalid-id' }).toArray();
docs.length.should.equal(0);
});
it('PUT /api/v2/authorization/roles fails when _id is missing', async function () {
const token = await getBearerToken(self.token.adminAll);
await request(self.app)
.put('/api/v2/authorization/roles')
.set('Authorization', 'Bearer ' + token)
.send({
name: 'api-security-role-missing-id',
permissions: ['api:entries:read'],
notes: 'should fail'
})
.expect(500);
const docs = await rolesCollection().find({ name: 'api-security-role-missing-id' }).toArray();
docs.length.should.equal(0);
});
it('PUT /api/v2/authorization/roles fails when _id is invalid', async function () {
const token = await getBearerToken(self.token.adminAll);
await request(self.app)
.put('/api/v2/authorization/roles')
.set('Authorization', 'Bearer ' + token)
.send({
_id: 'not-a-valid-objectid',
name: 'api-security-role-invalid-id',
permissions: ['api:entries:read'],
notes: 'should fail'
})
.expect(500);
const docs = await rolesCollection().find({ name: 'api-security-role-invalid-id' }).toArray();
docs.length.should.equal(0);
});
});
});
tests/api.shape-handling.test.js
+93 -9
View File
@@ -352,16 +352,12 @@ describe('API Shape Handling - Single Object vs Array Input', function () {
describe('Entries API - /api/entries/', function () {
beforeEach(function (done) {
self.ctx.entries().deleteMany({}, function () {
done();
});
beforeEach(async function () {
await self.ctx.entries().deleteMany({});
});
afterEach(function (done) {
self.ctx.entries().deleteMany({}, function () {
done();
});
afterEach(async function () {
await self.ctx.entries().deleteMany({});
});
it('POST accepts single SGV entry object', function (done) {
@@ -838,7 +834,95 @@ describe('API Shape Handling - Single Object vs Array Input', function () {
p.should.have.property('_id');
});
done();
});
});
});
});
describe('Legacy PUT update semantics', function () {
beforeEach(async function () {
await self.ctx.food().deleteMany({});
await self.ctx.activity().deleteMany({});
});
afterEach(async function () {
await self.ctx.food().deleteMany({});
await self.ctx.activity().deleteMany({});
});
it('PUT /api/food/ updates an existing record by _id when created_at changes', function (done) {
self.ctx.food.create({
name: 'API shape food',
category: 'Test',
carbs: 20,
protein: 10,
fat: 5
}, function (createErr, docs) {
if (createErr) return done(createErr);
var created = docs[0];
request(self.app)
.put('/api/food/')
.set('api-secret', known)
.send({
_id: created._id.toString(),
name: 'API shape food',
category: 'Test',
carbs: 25,
protein: 10,
fat: 5,
created_at: '2024-10-26T21:32:49.173Z'
})
.expect(200)
.end(function (err) {
if (err) return done(err);
self.ctx.food().find({ _id: created._id }).toArray()
.then(function (updatedDocs) {
updatedDocs.length.should.equal(1);
updatedDocs[0].carbs.should.equal(25);
updatedDocs[0].created_at.should.equal('2024-10-26T21:32:49.173Z');
done();
})
.catch(done);
});
});
});
it('PUT /api/activity/ updates an existing record by _id', function (done) {
self.ctx.activity.create([{
created_at: '2024-10-26T20:32:49.173Z',
heartrate: 80,
steps: 100,
activitylevel: 'walking'
}], function (createErr, docs) {
if (createErr) return done(createErr);
var created = docs[0];
request(self.app)
.put('/api/activity/')
.set('api-secret', known)
.send({
_id: created._id.toString(),
created_at: '2024-10-26T21:32:49.173Z',
heartrate: 95,
steps: 250,
activitylevel: 'running'
})
.expect(200)
.end(function (err) {
if (err) return done(err);
self.ctx.activity().find({ _id: created._id }).toArray()
.then(function (updatedDocs) {
updatedDocs.length.should.equal(1);
updatedDocs[0].heartrate.should.equal(95);
updatedDocs[0].steps.should.equal(250);
updatedDocs[0].activitylevel.should.equal('running');
done();
})
.catch(done);
});
});
});
});
});
tests/api.unauthorized.test.js
+4 -4
View File
@@ -38,12 +38,12 @@ describe('authed REST api', function ( ) {
this.archive.create(creating, done);
});
afterEach(function (done) {
this.archive( ).deleteMany({ }, done);
afterEach(async function () {
await this.archive( ).deleteMany({ });
});
after(function (done) {
this.archive( ).deleteMany({ }, done);
after(async function () {
await this.archive( ).deleteMany({ });
});
it('disallow unauthorized POST', function (done) {
tests/api3.storage.find.test.js
+44 -7
View File
@@ -3,13 +3,17 @@
require('should');
const find = require('../lib/api3/storage/mongoCollection/find');
const { ObjectId } = require('mongodb');
describe('API3 mongoCollection findMany', function () {
function createStubCollection (observed) {
describe('API3 mongoCollection find helpers', function () {
function createStubCursor (observed, docs) {
return {
find: function () {
return this;
},
project: function () {
return this;
},
sort: function () {
return this;
},
@@ -21,11 +25,17 @@ describe('API3 mongoCollection findMany', function () {
observed.skip = value;
return this;
},
project: function () {
return this;
},
toArray: function (callback) {
callback(null, []);
toArray: function () {
observed.toArrayCalls = (observed.toArrayCalls || 0) + 1;
return Promise.resolve(docs || []);
}
};
}
function createStubCollection (observed, docs) {
return {
find: function () {
return createStubCursor(observed, docs);
}
};
}
@@ -44,6 +54,7 @@ describe('API3 mongoCollection findMany', function () {
observed.limit.should.equal(5);
observed.skip.should.equal(2);
observed.toArrayCalls.should.equal(1);
result.should.eql([]);
});
@@ -63,4 +74,30 @@ describe('API3 mongoCollection findMany', function () {
observed.limit.should.equal(5);
observed.skip.should.equal(2);
});
it('normalizes findOne results when using promise-based cursors', async function () {
const observed = {};
const docId = new ObjectId();
const col = createStubCollection(observed, [{ _id: docId, type: 'sgv' }]);
const result = await find.findOne(col, docId.toString(), {});
observed.toArrayCalls.should.equal(1);
result.should.have.length(1);
result[0].should.have.property('identifier', docId.toString());
result[0].should.not.have.property('_id');
});
it('supports promise-based findOneFilter without normalization when requested', async function () {
const observed = {};
const docId = new ObjectId();
const col = createStubCollection(observed, [{ _id: docId, type: 'mbg' }]);
const result = await find.findOneFilter(col, { type: 'mbg' }, {}, { normalize: false });
observed.toArrayCalls.should.equal(1);
result.should.have.length(1);
result[0].should.have.property('_id');
result[0]._id.toString().should.equal(docId.toString());
});
});
tests/api3.storage.modify.test.js
+161
View File
@@ -0,0 +1,161 @@
'use strict';
require('should');
const { ObjectId } = require('mongodb');
const modify = require('../lib/api3/storage/mongoCollection/modify');
const MongoCollection = require('../lib/api3/storage/mongoCollection');
describe('API3 mongoCollection promise-based helpers', function () {
describe('modify helpers', function () {
it('insertOne uses promise-based collection methods and keeps existing behavior', async function () {
const doc = { _id: new ObjectId(), type: 'sgv' };
const insertedId = doc._id.toString();
const col = {
insertOne: function (receivedDoc) {
arguments.length.should.equal(1);
receivedDoc.should.equal(doc);
return Promise.resolve({ insertedId: receivedDoc._id });
}
};
const identifier = await modify.insertOne(col, doc);
identifier.should.equal(insertedId);
doc.should.not.have.property('_id');
});
it('replaceOne uses promise-based upsert without a callback', async function () {
const doc = { value: 42 };
const col = {
replaceOne: function (filter, receivedDoc, options) {
arguments.length.should.equal(3);
filter.should.eql({ $or: [{ identifier: 'record-1' }] });
receivedDoc.should.equal(doc);
options.should.eql({ upsert: true });
return Promise.resolve({ matchedCount: 1 });
}
};
const matchedCount = await modify.replaceOne(col, 'record-1', doc);
matchedCount.should.equal(1);
});
it('update and delete helpers use promise-based collection methods', async function () {
const col = {
updateOne: function (filter, update) {
arguments.length.should.equal(2);
filter.should.eql({ $or: [{ identifier: 'record-2' }] });
update.should.eql({ $set: { value: 84 } });
return Promise.resolve({ modifiedCount: 1 });
},
deleteOne: function (filter) {
arguments.length.should.equal(1);
filter.should.eql({ $or: [{ identifier: 'record-2' }] });
return Promise.resolve({ deletedCount: 1 });
},
deleteMany: function (filter) {
arguments.length.should.equal(1);
filter.should.eql({ $or: [{ kind: 'sgv' }, { kind: 'mbg' }] });
return Promise.resolve({ deletedCount: 2 });
}
};
const updated = await modify.updateOne(col, 'record-2', { value: 84 });
const deletedOne = await modify.deleteOne(col, 'record-2');
const deletedMany = await modify.deleteManyOr(col, [
{ field: 'kind', operator: 'eq', value: 'sgv' },
{ field: 'kind', operator: 'eq', value: 'mbg' }
]);
updated.should.eql({ updated: 1 });
deletedOne.should.eql({ deleted: 1 });
deletedMany.should.eql({ deleted: 2 });
});
});
describe('collection metadata helpers', function () {
function createCursor (observed, docs) {
return {
sort: function (spec) {
observed.sort = spec;
return this;
},
limit: function (value) {
observed.limit = value;
return this;
},
project: function (projection) {
observed.projection = projection;
return this;
},
toArray: function () {
observed.toArrayCalls = (observed.toArrayCalls || 0) + 1;
return Promise.resolve(docs);
}
};
}
function createStorage (observed, docs) {
const col = {
find: function () {
arguments.length.should.equal(0);
observed.findCalls = (observed.findCalls || 0) + 1;
return createCursor(observed, docs);
}
};
const ctx = {
store: {
collection: function (name) {
observed.collectionName = name;
return col;
},
ensureIndexes: function (receivedCol, fields) {
receivedCol.should.equal(col);
observed.indexFields = fields;
},
db: {
admin: function () {
return {
buildInfo: function () {
arguments.length.should.equal(0);
observed.buildInfoCalls = (observed.buildInfoCalls || 0) + 1;
return Promise.resolve({ version: '5.9.2' });
}
};
}
}
}
};
return new MongoCollection(ctx, {}, 'entries');
}
it('version uses promise-based buildInfo', async function () {
const observed = {};
const storage = createStorage(observed, []);
const version = await storage.version();
observed.collectionName.should.equal('entries');
observed.buildInfoCalls.should.equal(1);
version.should.eql({ storage: 'mongodb', version: '5.9.2' });
});
it('getLastModified uses promise-based cursor methods', async function () {
const observed = {};
const storage = createStorage(observed, [{ srvModified: 1234 }]);
const lastModified = await storage.getLastModified('srvModified');
observed.findCalls.should.equal(1);
observed.sort.should.eql({ srvModified: -1 });
observed.limit.should.equal(1);
observed.projection.should.eql({ srvModified: 1 });
observed.toArrayCalls.should.equal(1);
lastModified.should.eql({ srvModified: 1234 });
});
});
});
tests/cache-objectid-compat.test.js
+2 -2
View File
@@ -12,7 +12,7 @@
* On mongodb driver 3.x (v15.0.6), Object.keys(new ObjectID()) returned
* ['_bsontype','id'], making _.isEmpty return false (correct).
*
* On mongodb driver 5.x / mongodb-legacy (dev), Object.keys(new ObjectId())
* On mongodb driver 5.x, Object.keys(new ObjectId())
* returns [], making _.isEmpty return true (incorrect – treats valid
* ObjectId _id as empty).
*
@@ -21,7 +21,7 @@
*/
const _ = require('lodash');
const { ObjectId } = require('mongodb-legacy');
const { ObjectId } = require('mongodb');
const should = require('should');
describe('Cache ObjectId compatibility', function () {
tests/concurrent-writes.test.js
+14 -14
View File
@@ -306,16 +306,12 @@ describe('Concurrent Write Tests - MongoDB 5.x Compatibility', function () {
describe('Simultaneous POST requests to entries', function () {
beforeEach(function (done) {
self.ctx.entries().deleteMany({}, function () {
done();
});
beforeEach(async function () {
await self.ctx.entries().deleteMany({});
});
afterEach(function (done) {
self.ctx.entries().deleteMany({}, function () {
done();
});
afterEach(async function () {
await self.ctx.entries().deleteMany({});
});
it('handles 5 simultaneous single entry POSTs', function (done) {
@@ -422,9 +418,11 @@ describe('Concurrent Write Tests - MongoDB 5.x Compatibility', function () {
beforeEach(function (done) {
self.ctx.treatments.remove({ find: { created_at: { '$gte': '1999-01-01T00:00:00.000Z' } } }, function () {
self.ctx.devicestatus.remove({ find: { created_at: { '$gte': '1999-01-01T00:00:00.000Z' } } }, function () {
self.ctx.entries().deleteMany({}, function () {
done();
});
self.ctx.entries().deleteMany({})
.then(function () {
done();
})
.catch(done);
});
});
});
@@ -432,9 +430,11 @@ describe('Concurrent Write Tests - MongoDB 5.x Compatibility', function () {
afterEach(function (done) {
self.ctx.treatments.remove({ find: { created_at: { '$gte': '1999-01-01T00:00:00.000Z' } } }, function () {
self.ctx.devicestatus.remove({ find: { created_at: { '$gte': '1999-01-01T00:00:00.000Z' } } }, function () {
self.ctx.entries().deleteMany({}, function () {
done();
});
self.ctx.entries().deleteMany({})
.then(function () {
done();
})
.catch(done);
});
});
});
tests/dataloader.test.js
+66
View File
@@ -0,0 +1,66 @@
'use strict';
require('should');
const dataloaderInit = require('../lib/data/dataloader');
const createDData = require('../lib/data/ddata');
describe('dataloader', function () {
it('completes update when db.stats is promise-based', function (done) {
const ddata = createDData();
ddata.processTreatments = function () {};
const ctx = {
settings: {},
language: {
translate: function (value) { return value; }
},
cache: {
isEmpty: function () { return true; },
insertData: function (key, results) { return results; }
},
ddata: ddata,
entries: {
list: function (query, callback) { callback(null, []); }
},
treatments: {
list: function (query, callback) { callback(null, []); }
},
profile: {
last: function (callback) { callback(null, []); }
},
food: {
list: function (callback) { callback(null, []); }
},
devicestatus: {
list: function (query, callback) { callback(null, []); }
},
activity: {
list: function (query, callback) { callback(null, []); }
},
store: {
db: {
stats: function () {
return Promise.resolve({ dataSize: 123, indexSize: 456 });
}
}
}
};
const env = {
settings: {
isEnabled: function () { return false; },
units: 'mg/dl'
},
extendedSettings: {}
};
const loader = dataloaderInit(env, ctx);
loader.update(ddata, function (err) {
should.not.exist(err);
ddata.dbstats.should.eql({
dataSize: 123,
indexSize: 456
});
done();
});
});
});
tests/fixtures/test-guard.js
Vendored
+16 -2
View File
@@ -75,7 +75,14 @@ function guardedDeleteMany(collection, filter, callback) {
}
// Safe to proceed
return collection.deleteMany(filter, callback);
const promise = collection.deleteMany(filter);
if (callback) {
promise.then(
function onSuccess(result) { callback(null, result); },
function onError(err) { callback(err); }
);
}
return promise;
}
/**
@@ -95,7 +102,14 @@ function guardedDrop(collection, callback) {
throw err;
}
return collection.drop(callback);
const promise = collection.drop();
if (callback) {
promise.then(
function onSuccess(result) { callback(null, result); },
function onError(err) { callback(err); }
);
}
return promise;
}
/**
tests/issue-6923-legacy-uuid.test.js
+14 -17
View File
@@ -63,10 +63,8 @@ describe('Issue #6923: Legacy UUID override edit/delete', function () {
return self.ctx.store.collection(self.env.treatments_collection);
}
beforeEach(function (done) {
rawCollection().deleteMany({}, function () {
done();
});
beforeEach(async function () {
await rawCollection().deleteMany({});
});
/**
@@ -75,16 +73,19 @@ describe('Issue #6923: Legacy UUID override edit/delete', function () {
*/
function insertLegacyDoc (callback) {
var doc = Object.assign({}, LEGACY_OVERRIDE);
rawCollection().insertOne(doc, function (err) {
should.not.exist(err);
rawCollection().findOne({ _id: LEGACY_UUID }, function (err, stored) {
should.not.exist(err);
rawCollection().insertOne(doc)
.then(function () {
return rawCollection().findOne({ _id: LEGACY_UUID });
})
.then(function (stored) {
should.exist(stored, 'Legacy doc should exist after direct insert');
stored._id.should.equal(LEGACY_UUID);
should.not.exist(stored.identifier, 'Legacy doc must NOT have identifier field');
callback(stored);
})
.catch(function (err) {
should.not.exist(err);
});
});
}
describe('DELETE legacy UUID override via API', function () {
@@ -130,10 +131,8 @@ describe('Issue #6923: Legacy UUID override edit/delete', function () {
// Check the database after the server has had time to process the upsert
setTimeout(function () {
rawCollection().find({ eventType: 'Temporary Override' }).toArray(function (err, docs) {
try {
should.not.exist(err);
rawCollection().find({ eventType: 'Temporary Override' }).toArray()
.then(function (docs) {
docs.length.should.equal(1,
'PUT should update the existing legacy override, not create a duplicate. '
+ 'Found ' + docs.length + ' documents. '
@@ -143,10 +142,8 @@ describe('Issue #6923: Legacy UUID override edit/delete', function () {
);
done();
} catch (e) {
done(e);
}
});
})
.catch(done);
}, 5000);
});
});
tests/mongo-storage.retry.test.js
+134
View File
@@ -0,0 +1,134 @@
'use strict';
var should = require('should');
var mongodb = require('mongodb');
describe('mongo storage retry lifecycle', function () {
var originalMongoClient = mongodb.MongoClient;
var originalSetTimeout = global.setTimeout;
function setMongoClient(fakeMongoClient) {
Object.defineProperty(mongodb, 'MongoClient', {
configurable: true,
enumerable: true,
value: fakeMongoClient,
writable: true
});
}
afterEach(function () {
setMongoClient(originalMongoClient);
global.setTimeout = originalSetTimeout;
delete require.cache[require.resolve('../lib/storage/mongo-storage')];
});
it('closes failed retry clients and calls back only after a successful retry', function (done) {
var createdClients = [];
var connectAttempts = 0;
var retryDelays = [];
function FakeMongoClient() {
this.closed = 0;
createdClients.push(this);
}
FakeMongoClient.prototype.on = function () {};
FakeMongoClient.prototype.connect = function () {
connectAttempts += 1;
if (connectAttempts === 1) {
var err = new Error('server selection failed');
err.name = 'MongoServerSelectionError';
return Promise.reject(err);
}
return Promise.resolve();
};
FakeMongoClient.prototype.db = function () {
return {
databaseName: 'testdb',
command: function () {
return Promise.resolve({ authInfo: { authenticatedUserRoles: [] } });
},
collection: function (name) {
return { collectionName: name };
}
};
};
FakeMongoClient.prototype.close = function () {
this.closed += 1;
return Promise.resolve();
};
setMongoClient(FakeMongoClient);
global.setTimeout = function (fn, ms) {
retryDelays.push(ms);
Promise.resolve().then(fn);
return 1;
};
delete require.cache[require.resolve('../lib/storage/mongo-storage')];
var store = require('../lib/storage/mongo-storage');
var callbackCount = 0;
store({ storageURI: 'mongodb://example/testdb' }, function (err, db) {
callbackCount += 1;
should.not.exist(err);
should.exist(db);
callbackCount.should.equal(1);
connectAttempts.should.equal(2);
createdClients.length.should.equal(2);
createdClients[0].closed.should.equal(1);
createdClients[1].closed.should.equal(0);
retryDelays.should.eql([3000]);
db.db.databaseName.should.equal('testdb');
done();
}, true);
});
it('closes the client and reports authentication failure once', function (done) {
var createdClients = [];
var retryDelays = [];
function FakeMongoClient() {
this.closed = 0;
createdClients.push(this);
}
FakeMongoClient.prototype.on = function () {};
FakeMongoClient.prototype.connect = function () {
return Promise.reject(new Error('AuthenticationFailed: bad auth'));
};
FakeMongoClient.prototype.close = function () {
this.closed += 1;
return Promise.resolve();
};
setMongoClient(FakeMongoClient);
global.setTimeout = function (fn, ms) {
retryDelays.push(ms);
Promise.resolve().then(fn);
return 1;
};
delete require.cache[require.resolve('../lib/storage/mongo-storage')];
var store = require('../lib/storage/mongo-storage');
var callbackCount = 0;
store({ storageURI: 'mongodb://example/testdb' }, function (err, db) {
callbackCount += 1;
should.exist(err);
should.not.exist(db);
err.message.should.equal('MongoDB authentication failed! Double check the URL has the right username and password in MONGODB_URI.');
callbackCount.should.equal(1);
createdClients.length.should.equal(1);
createdClients[0].closed.should.equal(1);
retryDelays.should.eql([]);
done();
}, true);
});
});
tests/mongo-storage.test.js
+36 -1
View File
@@ -30,6 +30,42 @@ describe('mongo storage', function () {
});
});
it('Uses the default database from the connection string via the public client API', function (done) {
var store = require('../lib/storage/mongo-storage');
store(env, function (err, db) {
should.not.exist(err);
should.exist(db.db);
should.exist(db.client);
db.client.db().databaseName.should.equal(db.db.databaseName);
db.db.databaseName.should.equal('testdb');
done();
});
});
it('ensureIndexes uses createIndex without the legacy background option', function (done) {
var store = require('../lib/storage/mongo-storage');
var calls = [];
store(env, function (err, db) {
should.not.exist(err);
db.ensureIndexes({
collectionName: 'entries',
createIndex: function (field, options) {
calls.push({ field: field, options: options });
return Promise.resolve();
}
}, ['date']);
calls.length.should.equal(1);
calls[0].field.should.equal('date');
should.not.exist(calls[0].options);
done();
});
});
it('When no connection-string is given the storage-class should throw an error.', function (done) {
delete env.storageURI;
should.not.exist(env.storageURI);
@@ -58,4 +94,3 @@ describe('mongo storage', function () {
});
});
tests/sgv-devicestatus.test.js
+2 -2
View File
@@ -37,8 +37,8 @@ describe('Loop SGV Entry Upload Tests', function() {
});
});
beforeEach(function(done) {
self.ctx.entries().deleteMany({}, done);
beforeEach(async function() {
await self.ctx.entries().deleteMany({});
});
describe('TEST-SGV-001: Single SGV entry', function() {
tests/storage.shape-handling.test.js
+232 -95
View File
@@ -173,16 +173,12 @@ describe('Storage Layer Shape Handling - Direct Storage Tests', function () {
describe('Entries Storage - lib/server/entries.js', function () {
beforeEach(function (done) {
self.ctx.entries().deleteMany({}, function () {
done();
});
beforeEach(async function () {
await self.ctx.entries().deleteMany({});
});
afterEach(function (done) {
self.ctx.entries().deleteMany({}, function () {
done();
});
afterEach(async function () {
await self.ctx.entries().deleteMany({});
});
it('create() accepts single entry in array', function (done) {
@@ -218,16 +214,12 @@ describe('Storage Layer Shape Handling - Direct Storage Tests', function () {
describe('Profile Storage - lib/server/profile.js', function () {
beforeEach(function (done) {
self.ctx.profile().deleteMany({}, function () {
done();
});
beforeEach(async function () {
await self.ctx.profile().deleteMany({});
});
afterEach(function (done) {
self.ctx.profile().deleteMany({}, function () {
done();
});
afterEach(async function () {
await self.ctx.profile().deleteMany({});
});
it('create() accepts single profile object', function (done) {
@@ -347,13 +339,14 @@ describe('Storage Layer Shape Handling - Direct Storage Tests', function () {
self.ctx.profile.save(updated, function (saveErr) {
should.not.exist(saveErr);
self.ctx.profile().find({ _id: savedId }).toArray(function (findErr, docs) {
should.not.exist(findErr);
docs.length.should.equal(1);
docs[0].store.Default.dia.should.equal(4);
docs[0].created_at.should.equal('2024-10-26T21:32:49.173Z');
done();
});
self.ctx.profile().find({ _id: savedId }).toArray()
.then(function (docs) {
docs.length.should.equal(1);
docs[0].store.Default.dia.should.equal(4);
docs[0].created_at.should.equal('2024-10-26T21:32:49.173Z');
done();
})
.catch(done);
});
});
});
@@ -438,28 +431,25 @@ describe('Storage Layer Shape Handling - Direct Storage Tests', function () {
should.not.exist(err);
saved.created_at.should.equal('2020-01-01T00:00:00.000Z');
self.ctx.profile().find({ _id: saved._id }).toArray(function (findErr, docs) {
should.not.exist(findErr);
docs.length.should.equal(1);
docs[0].created_at.should.equal('2020-01-01T00:00:00.000Z');
done();
});
self.ctx.profile().find({ _id: saved._id }).toArray()
.then(function (docs) {
docs.length.should.equal(1);
docs[0].created_at.should.equal('2020-01-01T00:00:00.000Z');
done();
})
.catch(done);
});
});
});
describe('Food Storage - lib/server/food.js', function () {
beforeEach(function (done) {
self.ctx.food().deleteMany({}, function () {
done();
});
beforeEach(async function () {
await self.ctx.food().deleteMany({});
});
afterEach(function (done) {
self.ctx.food().deleteMany({}, function () {
done();
});
afterEach(async function () {
await self.ctx.food().deleteMany({});
});
it('create() accepts single food object', function (done) {
@@ -480,20 +470,59 @@ describe('Storage Layer Shape Handling - Direct Storage Tests', function () {
done();
});
});
it('save() updates an existing food by _id when created_at changes', function (done) {
self.ctx.food.create({
name: 'Test Food',
category: 'Test',
carbs: 20,
protein: 10,
fat: 5
}, function (err, createdDocs) {
should.not.exist(err);
should.exist(createdDocs);
createdDocs.should.be.an.Array();
createdDocs.length.should.equal(1);
var savedId = createdDocs[0]._id;
var updated = {
_id: savedId.toString(),
name: 'Updated Food',
category: 'Test',
carbs: 25,
protein: 10,
fat: 5,
created_at: '2024-10-26T21:32:49.173Z'
};
self.ctx.food.save(updated, function (saveErr, savedDocs) {
should.not.exist(saveErr);
should.exist(savedDocs);
savedDocs.should.be.an.Array();
savedDocs.length.should.equal(1);
self.ctx.food().find({ _id: savedId }).toArray()
.then(function (docs) {
docs.length.should.equal(1);
docs[0].name.should.equal('Updated Food');
docs[0].carbs.should.equal(25);
docs[0].created_at.should.equal('2024-10-26T21:32:49.173Z');
done();
})
.catch(done);
});
});
});
});
describe('Activity Storage - lib/server/activity.js', function () {
beforeEach(function (done) {
self.ctx.activity().deleteMany({}, function () {
done();
});
beforeEach(async function () {
await self.ctx.activity().deleteMany({});
});
afterEach(function (done) {
self.ctx.activity().deleteMany({}, function () {
done();
});
afterEach(async function () {
await self.ctx.activity().deleteMany({});
});
it('create() accepts array of activity objects (single object not supported)', function (done) {
@@ -511,6 +540,124 @@ describe('Storage Layer Shape Handling - Direct Storage Tests', function () {
done();
});
});
it('save() updates an existing activity by _id', function (done) {
self.ctx.activity.create([{
created_at: '2024-10-26T20:32:49.173Z',
heartrate: 80,
steps: 100,
activitylevel: 'walking'
}], function (err, createdDocs) {
should.not.exist(err);
should.exist(createdDocs);
createdDocs.length.should.equal(1);
should.exist(createdDocs[0]._id);
var savedId = createdDocs[0]._id;
var updated = {
_id: savedId.toString(),
created_at: '2024-10-26T21:32:49.173Z',
heartrate: 95,
steps: 250,
activitylevel: 'running'
};
self.ctx.activity.save(updated, function (saveErr, savedDoc) {
should.not.exist(saveErr);
should.exist(savedDoc);
savedDoc._id.toString().should.equal(savedId.toString());
self.ctx.activity().find({ _id: savedId }).toArray()
.then(function (docs) {
docs.length.should.equal(1);
docs[0].heartrate.should.equal(95);
docs[0].steps.should.equal(250);
docs[0].activitylevel.should.equal('running');
docs[0].created_at.should.equal('2024-10-26T21:32:49.173Z');
done();
})
.catch(done);
});
});
});
});
describe('Authorization Storage - lib/authorization/storage.js', function () {
function rolesCollection() {
return self.ctx.store.collection(self.env.authentication_collections_prefix + 'roles');
}
function subjectsCollection() {
return self.ctx.store.collection(self.env.authentication_collections_prefix + 'subjects');
}
beforeEach(async function () {
await rolesCollection().deleteMany({ name: 'mongo-save-role' });
await subjectsCollection().deleteMany({ name: 'mongo-save-subject' });
});
afterEach(async function () {
await rolesCollection().deleteMany({ name: 'mongo-save-role' });
await subjectsCollection().deleteMany({ name: 'mongo-save-subject' });
});
it('saveRole() updates an existing role without duplicating it', function (done) {
rolesCollection().insertOne({
name: 'mongo-save-role',
permissions: ['api:entries:read'],
notes: 'original',
created_at: '2024-10-26T20:32:49.173Z'
}).then(function (result) {
self.ctx.authorization.storage.saveRole({
_id: result.insertedId.toString(),
name: 'mongo-save-role',
permissions: ['api:entries:update'],
notes: 'updated',
created_at: '2024-10-26T21:32:49.173Z'
}, function (saveErr) {
should.not.exist(saveErr);
rolesCollection().find({ name: 'mongo-save-role' }).toArray()
.then(function (docs) {
docs.length.should.equal(1);
docs[0].permissions.should.deepEqual(['api:entries:update']);
docs[0].notes.should.equal('updated');
docs[0].created_at.should.equal('2024-10-26T21:32:49.173Z');
done();
})
.catch(done);
});
}).catch(done);
});
it('saveSubject() updates an existing subject without duplicating it', function (done) {
subjectsCollection().insertOne({
name: 'mongo-save-subject',
roles: ['readable'],
notes: 'original',
created_at: '2024-10-26T20:32:49.173Z'
}).then(function (result) {
self.ctx.authorization.storage.saveSubject({
_id: result.insertedId.toString(),
name: 'mongo-save-subject',
roles: ['admin'],
notes: 'updated',
created_at: '2024-10-26T21:32:49.173Z'
}, function (saveErr) {
should.not.exist(saveErr);
subjectsCollection().find({ name: 'mongo-save-subject' }).toArray()
.then(function (docs) {
docs.length.should.equal(1);
docs[0].roles.should.deepEqual(['admin']);
docs[0].notes.should.equal('updated');
docs[0].created_at.should.equal('2024-10-26T21:32:49.173Z');
done();
})
.catch(done);
});
}).catch(done);
});
});
});
@@ -533,71 +680,61 @@ describe('MongoDB insertOne vs insertMany Behavior', function () {
describe('Direct MongoDB operations - testing insertOne with array data', function () {
it('insertOne with object inserts correctly', function (done) {
it('insertOne with object inserts correctly', async function () {
var testCollection = self.ctx.store.collection('test_shape_handling');
testCollection.deleteMany({}, function () {
testCollection.insertOne({ type: 'test', value: 42 }, function (err, result) {
should.not.exist(err);
should.exist(result);
result.insertedId.should.be.ok();
await testCollection.deleteMany({});
var result = await testCollection.insertOne({ type: 'test', value: 42 });
should.exist(result);
result.insertedId.should.be.ok();
testCollection.find({}).toArray(function (err, docs) {
docs.length.should.equal(1);
docs[0].value.should.equal(42);
testCollection.deleteMany({}, done);
});
});
});
var docs = await testCollection.find({}).toArray();
docs.length.should.equal(1);
docs[0].value.should.equal(42);
await testCollection.deleteMany({});
});
it('insertOne with array creates single document containing array (NOT multiple docs)', function (done) {
it('insertOne with array creates single document containing array (NOT multiple docs)', async function () {
var testCollection = self.ctx.store.collection('test_shape_handling');
testCollection.deleteMany({}, function () {
var arrayData = [
{ type: 'test', value: 1 },
{ type: 'test', value: 2 },
{ type: 'test', value: 3 }
];
await testCollection.deleteMany({});
var arrayData = [
{ type: 'test', value: 1 },
{ type: 'test', value: 2 },
{ type: 'test', value: 3 }
];
testCollection.insertOne(arrayData, function (err, result) {
if (err) {
console.log('insertOne with array error:', err.message);
done();
} else {
testCollection.find({}).toArray(function (err, docs) {
console.log('Documents after insertOne with array:', JSON.stringify(docs, null, 2));
console.log('Number of documents:', docs.length);
try {
await testCollection.insertOne(arrayData);
} catch (err) {
console.log('insertOne with array error:', err.message);
await testCollection.deleteMany({});
return;
}
testCollection.deleteMany({}, done);
});
}
});
});
var docs = await testCollection.find({}).toArray();
console.log('Documents after insertOne with array:', JSON.stringify(docs, null, 2));
console.log('Number of documents:', docs.length);
await testCollection.deleteMany({});
});
it('insertMany with array creates multiple documents', function (done) {
it('insertMany with array creates multiple documents', async function () {
var testCollection = self.ctx.store.collection('test_shape_handling');
testCollection.deleteMany({}, function () {
var arrayData = [
{ type: 'test', value: 1 },
{ type: 'test', value: 2 },
{ type: 'test', value: 3 }
];
await testCollection.deleteMany({});
var arrayData = [
{ type: 'test', value: 1 },
{ type: 'test', value: 2 },
{ type: 'test', value: 3 }
];
testCollection.insertMany(arrayData, function (err, result) {
should.not.exist(err);
should.exist(result);
result.insertedCount.should.equal(3);
var result = await testCollection.insertMany(arrayData);
should.exist(result);
result.insertedCount.should.equal(3);
testCollection.find({}).toArray(function (err, docs) {
docs.length.should.equal(3);
testCollection.deleteMany({}, done);
});
});
});
var docs = await testCollection.find({}).toArray();
docs.length.should.equal(3);
await testCollection.deleteMany({});
});
});
});
tests/uuid-handling.test.js
+1 -2
View File
@@ -12,6 +12,7 @@
var request = require('supertest');
var should = require('should');
var ObjectID = require('mongodb').ObjectId;
var language = require('../lib/language')();
var api = require('../lib/api/');
@@ -243,7 +244,6 @@ describe('UUID_HANDLING=true', function() {
});
it('UUID-ON-003: ObjectId still works normally', function(done) {
var ObjectID = require('mongodb').ObjectId;
var testId = new ObjectID();
// Insert treatment with ObjectId
@@ -491,7 +491,6 @@ describe('UUID Edge Cases', function() {
});
it('UUID-EDGE-007: Valid ObjectId still works normally', function(done) {
var ObjectID = require('mongodb').ObjectId;
var testId = new ObjectID();
self.ctx.treatments.create([{
tests/websocket.shape-handling.test.js
+147 -15
View File
@@ -79,6 +79,10 @@ describe('WebSocket Shape Handling - dbAdd Single vs Array Input', function () {
return self.ctx.store.collection(self.env.treatments_collection);
}
function foodCollection() {
return self.ctx.food();
}
describe('dbAdd with treatments collection', function () {
beforeEach(function (done) {
@@ -278,8 +282,7 @@ describe('WebSocket Shape Handling - dbAdd Single vs Array Input', function () {
eventType: 'Note',
created_at: createdAt,
notes: 'legacy original'
}, function (insertErr) {
if (insertErr) return done(insertErr);
}).then(function () {
socket.emit('dbUpdate', {
collection: 'treatments',
@@ -293,7 +296,9 @@ describe('WebSocket Shape Handling - dbAdd Single vs Array Input', function () {
waitForConditionWithWarning({
condition: function (cb) {
treatmentsCollection().findOne({ _id: legacyId }, cb);
treatmentsCollection().findOne({ _id: legacyId })
.then(function (doc) { cb(null, doc); })
.catch(cb);
},
assertion: function (doc) {
should.exist(doc);
@@ -303,7 +308,7 @@ describe('WebSocket Shape Handling - dbAdd Single vs Array Input', function () {
operationName: 'verify websocket dbUpdate with custom string _id'
});
});
});
}).catch(done);
});
});
});
@@ -328,8 +333,7 @@ describe('WebSocket Shape Handling - dbAdd Single vs Array Input', function () {
eventType: 'Note',
created_at: createdAt,
notes: 'remove me'
}, function (insertErr) {
if (insertErr) return done(insertErr);
}).then(function () {
socket.emit('dbUpdateUnset', {
collection: 'treatments',
@@ -343,7 +347,9 @@ describe('WebSocket Shape Handling - dbAdd Single vs Array Input', function () {
waitForConditionWithWarning({
condition: function (cb) {
treatmentsCollection().findOne({ _id: legacyId }, cb);
treatmentsCollection().findOne({ _id: legacyId })
.then(function (doc) { cb(null, doc); })
.catch(cb);
},
assertion: function (doc) {
should.exist(doc);
@@ -353,7 +359,7 @@ describe('WebSocket Shape Handling - dbAdd Single vs Array Input', function () {
operationName: 'verify websocket dbUpdateUnset with custom string _id'
});
});
});
}).catch(done);
});
});
});
@@ -408,8 +414,7 @@ describe('WebSocket Shape Handling - dbAdd Single vs Array Input', function () {
eventType: 'Note',
created_at: createdAt,
notes: 'delete me'
}, function (insertErr) {
if (insertErr) return done(insertErr);
}).then(function () {
socket.emit('dbRemove', {
collection: 'treatments',
@@ -420,7 +425,9 @@ describe('WebSocket Shape Handling - dbAdd Single vs Array Input', function () {
waitForConditionWithWarning({
condition: function (cb) {
treatmentsCollection().findOne({ _id: legacyId }, cb);
treatmentsCollection().findOne({ _id: legacyId })
.then(function (doc) { cb(null, doc); })
.catch(cb);
},
assertion: function (doc) {
should.not.exist(doc);
@@ -429,7 +436,7 @@ describe('WebSocket Shape Handling - dbAdd Single vs Array Input', function () {
operationName: 'verify websocket dbRemove with custom string _id'
});
});
});
}).catch(done);
});
});
});
@@ -455,8 +462,7 @@ describe('WebSocket Shape Handling - dbAdd Single vs Array Input', function () {
eventType: 'Note',
created_at: originalCreatedAt,
notes: 'existing legacy note'
}, function (insertErr) {
if (insertErr) return done(insertErr);
}).then(function () {
socket.emit('dbAdd', {
collection: 'treatments',
@@ -473,7 +479,9 @@ describe('WebSocket Shape Handling - dbAdd Single vs Array Input', function () {
waitForConditionWithWarning({
condition: function (cb) {
treatmentsCollection().findOne({ _id: legacyId }, cb);
treatmentsCollection().findOne({ _id: legacyId })
.then(function (doc) { cb(null, doc); })
.catch(cb);
},
assertion: function (doc) {
should.exist(doc);
@@ -483,8 +491,132 @@ describe('WebSocket Shape Handling - dbAdd Single vs Array Input', function () {
operationName: 'verify websocket dbAdd dedupe with custom string _id'
});
});
}).catch(done);
});
});
});
describe('generic collection raw write watchpoints', function () {
beforeEach(async function () {
await foodCollection().deleteMany({});
});
it('dbAdd preserves custom string _id values for generic collections', function (done) {
connectAndAuthorize(function (err, socket, authResult) {
if (err) return done(err);
authResult.write.should.equal(true);
var legacyId = 'legacy-food-id-add';
socket.emit('dbAdd', {
collection: 'food',
data: {
_id: legacyId,
name: 'ws food',
carbs: 15
}
}, function (result) {
should.exist(result);
result.should.be.instanceof(Array);
result.length.should.equal(1);
result[0]._id.should.equal(legacyId);
waitForConditionWithWarning({
condition: function (cb) {
foodCollection().findOne({ _id: legacyId })
.then(function (doc) { cb(null, doc); })
.catch(cb);
},
assertion: function (doc) {
should.exist(doc);
doc.name.should.equal('ws food');
doc.carbs.should.equal(15);
},
done: done,
operationName: 'verify websocket dbAdd generic collection custom string _id'
});
});
});
});
it('dbUpdate supports custom string _id values for generic collections', function (done) {
connectAndAuthorize(function (err, socket, authResult) {
if (err) return done(err);
authResult.write.should.equal(true);
var legacyId = 'legacy-food-id-update';
foodCollection().insertOne({
_id: legacyId,
name: 'original food',
carbs: 10,
protein: 2
}).then(function () {
socket.emit('dbUpdate', {
collection: 'food',
_id: legacyId,
data: {
carbs: 18,
protein: 4
}
}, function (updateResult) {
should.exist(updateResult);
updateResult.result.should.equal('success');
waitForConditionWithWarning({
condition: function (cb) {
foodCollection().findOne({ _id: legacyId })
.then(function (doc) { cb(null, doc); })
.catch(cb);
},
assertion: function (doc) {
should.exist(doc);
doc.carbs.should.equal(18);
doc.protein.should.equal(4);
},
done: done,
operationName: 'verify websocket dbUpdate generic collection custom string _id'
});
});
}).catch(done);
});
});
it('dbRemove supports custom string _id values for generic collections', function (done) {
connectAndAuthorize(function (err, socket, authResult) {
if (err) return done(err);
authResult.write.should.equal(true);
var legacyId = 'legacy-food-id-remove';
foodCollection().insertOne({
_id: legacyId,
name: 'remove food',
carbs: 9
}).then(function () {
socket.emit('dbRemove', {
collection: 'food',
_id: legacyId
}, function (removeResult) {
should.exist(removeResult);
removeResult.result.should.equal('success');
waitForConditionWithWarning({
condition: function (cb) {
foodCollection().findOne({ _id: legacyId })
.then(function (doc) { cb(null, doc); })
.catch(cb);
},
assertion: function (doc) {
should.not.exist(doc);
},
done: done,
operationName: 'verify websocket dbRemove generic collection custom string _id'
});
});
}).catch(done);
});
});
});
});