From 35e5329d1a46588f54d16ce0c42dcfc0191f0d85 Mon Sep 17 00:00:00 2001 From: shaunwarman Date: Mon, 18 Aug 2025 13:44:19 -0500 Subject: [PATCH 1/3] feat: add smtp counts to user admin page for 1 hour, 24 hours and 72 hours --- app/controllers/web/admin/users.js | 72 +++++++++++++++++++++++++++--- app/views/admin/users/_table.pug | 34 +++++++++++++- 2 files changed, 100 insertions(+), 6 deletions(-) diff --git a/app/controllers/web/admin/users.js b/app/controllers/web/admin/users.js index cf1ad23427..637a1ac66c 100644 --- a/app/controllers/web/admin/users.js +++ b/app/controllers/web/admin/users.js @@ -12,7 +12,7 @@ const parser = require('mongodb-query-parser'); const { boolean } = require('boolean'); const _ = require('#helpers/lodash'); -const { Users } = require('#models'); +const { Users, Emails } = require('#models'); const config = require('#config'); const REGEX_BYTES = new RE2(/^((-|\+)?(\d+(?:\.\d+)?)) *(kb|mb|gb|tb|pb)$/i); @@ -52,7 +52,12 @@ async function list(ctx) { } } - const [users, itemCount] = await Promise.all([ + const now = new Date(); + const oneHourAgo = new Date(now - 60 * 60 * 1000); + const twentyFourHoursAgo = new Date(now - 24 * 60 * 60 * 1000); + const seventyTwoHoursAgo = new Date(now - 72 * 60 * 60 * 1000); + + const [users, itemCount, emailCounts] = await Promise.all([ // eslint-disable-next-line unicorn/no-array-callback-reference Users.find(query) .limit(ctx.query.limit) @@ -60,21 +65,78 @@ async function list(ctx) { .lean() .sort(ctx.query.sort || '-created_at') .exec(), - Users.countDocuments(query) + Users.countDocuments(query), + // Get SMTP outbound email counts per user with time-based breakdowns + Emails.aggregate([ + { + $match: { + // Only count delivered/sent emails (not failed/bounced) + status: { $in: ['delivered', 'deferred', 'sent'] } + } + }, + { + $group: { + _id: '$user', + totalEmails: { $sum: 1 }, + lastEmailAt: { $max: '$created_at' }, + // Count emails within time periods + emailsLast1Hour: { + $sum: { + $cond: [{ $gte: ['$created_at', oneHourAgo] }, 1, 0] + } + }, + emailsLast24Hours: { + $sum: { + $cond: [{ $gte: ['$created_at', twentyFourHoursAgo] }, 1, 0] + } + }, + emailsLast72Hours: { + $sum: { + $cond: [{ $gte: ['$created_at', seventyTwoHoursAgo] }, 1, 0] + } + } + } + } + ]) ]); + // Create a map for quick lookup of email counts + const emailCountMap = new Map(); + for (const count of emailCounts) { + emailCountMap.set(count._id.toString(), { + totalEmails: count.totalEmails, + lastEmailAt: count.lastEmailAt, + emailsLast1Hour: count.emailsLast1Hour, + emailsLast24Hours: count.emailsLast24Hours, + emailsLast72Hours: count.emailsLast72Hours + }); + } + + // Add email counts to each user + const usersWithEmailCounts = users.map((user) => ({ + ...user, + totalEmails: emailCountMap.get(user._id.toString())?.totalEmails || 0, + lastEmailAt: emailCountMap.get(user._id.toString())?.lastEmailAt || null, + emailsLast1Hour: + emailCountMap.get(user._id.toString())?.emailsLast1Hour || 0, + emailsLast24Hours: + emailCountMap.get(user._id.toString())?.emailsLast24Hours || 0, + emailsLast72Hours: + emailCountMap.get(user._id.toString())?.emailsLast72Hours || 0 + })); + const pageCount = Math.ceil(itemCount / ctx.query.limit); if (ctx.accepts('html')) return ctx.render('admin/users', { - users, + users: usersWithEmailCounts, pageCount, itemCount, pages: paginate.getArrayPages(ctx)(6, pageCount, ctx.query.page) }); const table = await ctx.render('admin/users/_table', { - users, + users: usersWithEmailCounts, pageCount, itemCount, pages: paginate.getArrayPages(ctx)(6, pageCount, ctx.query.page) diff --git a/app/views/admin/users/_table.pug b/app/views/admin/users/_table.pug index 2e0ec4f60f..4626544026 100644 --- a/app/views/admin/users/_table.pug +++ b/app/views/admin/users/_table.pug @@ -25,6 +25,16 @@ include ../../_pagination +sortHeader('max_quota_per_alias', 'Max Qouta', '#table-users') th(scope="col") +sortHeader('smtp_limit', 'SMTP Limit', '#table-users') + th(scope="col") + +sortHeader('totalEmails', 'Emails Sent', '#table-users') + th(scope="col") + +sortHeader('emailsLast1Hour', '1hr', '#table-users') + th(scope="col") + +sortHeader('emailsLast24Hours', '24hr', '#table-users') + th(scope="col") + +sortHeader('emailsLast72Hours', '72hr', '#table-users') + th(scope="col") + +sortHeader('lastEmailAt', 'Last Email', '#table-users') th(scope="col") +sortHeader('created_at', 'Created', '#table-users') th(scope="col") @@ -37,7 +47,7 @@ include ../../_pagination th.text-center.align-middle(scope="col")= t("Actions") tbody if users.length === 0 - td.alert.alert-info(colspan=passport && passport.otp ? "14" : "13")= t("No users exist for that search.") + td.alert.alert-info(colspan=passport && passport.otp ? "19" : "18")= t("No users exist for that search.") else each user in users tr @@ -116,6 +126,28 @@ include ../../_pagination data-toggle="tooltip", data-title=t("Update") ): i.fa.fa-fw.fa-save + td.align-middle.text-right + if user.totalEmails > 0 + a(href=l(`/admin/emails?user=${user._id}`), target="_blank")= user.totalEmails.toLocaleString() + else + = user.totalEmails + td.align-middle.text-right + span.badge( + class=user.emailsLast1Hour > 0 ? "badge-warning" : "badge-light" + )= user.emailsLast1Hour + td.align-middle.text-right + span.badge( + class=user.emailsLast24Hours > 0 ? "badge-info" : "badge-light" + )= user.emailsLast24Hours + td.align-middle.text-right + span.badge( + class=user.emailsLast72Hours > 0 ? "badge-primary" : "badge-light" + )= user.emailsLast72Hours + td.align-middle + if user.lastEmailAt + .dayjs(data-time=new Date(user.lastEmailAt).getTime())= dayjs(user.lastEmailAt).tz(user.timezone === 'Etc/Unknown' ? 'UTC' : user.timezone).format("M/D/YY h:mm A z") + else + = "Never" td.align-middle.dayjs( data-time=new Date(user.created_at).getTime() )= dayjs(user.created_at).tz(user.timezone === 'Etc/Unknown' ? 'UTC' : user.timezone).format("M/D/YY h:mm A z") From aff32103c11cf6f14f6226e34a8d8e6cd498d347 Mon Sep 17 00:00:00 2001 From: shaunwarman Date: Mon, 18 Aug 2025 20:25:16 -0500 Subject: [PATCH 2/3] fix: move to counter approach and some query optimizations --- app/controllers/web/admin/users.js | 70 ++-------------- app/models/emails.js | 24 ++++++ app/models/users.js | 34 ++++++++ jobs/index.js | 6 ++ jobs/update-smtp-counters.js | 128 +++++++++++++++++++++++++++++ 5 files changed, 201 insertions(+), 61 deletions(-) create mode 100644 jobs/update-smtp-counters.js diff --git a/app/controllers/web/admin/users.js b/app/controllers/web/admin/users.js index 637a1ac66c..95c5482752 100644 --- a/app/controllers/web/admin/users.js +++ b/app/controllers/web/admin/users.js @@ -12,7 +12,7 @@ const parser = require('mongodb-query-parser'); const { boolean } = require('boolean'); const _ = require('#helpers/lodash'); -const { Users, Emails } = require('#models'); +const { Users } = require('#models'); const config = require('#config'); const REGEX_BYTES = new RE2(/^((-|\+)?(\d+(?:\.\d+)?)) *(kb|mb|gb|tb|pb)$/i); @@ -52,12 +52,7 @@ async function list(ctx) { } } - const now = new Date(); - const oneHourAgo = new Date(now - 60 * 60 * 1000); - const twentyFourHoursAgo = new Date(now - 24 * 60 * 60 * 1000); - const seventyTwoHoursAgo = new Date(now - 72 * 60 * 60 * 1000); - - const [users, itemCount, emailCounts] = await Promise.all([ + const [users, itemCount] = await Promise.all([ // eslint-disable-next-line unicorn/no-array-callback-reference Users.find(query) .limit(ctx.query.limit) @@ -65,64 +60,17 @@ async function list(ctx) { .lean() .sort(ctx.query.sort || '-created_at') .exec(), - Users.countDocuments(query), - // Get SMTP outbound email counts per user with time-based breakdowns - Emails.aggregate([ - { - $match: { - // Only count delivered/sent emails (not failed/bounced) - status: { $in: ['delivered', 'deferred', 'sent'] } - } - }, - { - $group: { - _id: '$user', - totalEmails: { $sum: 1 }, - lastEmailAt: { $max: '$created_at' }, - // Count emails within time periods - emailsLast1Hour: { - $sum: { - $cond: [{ $gte: ['$created_at', oneHourAgo] }, 1, 0] - } - }, - emailsLast24Hours: { - $sum: { - $cond: [{ $gte: ['$created_at', twentyFourHoursAgo] }, 1, 0] - } - }, - emailsLast72Hours: { - $sum: { - $cond: [{ $gte: ['$created_at', seventyTwoHoursAgo] }, 1, 0] - } - } - } - } - ]) + Users.countDocuments(query) ]); - // Create a map for quick lookup of email counts - const emailCountMap = new Map(); - for (const count of emailCounts) { - emailCountMap.set(count._id.toString(), { - totalEmails: count.totalEmails, - lastEmailAt: count.lastEmailAt, - emailsLast1Hour: count.emailsLast1Hour, - emailsLast24Hours: count.emailsLast24Hours, - emailsLast72Hours: count.emailsLast72Hours - }); - } - - // Add email counts to each user + // Use the optimized user model counters instead of aggregation const usersWithEmailCounts = users.map((user) => ({ ...user, - totalEmails: emailCountMap.get(user._id.toString())?.totalEmails || 0, - lastEmailAt: emailCountMap.get(user._id.toString())?.lastEmailAt || null, - emailsLast1Hour: - emailCountMap.get(user._id.toString())?.emailsLast1Hour || 0, - emailsLast24Hours: - emailCountMap.get(user._id.toString())?.emailsLast24Hours || 0, - emailsLast72Hours: - emailCountMap.get(user._id.toString())?.emailsLast72Hours || 0 + totalEmails: user.smtp_emails_sent_total || 0, + lastEmailAt: user.smtp_last_email_sent_at || null, + emailsLast1Hour: user.smtp_emails_sent_1h || 0, + emailsLast24Hours: user.smtp_emails_sent_24h || 0, + emailsLast72Hours: user.smtp_emails_sent_72h || 0 })); const pageCount = Math.ceil(itemCount / ctx.query.limit); diff --git a/app/models/emails.js b/app/models/emails.js index 1cf7a4a58b..14831a04b5 100644 --- a/app/models/emails.js +++ b/app/models/emails.js @@ -1156,6 +1156,30 @@ Emails.post('save', async function (email, next) { } }); +// Update user SMTP counters when email is successfully sent +Emails.post('save', async function (email) { + // Only update counters for new emails that are sent/delivered + if (!email._isNew || !['sent', 'delivered'].includes(email.status)) return; + + try { + // Increment the user's SMTP counters + await Users.findByIdAndUpdate(email.user, { + $inc: { + smtp_emails_sent_1h: 1, + smtp_emails_sent_24h: 1, + smtp_emails_sent_72h: 1, + smtp_emails_sent_total: 1 + }, + $set: { + smtp_last_email_sent_at: new Date() + } + }); + } catch (err) { + // Log error but don't fail the email save + logger.error(err, { email_id: email._id, user_id: email.user }); + } +}); + Emails.statics.getMessage = async function (obj, returnString = false) { if (Buffer.isBuffer(obj)) { if (returnString) return obj.toString(); diff --git a/app/models/users.js b/app/models/users.js index 46d0dd995a..5b299ab3d8 100644 --- a/app/models/users.js +++ b/app/models/users.js @@ -231,6 +231,40 @@ object[config.userFields.smtpLimit] = { max: 100000 }; +// SMTP count tracking for admin dashboard +object.smtp_emails_sent_1h = { + type: Number, + default: 0, + min: 0, + index: true +}; + +object.smtp_emails_sent_24h = { + type: Number, + default: 0, + min: 0, + index: true +}; + +object.smtp_emails_sent_72h = { + type: Number, + default: 0, + min: 0, + index: true +}; + +object.smtp_emails_sent_total = { + type: Number, + default: 0, + min: 0, + index: true +}; + +object.smtp_last_email_sent_at = { + type: Date, + index: true +}; + // Custom receipt email object[config.userFields.receiptEmail] = { type: String, diff --git a/jobs/index.js b/jobs/index.js index 83bb136a0b..ddc6edccb2 100644 --- a/jobs/index.js +++ b/jobs/index.js @@ -241,6 +241,12 @@ let jobs = [ interval: '1d', timeout: 0 }, + // update SMTP counters every hour + { + name: 'update-smtp-counters', + interval: '1h', + timeout: 0 + }, // session management { name: 'session-management', diff --git a/jobs/update-smtp-counters.js b/jobs/update-smtp-counters.js new file mode 100644 index 0000000000..e852c08870 --- /dev/null +++ b/jobs/update-smtp-counters.js @@ -0,0 +1,128 @@ +/** + * Copyright (c) Forward Email LLC + * SPDX-License-Identifier: BUSL-1.1 + */ + +// eslint-disable-next-line import/no-unassigned-import +require('#config/env'); + +const process = require('node:process'); +const { parentPort } = require('node:worker_threads'); + +// eslint-disable-next-line import/no-unassigned-import +require('#config/mongoose'); + +const Graceful = require('@ladjs/graceful'); +const mongoose = require('mongoose'); + +const Emails = require('#models/emails'); +const Users = require('#models/users'); +const logger = require('#helpers/logger'); +const monitorServer = require('#helpers/monitor-server'); +const setupMongoose = require('#helpers/setup-mongoose'); + +monitorServer(); + +const graceful = new Graceful({ + mongooses: [mongoose], + logger +}); + +graceful.listen(); + +(async () => { + await setupMongoose(logger); + + try { + const now = new Date(); + const oneHourAgo = new Date(now - 60 * 60 * 1000); + const twentyFourHoursAgo = new Date(now - 24 * 60 * 60 * 1000); + const seventyTwoHoursAgo = new Date(now - 72 * 60 * 60 * 1000); + + logger.info('Starting SMTP counter update job'); + + // Get SMTP counts per user with time-based breakdowns + const emailCounts = await Emails.aggregate([ + { + $match: { + // Only count delivered/sent emails (not failed/bounced) + status: { $in: ['delivered', 'deferred', 'sent'] } + } + }, + { + $group: { + _id: '$user', + totalEmails: { $sum: 1 }, + lastEmailAt: { $max: '$created_at' }, + // Count emails within time periods + emailsLast1Hour: { + $sum: { + $cond: [{ $gte: ['$created_at', oneHourAgo] }, 1, 0] + } + }, + emailsLast24Hours: { + $sum: { + $cond: [{ $gte: ['$created_at', twentyFourHoursAgo] }, 1, 0] + } + }, + emailsLast72Hours: { + $sum: { + $cond: [{ $gte: ['$created_at', seventyTwoHoursAgo] }, 1, 0] + } + } + } + } + ]); + + logger.info(`Found email counts for ${emailCounts.length} users`); + + // Create bulk operations to update all users + const bulkOperations = []; + + // First, reset all counters to 0 for all users + bulkOperations.push({ + updateMany: { + filter: {}, + update: { + $set: { + smtp_emails_sent_1h: 0, + smtp_emails_sent_24h: 0, + smtp_emails_sent_72h: 0, + smtp_emails_sent_total: 0, + smtp_last_email_sent_at: null + } + } + } + }); + + // Then update users who have email counts + for (const count of emailCounts) { + bulkOperations.push({ + updateOne: { + filter: { _id: count._id }, + update: { + $set: { + smtp_emails_sent_1h: count.emailsLast1Hour, + smtp_emails_sent_24h: count.emailsLast24Hours, + smtp_emails_sent_72h: count.emailsLast72Hours, + smtp_emails_sent_total: count.totalEmails, + smtp_last_email_sent_at: count.lastEmailAt + } + } + } + }); + } + + if (bulkOperations.length > 0) { + await Users.bulkWrite(bulkOperations, { ordered: false }); + logger.info(`Updated SMTP counters for all users`); + } + + logger.info('SMTP counter update job completed successfully'); + } catch (err) { + await logger.error(err); + } + + if (parentPort) parentPort.postMessage('done'); + else process.exit(0); +})(); From e04de694c944930151ea655983ee3ab8c6cec19a Mon Sep 17 00:00:00 2001 From: shaunwarman Date: Tue, 19 Aug 2025 21:18:20 -0500 Subject: [PATCH 3/3] fix: pr feedback --- app/controllers/web/admin/users.js | 14 ++---------- app/views/admin/users/_table.pug | 34 ++++++++++++++++-------------- 2 files changed, 20 insertions(+), 28 deletions(-) diff --git a/app/controllers/web/admin/users.js b/app/controllers/web/admin/users.js index 95c5482752..cf1ad23427 100644 --- a/app/controllers/web/admin/users.js +++ b/app/controllers/web/admin/users.js @@ -63,28 +63,18 @@ async function list(ctx) { Users.countDocuments(query) ]); - // Use the optimized user model counters instead of aggregation - const usersWithEmailCounts = users.map((user) => ({ - ...user, - totalEmails: user.smtp_emails_sent_total || 0, - lastEmailAt: user.smtp_last_email_sent_at || null, - emailsLast1Hour: user.smtp_emails_sent_1h || 0, - emailsLast24Hours: user.smtp_emails_sent_24h || 0, - emailsLast72Hours: user.smtp_emails_sent_72h || 0 - })); - const pageCount = Math.ceil(itemCount / ctx.query.limit); if (ctx.accepts('html')) return ctx.render('admin/users', { - users: usersWithEmailCounts, + users, pageCount, itemCount, pages: paginate.getArrayPages(ctx)(6, pageCount, ctx.query.page) }); const table = await ctx.render('admin/users/_table', { - users: usersWithEmailCounts, + users, pageCount, itemCount, pages: paginate.getArrayPages(ctx)(6, pageCount, ctx.query.page) diff --git a/app/views/admin/users/_table.pug b/app/views/admin/users/_table.pug index 4626544026..f5ef5aaea1 100644 --- a/app/views/admin/users/_table.pug +++ b/app/views/admin/users/_table.pug @@ -26,15 +26,15 @@ include ../../_pagination th(scope="col") +sortHeader('smtp_limit', 'SMTP Limit', '#table-users') th(scope="col") - +sortHeader('totalEmails', 'Emails Sent', '#table-users') + +sortHeader('smtp_emails_sent_total', 'Emails Sent', '#table-users') th(scope="col") - +sortHeader('emailsLast1Hour', '1hr', '#table-users') + +sortHeader('smtp_emails_sent_1h', '1hr', '#table-users') th(scope="col") - +sortHeader('emailsLast24Hours', '24hr', '#table-users') + +sortHeader('smtp_emails_sent_24h', '24hr', '#table-users') th(scope="col") - +sortHeader('emailsLast72Hours', '72hr', '#table-users') + +sortHeader('smtp_emails_sent_72h', '72hr', '#table-users') th(scope="col") - +sortHeader('lastEmailAt', 'Last Email', '#table-users') + +sortHeader('smtp_last_email_sent_at', 'Last Email', '#table-users') th(scope="col") +sortHeader('created_at', 'Created', '#table-users') th(scope="col") @@ -127,25 +127,27 @@ include ../../_pagination data-title=t("Update") ): i.fa.fa-fw.fa-save td.align-middle.text-right - if user.totalEmails > 0 - a(href=l(`/admin/emails?user=${user._id}`), target="_blank")= user.totalEmails.toLocaleString() + if user.smtp_emails_sent_total > 0 + a(href=l(`/admin/emails?user=${user._id}`), target="_blank")= user.smtp_emails_sent_total.toLocaleString() else - = user.totalEmails + = user.smtp_emails_sent_total || 0 td.align-middle.text-right span.badge( - class=user.emailsLast1Hour > 0 ? "badge-warning" : "badge-light" - )= user.emailsLast1Hour + class=user.smtp_emails_sent_1h > 0 ? "badge-warning" : "badge-light" + )= user.smtp_emails_sent_1h || 0 td.align-middle.text-right span.badge( - class=user.emailsLast24Hours > 0 ? "badge-info" : "badge-light" - )= user.emailsLast24Hours + class=user.smtp_emails_sent_24h > 0 ? "badge-info" : "badge-light" + )= user.smtp_emails_sent_24h || 0 td.align-middle.text-right span.badge( - class=user.emailsLast72Hours > 0 ? "badge-primary" : "badge-light" - )= user.emailsLast72Hours + class=user.smtp_emails_sent_72h > 0 ? "badge-primary" : "badge-light" + )= user.smtp_emails_sent_72h || 0 td.align-middle - if user.lastEmailAt - .dayjs(data-time=new Date(user.lastEmailAt).getTime())= dayjs(user.lastEmailAt).tz(user.timezone === 'Etc/Unknown' ? 'UTC' : user.timezone).format("M/D/YY h:mm A z") + if user.smtp_last_email_sent_at + .dayjs( + data-time=new Date(user.smtp_last_email_sent_at).getTime() + )= dayjs(user.smtp_last_email_sent_at).tz(user.timezone === 'Etc/Unknown' ? 'UTC' : user.timezone).format("M/D/YY h:mm A z") else = "Never" td.align-middle.dayjs(