Compare commits

..

3 Commits

Author SHA1 Message Date
Phillip Thelen bbf0e1609d Improve rate limiter options 2026-05-07 15:43:55 +02:00
Phillip Thelen 95e2247050 Improve rate limit handling 2026-05-06 13:19:02 +02:00
Fiz 1178da3a26 fix(quests): stuck "you were invited" banner after accept/reject (#15647)
wrap the conditional group update and the user save in a single transaction so a failure                                                                                                              between them can't leave members[uid] and RSVPNeeded out of sync
2026-05-05 11:09:30 -05:00
10 changed files with 210 additions and 112 deletions
+25 -12
View File
@@ -32,7 +32,8 @@ describe('rateLimiter middleware', () => {
it('is disabled when the env var is not defined', () => {
nconfGetStub.withArgs('RATE_LIMITER_ENABLED').returns(undefined);
const attachRateLimiter = requireAgain(pathToRateLimiter).default;
const setupRateLimiter = requireAgain(pathToRateLimiter).default;
const attachRateLimiter = setupRateLimiter();
attachRateLimiter(req, res, next);
expect(next).to.have.been.calledOnce;
@@ -43,7 +44,8 @@ describe('rateLimiter middleware', () => {
it('is disabled when the env var is an not "true"', () => {
nconfGetStub.withArgs('RATE_LIMITER_ENABLED').returns('false');
const attachRateLimiter = requireAgain(pathToRateLimiter).default;
const setupRateLimiter = requireAgain(pathToRateLimiter).default;
const attachRateLimiter = setupRateLimiter();
attachRateLimiter(req, res, next);
expect(next).to.have.been.calledOnce;
@@ -55,7 +57,8 @@ describe('rateLimiter middleware', () => {
it('does not throw when there are available points', async () => {
nconfGetStub.withArgs('RATE_LIMITER_ENABLED').returns('true');
nconfGetStub.withArgs('RATE_LIMITER_IP_COST').returns(1);
const attachRateLimiter = requireAgain(pathToRateLimiter).default;
const setupRateLimiter = requireAgain(pathToRateLimiter).default;
const attachRateLimiter = setupRateLimiter();
await attachRateLimiter(req, res, next);
expect(next).to.have.been.calledOnce;
@@ -77,7 +80,8 @@ describe('rateLimiter middleware', () => {
sandbox.stub(RateLimiterMemory.prototype, 'consume')
.returns(Promise.reject(new Error('Unknown error.')));
const attachRateLimiter = requireAgain(pathToRateLimiter).default;
const setupRateLimiter = requireAgain(pathToRateLimiter).default;
const attachRateLimiter = setupRateLimiter();
await attachRateLimiter(req, res, next);
expect(next).to.have.been.calledOnce;
@@ -92,7 +96,9 @@ describe('rateLimiter middleware', () => {
it('does not throw when LIVELINESS_PROBE_KEY is correct', async () => {
nconfGetStub.withArgs('RATE_LIMITER_ENABLED').returns('true');
nconfGetStub.withArgs('LIVELINESS_PROBE_KEY').returns('abc');
const attachRateLimiter = requireAgain(pathToRateLimiter).default;
const setupRateLimiter = requireAgain(pathToRateLimiter).default;
const attachRateLimiter = setupRateLimiter();
req.query.liveliness = 'abc';
await attachRateLimiter(req, res, next);
@@ -107,7 +113,8 @@ describe('rateLimiter middleware', () => {
nconfGetStub.withArgs('RATE_LIMITER_ENABLED').returns('true');
nconfGetStub.withArgs('LIVELINESS_PROBE_KEY').returns('abc');
nconfGetStub.withArgs('RATE_LIMITER_IP_COST').returns(1);
const attachRateLimiter = requireAgain(pathToRateLimiter).default;
const setupRateLimiter = requireAgain(pathToRateLimiter).default;
const attachRateLimiter = setupRateLimiter();
req.query.liveliness = 'das';
await attachRateLimiter(req, res, next);
@@ -124,7 +131,8 @@ describe('rateLimiter middleware', () => {
nconfGetStub.withArgs('RATE_LIMITER_ENABLED').returns('true');
nconfGetStub.withArgs('LIVELINESS_PROBE_KEY').returns(undefined);
nconfGetStub.withArgs('RATE_LIMITER_IP_COST').returns(1);
const attachRateLimiter = requireAgain(pathToRateLimiter).default;
const setupRateLimiter = requireAgain(pathToRateLimiter).default;
const attachRateLimiter = setupRateLimiter();
await attachRateLimiter(req, res, next);
@@ -140,7 +148,8 @@ describe('rateLimiter middleware', () => {
nconfGetStub.withArgs('RATE_LIMITER_ENABLED').returns('true');
nconfGetStub.withArgs('LIVELINESS_PROBE_KEY').returns('');
nconfGetStub.withArgs('RATE_LIMITER_IP_COST').returns(1);
const attachRateLimiter = requireAgain(pathToRateLimiter).default;
const setupRateLimiter = requireAgain(pathToRateLimiter).default;
const attachRateLimiter = setupRateLimiter();
req.query.liveliness = '';
await attachRateLimiter(req, res, next);
@@ -156,7 +165,8 @@ describe('rateLimiter middleware', () => {
it('throws when there are no available points remaining', async () => {
nconfGetStub.withArgs('RATE_LIMITER_ENABLED').returns('true');
nconfGetStub.withArgs('RATE_LIMITER_IP_COST').returns(1);
const attachRateLimiter = requireAgain(pathToRateLimiter).default;
const setupRateLimiter = requireAgain(pathToRateLimiter).default;
const attachRateLimiter = setupRateLimiter();
// call for 31 times
for (let i = 0; i < 31; i += 1) {
@@ -180,7 +190,8 @@ describe('rateLimiter middleware', () => {
it('uses the user id if supplied or the ip address', async () => {
nconfGetStub.withArgs('RATE_LIMITER_ENABLED').returns('true');
nconfGetStub.withArgs('RATE_LIMITER_IP_COST').returns(1);
const attachRateLimiter = requireAgain(pathToRateLimiter).default;
const setupRateLimiter = requireAgain(pathToRateLimiter).default;
const attachRateLimiter = setupRateLimiter();
req.ip = 1;
await attachRateLimiter(req, res, next);
@@ -210,7 +221,8 @@ describe('rateLimiter middleware', () => {
it('applies increased cost for registration calls with and without user id', async () => {
nconfGetStub.withArgs('RATE_LIMITER_ENABLED').returns('true');
nconfGetStub.withArgs('RATE_LIMITER_REGISTRATION_COST').returns(3);
const attachRateLimiter = requireAgain(pathToRateLimiter).default;
const setupRateLimiter = requireAgain(pathToRateLimiter).default;
const attachRateLimiter = setupRateLimiter();
req.path = '/api/v4/user/auth/local/register';
req.ip = 1;
@@ -241,7 +253,8 @@ describe('rateLimiter middleware', () => {
it('applies increased cost for unauthenticated API calls', async () => {
nconfGetStub.withArgs('RATE_LIMITER_ENABLED').returns('true');
nconfGetStub.withArgs('RATE_LIMITER_IP_COST').returns(10);
const attachRateLimiter = requireAgain(pathToRateLimiter).default;
const setupRateLimiter = requireAgain(pathToRateLimiter).default;
const attachRateLimiter = setupRateLimiter();
req.ip = 1;
await attachRateLimiter(req, res, next);
@@ -91,6 +91,23 @@ describe('POST /groups/:groupId/quests/accept', () => {
expect(partyMembers[0].party.quest.RSVPNeeded).to.be.false;
});
it('heals stuck RSVPNeeded when group already has the user accepted', async () => {
await leader.post(`/groups/${questingGroup._id}/quests/invite/${PET_QUEST}`);
await partyMembers[0].post(`/groups/${questingGroup._id}/quests/accept`);
await partyMembers[0].updateOne({ 'party.quest.RSVPNeeded': true });
await partyMembers[0].sync();
expect(partyMembers[0].party.quest.RSVPNeeded).to.be.true;
const res = await partyMembers[0].post(`/groups/${questingGroup._id}/quests/accept`);
expect(res).to.exist;
await partyMembers[0].sync();
await questingGroup.sync();
expect(partyMembers[0].party.quest.RSVPNeeded).to.equal(false);
expect(questingGroup.quest.members[partyMembers[0]._id]).to.equal(true);
});
it('does not accept invite for a quest already underway', async () => {
await leader.post(`/groups/${questingGroup._id}/quests/invite/${PET_QUEST}`);
await partyMembers[0].post(`/groups/${questingGroup._id}/quests/accept`);
@@ -100,6 +100,23 @@ describe('POST /groups/:groupId/quests/reject', () => {
expect(partyMembers[0].party.quest.RSVPNeeded).to.be.false;
});
it('heals stuck RSVPNeeded when group already has the user rejected', async () => {
await leader.post(`/groups/${questingGroup._id}/quests/invite/${PET_QUEST}`);
await partyMembers[0].post(`/groups/${questingGroup._id}/quests/reject`);
await partyMembers[0].updateOne({ 'party.quest.RSVPNeeded': true });
await partyMembers[0].sync();
expect(partyMembers[0].party.quest.RSVPNeeded).to.be.true;
const res = await partyMembers[0].post(`/groups/${questingGroup._id}/quests/reject`);
expect(res).to.exist;
await partyMembers[0].sync();
await questingGroup.sync();
expect(partyMembers[0].party.quest.RSVPNeeded).to.equal(false);
expect(questingGroup.quest.members[partyMembers[0]._id]).to.equal(false);
});
it('return an error when a user rejects an invite already accepted', async () => {
await leader.post(`/groups/${questingGroup._id}/quests/invite/${PET_QUEST}`);
await partyMembers[0].post(`/groups/${questingGroup._id}/quests/accept`);
+44 -12
View File
@@ -198,16 +198,32 @@ api.acceptQuest = {
if (group.type !== 'party') throw new NotAuthorized(res.t('guildQuestsNotSupported'));
if (!group.quest.key) throw new NotFound(res.t('questInviteNotFound'));
if (group.quest.active) throw new NotAuthorized(res.t('questAlreadyStartedFriendly'));
if (group.quest.members[user._id]) throw new BadRequest(res.t('questAlreadyAccepted'));
const acceptedSuccessfully = await group.handleQuestInvitation(user, true);
if (group.quest.members[user._id] === true) {
if (user.party.quest.RSVPNeeded) {
user.party.quest.RSVPNeeded = false;
await user.save();
res.respond(200, group.quest);
return;
}
throw new BadRequest(res.t('questAlreadyAccepted'));
}
if (group.quest.members[user._id] === false) {
throw new BadRequest(res.t('questAlreadyAccepted'));
}
let acceptedSuccessfully = false;
await Group.db.transaction(async session => {
acceptedSuccessfully = await group.handleQuestInvitation(user, true, session);
if (!acceptedSuccessfully) return;
user.party.quest.RSVPNeeded = false;
await user.save({ session });
});
if (!acceptedSuccessfully) {
throw new NotAuthorized(res.t('questAlreadyAccepted'));
}
user.party.quest.RSVPNeeded = false;
await user.save();
if (canStartQuestAutomatically(group)) {
await group.startQuest(user);
}
@@ -251,18 +267,34 @@ api.rejectQuest = {
if (group.type !== 'party') throw new NotAuthorized(res.t('guildQuestsNotSupported'));
if (!group.quest.key) throw new NotFound(res.t('questInvitationDoesNotExist'));
if (group.quest.active) throw new NotAuthorized(res.t('questAlreadyStartedFriendly'));
if (group.quest.members[user._id]) throw new BadRequest(res.t('questAlreadyAccepted'));
if (group.quest.members[user._id] === false) throw new BadRequest(res.t('questAlreadyRejected'));
const rejectedSuccessfully = await group.handleQuestInvitation(user, false);
if (group.quest.members[user._id] === true) {
throw new BadRequest(res.t('questAlreadyAccepted'));
}
if (group.quest.members[user._id] === false) {
if (user.party.quest.RSVPNeeded) {
user.party.quest = Group.cleanQuestUser(user.party.quest.progress);
user.markModified('party.quest');
await user.save();
res.respond(200, group.quest);
return;
}
throw new BadRequest(res.t('questAlreadyRejected'));
}
let rejectedSuccessfully = false;
await Group.db.transaction(async session => {
rejectedSuccessfully = await group.handleQuestInvitation(user, false, session);
if (!rejectedSuccessfully) return;
user.party.quest = Group.cleanQuestUser(user.party.quest.progress);
user.markModified('party.quest');
await user.save({ session });
});
if (!rejectedSuccessfully) {
throw new NotAuthorized(res.t('questAlreadyRejected'));
}
user.party.quest = Group.cleanQuestUser(user.party.quest.progress);
user.markModified('party.quest');
await user.save();
if (canStartQuestAutomatically(group)) {
await group.startQuest(user);
}
+4 -2
View File
@@ -27,7 +27,6 @@ import {
moveTask,
setNextDue,
requiredGroupFields,
normalizeDailyStartDate,
} from '../../libs/tasks/utils';
import common from '../../../common';
import { apiError } from '../../libs/apiError';
@@ -649,10 +648,13 @@ api.updateTask = {
task.group.managerNotes = sanitizedObj.managerNotes;
}
// For daily tasks, update start date based on timezone to maintain consistency
if (task.type === 'daily'
&& task.startDate
) {
task.startDate = normalizeDailyStartDate(task.startDate, user);
task.startDate = moment(task.startDate).utcOffset(
-user.preferences.timezoneOffset,
).startOf('day').toDate();
// If the daily task was set to repeat monthly on a day of the month, and the start date was
// updated, the task will then need to be updated to repeat on the same day of the month as
+7 -2
View File
@@ -1,3 +1,4 @@
import moment from 'moment';
import cloneDeep from 'lodash/cloneDeep';
import compact from 'lodash/compact';
import forEach from 'lodash/forEach';
@@ -8,7 +9,6 @@ import {
setNextDue,
validateTaskAlias,
requiredGroupFields,
normalizeDailyStartDate,
} from './utils';
import { model as Challenge } from '../../models/challenge';
import { model as Group } from '../../models/group';
@@ -80,8 +80,13 @@ async function createTasks (req, res, options = {}) {
}
}
// set startDate to midnight in the user's timezone
if (taskType === 'daily') {
newTask.startDate = normalizeDailyStartDate(newTask.startDate, user);
const awareStartDate = moment(newTask.startDate).utcOffset(-user.preferences.timezoneOffset);
if (awareStartDate.format('HMsS') !== '0000') {
awareStartDate.startOf('day');
newTask.startDate = awareStartDate.toDate();
}
}
setNextDue(newTask, user);
-15
View File
@@ -59,21 +59,6 @@ export function moveTask (order, taskId, to) {
}
}
export function normalizeDailyStartDate (date, user) {
if (!date) return date;
const utcView = moment.utc(date);
const looksLikeMidnightLocal = utcView.second() === 0
&& utcView.millisecond() === 0
&& [0, 15, 30, 45].includes(utcView.minute());
if (looksLikeMidnightLocal) {
return new Date(date);
}
return moment(date)
.utcOffset(-(user.preferences.timezoneOffset || 0))
.startOf('day')
.toDate();
}
export function setNextDue (task, user, dueDateOption) {
if (task.type !== 'daily') return;
+17 -3
View File
@@ -1,8 +1,9 @@
import nconf from 'nconf';
import express from 'express';
import expressValidator from 'express-validator';
import path from 'path';
import setupBody from './setupBody';
import rateLimiter from './rateLimiter';
import setupRateLimiter from './rateLimiter';
import setupExpress from '../libs/setupExpress';
import * as routes from '../libs/routes';
@@ -10,6 +11,10 @@ const API_V3_CONTROLLERS_PATH = path.join(__dirname, '/../controllers/api-v3/');
const API_V4_CONTROLLERS_PATH = path.join(__dirname, '/../controllers/api-v4/');
const TOP_LEVEL_CONTROLLERS_PATH = path.join(__dirname, '/../controllers/top-level/');
const RATE_LIMITER_V4_POINTS = nconf.get('RATE_LIMITER_V4_POINTS') || 200;
const RATE_LIMITER_V4_REGISTRATION_COST = nconf.get('RATE_LIMITER_V4_REGISTRATION_COST') || 15;
const RATE_LIMITER_V4_LOGIN_COST = nconf.get('RATE_LIMITER_V4_LOGIN_COST') || 10;
const app = express();
// re-set the view options because they are not inherited from the top level app
@@ -25,7 +30,10 @@ app.use('/', topLevelRouter);
const v3Router = express.Router(); // eslint-disable-line new-cap
routes.walkControllers(v3Router, API_V3_CONTROLLERS_PATH);
app.use('/api/v3', rateLimiter, v3Router);
const v3RateLimiter = setupRateLimiter({
keyPrefix: 'api-v3',
});
app.use('/api/v3', v3RateLimiter, v3Router);
// API v4 proxies API v3 routes by default.
// It can also disable or override v3 routes
@@ -49,6 +57,12 @@ const v4RouterOverrides = [
const v4Router = express.Router(); // eslint-disable-line new-cap
routes.walkControllers(v4Router, API_V3_CONTROLLERS_PATH, v4RouterOverrides);
routes.walkControllers(v4Router, API_V4_CONTROLLERS_PATH);
app.use('/api/v4', v4Router);
const v4RateLimiter = setupRateLimiter({
keyPrefix: 'api-v4',
points: RATE_LIMITER_V4_POINTS,
registrationCost: RATE_LIMITER_V4_REGISTRATION_COST,
loginCost: RATE_LIMITER_V4_LOGIN_COST,
});
app.use('/api/v4', v4RateLimiter, v4Router);
export default app;
+75 -65
View File
@@ -23,55 +23,40 @@ const REDIS_HOST = nconf.get('REDIS_HOST');
const REDIS_PASSWORD = nconf.get('REDIS_PASSWORD');
const REDIS_PORT = nconf.get('REDIS_PORT');
const LIVELINESS_PROBE_KEY = nconf.get('LIVELINESS_PROBE_KEY');
const REGISTRATION_COST = nconf.get('RATE_LIMITER_REGISTRATION_COST') || 5;
const BASE_POINTS = nconf.get('RATE_LIMITER_BASE_POINTS') || 30;
const BASE_DURATION = nconf.get('RATE_LIMITER_BASE_DURATION') || 60;
const REGISTRATION_COST = nconf.get('RATE_LIMITER_REGISTRATION_COST') || 10;
const LOGIN_COST = nconf.get('RATE_LIMITER_LOGIN_COST') || 10;
const IP_RATE_LIMIT_COST = nconf.get('RATE_LIMITER_IP_COST') || 5;
let redisClient;
let rateLimiter;
const rateLimiterOpts = {
keyPrefix: 'api-v3',
points: 30, // 30 requests
duration: 60, // per 1 minute by User ID or IP
};
if (RATE_LIMITER_ENABLED && !IS_TEST) {
redisClient = redis.createClient({
host: REDIS_HOST,
password: REDIS_PASSWORD,
port: REDIS_PORT,
enable_offline_queue: false,
});
if (RATE_LIMITER_ENABLED) {
if (IS_TEST) {
rateLimiter = new RateLimiterMemory({
...rateLimiterOpts,
});
} else {
redisClient = redis.createClient({
host: REDIS_HOST,
password: REDIS_PASSWORD,
port: REDIS_PORT,
enable_offline_queue: false,
});
redisClient.on('ready', () => {
SERVER_STATUS.REDIS = true;
});
redisClient.on('ready', () => {
SERVER_STATUS.REDIS = true;
});
redisClient.on('reconnecting', () => {
SERVER_STATUS.REDIS = false;
});
redisClient.on('reconnecting', () => {
SERVER_STATUS.REDIS = false;
});
redisClient.on('error', error => {
logger.error(error, 'Redis Error');
});
rateLimiter = new RateLimiterRedis({
...rateLimiterOpts,
storeClient: redisClient,
});
}
redisClient.on('error', error => {
logger.error(error, 'Redis Error');
});
} else {
SERVER_STATUS.REDIS = true;
}
function setResponseHeaders (res, rateLimiterRes) {
function setResponseHeaders (res, points, rateLimiterRes) {
const headers = {
'X-RateLimit-Limit': rateLimiterOpts.points,
'X-RateLimit-Limit': points,
'X-RateLimit-Remaining': rateLimiterRes.remainingPoints,
'X-RateLimit-Reset': new Date(Date.now() + rateLimiterRes.msBeforeNext),
};
@@ -83,34 +68,59 @@ function setResponseHeaders (res, rateLimiterRes) {
res.set(headers);
}
export default function rateLimiterMiddleware (req, res, next) {
if (!RATE_LIMITER_ENABLED) return next();
if (LIVELINESS_PROBE_KEY && req.query.liveliness === LIVELINESS_PROBE_KEY) return next();
const userId = req.header('x-api-user');
let cost = 1;
if (req.path === '/api/v4/user/auth/local/register' || req.path === '/api/v3/user/auth/local/register') {
cost = REGISTRATION_COST;
} else if (!userId) {
cost = IP_RATE_LIMIT_COST;
export default function setupRateLimiter (options = {}) {
const rateLimiterOpts = {
keyPrefix: options.keyPrefix || 'api',
points: options.points || BASE_POINTS, // 30 requests
duration: options.duration || BASE_DURATION, // per 1 minute by User ID or IP
};
let rateLimiter;
if (!RATE_LIMITER_ENABLED) {
return (req, res, next) => next();
}
return rateLimiter.consume(userId || req.ip, cost)
.then(rateLimiterRes => {
setResponseHeaders(res, rateLimiterRes);
return next();
})
.catch(rateLimiterRes => {
if (rateLimiterRes instanceof RateLimiterRes) {
setResponseHeaders(res, rateLimiterRes);
return next(new TooManyRequests(apiError('clientRateLimited')));
}
// In case of an unhandled error we skip the middleware as it could mean
// , for example, that the connection to the redis database is not working.
// We do not want to block all requests in these cases.
logger.error(rateLimiterRes, 'Rate Limiter Error');
return next();
if (IS_TEST) {
rateLimiter = new RateLimiterMemory({
...rateLimiterOpts,
});
} else {
rateLimiter = new RateLimiterRedis({
...rateLimiterOpts,
storeClient: redisClient,
});
}
return function rateLimiterMiddleware (req, res, next) {
if (!RATE_LIMITER_ENABLED) return next();
if (LIVELINESS_PROBE_KEY && req.query.liveliness === LIVELINESS_PROBE_KEY) return next();
const userId = req.header('x-api-user');
let cost = 1;
if (req.path.indexOf('/user/auth/local/register') > 0) {
cost = options.registrationCost || REGISTRATION_COST;
} else if (req.path.indexOf('/user/auth/local/login') > 0) {
cost = options.loginCost || LOGIN_COST;
} else if (req.path.indexOf('/user/auth/verify-username') > 0) {
cost = 1; // Verifying username might happen multiple times during typing
} else if (!userId) {
cost = options.ipRateLimitCost || IP_RATE_LIMIT_COST;
}
return rateLimiter.consume(userId || req.ip, cost)
.then(rateLimiterRes => {
setResponseHeaders(res, rateLimiterOpts.points, rateLimiterRes);
return next();
})
.catch(rateLimiterRes => {
if (rateLimiterRes instanceof RateLimiterRes) {
setResponseHeaders(res, rateLimiterOpts.points, rateLimiterRes);
return next(new TooManyRequests(apiError('clientRateLimited')));
}
// In case of an unhandled error we skip the middleware as it could mean
// , for example, that the connection to the redis database is not working.
// We do not want to block all requests in these cases.
logger.error(rateLimiterRes, 'Rate Limiter Error');
return next();
});
};
}
+4 -1
View File
@@ -654,7 +654,8 @@ schema.methods.sendChat = async function sendChat (options = {}) {
return newChatMessage;
};
schema.methods.handleQuestInvitation = async function handleQuestInvitation (user, accept) {
// eslint-disable-next-line max-len
schema.methods.handleQuestInvitation = async function handleQuestInvitation (user, accept, session) {
if (!user) throw new InternalServerError('Must provide user to handle quest invitation');
if (accept !== true && accept !== false) throw new InternalServerError('Must provide accept param handle quest invitation');
@@ -662,12 +663,14 @@ schema.methods.handleQuestInvitation = async function handleQuestInvitation (use
// to prevent multiple concurrent requests overriding updates
// see https://github.com/HabitRPG/habitica/issues/11398
const Group = this.constructor;
const options = session ? { session } : {};
const result = await Group.updateOne(
{
_id: this._id,
[`quest.members.${user._id}`]: { $type: 10 }, // match BSON Type Null (type number 10)
},
{ $set: { [`quest.members.${user._id}`]: accept } },
options,
).exec();
if (result.modifiedCount) {