start building out more sophisticated rate limiting

test/api/unit/middlewares/rateLimiter.test.js

@@ -10,7 +10,7 @@ import { TooManyRequests } from '../../../../website/server/libs/errors';
 import { apiError } from '../../../../website/server/libs/apiError';
 import logger from '../../../../website/server/libs/logger';
 
-describe('rateLimiter middleware', () => {
+describe.only('rateLimiter middleware', () => {
   const pathToRateLimiter = '../../../../website/server/middlewares/rateLimiter';
 
   let res; let req; let next; let nconfGetStub;
@@ -54,6 +54,7 @@ describe('rateLimiter middleware', () => {
 
   it('does not throw when there are available points', async () => {
     nconfGetStub.withArgs('RATE_LIMITER_ENABLED').returns('true');
+    nconfGetStub.withArgs('RATE_LIMITER_IP_COST').returns(1);
     const attachRateLimiter = requireAgain(pathToRateLimiter).default;
     await attachRateLimiter(req, res, next);
 
@@ -71,6 +72,7 @@ describe('rateLimiter middleware', () => {
 
   it('does not throw when an unknown error is thrown by the rate limiter', async () => {
     nconfGetStub.withArgs('RATE_LIMITER_ENABLED').returns('true');
+    nconfGetStub.withArgs('RATE_LIMITER_IP_COST').returns(1);
     sandbox.stub(logger, 'error');
     sandbox.stub(RateLimiterMemory.prototype, 'consume')
       .returns(Promise.reject(new Error('Unknown error.')));
@@ -104,6 +106,7 @@ describe('rateLimiter middleware', () => {
   it('limits when LIVELINESS_PROBE_KEY is incorrect', async () => {
     nconfGetStub.withArgs('RATE_LIMITER_ENABLED').returns('true');
     nconfGetStub.withArgs('LIVELINESS_PROBE_KEY').returns('abc');
+    nconfGetStub.withArgs('RATE_LIMITER_IP_COST').returns(1);
     const attachRateLimiter = requireAgain(pathToRateLimiter).default;
 
     req.query.liveliness = 'das';
@@ -120,6 +123,7 @@ describe('rateLimiter middleware', () => {
   it('limits when LIVELINESS_PROBE_KEY is not set', async () => {
     nconfGetStub.withArgs('RATE_LIMITER_ENABLED').returns('true');
     nconfGetStub.withArgs('LIVELINESS_PROBE_KEY').returns(undefined);
+    nconfGetStub.withArgs('RATE_LIMITER_IP_COST').returns(1);
     const attachRateLimiter = requireAgain(pathToRateLimiter).default;
 
     await attachRateLimiter(req, res, next);
@@ -135,6 +139,7 @@ describe('rateLimiter middleware', () => {
   it('throws when LIVELINESS_PROBE_KEY is blank', async () => {
     nconfGetStub.withArgs('RATE_LIMITER_ENABLED').returns('true');
     nconfGetStub.withArgs('LIVELINESS_PROBE_KEY').returns('');
+    nconfGetStub.withArgs('RATE_LIMITER_IP_COST').returns(1);
     const attachRateLimiter = requireAgain(pathToRateLimiter).default;
 
     req.query.liveliness = '';
@@ -150,6 +155,7 @@ describe('rateLimiter middleware', () => {
 
   it('throws when there are no available points remaining', async () => {
     nconfGetStub.withArgs('RATE_LIMITER_ENABLED').returns('true');
+    nconfGetStub.withArgs('RATE_LIMITER_IP_COST').returns(1);
     const attachRateLimiter = requireAgain(pathToRateLimiter).default;
 
     // call for 31 times
@@ -173,6 +179,7 @@ describe('rateLimiter middleware', () => {
 
   it('uses the user id if supplied or the ip address', async () => {
     nconfGetStub.withArgs('RATE_LIMITER_ENABLED').returns('true');
+    nconfGetStub.withArgs('RATE_LIMITER_IP_COST').returns(1);
     const attachRateLimiter = requireAgain(pathToRateLimiter).default;
 
     req.ip = 1;
@@ -199,4 +206,71 @@ describe('rateLimiter middleware', () => {
       'X-RateLimit-Reset': sinon.match(Date),
     });
   });
+
+  it('applies increased cost for registration calls with and without user id', async () => {
+    nconfGetStub.withArgs('RATE_LIMITER_ENABLED').returns('true');
+    nconfGetStub.withArgs('RATE_LIMITER_REGISTRATION_COST').returns(3);
+    const attachRateLimiter = requireAgain(pathToRateLimiter).default;
+    req.path = '/api/v4/user/auth/local/register';
+
+    req.ip = 1;
+    await attachRateLimiter(req, res, next);
+
+    req.headers['x-api-user'] = 'user-1';
+    await attachRateLimiter(req, res, next);
+    await attachRateLimiter(req, res, next);
+
+    // user id an ip are counted as separate sources
+    expect(res.set).to.have.been.calledWithMatch({
+      'X-RateLimit-Limit': 30,
+      'X-RateLimit-Remaining': 27, // 2 calls with user id
+      'X-RateLimit-Reset': sinon.match(Date),
+    });
+
+    req.headers['x-api-user'] = undefined;
+    await attachRateLimiter(req, res, next);
+    await attachRateLimiter(req, res, next);
+
+    expect(res.set).to.have.been.calledWithMatch({
+      'X-RateLimit-Limit': 30,
+      'X-RateLimit-Remaining': 24, // 3 calls with only ip
+      'X-RateLimit-Reset': sinon.match(Date),
+    });
+  });
+
+  it('applies increased cost for unauthenticated API calls', async () => {
+    nconfGetStub.withArgs('RATE_LIMITER_ENABLED').returns('true');
+    nconfGetStub.withArgs('RATE_LIMITER_IP_COST').returns(10);
+    const attachRateLimiter = requireAgain(pathToRateLimiter).default;
+
+    req.ip = 1;
+    await attachRateLimiter(req, res, next);
+    await attachRateLimiter(req, res, next);
+
+    expect(res.set).to.have.been.calledWithMatch({
+      'X-RateLimit-Limit': 30,
+      'X-RateLimit-Remaining': 10,
+      'X-RateLimit-Reset': sinon.match(Date),
+    });
+  });
+
+  describe('authentication rate limiting', async () => {
+    it('applies cost for failed login attempts', async () => {
+      nconfGetStub.withArgs('RATE_LIMITER_ENABLED').returns('true');
+      nconfGetStub.withArgs('RATE_LIMITER_IP_COST').returns(1);
+      const attachRateLimiter = requireAgain(pathToRateLimiter).default;
+
+      req.path = '/api/v4/user/auth/local/login';
+
+      req.ip = 1;
+      await attachRateLimiter(req, res, next);
+      await attachRateLimiter(req, res, next);
+
+      expect(res.set).to.have.been.calledWithMatch({
+        'X-RateLimit-Limit': 30,
+        'X-RateLimit-Remaining': 28,
+        'X-RateLimit-Reset': sinon.match(Date),
+      });
+    });
+  });
 });

website/server/middlewares/appRoutes.js

@@ -3,7 +3,7 @@ import expressValidator from 'express-validator';
 import path from 'path';
 import analytics from './analytics';
 import setupBody from './setupBody';
-import rateLimiter from './rateLimiter';
+import { middleware as rateLimiter } from './rateLimiter';
 import setupExpress from '../libs/setupExpress';
 import * as routes from '../libs/routes';
 
@@ -51,6 +51,6 @@ const v4RouterOverrides = [
 const v4Router = express.Router(); // eslint-disable-line new-cap
 routes.walkControllers(v4Router, API_V3_CONTROLLERS_PATH, v4RouterOverrides);
 routes.walkControllers(v4Router, API_V4_CONTROLLERS_PATH);
-app.use('/api/v4', v4Router);
+app.use('/api/v4', rateLimiter, v4Router);
 
 export default app;

website/server/middlewares/errorHandler.js

@@ -11,7 +11,11 @@ import {
   InternalServerError,
 } from '../libs/errors';
 
-export default function errorHandler (err, req, res, next) { // eslint-disable-line no-unused-vars
+import {
+  rateLimitErrors,
+} from './rateLimiter';
+
+export default async function errorHandler (err, req, res, next) {
   // In case of a CustomError class, use it's data
   // Otherwise try to identify the type of error (mongoose validation, mongodb unique, ...)
   // If we can't identify it, respond with a generic 500 error
@@ -85,6 +89,11 @@ export default function errorHandler (err, req, res, next) { // eslint-disable-l
     jsonRes.errors = responseErr.errors;
   }
 
+  const rateLimitRes = await rateLimitErrors(req, res, next);
+
+  if (rateLimitRes) {
+    return rateLimitRes.status(429);
+  }
   // In some occasions like when invalid JSON is supplied `res.respond` might be not yet available,
   // in this case we use the standard res.status(...).json(...)
   return res.status(responseErr.httpCode).json(jsonRes);

website/server/middlewares/rateLimiter.js

@@ -18,13 +18,28 @@ import { apiError } from '../libs/apiError';
 
 const IS_TEST = nconf.get('IS_TEST');
 const RATE_LIMITER_ENABLED = nconf.get('RATE_LIMITER_ENABLED') === 'true';
+const RATE_LIMITER_IN_MEMORY = nconf.get('RATE_LIMITER_IN_MEMORY') === 'true';
 const REDIS_HOST = nconf.get('REDIS_HOST');
 const REDIS_PASSWORD = nconf.get('REDIS_PASSWORD');
 const REDIS_PORT = nconf.get('REDIS_PORT');
 const LIVELINESS_PROBE_KEY = nconf.get('LIVELINESS_PROBE_KEY');
+const REGISTRATION_COST = nconf.get('RATE_LIMITER_REGISTRATION_COST') || 5;
+const IP_RATE_LIMIT_COST = nconf.get('RATE_LIMITER_IP_COST') || 2;
+
+const REGISTER_CALLS = [
+  '/api/v4/user/auth/local/register',
+  '/api/v3/user/auth/local/register',
+];
+const AUTH_CALLS = [
+  '/api/v4/user/auth/local/register',
+  '/api/v3/user/auth/local/register',
+  '/api/v4/user/auth/local/login',
+  '/api/v3/user/auth/local/login',
+];
 
 let redisClient;
 let rateLimiter;
+let authLimiter;
 
 const rateLimiterOpts = {
   keyPrefix: 'api-v3',
@@ -32,11 +47,21 @@ const rateLimiterOpts = {
   duration: 60, // per 1 minute by User ID or IP
 };
 
+const authLimiterOpts = {
+  keyPrefix: 'api-auth',
+  points: 10,
+  duration: 60 * 60 * 24,
+  blockDuration: 60 * 60 * 24,
+};
+
 if (RATE_LIMITER_ENABLED) {
-  if (IS_TEST) {
+  if (IS_TEST || RATE_LIMITER_IN_MEMORY) {
     rateLimiter = new RateLimiterMemory({
       ...rateLimiterOpts,
     });
+    authLimiter = new RateLimiterMemory({
+      ...authLimiterOpts,
+    });
   } else {
     redisClient = redis.createClient({
       host: REDIS_HOST,
@@ -52,13 +77,18 @@ if (RATE_LIMITER_ENABLED) {
     rateLimiter = new RateLimiterRedis({
       ...rateLimiterOpts,
       storeClient: redisClient,
+      inMemoryBlockOnConsumed: 10,
+    });
+    authLimiter = new RateLimiterRedis({
+      ...authLimiterOpts,
+      storeClient: redisClient,
     });
   }
 }
 
-function setResponseHeaders (res, rateLimiterRes) {
+function setResponseHeaders (res, rateLimiterRes, limit) {
   const headers = {
-    'X-RateLimit-Limit': rateLimiterOpts.points,
+    'X-RateLimit-Limit': limit || rateLimiterOpts.points,
     'X-RateLimit-Remaining': rateLimiterRes.remainingPoints,
     'X-RateLimit-Reset': new Date(Date.now() + rateLimiterRes.msBeforeNext),
   };
@@ -70,20 +100,46 @@ function setResponseHeaders (res, rateLimiterRes) {
   res.set(headers);
 }
 
-export default function rateLimiterMiddleware (req, res, next) {
+export async function middleware (req, res, next) {
   if (!RATE_LIMITER_ENABLED) return next();
   if (LIVELINESS_PROBE_KEY && req.query.liveliness === LIVELINESS_PROBE_KEY) return next();
 
   const userId = req.header('x-api-user');
 
-  return rateLimiter.consume(userId || req.ip)
-    .then(rateLimiterRes => {
-      setResponseHeaders(res, rateLimiterRes);
-      return next();
+  let cost = 1;
+  let isAuth = false;
+  if (AUTH_CALLS.indexOf(req.path) !== -1) {
+    isAuth = true;
+    let retrySecs = 0;
+    const authRateLimiterRes = await authLimiter.get(req.ip);
+
+    if (authRateLimiterRes !== null && authRateLimiterRes.consumedPoints > 10) {
+      retrySecs = Math.round(authRateLimiterRes.msBeforeNext / 1000) || 1;
+    }
+    if (retrySecs > 0) {
+      setResponseHeaders(res, authRateLimiterRes, authLimiterOpts.points);
+      return next(new TooManyRequests(apiError('clientRateLimited')));
+    }
+    if (REGISTER_CALLS.indexOf(req.path) !== -1) {
+      cost = REGISTRATION_COST;
+    }
+  } else if (!userId) {
+    cost *= IP_RATE_LIMIT_COST;
+  }
+  return rateLimiter.consume(userId || req.ip, cost)
+    .then(async rateLimiterRes => {
+      const r = next();
+      if (isAuth) {
+        const authRateLimiterRes = await authLimiter.consume(req.ip);
+        setResponseHeaders(res, authRateLimiterRes, authLimiterOpts.points);
+      } else {
+        setResponseHeaders(res, rateLimiterRes, rateLimiterOpts.points);
+      }
+      return r;
     })
     .catch(rateLimiterRes => {
       if (rateLimiterRes instanceof RateLimiterRes) {
-        setResponseHeaders(res, rateLimiterRes);
+        setResponseHeaders(res, rateLimiterRes, rateLimiterOpts.points);
         return next(new TooManyRequests(apiError('clientRateLimited')));
       }
 
@@ -94,3 +150,18 @@ export default function rateLimiterMiddleware (req, res, next) {
       return next();
     });
 }
+
+export async function rateLimitErrors (req, res, next) {
+  if (AUTH_CALLS.indexOf(req.path) !== -1) {
+    try {
+      const authRateLimiterRes = await authLimiter.consume(req.ip);
+      setResponseHeaders(res, authRateLimiterRes, authLimiterOpts.points);
+    } catch (rateLimiterRes) {
+      if (rateLimiterRes instanceof RateLimiterRes) {
+        setResponseHeaders(res, rateLimiterRes, authLimiterOpts.points);
+        return next(new TooManyRequests(apiError('clientRateLimited')));
+      }
+    }
+  }
+  return undefined;
+}