Remove non specified fields to reduce noise

We were getting too many random or obscure headers and unexpected params
which was causing our elasticsearch indexes to run out of fields.

Author: djfarrelly

lib/utils.js

@@ -36,31 +36,53 @@ const removeTokensFromUrl = function(url) {
 };
 module.exports.removeTokensFromUrl = removeTokensFromUrl;
 
+/**
+ * pick
+ * Given an object and array of keys, return an object with only the specified fields
+ *
+ * @param {Object} obj
+ * @param {Array} keys
+ * @return {Object}
+ */
+const pick = function(obj, keys = []) {
+  return keys.reduce((picked, key) => {
+    if (key in obj) {
+      picked[key] = obj[key];
+    }
+    return picked;
+  }, {});
+};
+module.exports.pick = pick;
+
 
+const headersToLog = [
+  'user-agent',
+  'referer',
+  'cache-control',
+];
 /**
  * getRequestDataToLog
  * Given a http request object, return the proper structure to log
  *
  * @param {http.ClientRequest} req
  * @return {Object}
  */
-const getRequestDataToLog = function(req) {
+const getRequestDataToLog = function(req, params = []) {
   const cleanUrl = removeTokensFromUrl(req.url);
   const query = req.query || parse(req.url, true).query;
 
   const request = {
     url: cleanUrl,
     method: req.method,
-    params: removeTokensFromQuery(query),
+    params: removeTokensFromQuery(pick(query, params)),
     connection: {
       remoteAddress: req.connection.remoteAddress,
       remotePort: req.connection.remotePort,
     },
   };
 
-  // Copy headers, removing cookies from the noise
-  request.headers = Object.assign({}, req.headers);
-  delete request.headers.cookie;
+  // Copy only headers we wish to log
+  request.headers = pick(Object.assign({}, req.headers), headersToLog);
 
   return request;
 };

test/testUtils.js

@@ -4,6 +4,7 @@ const { assert } = require('chai');
 const {
   removeTokensFromQuery,
   removeTokensFromUrl,
+  pick,
   getRequestDataToLog,
 } = require('../lib/utils');
 
@@ -52,8 +53,44 @@ describe('utils', () => {
 
   });
 
+  describe('pick', () => {
+
+    it('should only return object with specified keys', () => {
+      const obj = {
+        good: 'should remain',
+        bad: 'should be gone',
+      };
+      const picked = pick(obj, ['good']);
+      assert.deepEqual(picked, { good: 'should remain' });
+    });
+
+  });
+
   describe('getRequestDataToLog', () => {
 
+    it('should filter out headers', () => {
+      const req = {
+        url: '/path?key=val',
+        method: 'GET',
+        connection: {
+          remoteAddress: '::ffff:172.18.0.12',
+          remotePort: 40234,
+        },
+        headers: {
+          'user-agent': 'Mozilla/5.0',
+          referer: 'https://twitter.com',
+          'cache-control': 'no-cache',
+          'x-something-bogus': false,
+        }
+      };
+      const data = getRequestDataToLog(req);
+      assert.deepEqual(data.headers, {
+        'user-agent': 'Mozilla/5.0',
+        referer: 'https://twitter.com',
+        'cache-control': 'no-cache',
+      });
+    });
+
     it('should grab the data we want to track', () => {
       const req = {
         url: '/path?key=val',
@@ -75,19 +112,35 @@ describe('utils', () => {
       assert.deepEqual(data, {
         url: '/path?key=val',
         method: 'GET',
-        params: {
-          key: 'val',
-        },
         connection: {
           remoteAddress: '::ffff:172.18.0.12',
           remotePort: 40234,
         },
         headers: {
           'user-agent': 'Mozilla/5.0',
         },
+        params: {},
       });
     });
 
+    it('should filter out params that are not specified', () => {
+      const req = {
+        url: '/path?keep=1&remove=0',
+        method: 'GET',
+        query: {
+          keep: 1,
+          remove: 0,
+        },
+        connection: {
+          remoteAddress: '::ffff:172.18.0.12',
+          remotePort: 40234,
+        },
+        headers: {},
+      };
+      const data = getRequestDataToLog(req, ['keep']);
+      assert.deepEqual(data.params, { keep: 1 });
+    });
+
     it('should handle requests with unparsed query objects', () => {
       const req = {
         url: '/path?key=val',
@@ -100,7 +153,7 @@ describe('utils', () => {
           cookie: 'ok',
         },
       };
-      const data = getRequestDataToLog(req);
+      const data = getRequestDataToLog(req, ['key']);
       assert.equal(data.params.key, 'val');
     });