add leaderboard caching and pagination

Author: ginkoid

app.js

@@ -1,6 +1,8 @@
 const path = require('path')
 const express = require('express')
 
+require('./server/leaderboard')
+
 const app = express()
 
 app.use(express.raw({
@@ -11,5 +13,12 @@ app.use('/api/v1', require('./server/api'))
 
 const staticPath = path.join(__dirname, '/build')
 app.use(express.static(staticPath, { extensions: ['html'] }))
+app.use((req, res, next) => {
+  if (req.method !== 'GET') {
+    next()
+    return
+  }
+  res.sendFile(path.join(staticPath, 'index.html'))
+})
 
 module.exports = app

config/index.js

@@ -12,5 +12,6 @@ module.exports = {
     eligible: 0,
     ineligible: 1
   },
-  loginTimeout: 10 * 60 * 1000
+  loginTimeout: 10 * 60 * 1000,
+  leaderboardUpdateInterval: 10 * 1000
 }

migrations/1581840671046_add-solve-createdat.js

@@ -0,0 +1,9 @@
+exports.up = function (pgm) {
+  pgm.addColumns('solves', {
+    createdat: { type: 'timestamp', notNull: true }
+  })
+}
+
+exports.down = function (pgm) {
+  pgm.dropColumns('solves', ['createdat'])
+}

server/api/index.js

@@ -29,21 +29,23 @@ const routeValidators = routes.map((route) => {
   return ret
 })
 
+const makeSendResponse = (res) => (responseKind, data = null) => {
+  const response = responseList[responseKind]
+  if (response === undefined) {
+    throw new Error(`unknown response ${responseKind}`)
+  }
+  res.set('content-type', 'application/json')
+  res.status(response.status)
+  res.send(JSON.stringify({
+    kind: responseKind,
+    message: response.message,
+    data
+  }))
+}
+
 routes.forEach((route, i) => {
   router[route.method](route.path, async (req, res) => {
-    const sendResponse = (responseKind, data = null) => {
-      const response = responseList[responseKind]
-      if (response === undefined) {
-        throw new Error(`unknown response ${responseKind}`)
-      }
-      res.set('content-type', 'application/json')
-      res.status(response.status)
-      res.send(JSON.stringify({
-        kind: responseKind,
-        message: response.message,
-        data
-      }))
-    }
+    const sendResponse = makeSendResponse(res)
 
     if (req.body instanceof Buffer) {
       try {
@@ -100,4 +102,8 @@ routes.forEach((route, i) => {
   })
 })
 
+router.use((req, res) => {
+  makeSendResponse(res)(responses.badEndpoint)
+})
+
 module.exports = router

server/api/leaderboard.js

@@ -1,65 +1,34 @@
-const db = require('../database')
-const challenges = require('../challenges')
 const { responses } = require('../responses')
-
-const util = require('../util')
+const cache = require('../cache')
 
 module.exports = {
   method: 'get',
   path: '/leaderboard',
   requireAuth: false,
-  handler: async ({ req }) => {
-    const solveAmount = {}
-    const challengeValues = {}
-    const userSolves = {}
-    const userScores = []
-
-    const solves = await db.solves.getAllSolves()
-    const users = await db.users.getAllUsers()
-
-    for (let i = 0; i < solves.length; i++) {
-      // Accumulate in solveAmount
-      if (!(solves[i].challengeid in solveAmount)) {
-        solveAmount[solves[i].challengeid] = 1
-      } else {
-        solveAmount[solves[i].challengeid] += 1
-      }
-      // Store which challenges each user solved for later
-      if (!(solves[i].userid in userSolves)) {
-        userSolves[solves[i].userid] = [solves[i].challengeid]
-      } else {
-        userSolves[solves[i].userid].push(solves[i].challengeid)
-      }
-    }
-
-    const allChallenges = challenges.getAllChallenges()
-
-    for (let i = 0; i < allChallenges.length; i++) {
-      const challenge = allChallenges[i]
-      if (!(challenge.id in solveAmount)) {
-        // There are currently no solves
-        challengeValues[challenge.id] = util.scores.getScore('dynamic', challenge.points.min, challenge.points.max, 0)
-      } else {
-        challengeValues[challenge.id] = util.scores.getScore('dynamic', challenge.points.min, challenge.points.max, solveAmount[challenge.id])
-      }
-    }
-
-    for (let i = 0; i < users.length; i++) {
-      if (!(users[i].userid in userSolves)) {
-        // The user has not solved anything
-        userScores.push([users[i].name, 0])
-      } else {
-        let currScore = 0
-        for (let j = 0; j < userSolves[users[i].userid].length; j++) {
-          // Add the score for the specific solve loaded fr om the challengeValues array using ids
-          currScore += challengeValues[userSolves[users[i].userid][j]]
+  schema: {
+    query: {
+      type: 'object',
+      properties: {
+        limit: {
+          type: 'string'
+        },
+        offset: {
+          type: 'string'
         }
-        userScores.push([users[i].name, currScore])
-      }
+      },
+      required: ['limit', 'offset']
     }
-
-    const sortedUsers = userScores.sort((a, b) => b[1] - a[1])
-
-    return [responses.goodLeaderboard, sortedUsers]
+  },
+  handler: async ({ req }) => {
+    const limit = parseInt(req.query.limit)
+    const offset = parseInt(req.query.offset)
+    if (limit < 0 || offset < 0) {
+      return responses.badBody
+    }
+    const result = await cache.leaderboard.getRange({
+      start: offset,
+      end: offset + limit
+    })
+    return [responses.goodLeaderboard, result]
   }
 }

server/api/submitflag.js

@@ -38,7 +38,7 @@ module.exports = {
       if (submittedFlag === challenge.flag) {
         const solved = await db.solves.getSolvesByUserIdAndChallId({ userid: uuid, challengeid: challengeid })
         if (solved === undefined) {
-          db.solves.newSolve({ id: uuidv4(), challengeid: challengeid, userid: uuid })
+          db.solves.newSolve({ id: uuidv4(), challengeid: challengeid, userid: uuid, createdat: new Date() })
           return responses.goodFlag
         } else {
           return responses.alreadySolved

server/auth/token.js

@@ -14,7 +14,7 @@ const tokenKinds = {
 const tokenExpiries = {
   [tokenKinds.auth]: Infinity,
   [tokenKinds.team]: Infinity,
-  [tokenKinds.verify]: 1000 * 60 * 10
+  [tokenKinds.verify]: config.loginTimeout
 }
 
 const timeNow = () => Math.floor(Date.now() / 1000)

server/cache/client.js

@@ -0,0 +1,8 @@
+const redis = require('redis')
+const config = require('../../config')
+
+const client = redis.createClient({
+  url: config.redisUrl
+})
+
+module.exports = client

server/cache/index.js

@@ -1,3 +1,4 @@
 module.exports = {
-  login: require('./login')
+  login: require('./login'),
+  leaderboard: require('./leaderboard')
 }

server/cache/leaderboard.js

@@ -0,0 +1,34 @@
+const { promisify } = require('util')
+const client = require('./client')
+
+const redisEval = promisify(client.eval.bind(client))
+const redisLrange = promisify(client.lrange.bind(client))
+
+const setLeaderboard = async (leaderboard) => {
+  if (leaderboard.length === 0) {
+    return
+  }
+  await redisEval(
+    'redis.call("DEL", KEYS[1]); redis.call("RPUSH", KEYS[1], unpack(cjson.decode(ARGV[1])))',
+    1,
+    'leaderboard',
+    JSON.stringify(leaderboard.flat())
+  )
+}
+
+const getRange = async ({ start, end }) => {
+  let redisResult = []
+  try {
+    redisResult = await redisLrange('leaderboard', start * 3, end * 3 - 1)
+  } catch (e) {}
+  const result = []
+  for (let i = 0; i < redisResult.length / 3; i++) {
+    result.push([redisResult[i], redisResult[i + 1], parseInt(redisResult[i + 2])])
+  }
+  return result
+}
+
+module.exports = {
+  setLeaderboard,
+  getRange
+}

server/cache/login.js

@@ -1,10 +1,6 @@
 const { promisify } = require('util')
-const redis = require('redis')
 const config = require('../../config')
-
-const client = redis.createClient({
-  url: config.redisUrl
-})
+const client = require('./client')
 
 const prefixes = {
   login: 'l'

server/database/solves.js

@@ -2,19 +2,19 @@ const db = require('./db')
 
 const ret = {
   getAllSolves: () => {
-    return db.query('SELECT * FROM solves')
+    return db.query('SELECT * FROM solves ORDER BY createdat DESC')
       .then(res => res.rows)
   },
   getSolvesByUserId: ({ userid }) => {
-    return db.query('SELECT * FROM solves WHERE userid = $1', [userid])
+    return db.query('SELECT * FROM solves WHERE userid = $1 ORDER BY createdat DESC', [userid])
       .then(res => res.rows)
   },
   getSolvesByUserIdAndChallId: ({ userid, challengeid }) => {
-    return db.query('SELECT * FROM solves WHERE userid = $1 AND challengeid = $2', [userid, challengeid])
+    return db.query('SELECT * FROM solves WHERE userid = $1 AND challengeid = $2 ORDER BY createdat DESC', [userid, challengeid])
       .then(res => res.rows[0])
   },
-  newSolve: ({ id, userid, challengeid }) => {
-    return db.query('INSERT INTO solves (id, challengeid, userid) VALUES ($1, $2, $3) RETURNING *', [id, challengeid, userid])
+  newSolve: ({ id, userid, challengeid, createdat }) => {
+    return db.query('INSERT INTO solves (id, challengeid, userid, createdat) VALUES ($1, $2, $3, $4) RETURNING *', [id, challengeid, userid, createdat])
       .then(res => res.rows[0])
   },
   removeSolvesByUserId: ({ userid }) => {

server/leaderboard/calculate.js

@@ -0,0 +1,59 @@
+const { workerData, parentPort } = require('worker_threads')
+const util = require('../util')
+
+const { solves, users, allChallenges } = workerData
+
+const solveAmount = new Map()
+const challengeValues = new Map()
+const userSolves = new Map()
+const userLastSolves = new Map()
+const userScores = []
+
+for (let i = 0; i < solves.length; i++) {
+  // Accumulate in solveAmount
+  const challId = solves[i].challengeid
+  const userId = solves[i].userid
+  if (!(challId in solveAmount)) {
+    solveAmount.set(challId, 1)
+  } else {
+    solveAmount.set(challId, solveAmount.get(challId) + 1)
+  }
+  // Store which challenges each user solved for later
+  if (!(userId in userSolves)) {
+    userSolves.set(userId, [challId])
+    userLastSolves.set(userId, solves[i].createdat)
+  } else {
+    userSolves.get(solves[i].userid).push(challId)
+  }
+}
+
+for (let i = 0; i < allChallenges.length; i++) {
+  const challenge = allChallenges[i]
+  if (!(challenge.id in solveAmount)) {
+    // There are currently no solves
+    challengeValues.set(challenge.id, util.scores.getScore('dynamic', challenge.points.min, challenge.points.max, 0))
+  } else {
+    challengeValues.set(challenge.id, util.scores.getScore('dynamic', challenge.points.min, challenge.points.max, solveAmount.get(challenge.id)))
+  }
+}
+
+for (let i = 0; i < users.length; i++) {
+  const userId = users[i].userid
+  let currScore = 0
+  const solvedChalls = userSolves.get(userId)
+  for (let j = 0; j < solvedChalls.length; j++) {
+    // Add the score for the specific solve loaded fr om the challengeValues array using ids
+    currScore += challengeValues.get(solvedChalls[j])
+  }
+  userScores.push([userId, users[i].name, currScore, userLastSolves.get(userId)])
+}
+
+const sortedUsers = userScores.sort((a, b) => {
+  const scoreCompare = b[2] - a[2]
+  if (scoreCompare !== 0) {
+    return scoreCompare
+  }
+  return a[3] - b[3]
+}).map((user) => user.slice(0, 3))
+
+parentPort.postMessage(sortedUsers)

server/leaderboard/index.js

@@ -0,0 +1,30 @@
+const path = require('path')
+const { Worker } = require('worker_threads')
+const database = require('../database')
+const challenges = require('../challenges')
+const cache = require('../cache')
+const config = require('../../config')
+
+const fetchData = async () => {
+  const [solves, users] = await Promise.all([
+    database.solves.getAllSolves(),
+    database.users.getAllUsers()
+  ])
+  return {
+    solves,
+    users,
+    allChallenges: challenges.getAllChallenges()
+  }
+}
+
+const runUpdate = async () => {
+  const worker = new Worker(path.join(__dirname, 'calculate.js'), {
+    workerData: await fetchData()
+  })
+  worker.once('message', (data) => {
+    cache.leaderboard.setLeaderboard(data)
+  })
+}
+
+setInterval(runUpdate, config.leaderboardUpdateInterval)
+runUpdate()

server/responses/index.js

@@ -63,6 +63,10 @@ const responseList = {
     status: 400,
     message: 'The request JSON body is malformed.'
   },
+  badEndpoint: {
+    status: 404,
+    message: 'The request endpoint could not be found.'
+  },
   errorInternal: {
     status: 500,
     message: 'An internal error occurred.'