Merge pull request #120 from clojureverse/oxalorg/feature-db-stats

Message volume statistics

Author: plexus

repl/stats_from_indexer.clj

@@ -0,0 +1,84 @@
+(ns repl-sessions.stats-from-indexer
+  (:require [clojurians-log.time-util :as time-util])
+  (:import (java.time LocalDate)))
+
+;; Fetching all messages for a given channel and day is really fast, but we also
+;; want to know the list of all channels for a given day, and how many messages
+;; each had, and which day was the previous or next day that had messages, and
+;; these queries are really slow, since they basically need to traverse all
+;; messages.
+
+;; To work around that we build up these "indexes" and keep them in an atom. We
+;; have an "indexer" component that rebuilds them regularly.
+
+(keys @clojurians-log.db.queries/!indexes)
+;; => (:chan-day-cnt :day-chan-cnt :chan-id->name :chan-name->id)
+
+(defn day-chan-cnt []
+  (:day-chan-cnt @clojurians-log.db.queries/!indexes))
+
+;; For example day-chan-cnt groups first on day, then on channel, and then shows
+;; the count of messages.
+
+(day-chan-cnt)
+;;=>
+{"2018-01-28" {"C03S1KBA2" 1},
+ "2018-02-02" {"C064BA6G2" 28,
+               "C099W16KZ" 19,
+               "C0617A8PQ" 51,
+               ,,,}}
+
+;; So we can easily sum up all messages for a given day.
+
+(defn day-total [day]
+  (apply + (vals (get (day-chan-cnt) day))))
+
+(day-total "2018-02-02")
+;; => 887
+
+;; If we want to query this we just need to extrapolate to a range of days. We
+;; use clojure.java-time elsewhere but actually the java.time API is quite nice
+;; and nowadays I tend to use it directly. A simple date (so year+month+day)
+;; without any time or timezone information is represented as a
+;; java.time.LocalDate.
+
+;; Good example here of recursion and lazy-seq. Note that the lazy-seq is
+;; optional here, you can remove it and still get a valid result, it would just
+;; be eager instead of lazy.
+
+;; There are obviously more ways to write this, for instance with loop/recur.
+;; This use of recursion + cons is a very "classic lisp" approach.
+
+(defn range-of-local-dates [^LocalDate ld1 ^LocalDate ld2]
+  (when (.isBefore ld1 ld2)
+    (cons ld1 (lazy-seq (range-of-local-dates (.plusDays ld1 1) ld2)))))
+
+;; A bit more clojure-y, use vectors with [year month day], return strings
+;; like "2018-02-02", since that is what we have in the indexes. Note that just
+;; calling `str` on any java.time class usually gives a nicely formatted result.
+
+(defn range-of-days [[y1 m1 d1] [y2 m2 d2]]
+  (map str
+       (range-of-local-dates
+        (java.time.LocalDate/of y1 m1 d1)
+        (java.time.LocalDate/of y2 m2 d2))))
+
+;; So this is what that looks like now. I made the range half-open (not
+;; including the end date), might make more sense to make it inclusive.
+
+(range-of-days [2018 2 2] [2018 2 5])
+;; => ("2018-02-02" "2018-02-03" "2018-02-04")
+
+;; So now we can grab the numbers for these days and sum them up. This is a
+;; textbook example of where a transducer works nicely. Check the LI episode for
+;; transducers if you haven't seen this before. Obviously there are other ways
+;; to write this too, like a simple (apply + (map ...))
+
+(defn days-total [days]
+  (transduce (map day-total) + 0 days))
+
+;; And there you go
+
+(days-total
+ (range-of-days [2018 2 2] [2018 2 5]))
+;; => 1913

resources/public/js/stats.js

@@ -0,0 +1,25 @@
+var ctx = document.getElementById('message-chart');
+
+var messageJson = document.getElementById('message-data').innerHTML;
+var messageData = JSON.parse(messageJson.replace(/"/g,'"'));
+var labels = messageData.map(function(elem) {
+  return elem["day"];
+})
+var data = messageData.map(function(elem) {
+  return elem["msg-count"];
+})
+
+var messageChart = new Chart(ctx, {
+  type: 'line',
+  data: {
+      labels: labels,
+      datasets: [{
+          label: 'No. of slack messages',
+          fill: false,
+          data: data,
+          backgroundColor: 'rgba(255, 99, 132, 0.2)',
+          borderColor: 'rgba(54, 162, 235, 1)',
+          borderWidth: 2
+      }]
+  },
+});

src/clojurians_log/db/queries.clj

@@ -137,6 +137,24 @@
                [?emoji :emoji/url ?url]]
              db)))
 
+(defn unique-users-between-days [db from-day to-day]
+  (d/q '[:find (count ?user)
+         :in $ ?from-day ?to-day
+         :where
+         [?msg :message/user ?user]
+         [?msg :message/day ?day]
+         [(>= ?day ?from-day)]
+         [(<= ?day ?to-day)]]
+       db
+       from-day
+       to-day))
+
+(defn message-stats-between-days [from-day to-day]
+  (letfn [(day-chan-cnt [] (:day-chan-cnt @!indexes))
+          (day-total [day] (apply + (vals (get (day-chan-cnt) day))))
+          (days-total [days] (transduce (map day-total) + 0 days))]
+    (mapv #(hash-map :day % :msg-count (day-total %)) (time-util/range-of-days from-day to-day))))
+
 #_
 (doseq [v [#'clojurians-log.db.queries/user-names
            #'clojurians-log.db.queries/channel

src/clojurians_log/routes.clj

@@ -144,6 +144,15 @@
         views/sitemap
         response/xml-render)))
 
+(defn message-stats-route [{:keys [endpoint] :as request}]
+  (let [config @(get-in endpoint [:config :value])
+        {:keys [from-date to-date]} (:path-params request)]
+    (-> request
+        make-context
+        (assoc :data/message-stats (queries/message-stats-between-days from-date to-date))
+        views/message-stats-page
+        response/render)))
+
 (def routes
   [["/" {:name :clojurians-log.routes/index
          :get index-route}]
@@ -158,4 +167,5 @@
    ["/{channel}/{date}" {:name :clojurians-log.routes/channel-date,
                          :get log-route}]
    ["/{channel}/{date}/{ts}" {:name :clojurians-log.routes/message,
-                              :get log-route}]])
+                              :get log-route}]
+   ["/_/stats/{from-date}/{to-date}" {:name :clojurians-log.routes/message-stats :get message-stats-route}]])

src/clojurians_log/time_util.clj

@@ -2,7 +2,7 @@
   (:require [java-time :as jt]
             [java-time.local :as jt.l]
             [clojure.string :as str])
-  (:import [java.time Instant]
+  (:import [java.time Instant LocalDate]
            [java.time.format DateTimeFormatter]))
 
 (defn ts->inst
@@ -81,3 +81,14 @@
          ret# ~expr]
      (prn (str ~label ": " (/ (double (- (. System (nanoTime)) start#)) 1000000.0) " msecs"))
      ret#))
+
+(defn range-of-local-dates [^LocalDate ld1 ^LocalDate ld2]
+  (when (.isBefore ld1 (.plusDays ld2 1))
+    (cons ld1 (lazy-seq (range-of-local-dates (.plusDays ld1 1) ld2)))))
+
+(defn range-of-days 
+  "Takes 2 day values as a strings and returns range of all days between them inclusive of both"
+  [from-day to-day]
+  (let [ld1 (java.time.LocalDate/parse from-day)
+        ld2 (java.time.LocalDate/parse to-day)]
+    (map str (range-of-local-dates ld1 ld2))))

src/clojurians_log/views.clj

@@ -2,6 +2,7 @@
   (:require [hiccup2.core :as hiccup]
             [cemerick.url :refer [url]]
             [clojurians-log.time-util :as cl.tu]
+            [clojure.data.json :as json]
             [clojure.string :as str]
             [clojurians-log.slack-messages :as slack-messages]
             [reitit.core]
@@ -16,6 +17,11 @@
   [messages ts]
   (some #(when (= (:message/ts %) ts) %) messages))
 
+(defn jsfile [path]
+  (when-let [f (io/file (io/resource (str "public" path)))]
+    (let [ts (.lastModified f)]
+      [:script
+       {:src (str path "?version=" ts)}])))
 
 (defn stylesheet [path]
   (when-let [f (io/file (io/resource (str "public" path)))]
@@ -286,6 +292,43 @@
                               :date day}))]
         [:lastmod day]]))])
 
+
+(defn- page-head-stats [{:data/keys [title]}]
+  [:head
+   [:meta {:charset "utf-8"}]
+   [:meta {:http-equiv "X-UA-Compatible" :content "IE=edge"}]
+   [:meta {:content="width=device-width, initial-scale=1" :name "viewport"}]
+   [:title title]
+   [:link {:href "https://unpkg.com/sakura.css/css/sakura.css"
+           :rel "stylesheet"
+           :type "text/css"}]
+   (stylesheet "/css/gh-fork-ribbon.min.css")
+   [:script {:src "https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.9.4/Chart.min.js"}]
+   ])
+
+(defn- message-stats-page-html [{:data/keys [message-stats] :as context}]
+  [:html
+   (page-head-stats context)
+   [:body
+    [:div
+     [:h4 "Slack message stats"]
+     [:p 
+      [:strong {:style {:border-bottom "1px solid black"}} (:day (first message-stats))]
+      " to "
+      [:strong {:style {:border-bottom "1px solid black"}} (:day (last message-stats))]]
+     [:div {:width "100%"}
+      [:canvas#message-chart]]
+     [:h4 "Total message count: " (reduce #(+ %1 (:msg-count %2)) 0 message-stats)]
+     [:table
+      [:thead
+       [:tr
+        [:th "Day"]
+        [:th "Message count"]]]
+      [:tbody
+       (for [day-stat message-stats] [:tr [:td (:day day-stat)] [:td (:msg-count day-stat)]])]]]
+    [:script#message-data {:type "application/json"} (json/write-str message-stats)]
+    (jsfile "/js/stats.js")]])
+
 (defn log-page [context]
   (assoc context :response/html (log-page-html context)))
 
@@ -300,3 +343,6 @@
 
 (defn sitemap [context]
   (assoc context :response/xml (sitemap-xml context)))
+
+(defn message-stats-page [context]
+  (assoc context :response/html (message-stats-page-html context)))