From aa368614b1a90f83560352535e3494059a6bae15 Mon Sep 17 00:00:00 2001
From: Jocelyn Badgley <joc@twipped.com>
Date: Sat, 20 May 2023 13:55:18 -0700
Subject: [PATCH] Created a few helper scripts to dump some refined analytics

---
 analytics/queries/referrers.js | 59 ++++++++++++++++++++++++++++++++++
 analytics/queries/visits.js    | 27 ++++++++++++++++
 2 files changed, 86 insertions(+)
 create mode 100644 analytics/queries/referrers.js
 create mode 100644 analytics/queries/visits.js

diff --git a/analytics/queries/referrers.js b/analytics/queries/referrers.js
new file mode 100644
index 0000000..0cde967
--- /dev/null
+++ b/analytics/queries/referrers.js
@@ -0,0 +1,59 @@
+const path = require('path');
+const { open: opensql } = require('sqlite');
+const sqlite3 = require('sqlite3');
+const sql = require('../sql-tag'); 
+
+(async () => {
+  // open the database
+  const db = await opensql({
+    filename: path.resolve(__dirname, '..', 'database.sqlite'),
+    driver: sqlite3.Database,
+  });
+
+  await db.exec(sql`
+    CREATE INDEX IF NOT EXISTS entries ON records (
+      referrer_host
+    );
+  `);
+
+  const rows = await db.all(sql`
+    SELECT referrer_host, count(DISTINCT IFNULL(tid, ip)) as tids
+    FROM records
+    WHERE date(dts) > date('now', '-12 month')
+    AND referrer_host != 'genderdysphoria.fyi'
+    AND referrer_host != 'www.genderdysphoria.fyi'
+    GROUP BY referrer_host
+    ORDER BY tids DESC;
+  `);
+
+  const hosts = new Map();
+  for (const {referrer_host, tids} of rows) {
+    const host = matchHost(referrer_host);
+    const row = hosts.get(host) || { referrer_host, count: 0 };
+    const current = row.count || 0;
+    hosts.set(host, { host, referrer_host: row.referrer_host, count: current + tids });
+  }
+
+  let results = Array.from(hosts.values(), ({ host, referrer_host, count }) => [referrer_host, count])
+  results = results.sort((a,b) => b[1] - a[1]);
+  results = results.slice(0, 100);
+
+  // console.table(results);
+  for (const [host, count] of results) {
+    process.stdout.write(host.padEnd(38, ' ') + count + '\n');
+  }
+
+})().catch(console.error);
+
+const MATCH = /^t\.co$|(?:[^.]+)(?=(?:\.com?)?\.[A-za-z]{2,}$)/;
+function matchHost (input) {
+  if (input === 'com.andrewshu.android.reddit') return 'reddit';
+  if (input.startsWith('com.laurencedawson.reddit_sync')) return 'reddit';
+  if (input === 'genderdysphoria-fyi.translate.goog') return 'google';
+  if (input.endsWith('.messenger.com')) return 'facebook';
+  if (input.startsWith('com.')) {
+    input = input.split('.').reverse().join('.');
+  }
+  const [host] = input.match(MATCH) || [input];
+  return host;
+}
\ No newline at end of file
diff --git a/analytics/queries/visits.js b/analytics/queries/visits.js
new file mode 100644
index 0000000..a41eb41
--- /dev/null
+++ b/analytics/queries/visits.js
@@ -0,0 +1,27 @@
+const path = require('path');
+const { open: opensql } = require('sqlite');
+const sqlite3 = require('sqlite3');
+const sql = require('../sql-tag'); 
+
+(async () => {
+  // open the database
+  const db = await opensql({
+    filename: path.resolve(__dirname, '..', 'database.sqlite'),
+    driver: sqlite3.Database,
+  });
+
+  const rows = await db.all(sql`
+    SELECT
+        date(dts) as day,
+        count(DISTINCT IFNULL(tid, ip)) as tids
+    FROM records
+    WHERE date(dts) > date('now', '-12 month')
+    GROUP BY date(dts);
+  `);
+
+  // console.table(results);
+  for (const { day, tids } of rows) {
+    process.stdout.write(day + '\t' + tids + '\n');
+  }
+
+})().catch(console.error);