zo/GenderDysphoria.fyi
1
0
Fork 0
mirror of https://github.com/GenderDysphoria/GenderDysphoria.fyi.git synced 2025-01-30 23:06:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Analytics tweaks

Browse Source
This commit is contained in:
Jocelyn Badgley (Twipped) 2021-05-07 11:10:29 -07:00
parent 3689c12667
commit 39a1aa2f6f
4 changed files with 58 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -14,7 +14,7 @@ node_modules
/terraform/*.tfstate*
/terraform/.terraform
/terraform/files/*.zip
/analytics/RAW*
/analytics/*.gz
/analytics/combined.log
/analytics/database*
/assets.json

4
analytics/package.json
View File

@ -4,9 +4,10 @@
"description": "",
"main": "index.js",
"scripts": {
"start": "node index.js",
"sync": "aws s3 sync s3://gdbible-analytics/RAW ./RAW",
"empty": "aws s3 rm s3://gdbible-analytics/RAW --recursive",
"combine": "find RAW -name '*.gz' -exec cat '{}' ';'| zcat | sed '/^#/ d' > combined.log"
"combine": "find RAW_ -name '*.gz' -exec cat '{}' ';' > combined.log"
},
"author": "Jocelyn Badgley <joc@twipped.com> (http://twipped.com/)",
"license": "MIT",
@ -15,6 +16,7 @@
"date-fns": "~2.9.0",
"glob-stream": "~6.1.0",
"named-placeholders": "~1.1.2",
"readable-stream": "~3.6.0",
"split2": "~3.2.2",
"sqlite": "~4.0.19",
"sqlite3": "~5.0.2",

12
analytics/queries.sql
View File

@ -14,7 +14,17 @@ HAVING total > 5;
SELECT referrer_host, count(DISTINCT IFNULL(tid, ip)) as tids, referrer
FROM records
GROUP BY referrer_host;
WHERE date(dts) > date('now', '-1 month')
AND referrer_host != 'genderdysphoria.fyi'
GROUP BY referrer_host
ORDER BY tids DESC;
SELECT referrer_host, count(DISTINCT IFNULL(tid, ip)) as tids, referrer
FROM records
WHERE date(dts) > date('now', '-1 day')
AND INSTR(referrer_host, 'tiktok')
GROUP BY referrer_host
ORDER BY tids DESC;
SELECT COUNT(IFNULL(tid,ip)) as total, referrer
FROM records

43
analytics/sync.js Normal file
View File

@ -0,0 +1,43 @@
var Readable = require('readable-stream').Readable;
function identity (_in) {
return _in;
}
/**
* Returns a stream for any paged AWS function
* you can optionally provide a mapping function
* like S3::listObjectsV2()
*
* @param {function} req - a non executed AWS function
* @param {function} fn - a function that selects/maps the results
* @param {object} opts - stream options
*/
function s3PageStream (req, fn, opts) {
opts = Object.assign({}, opts, { read, objectMode: true });
if (!fn) fn = identity;
var stream = new Readable(opts);
return stream;
function read () {
if (!req) return;
var _req = req;
req = null; // poor man's once!
_req.send(page_handler);
}
function page_handler (e, data) {
if (e) return stream.destroy(e);
data.Contents.forEach((obj) => {
stream.push(fn(obj));
});
var nextPage = this.hasNextPage() ? this.nextPage() : null;
if (nextPage) nextPage.send(page_handler);
else stream.push(null);
}
}