mirror of
https://github.com/GenderDysphoria/GenderDysphoria.fyi.git
synced 2025-01-31 07:16:17 +00:00
110 lines
3.3 KiB
JavaScript
110 lines
3.3 KiB
JavaScript
const fs = require('fs-extra');
|
|
const { resolve } = require('./resolve');
|
|
const { chunk, uniq, difference } = require('lodash');
|
|
const Twitter = require('twitter-lite');
|
|
const log = require('fancy-log');
|
|
const tweetparse = require('./lib/tweetparse');
|
|
const { hasOwn } = require('./lib/util');
|
|
|
|
module.exports = exports = async function () {
|
|
const tc = new TwitterClient();
|
|
await tc.initialize();
|
|
return tc;
|
|
};
|
|
|
|
|
|
class TwitterClient {
|
|
|
|
async initialize () {
|
|
const [ lookup, backup, cache ] = await Promise.all([
|
|
fs.readJson(resolve('twitter-config.json')).catch(() => null)
|
|
.then(makeFetcher),
|
|
fs.readJson(resolve('twitter-backup.json')).catch(() => ({})),
|
|
fs.readJson(resolve('twitter-cache.json')).catch(() => ({})),
|
|
]);
|
|
|
|
this._lookup = lookup;
|
|
this._backupData = backup;
|
|
this._cache = cache;
|
|
this._presentTweets = Object.keys(cache);
|
|
}
|
|
|
|
async write () {
|
|
return Promise.all([
|
|
fs.writeFile(resolve('twitter-cache.json'), JSON.stringify(this._cache, null, 2)),
|
|
fs.writeFile(resolve('twitter-backup.json'), JSON.stringify(this._backupData, null, 2)),
|
|
]);
|
|
}
|
|
|
|
async get (tweetids) {
|
|
if (!Array.isArray(tweetids)) tweetids = [ tweetids ];
|
|
|
|
tweetids = uniq(tweetids.map(parseTweetId));
|
|
|
|
let tweetsNeeded = this._missing(tweetids).filter(Boolean);
|
|
|
|
while (tweetsNeeded.length) {
|
|
// log('Fetching tweets: ' + tweetsNeeded.join(', '));
|
|
// const arriving = await Promise.all(chunk(tweetsNeeded, 99).map(this._lookup));
|
|
const arriving = [];
|
|
const tweetsRequested = tweetsNeeded;
|
|
tweetsNeeded = [];
|
|
const loaded = [];
|
|
for (const tweet of arriving.flat(1)) {
|
|
if (tweet.quoted_status_id_str && !this._cache[tweet.quoted_status_id_str]) {
|
|
tweetsNeeded.push(tweet.quoted_status_id_str);
|
|
}
|
|
|
|
this._backupData[tweet.id_str] = tweet;
|
|
this._cache[tweet.id_str] = tweetparse(tweet);
|
|
loaded.push(tweet.id_str);
|
|
}
|
|
|
|
const absent = difference(tweetsRequested, loaded);
|
|
for (const id of absent) {
|
|
if (!id) continue;
|
|
if (!hasOwn(this._backupData, id)) {
|
|
log.error('Could not find tweet ' + id);
|
|
continue;
|
|
}
|
|
const tweet = this._backupData[id];
|
|
|
|
if (tweet) {
|
|
log('Pulled tweet from backup ' + id);
|
|
this._cache[id] = tweetparse(this._backupData[id]);
|
|
} else {
|
|
this._cache[id] = false;
|
|
}
|
|
}
|
|
}
|
|
|
|
return tweetids.map((id) => this._cache[id] || null);
|
|
}
|
|
|
|
_missing (tweetids) {
|
|
return difference(tweetids, this._presentTweets);
|
|
}
|
|
}
|
|
|
|
|
|
function makeFetcher (config) {
|
|
if (!config) return () => [];
|
|
const client = new Twitter(config);
|
|
return (tweetids) => client
|
|
.get('statuses/lookup', { id: tweetids.join(','), tweet_mode: 'extended' })
|
|
// .then((r) => { console.log({r}); return r; })
|
|
.catch((e) => { log.error(e); return []; });
|
|
}
|
|
|
|
const tweeturl = /https?:\/\/twitter\.com\/(?:#!\/)?(?:\w+)\/status(?:es)?\/(\d+)/i;
|
|
const tweetidcheck = /^\d+$/;
|
|
function parseTweetId (tweetid) {
|
|
// we can't trust an id that isn't a string
|
|
if (typeof tweetid !== 'string') return false;
|
|
|
|
const match = tweetid.match(tweeturl);
|
|
if (match) return match[1];
|
|
if (tweetid.match(tweetidcheck)) return tweetid;
|
|
return false;
|
|
}
|