Pulled in the twitter content backup functionality from curvyandtrans.com

Sadly, lost the images from one of Emmy_Zje's deleted tweets.

build/actions.js

@@ -12,16 +12,17 @@ const actions = {
     return readFile(input);
   },
 
-  async transcode ({ input, output }) {
+  async transcode ({ input, output, duplicate }) {
     const result = await actions.image({
       input,
       output,
+      duplicate,
       format: 'jpeg',
     });
     return result;
   },
 
-  async fetch ({ input, output }) {
+  async fetch ({ input, output, duplicate }) {
     const res = await fetch(input);
     if (res.status !== 200) {
       throw new Error(`File could not be fetched (${res.status}): "${input}"`);
@@ -30,16 +31,24 @@ const actions = {
     output = resolve(output);
     await fs.ensureDir(path.dirname(output));
     await fs.writeFile(output, body);
+    if (duplicate) {
+      await fs.ensureDir(resolve(path.dirname(duplicate)));
+      await fs.writeFile(duplicate, body);
+    }
     return body;
   },
 
-  async write ({ output, content }) {
+  async write ({ output, content, duplicate }) {
     if (!content) {
       throw new TypeError('Got an empty write?' + output);
     }
     output = resolve(output);
     await fs.ensureDir(path.dirname(output));
     await fs.writeFile(output, content);
+    if (duplicate) {
+      await fs.ensureDir(resolve(path.dirname(duplicate)));
+      await fs.writeFile(duplicate, content);
+    }
     return Buffer.from(content);
   },
 
@@ -157,6 +166,10 @@ const actions = {
     let result = await Promise.fromCallback((cb) => gmfile.toBuffer(cb));
     if (options.format === 'ico') result = await ico(result);
     await fs.writeFile(output, result);
+    if (options.duplicate) {
+      await fs.ensureDir(resolve(path.dirname(options.duplicate)));
+      await fs.writeFile(options.duplicate, result);
+    }
 
     return result;
   },

build/cache.js

@@ -72,10 +72,10 @@ module.exports = exports = class Manifest {
 
   async get (task) {
     const hash = this.hash(task);
-    const { input, output } = task;
+    const { input, output, cache: altCachePath } = task;
     const ext = path.extname(task.output);
     const local = !task.input.includes('://');
-    const cached = path.join(CACHE, hash + ext);
+    var cached = path.join(CACHE, hash + ext);
     const result = {
       iTime: 0,
       iRev: null,
@@ -85,9 +85,15 @@ module.exports = exports = class Manifest {
       action: task.action.name,
       input,
       output,
+      duplicate: altCachePath,
       mode: 'new',
     };
 
+    var acTime;
+    if (altCachePath) {
+      acTime = await this.stat(altCachePath);
+    }
+
     const [ iTime, oTime, cTime, iRev ] = await Promise.all([
       local && this.stat(input),
       this.stat(output),
@@ -148,7 +154,17 @@ module.exports = exports = class Manifest {
     }
 
     result.mode = 'cached';
+
+    if (acTime && acTime > cTime) {
+      result.cache = await readFile(altCachePath);
+    } else {
       result.cache = await readFile(cached);
+      if (altCachePath && !acTime) {
+        await fs.ensureDir(resolve(path.dirname(altCachePath)));
+        await fs.writeFile(resolve(altCachePath), result.cache);
+      }
+    }
+
     return result;
   }
 
@@ -157,7 +173,7 @@ module.exports = exports = class Manifest {
     if (task.nocache || !task.action.name) return null;
 
     const hash = this.hash(task);
-    const { input, output } = task;
+    const { input, output, cache: altCachePath } = task;
     const local = !task.input.includes('://');
 
     const [ iTime, iRev ] = await Promise.all([
@@ -175,6 +191,7 @@ module.exports = exports = class Manifest {
       output,
       oTime: Math.floor(lastSeen / 1000),
       lastSeen,
+      duplicate: altCachePath,
     };
 
     if (record.revPath) this.revManifest[output] = record.revPath;
@@ -185,17 +202,22 @@ module.exports = exports = class Manifest {
 
   async set (task, result, lastSeen = new Date()) {
     const hash = this.hash(task);
-    const { input, output } = task;
+    const { input, output, cache: altCachePath } = task;
     const nocache = task.nocache || task.action.name === 'copy';
     const ext = path.extname(task.output);
     const local = !task.input.includes('://');
     const cached = path.join(CACHE, hash + ext);
     const oRev = revHash(result);
 
+    if (result && altCachePath) {
+      await fs.ensureDir(resolve(path.dirname(altCachePath)));
+    }
+
     const [ iTime, iRev ] = await Promise.all([
       local && this.stat(input),
       local && this.compareBy.inputRev && this.revFile(input),
       result && !nocache && fs.writeFile(resolve(cached), result),
+      result && altCachePath && fs.writeFile(resolve(altCachePath), result),
     ]);
 
     const record = {
@@ -208,6 +230,7 @@ module.exports = exports = class Manifest {
       oTime: Math.floor(lastSeen / 1000),
       oRev,
       lastSeen,
+      duplicate: altCachePath,
       revPath: revPath(output, oRev),
     };
 

build/lib/tweetparse.js

@@ -1,5 +1,6 @@
 var twemoji = require('twemoji' );
 const { deepPick, has } = require('./util');
+const path = require('path');
 
 const schema = {
   id_str: true,
@@ -23,6 +24,7 @@ const schema = {
     } ] },
   } ] },
   media: true,
+  in_reply_to_status_id_str: true,
 };
 
 var entityProcessors = {
@@ -94,7 +96,8 @@ module.exports = exports = function (tweets) {
 
     tweet.user.avatar = {
       input: tweet.user.profile_image_url_https,
-      output: 'tweets/' + tweet.user.screen_name + '.jpg',
+      output: `tweets/${tweet.user.screen_name}.jpg`,
+      cache: `twitter-avatars/${tweet.user.screen_name}.jpg`,
     };
 
     tweet.media = [
@@ -113,7 +116,22 @@ module.exports = exports = function (tweets) {
     }
 
     if (has(tweet, 'entities.media') && has(tweet, 'extended_entities.media')) {
-      tweet.entities.media = tweet.extended_entities.media;
+      tweet.entities.media = tweet.extended_entities.media.map((media) => {
+        media = { ...media };
+        if (media.media_url_https) {
+          const mediaItem = {
+            input: media.media_url_https,
+            output: `tweets/${tweet.id_str}/${path.basename(media.media_url_https)}`,
+            cache: `twitter-entities/${tweet.id_str}/${path.basename(media.media_url_https)}`,
+          };
+          if (media.type === 'photo') mediaItem.input += '?name=medium';
+          tweet.media.push(mediaItem);
+          media.media_url_https = '/' + mediaItem.output;
+        }
+
+        return media;
+      });
+
       delete tweet.extended_entities;
     }
 

build/page-tweets.js

@@ -1,4 +1,4 @@
-const { chunk, uniq, difference } = require('lodash');
+const { chunk, uniq, uniqBy, difference } = require('lodash');
 const fs = require('fs-extra');
 const { resolve } = require('./resolve');
 const log = require('fancy-log');
@@ -65,7 +65,7 @@ module.exports = exports = async function tweets (pages) {
 
   /* Apply Tweets to Pages **************************************************/
 
-  const twitterMedia = [];
+  var twitterMedia = [];
 
   function attachTweet (dict, tweetid) {
     if (!hasOwn(twitterCache, tweetid) && twitterBackup[tweetid]) {
@@ -91,6 +91,8 @@ module.exports = exports = async function tweets (pages) {
     }, {});
   }
 
+  twitterMedia = uniqBy(twitterMedia, 'output');
+
   await Promise.all([
     fs.writeFile(resolve('twitter-media.json'),  JSON.stringify(twitterMedia,  null, 2)),
     fs.writeFile(resolve('twitter-cache.json'),  JSON.stringify(twitterCache,  null, 2)),

build/twitter-client.js

@@ -0,0 +1,107 @@
+const fs = require('fs-extra');
+const { resolve } = require('./resolve');
+const { chunk, uniq, difference } = require('lodash');
+const Twitter = require('twitter-lite');
+const log = require('fancy-log');
+const tweetparse = require('./lib/tweetparse');
+const { hasOwn } = require('./lib/util');
+
+module.exports = exports = async function () {
+  const tc = new TwitterClient();
+  await tc.initialize();
+  return tc;
+};
+
+
+class TwitterClient {
+
+  async initialize () {
+    const [ lookup, backup, cache ] = await Promise.all([
+      fs.readJson(resolve('twitter-config.json')).catch(() => null)
+        .then(makeFetcher),
+      fs.readJson(resolve('twitter-backup.json')).catch(() => ({})),
+      fs.readJson(resolve('twitter-cache.json')).catch(() => ({})),
+    ]);
+
+    this._lookup = lookup;
+    this._backupData = backup;
+    this._cache = cache;
+    this._presentTweets = Object.keys(cache);
+  }
+
+  async write () {
+    return Promise.all([
+      fs.writeFile(resolve('twitter-cache.json'),  JSON.stringify(this._cache,  null, 2)),
+      fs.writeFile(resolve('twitter-backup.json'), JSON.stringify(this._backupData, null, 2)),
+    ]);
+  }
+
+  async get (tweetids) {
+    if (!Array.isArray(tweetids)) tweetids = [ tweetids ];
+
+    tweetids = uniq(tweetids.map(parseTweetId));
+
+    let tweetsNeeded = this._missing(tweetids);
+
+    while (tweetsNeeded.length) {
+      log('Fetching tweets: ' + tweetsNeeded.join(', '));
+      const arriving = await Promise.all(chunk(tweetsNeeded, 99).map(this._lookup));
+      const tweetsRequested = tweetsNeeded;
+      tweetsNeeded = [];
+      const loaded = [];
+      for (const tweet of arriving.flat(1)) {
+        if (tweet.quoted_status_id_str && !this._cache[tweet.quoted_status_id_str]) {
+          tweetsNeeded.push(tweet.quoted_status_id_str);
+        }
+
+        this._backupData[tweet.id_str] = tweet;
+        this._cache[tweet.id_str] = tweetparse(tweet);
+        loaded.push(tweet.id_str);
+      }
+
+      const absent = difference(tweetsRequested, loaded);
+      for (const id of absent) {
+        if (!hasOwn(this._backupData, id)) {
+          log.error('Could not find tweet ' + id);
+          continue;
+        }
+        const tweet = this._backupData[id];
+
+        if (tweet) {
+          log('Pulled tweet from backup ' + id);
+          this._cache[id] = tweetparse(this._backupData[id]);
+        } else {
+          this._cache[id] = false;
+        }
+      }
+    }
+
+    return tweetids.map((id) => this._cache[id] || null);
+  }
+
+  _missing (tweetids) {
+    return difference(tweetids, this._presentTweets);
+  }
+}
+
+
+function makeFetcher (config) {
+  if (!config) return () => [];
+  const client = new Twitter(config);
+  return (tweetids) => client
+    .get('statuses/lookup', { id: tweetids.join(','), tweet_mode: 'extended' })
+    // .then((r) => { console.log({r}); return r; })
+    .catch((e) => { log.error(e); return []; });
+}
+
+const tweeturl = /https?:\/\/twitter\.com\/(?:#!\/)?(?:\w+)\/status(?:es)?\/(\d+)/i;
+const tweetidcheck = /^\d+$/;
+function parseTweetId (tweetid) {
+  // we can't trust an id that isn't a string
+  if (typeof tweetid !== 'string') return false;
+
+  const match = tweetid.match(tweeturl);
+  if (match) return match[1];
+  if (tweetid.match(tweetidcheck)) return tweetid;
+  return false;
+}

build/twitter-thread.js

@@ -0,0 +1,34 @@
+
+const twitterClient = require('./twitter-client');
+
+module.exports = exports = async function loadThread (tweetid) {
+  const tc = await twitterClient();
+
+  async function quoteds (tweet) {
+    if (!tweet.quoted_status_id_str) return [];
+    const [ qt ] = await tc.get(tweet.quoted_status_id_str);
+    if (!qt) return [];
+    return [ qt.id_str, ...(await quoteds(qt)) ];
+  }
+
+  const embeds = [];
+  const dependencies = [];
+  let id = tweetid;
+  do {
+    const [ tweet ] = await tc.get(id);
+    if (!tweet) break;
+    embeds.unshift(tweet.id_str);
+    dependencies.unshift(tweet.id_str);
+
+    if (tweet.quoted_status_id_str) {
+      const qts = await quoteds(tweet);
+      if (qts.length) dependencies.unshift(...qts);
+    }
+
+    id = tweet.in_reply_to_status_id_str;
+  } while (id);
+
+  await tc.write();
+
+  return [ embeds, dependencies ];
+};