More overhaul, image tasks and content assets now come from the same code.

gulp/content/files.js

@@ -1,95 +1,249 @@
 const path = require('path');
 const glob = require('../lib/glob');
-const memoize = require('memoizepromise');
+const getImageDimensions = require('../lib/dimensions');
-const getDimensions = require('../lib/dimensions');
+const getVideoDimensions = require('get-video-dimensions');
-const { keyBy } = require('lodash');
+const { keyBy, pick, filter, get, set, memoize } = require('lodash');
+const actions = require('../imgflow/actions');
+
+const ROOT = path.resolve(__dirname, '../..');
+
+function resolve (...args) {
+  args = args.filter(Boolean);
+  let fpath = args.shift();
+  if (!fpath) return ROOT;
+  if (fpath[0] === '/') fpath = fpath.slice(1);
+  return path.resolve(ROOT, fpath, ...args);
+}
+
+
+module.exports = exports = async function findAssets () {
+  const files = await glob('pages/**/*.{jpeg,jpg,png,gif,mp4}', { cwd: ROOT });
+  const map = {};
+  const assets = (await Promise.all(files.map(async (filepath) => {
+    const asset = new Asset(path.relative(ROOT, filepath));
+    await asset.load();
+    set(map, [ ...asset.base.split('/'), asset.name ], asset);
+    return asset;
+  }))).filter(Boolean);
+
+  Object.freeze(map);
+
+  function within (dir) {
+    const subset = filter(assets, { dir });
+    return {
+      get titlecard () {
+        return get(filter(subset, { name: 'titlecard' }), [ 0, 'url' ]);
+      },
+      get assets () {
+        return keyBy(subset.map((a) => a.webready()), 'name');
+      },
+      get all () {
+        return [ ...subset ];
+      },
+    };
+  }
+
+  return {
+    map,
+    for: memoize(within),
+    get tasks () {
+      return assets.map((a) => a.tasks()).flat(1);
+    },
+    get all () {
+      return [ ...assets ];
+    },
+  };
+};
+
+
+const JPG  = '.jpg';
+const JPEG = '.jpeg';
+const PNG  = '.png';
+const GIF  = '.gif';
+const MP4  = '.mp4';
+const M4V  = '.m4v';
+
+const FILETYPE = {
+  [JPG]:  'jpeg',
+  [JPEG]: 'jpeg',
+  [PNG]:  'png',
+  [GIF]:  'gif',
+  [MP4]:  'mp4',
+  [M4V]:  'mp4',
+};
 
 const RESOLUTIONS = [ 2048, 1024, 768, 576, 300, 100 ];
 
-module.exports = exports = function () {
-  return memoize(async (cwd, siteDir) => {
-    const imageFiles = (await glob('{*,_images/*}.{jpeg,jpg,png,gif,mp4}', { cwd }));
 
-    const images = (await Promise.all(imageFiles.map(async (imgpath) => {
+class Asset {
 
-      const ext = path.extname(imgpath);
+  constructor (filepath) {
-      let basename = path.basename(imgpath, ext);
+    const file = path.parse(filepath);
+    let { base: basename, name } = file;
 
-      if (basename === 'titlecard') return;
+    this.preprocessed = false;
-
+    if (name[0] === '_') {
-      if (ext === '.mp4') {
+      this.preprocessed = true;
-        return {
+      file.name = name = name.slice(1);
-          name: basename,
+      file.basename = basename = basename.slice(1);
-          type: 'movie',
-          full: path.join(siteDir, `${basename}${ext}`),
-        };
     }
 
-      const dimensions = await getDimensions(path.resolve(cwd, imgpath));
+    this.type = FILETYPE[file.ext] || file.ext.slice(1);
-      const { width, height } = dimensions;
+    if ([ JPG, JPEG, PNG, GIF ].includes(file.ext)) {
-      dimensions.ratioH = Math.round((height / width) * 100);
+      this.kind = 'image';
-      dimensions.ratioW = Math.round((width / height) * 100);
+    } else if ([ MP4, M4V ].includes(file.ext)) {
-      if (dimensions.ratioH > 100) {
+      this.kind = 'video';
-        dimensions.orientation = 'tall';
-      } else if (dimensions.ratioH === 100) {
-        dimensions.orientation = 'square';
     } else {
-        dimensions.orientation = 'wide';
+      this.kind = 'raw';
     }
 
-      const filetype = {
+    // remove the pages root and any _images segment from the dir
-        '.jpeg': 'jpeg',
+    const dir = file.dir.split('/');
-        '.jpg':  'jpeg',
+    if (dir[0] === 'pages') dir.shift();
-        '.png':  'png',
+    const i = dir.indexOf('_images');
-        '.gif':  'gif',
+    if (i > -1) dir.splice(i, 1);
-      }[ext];
 
-      if (basename[0] === '_') {
+    this.input    = resolve(filepath);           // /local/path/to/pages/file.ext
-        basename = basename.slice(1);
+    this.cwd      = resolve(file.dir);           // /local/path/to/pages/, pages/folder, pages/folder/subfolder
-        return {
+    this.base     = path.join(...dir);           // '', 'folder', 'folder/subfolder'
-          name: basename,
+    this.dir      = path.join('/', ...dir);      // /, /folder, /folder/subfolder
-          type: 'image',
+    this.name     = name;                        // index, fileA, fileB
-          sizes: [
+    this.basename = basename;                    // index.ext, fileA.ext, fileB.ext
-            {
+    this.dest     = path.join('dist/', ...dir);  // dist/, dist/folder, dist/folder/subfolder
-              url: path.join(siteDir, `${basename}${ext}`),
+    this.ext      = file.ext;
-              width: dimensions.width,
+
-              height: dimensions.height,
+    this.out = path.join(this.dest, `${this.name}${this.preprocessed ? this.ext : '.' + this.type}`);
-            },
+    this.url = path.join(this.dir,  `${this.name}${this.preprocessed ? this.ext : '.' + this.type}`);
-          ],
+  }
+
+  load () {
+    switch (this.kind) {
+    case 'video': return this.loadVideo();
+    case 'image': return this.loadImage();
+    default:
+    }
+  }
+
+  async loadImage () {
+
+    const { width, height } = await getImageDimensions(this.input);
+
+    const ratioH = Math.round((height / width) * 100);
+    const ratioW = Math.round((width / height) * 100);
+    let orientation = 'wide';
+    if (ratioH > 100) {
+      orientation = 'tall';
+    } else if (ratioH === 100) {
+      orientation = 'square';
+    }
+
+    this.dimensions = {
+      width,
+      height,
+      ratioH,
+      ratioW,
+      orientation,
     };
-      }
 
-      const sizes = [
+    if (this.preprocessed) {
+      this.sizes = [ {
+        output: resolve(this.out),
+        url: this.url,
+        width,
+        height,
+      } ];
+    } else {
+      this.sizes = [
         {
-          url: path.join(siteDir, `${basename}.${filetype}`),
+          output: resolve(this.out),
-          width: dimensions.width,
+          url: this.url,
-          height: dimensions.height,
+          width,
+          height,
         },
       ];
 
       for (const w of RESOLUTIONS) {
-        if (w > dimensions.width) continue;
+        if (w > width) continue;
-        sizes.push({
+        this.sizes.push({
-          url: path.join(siteDir, `${basename}.${w}w.${filetype}`),
+          output: resolve(this.dest, `${this.name}.${w}w.${this.type}`),
+          url: path.join(this.dir,  `${this.name}.${w}w.${this.type}`),
           width: w,
-          height: Math.ceil((w / dimensions.width) * dimensions.height),
+          height: Math.ceil((w / width) * height),
         });
       }
 
-      sizes.reverse();
+      this.sizes.reverse();
+    }
 
+    return this;
+  }
+
+  async loadVideo () {
+    const { width, height } = await getVideoDimensions(this.input);
+
+    const ratioH = Math.round((height / width) * 100);
+    const ratioW = Math.round((width / height) * 100);
+    let orientation = 'wide';
+    if (ratioH > 100) {
+      orientation = 'tall';
+    } else if (ratioH === 100) {
+      orientation = 'square';
+    }
+
+    this.dimensions = {
+      width,
+      height,
+      ratioH,
+      ratioW,
+      orientation,
+    };
+
+    this.sizes = [ {
+      output: resolve(this.dest, this.basename),
+      url: path.join(this.dir, this.basename),
+      width,
+      height,
+    } ];
+
+    return this;
+  }
+
+  toJson () {
+    return pick(this, [
+      'preprocessed',
+      'type',
+      'kind',
+      'input',
+      'cwd',
+      'base',
+      'dir',
+      'name',
+      'basename',
+      'dest',
+      'ext',
+      'dimensions',
+    ]);
+  }
+
+  webready () {
+    const { kind, name } = this;
     return {
-        name: basename,
+      kind,
-        type: 'image',
+      name,
-        sizes,
+      sizes: this.sizes.map((s) => pick(s, [ 'url', 'width', 'height' ])),
     };
-    }))).filter(Boolean);
+  }
 
-    const titlecard = (await glob('titlecard.{jpeg,jpg,png,gif}', { cwd }))[0];
+  tasks () {
+    return this.sizes.map(({ output, width }) => ({
+      input: this.input,
+      output,
+      format: this.preprocessed ? undefined : this.type,
+      width:  this.preprocessed ? undefined : width,
+      action: this.preprocessed ? actions.copy : actions.image,
+    }));
+  }
 
-    return {
+}
-      images: keyBy(images, 'name'),
+
-      titlecard: titlecard ? path.join(siteDir, titlecard) : '/images/titlecard.png',
+exports.Asset = Asset;
-    };
-  });
-};

gulp/content/index.js

@@ -8,28 +8,28 @@ const tweetparse = require('../lib/tweetparse');
 const getEngines = require('./renderers');
 const Twitter = require('twitter-lite');
 const Page = require('./page');
-const createFileLoader = require('./files');
+const createAssetLoader = require('./files');
 
 const ROOT = path.resolve(__dirname, '../..');
 
 exports.parse = async function parsePageContent () {
-  const [ files, twitter, twitterBackup, twitterCache ] = await Promise.all([
+  const [ files, twitter, twitterBackup, twitterCache, Assets ] = await Promise.all([
     glob('pages/**/*.{md,hbs,html,xml}', { cwd: ROOT }),
     fs.readJson(resolve('twitter-config.json')).catch(() => null)
       .then(getTwitterClient),
     fs.readJson(resolve('twitter-backup.json')).catch(() => ({})),
     fs.readJson(resolve('twitter-cache.json')).catch(() => ({})),
+    createAssetLoader(),
   ]);
 
 
   let tweetsNeeded = [];
   const tweetsPresent = Object.keys(twitterCache);
-  const artifactLoader = createFileLoader();
 
   let pages = await Promise.map(files, async (filepath) => {
     const page = new Page(filepath);
     if (!page.input) return;
-    await page.load({ artifactLoader });
+    await page.load({ Assets });
 
     if (page.tweets.length) {
       const missing = difference(page.tweets, tweetsPresent);

gulp/content/page.js

@@ -57,13 +57,20 @@ module.exports = exports = class Page  {
     // this is not a page file
     if (![ MD, HBS, HTML, XML ].includes(ext)) return false;
 
-    this.input = resolve(filepath);                     // /local/path/to/pages/folder/file.ext
+    // remove the pages root and any _images segment from the dir
+    const dir = file.dir.split('/');
+    if (dir[0] === 'pages') dir.shift();
+    const i = dir.indexOf('_images');
+    if (i > -1) dir.splice(i, 1);
+
+    this.input    = resolve(filepath);           // /local/path/to/pages/file.ext
     this.cwd      = resolve(file.dir);           // /local/path/to/pages/, pages/folder, pages/folder/subfolder
-    this.base = file.dir.replace(/^pages\/?/, '');       // '', 'folder', 'folder/subfolder'
+    this.base     = path.join(...dir);           // '', 'folder', 'folder/subfolder'
-    this.dir = file.dir.replace(/^pages\/?/, '/');       // /, /folder, /folder/subfolder
+    this.dir      = path.join('/', ...dir);      // /, /folder, /folder/subfolder
     this.name     = name;                        // index, fileA, fileB
     this.basename = basename;                    // index.ext, fileA.ext, fileB.ext
-    this.dest = file.dir.replace(/^pages\/?/, 'dist/');  // dist/, dist/folder, dist/folder/subfolder
+    this.dest     = path.join('dist/', ...dir);  // dist/, dist/folder, dist/folder/subfolder
+    this.ext      = file.ext;
 
     var isIndexPage = (name === 'index');
     var isCleanUrl = [ HBS, MD ].includes(ext);
@@ -102,13 +109,14 @@ module.exports = exports = class Page  {
 
   }
 
-  async load ({ artifactLoader }) {
+  async load ({ Assets }) {
-    const [ raw, { ctime, mtime }, { images, titlecard } ] = await Promise.all([
+    const [ raw, { ctime, mtime } ] = await Promise.all([
       fs.readFile(this.input).catch(() => null),
       fs.stat(this.input).catch(() => {}),
-      artifactLoader(this.cwd, this.dir),
     ]);
 
+    const { titlecard, assets } = Assets.for(this.dir);
+
     // empty file
     if (!raw || !ctime) {
       log.error('Could not load page: ' + this.filepath);
@@ -124,7 +132,7 @@ module.exports = exports = class Page  {
 
     this.source = body;
     this.meta = meta;
-    this.images = images;
+    this.images = assets;
     this.titlecard = titlecard;
     this.tweets = (meta.tweets || []).map(parseTweetId);
     this.dateCreated = meta.date && new Date(meta.date) || ctime;
@@ -151,6 +159,7 @@ module.exports = exports = class Page  {
       'base',
       'dir',
       'name',
+      'ext',
       'basename',
       'dest',
       'out',
@@ -159,6 +168,7 @@ module.exports = exports = class Page  {
       'engine',
       'source',
       'images',
+      'assets',
       'titlecard',
       'tweets',
       'classes',

gulp/imgflow/index.js

@@ -1,130 +1,47 @@
 const path = require('path');
-const glob = require('../lib/glob');
+const { uniqBy } = require('lodash');
-const { sortBy, uniqBy } = require('lodash');
 const Promise = require('bluebird');
 const fs = require('fs-extra');
-const log = require('fancy-log');
 const actions = require('./actions');
-const getDimensions = require('../lib/dimensions');
+const createAssetLoader = require('../content/files');
 
-const CWD = path.resolve(__dirname, '../..');
+
-const PAGES = path.join(CWD, 'pages');
+const ROOT = path.resolve(__dirname, '../..');
-const SOURCE = path.resolve(CWD, 'pages/**/*.{jpeg,jpg,png,gif,mp4}');
-const MANIFEST_PATH = path.resolve(CWD, 'if-manifest.json');
-const REV_MANIFEST_PATH = path.resolve(CWD, 'rev-manifest.json');
-const MEDIA_INDEX = path.resolve(CWD, 'twitter-media.json');
 const CACHE = 'if-cache';
-const revHash = require('rev-hash');
-const revPath = require('rev-path');
 
+const { changed, execute } = require('./pipeline');
 
-const LOG = {
+function resolve (...args) {
-  new:    true,
+  args = args.filter(Boolean);
-  update: true,
+  let fpath = args.shift();
-  skip:   true,
+  if (!fpath) return ROOT;
-  rebuild:  true,
+  if (fpath[0] === '/') fpath = fpath.slice(1);
-  cached: false,
+  return path.resolve(ROOT, fpath, ...args);
-  copy: false,
+}
-};
 
 module.exports = exports = async function postImages ({ rev = false }) {
 
-  var manifest;
+  const [ manifest, { tasks } ] = await Promise.all([
-  try {
+    fs.readJson(resolve('if-manifest.json')).catch(() => ({})),
-    manifest = JSON.parse(await fs.readFile(MANIFEST_PATH));
+    createAssetLoader(),
-  } catch (e) {
+    fs.ensureDir(resolve(CACHE)),
-    manifest = {};
+  ]);
-  }
 
-  await fs.ensureDir(path.resolve(CWD, CACHE));
+  const filtered = await changed(manifest, tasks);
-
-  const allfiles = (await glob(SOURCE));
-  const tasks = [];
-
-  for (const filepath of allfiles) {
-    const input = path.relative(CWD, filepath);
-    const output = path.relative(PAGES, filepath).replace('/_images', '');
-    const file = path.parse(output);
-    // console.log(input, output);
-
-    // is a titlecard image or a video
-    if (file.name === 'titlecard' || file.ext === '.mp4') {
-      tasks.push({
-        input,
-        output,
-        action: actions.copy,
-      });
-      continue;
-    }
-
-    // is a file we've pre-sized and do not want processed
-    if (file.name[0] === '_') {
-      tasks.push({
-        input,
-        output: path.format({ ...file, base: file.base.substring(1) }),
-        action: actions.copy,
-      });
-      continue;
-    }
-
-    const format = {
-      '.jpeg': 'jpeg',
-      '.jpg':  'jpeg',
-      '.png':  'png',
-      '.gif':  'gif',
-    }[file.ext];
-
-    if (!format) throw new Error('Got an unexpected format: ' + file.ext);
-
-    const dimensions = await getDimensions(filepath);
-
-    tasks.push({
-      input: filepath,
-      output: `${file.dir}/${file.name}.${format}`,
-      format,
-      action: actions.image,
-    });
-
-    for (const w of [ 2048, 1024, 768, 576, 300, 100 ]) {
-      if (w > dimensions.width) continue;
-      tasks.push({
-        input: filepath,
-        output: `${file.dir}/${file.name}.${w}w.${format}`,
-        format,
-        width: w,
-        action: actions.image,
-      });
-    }
-
-  }
-
-  const filtered = await filter(manifest, tasks);
   await execute(manifest, filtered, rev);
 };
 
 exports.prod = function imagesProd () { return exports({ rev: true }); };
 
 exports.twitter = async function twitterImages ({ rev = false }) {
-  await fs.ensureDir(path.resolve(CWD, CACHE));
+  const [ manifest, media ] = await Promise.all([
+    fs.readJson(resolve('if-manifest.json')).catch(() => ({})),
+    fs.readJson(resolve('twitter-media.json')).catch(() => ([])),
+    fs.ensureDir(resolve(CACHE)),
+  ]);
 
-  var manifest;
+  const tasks = uniqBy(media, 'output').map((m) => ({ ...m, action: actions.fetch }));
-  try {
+  const filtered = await changed(manifest, tasks);
-    manifest = JSON.parse(await fs.readFile(MANIFEST_PATH));
-  } catch (e) {
-    manifest = {};
-  }
-
-  var media;
-  try {
-    media = JSON.parse(await fs.readFile(MEDIA_INDEX));
-  } catch (e) {
-    media = [];
-  }
-
-  media = uniqBy(media, 'output');
-
-  const tasks = media.map((m) => ({ ...m, action: actions.fetch }));
-  const filtered = await filter(manifest, tasks);
   await execute(manifest, filtered, rev);
 };
 
@@ -132,16 +49,11 @@ exports.twitter.prod = function imagesProd () { return exports.twitter({ rev: tr
 
 
 exports.favicon = async function favicon ({ rev = false }) {
-  await fs.ensureDir(path.resolve(CWD, CACHE));
+  const input = resolve('favicon.png');
-
+  const [ manifest ] = await Promise.all([
-  const input = path.resolve(CWD, 'favicon.png');
+    fs.readJson(resolve('if-manifest.json')).catch(() => ({})),
-
+    fs.ensureDir(resolve(CACHE)),
-  var manifest;
+  ]);
-  try {
-    manifest = JSON.parse(await fs.readFile(MANIFEST_PATH));
-  } catch (e) {
-    manifest = {};
-  }
 
   const tasks = [ 32, 57, 64, 76, 96, 114, 120, 128, 144, 152, 180, 192, 196, 228 ].map((width) => ({
     input,
@@ -158,169 +70,9 @@ exports.favicon = async function favicon ({ rev = false }) {
     action: actions.image,
   });
 
-  const filtered = await filter(manifest, tasks);
+  const filtered = await changed(manifest, tasks);
   await execute(manifest, filtered, rev);
 };
 
 exports.favicon.prod = function imagesProd () { return exports.favicon({ rev: true }); };
 
-
-async function filter (manifest, tasks) {
-  const statMap = new Map();
-  async function stat (f) {
-    if (statMap.has(f)) return statMap.get(f);
-
-    const p = fs.stat(path.resolve(CWD, f))
-      .catch(() => null)
-      .then((stats) => (stats && Math.floor(stats.mtimeMs / 1000)));
-
-    statMap.set(f, p);
-    return p;
-  }
-
-  return Promise.filter(tasks, async (task) => {
-
-    const local = task.input.slice(0, 4) !== 'http';
-    const hash = task.action.name + '.' + revHash(task.input) + '|' + revHash(task.output);
-    const cachePath = path.join(CACHE, `${hash}${path.extname(task.output)}`);
-    const [ inTime, outTime, cachedTime ] = await Promise.all([
-      local && stat(path.resolve(CWD, task.input)),
-      stat(path.resolve(CWD, 'dist', task.output)),
-      stat(path.resolve(CWD, cachePath)),
-    ]);
-
-    task.manifest = manifest[hash];
-    task.hash = hash;
-    task.cache = cachePath;
-
-    // how did this happen?
-    if (local && !inTime) {
-      log.error('Input file could not be found?', task.input);
-      return false;
-    }
-
-    // never seen this file before
-    if (!task.manifest) {
-      task.apply = {
-        hash,
-        input: task.input,
-        output: task.output,
-        mtime: inTime,
-      };
-      task.log = [ 'new', task.input, task.output, hash ];
-      return true;
-    }
-
-    // file modification time does not match last read, rebuild
-    if (local && inTime > task.manifest.mtime) {
-      task.log = [ 'update', task.input, task.output ];
-      task.apply = {
-        mtime: inTime,
-      };
-      return true;
-    }
-
-    task.apply = {
-      mtime: local ? inTime : Math.floor(Date.now() / 1000),
-    };
-
-    // target file exists, nothing to do
-    if (outTime) {
-      return false;
-      // task.log = [ 'skip', task.input, task.output, inTime, task.manifest.mtime ];
-      // task.action = null;
-      // return true;
-    }
-
-    // file exists in the cache, change the task to a copy action
-    if (cachedTime) {
-      task.log = [ 'cached', task.input, task.output ];
-      task.action = actions.copy;
-      task.input = cachePath;
-      return true;
-    }
-
-    // task is a file copy
-    if (task.action === actions.copy) {
-      task.log = [ 'copy', task.input, task.output ];
-      return true;
-    }
-
-    // file does not exist in cache, build it
-    task.log = [ 'rebuild', task.input, task.output ];
-    return true;
-  });
-}
-
-
-async function execute (manifest, tasks, rev) {
-  const lastSeen = Math.floor(Date.now() / 1000);
-  const revManifest = {};
-
-  let writeCounter = 0;
-  let lastWriteTime = 0;
-  async function writeManifest (force) {
-    if (!force && rev) return; // disable interim writes during prod builds.
-    if (!force && ++writeCounter % 100) return;
-    const now = Date.now();
-    if (!force && now - lastWriteTime < 10000) return;
-    lastWriteTime = now;
-    await fs.writeFile(MANIFEST_PATH, JSON.stringify(manifest, null, 2));
-  }
-
-  await Promise.map(sortBy(tasks, [ 'input', 'output' ]), async (task) => {
-    const output = path.resolve(CWD, 'dist', task.output);
-
-    const result = task.action && await task.action({ ...task, output });
-    const apply = task.apply || {};
-    if (task.log && LOG[task.log[0]]) log.info(...task.log);
-    apply.lastSeen = lastSeen;
-    apply.lastSeenHuman = new Date();
-
-    if (!result) log('Nothing happened?', task);
-
-    const rhash = result && revHash(result);
-    const hashedPath = revPath(task.output, rhash);
-    apply.revHash = rhash;
-    apply.revPath = hashedPath;
-
-    if (rev && rhash) {
-      const rOutPath = task.output;
-      const rNewPath = hashedPath;
-
-      revManifest[rOutPath] = rNewPath;
-
-      await fs.copy(output, path.resolve(CWD, 'dist', hashedPath));
-    }
-
-    manifest[task.hash] = { ...manifest[task.hash], ...apply };
-    await writeManifest();
-
-  }, { concurrency: rev ? 20 : 10 });
-
-  // filter unseen files from history
-  // manifest = omitBy(manifest, (m) => m.lastSeen !== lastSeen);
-
-  await writeManifest(true);
-
-  if (rev) {
-    let originalManifest = {};
-    try {
-      if (await fs.exists(REV_MANIFEST_PATH)) {
-        originalManifest = JSON.parse(await fs.readFile(REV_MANIFEST_PATH));
-      }
-    } catch (e) {
-      // do nothing
-    }
-
-    Object.assign(originalManifest, revManifest);
-
-    await fs.writeFile(REV_MANIFEST_PATH, JSON.stringify(originalManifest, null, 2));
-  }
-
-}
-
-if (require.main === module) {
-  exports().catch(console.error).then(() => process.exit()); // eslint-disable-line
-}
-

gulp/imgflow/pipeline.js

@@ -0,0 +1,179 @@
+const path = require('path');
+const { sortBy } = require('lodash');
+const Promise = require('bluebird');
+const fs = require('fs-extra');
+const log = require('fancy-log');
+const actions = require('./actions');
+const revHash = require('rev-hash');
+const revPath = require('rev-path');
+
+const CWD = path.resolve(__dirname, '../..');
+const PAGES = path.join(CWD, 'pages');
+const SOURCE = path.resolve(PAGES, '**/*.{jpeg,jpg,png,gif,mp4}');
+const MANIFEST_PATH = path.resolve(CWD, 'if-manifest.json');
+const REV_MANIFEST_PATH = path.resolve(CWD, 'rev-manifest.json');
+const MEDIA_INDEX = path.resolve(CWD, 'twitter-media.json');
+const CACHE = 'if-cache';
+
+const LOG = {
+  new:     true,
+  update:  true,
+  skip:    true,
+  rebuild: true,
+  cached:  false,
+  copy:    false,
+};
+
+exports.changed = async function changed (manifest, tasks) {
+  const statMap = new Map();
+  async function stat (f) {
+    if (statMap.has(f)) return statMap.get(f);
+
+    const p = fs.stat(path.resolve(CWD, f))
+      .catch(() => null)
+      .then((stats) => (stats && Math.floor(stats.mtimeMs / 1000)));
+
+    statMap.set(f, p);
+    return p;
+  }
+
+  return Promise.filter(tasks, async (task) => {
+
+    const local = task.input.slice(0, 4) !== 'http';
+    const hash = task.action.name + '.' + revHash(task.input) + '|' + revHash(task.output);
+    const cachePath = path.join(CACHE, `${hash}${path.extname(task.output)}`);
+    const [ inTime, outTime, cachedTime ] = await Promise.all([
+      local && stat(path.resolve(CWD, task.input)),
+      stat(path.resolve(CWD, 'dist', task.output)),
+      stat(path.resolve(CWD, cachePath)),
+    ]);
+
+    task.manifest = manifest[hash];
+    task.hash = hash;
+    task.cache = cachePath;
+
+    // how did this happen?
+    if (local && !inTime) {
+      log.error('Input file could not be found?', task.input);
+      return false;
+    }
+
+    // never seen this file before
+    if (!task.manifest) {
+      task.apply = {
+        hash,
+        input: task.input,
+        output: task.output,
+        mtime: inTime,
+      };
+      task.log = [ 'new', task.input, task.output, hash ];
+      return true;
+    }
+
+    // file modification time does not match last read, rebuild
+    if (local && inTime > task.manifest.mtime) {
+      task.log = [ 'update', task.input, task.output ];
+      task.apply = {
+        mtime: inTime,
+      };
+      return true;
+    }
+
+    task.apply = {
+      mtime: local ? inTime : Math.floor(Date.now() / 1000),
+    };
+
+    // target file exists, nothing to do
+    if (outTime) {
+      return false;
+      // task.log = [ 'skip', task.input, task.output, inTime, task.manifest.mtime ];
+      // task.action = null;
+      // return true;
+    }
+
+    // file exists in the cache, change the task to a copy action
+    if (cachedTime) {
+      task.log = [ 'cached', task.input, task.output ];
+      task.action = actions.copy;
+      task.input = cachePath;
+      return true;
+    }
+
+    // task is a file copy
+    if (task.action === actions.copy) {
+      task.log = [ 'copy', task.input, task.output ];
+      return true;
+    }
+
+    // file does not exist in cache, build it
+    task.log = [ 'rebuild', task.input, task.output ];
+    return true;
+  });
+};
+
+
+exports.execute = async function execute (manifest, tasks, rev) {
+  const lastSeen = Math.floor(Date.now() / 1000);
+  const revManifest = {};
+
+  let writeCounter = 0;
+  let lastWriteTime = 0;
+  async function writeManifest (force) {
+    if (!force && rev) return; // disable interim writes during prod builds.
+    if (!force && ++writeCounter % 100) return;
+    const now = Date.now();
+    if (!force && now - lastWriteTime < 10000) return;
+    lastWriteTime = now;
+    await fs.writeFile(MANIFEST_PATH, JSON.stringify(manifest, null, 2));
+  }
+
+  await Promise.map(sortBy(tasks, [ 'input', 'output' ]), async (task) => {
+    const output = path.resolve(CWD, 'dist', task.output);
+
+    const result = task.action && await task.action({ ...task, output });
+    const apply = task.apply || {};
+    if (task.log && LOG[task.log[0]]) log.info(...task.log);
+    apply.lastSeen = lastSeen;
+    apply.lastSeenHuman = new Date();
+
+    if (!result) log('Nothing happened?', task);
+
+    const rhash = result && revHash(result);
+    const hashedPath = revPath(task.output, rhash);
+    apply.revHash = rhash;
+    apply.revPath = hashedPath;
+
+    if (rev && rhash) {
+      const rOutPath = task.output;
+      const rNewPath = hashedPath;
+
+      revManifest[rOutPath] = rNewPath;
+
+      await fs.copy(output, path.resolve(CWD, 'dist', hashedPath));
+    }
+
+    manifest[task.hash] = { ...manifest[task.hash], ...apply };
+    await writeManifest();
+
+  }, { concurrency: rev ? 20 : 10 });
+
+  // filter unseen files from history
+  // manifest = omitBy(manifest, (m) => m.lastSeen !== lastSeen);
+
+  await writeManifest(true);
+
+  if (rev) {
+    let originalManifest = {};
+    try {
+      if (await fs.exists(REV_MANIFEST_PATH)) {
+        originalManifest = JSON.parse(await fs.readFile(REV_MANIFEST_PATH));
+      }
+    } catch (e) {
+      // do nothing
+    }
+
+    Object.assign(originalManifest, revManifest);
+
+    await fs.writeFile(REV_MANIFEST_PATH, JSON.stringify(originalManifest, null, 2));
+  }
+};

package-lock.json

@@ -522,6 +522,12 @@
       "integrity": "sha512-uMgjozySS8adZZYePpaWs8cxB9/kdzmpX6SgJZ+wbz1K5eYk5QMYDVJaZKhxyIHUdnnJkfR7SVgStgH7LkGUyg==",
       "dev": true
     },
+    "any-promise": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz",
+      "integrity": "sha1-q8av7tzqUugJzcA3au0845Y10X8=",
+      "dev": true
+    },
     "anymatch": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-2.0.0.tgz",
@@ -4005,6 +4011,15 @@
       "integrity": "sha1-3BXKHGcjh8p2vTesCjlbogQqLCg=",
       "dev": true
     },
+    "get-video-dimensions": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/get-video-dimensions/-/get-video-dimensions-1.0.0.tgz",
+      "integrity": "sha1-/H5ayBw5JEH1uG1Q3XeDiptTFHo=",
+      "dev": true,
+      "requires": {
+        "mz": "^1.2.1"
+      }
+    },
     "getpass": {
       "version": "0.1.7",
       "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz",
@@ -6411,6 +6426,17 @@
       "integrity": "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==",
       "dev": true
     },
+    "mz": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/mz/-/mz-1.3.0.tgz",
+      "integrity": "sha1-BvCT/dmVagbTfhsegTROJ0eMQvA=",
+      "dev": true,
+      "requires": {
+        "native-or-bluebird": "1",
+        "thenify": "3",
+        "thenify-all": "1"
+      }
+    },
     "nan": {
       "version": "2.14.0",
       "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.0.tgz",
@@ -6436,6 +6462,12 @@
         "to-regex": "^3.0.1"
       }
     },
+    "native-or-bluebird": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/native-or-bluebird/-/native-or-bluebird-1.2.0.tgz",
+      "integrity": "sha1-OcR7/Xgl0fuf+tMiEK4l2q3xAck=",
+      "dev": true
+    },
     "natural-compare": {
       "version": "1.4.0",
       "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz",
@@ -9033,6 +9065,24 @@
       "integrity": "sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=",
       "dev": true
     },
+    "thenify": {
+      "version": "3.3.0",
+      "resolved": "https://registry.npmjs.org/thenify/-/thenify-3.3.0.tgz",
+      "integrity": "sha1-5p44obq+lpsBCCB5eLn2K4hgSDk=",
+      "dev": true,
+      "requires": {
+        "any-promise": "^1.0.0"
+      }
+    },
+    "thenify-all": {
+      "version": "1.6.0",
+      "resolved": "https://registry.npmjs.org/thenify-all/-/thenify-all-1.6.0.tgz",
+      "integrity": "sha1-GhkY1ALY/D+Y+/I02wvMjMEOlyY=",
+      "dev": true,
+      "requires": {
+        "thenify": ">= 3.1.0 < 4"
+      }
+    },
     "through": {
       "version": "2.3.8",
       "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz",

package.json

@@ -40,6 +40,7 @@
     "forever": "~2.0.0",
     "front-matter": "~3.1.0",
     "fs-extra": "~8.1.0",
+    "get-video-dimensions": "~1.0.0",
     "glob": "~7.1.6",
     "gm": "~1.23.1",
     "gulp": "~4.0.2",