diff --git a/gulp/content/files.js b/gulp/content/files.js index d066ce0..0dd5e48 100644 --- a/gulp/content/files.js +++ b/gulp/content/files.js @@ -1,95 +1,249 @@ const path = require('path'); const glob = require('../lib/glob'); -const memoize = require('memoizepromise'); -const getDimensions = require('../lib/dimensions'); -const { keyBy } = require('lodash'); +const getImageDimensions = require('../lib/dimensions'); +const getVideoDimensions = require('get-video-dimensions'); +const { keyBy, pick, filter, get, set, memoize } = require('lodash'); +const actions = require('../imgflow/actions'); + +const ROOT = path.resolve(__dirname, '../..'); + +function resolve (...args) { + args = args.filter(Boolean); + let fpath = args.shift(); + if (!fpath) return ROOT; + if (fpath[0] === '/') fpath = fpath.slice(1); + return path.resolve(ROOT, fpath, ...args); +} + + +module.exports = exports = async function findAssets () { + const files = await glob('pages/**/*.{jpeg,jpg,png,gif,mp4}', { cwd: ROOT }); + const map = {}; + const assets = (await Promise.all(files.map(async (filepath) => { + const asset = new Asset(path.relative(ROOT, filepath)); + await asset.load(); + set(map, [ ...asset.base.split('/'), asset.name ], asset); + return asset; + }))).filter(Boolean); + + Object.freeze(map); + + function within (dir) { + const subset = filter(assets, { dir }); + return { + get titlecard () { + return get(filter(subset, { name: 'titlecard' }), [ 0, 'url' ]); + }, + get assets () { + return keyBy(subset.map((a) => a.webready()), 'name'); + }, + get all () { + return [ ...subset ]; + }, + }; + } + + return { + map, + for: memoize(within), + get tasks () { + return assets.map((a) => a.tasks()).flat(1); + }, + get all () { + return [ ...assets ]; + }, + }; +}; + + +const JPG = '.jpg'; +const JPEG = '.jpeg'; +const PNG = '.png'; +const GIF = '.gif'; +const MP4 = '.mp4'; +const M4V = '.m4v'; + +const FILETYPE = { + [JPG]: 'jpeg', + [JPEG]: 'jpeg', + [PNG]: 'png', + [GIF]: 'gif', + [MP4]: 'mp4', + [M4V]: 'mp4', +}; const RESOLUTIONS = [ 2048, 1024, 768, 576, 300, 100 ]; -module.exports = exports = function () { - return memoize(async (cwd, siteDir) => { - const imageFiles = (await glob('{*,_images/*}.{jpeg,jpg,png,gif,mp4}', { cwd })); - const images = (await Promise.all(imageFiles.map(async (imgpath) => { +class Asset { - const ext = path.extname(imgpath); - let basename = path.basename(imgpath, ext); + constructor (filepath) { + const file = path.parse(filepath); + let { base: basename, name } = file; - if (basename === 'titlecard') return; + this.preprocessed = false; + if (name[0] === '_') { + this.preprocessed = true; + file.name = name = name.slice(1); + file.basename = basename = basename.slice(1); + } - if (ext === '.mp4') { - return { - name: basename, - type: 'movie', - full: path.join(siteDir, `${basename}${ext}`), - }; - } + this.type = FILETYPE[file.ext] || file.ext.slice(1); + if ([ JPG, JPEG, PNG, GIF ].includes(file.ext)) { + this.kind = 'image'; + } else if ([ MP4, M4V ].includes(file.ext)) { + this.kind = 'video'; + } else { + this.kind = 'raw'; + } - const dimensions = await getDimensions(path.resolve(cwd, imgpath)); - const { width, height } = dimensions; - dimensions.ratioH = Math.round((height / width) * 100); - dimensions.ratioW = Math.round((width / height) * 100); - if (dimensions.ratioH > 100) { - dimensions.orientation = 'tall'; - } else if (dimensions.ratioH === 100) { - dimensions.orientation = 'square'; - } else { - dimensions.orientation = 'wide'; - } + // remove the pages root and any _images segment from the dir + const dir = file.dir.split('/'); + if (dir[0] === 'pages') dir.shift(); + const i = dir.indexOf('_images'); + if (i > -1) dir.splice(i, 1); - const filetype = { - '.jpeg': 'jpeg', - '.jpg': 'jpeg', - '.png': 'png', - '.gif': 'gif', - }[ext]; + this.input = resolve(filepath); // /local/path/to/pages/file.ext + this.cwd = resolve(file.dir); // /local/path/to/pages/, pages/folder, pages/folder/subfolder + this.base = path.join(...dir); // '', 'folder', 'folder/subfolder' + this.dir = path.join('/', ...dir); // /, /folder, /folder/subfolder + this.name = name; // index, fileA, fileB + this.basename = basename; // index.ext, fileA.ext, fileB.ext + this.dest = path.join('dist/', ...dir); // dist/, dist/folder, dist/folder/subfolder + this.ext = file.ext; - if (basename[0] === '_') { - basename = basename.slice(1); - return { - name: basename, - type: 'image', - sizes: [ - { - url: path.join(siteDir, `${basename}${ext}`), - width: dimensions.width, - height: dimensions.height, - }, - ], - }; - } + this.out = path.join(this.dest, `${this.name}${this.preprocessed ? this.ext : '.' + this.type}`); + this.url = path.join(this.dir, `${this.name}${this.preprocessed ? this.ext : '.' + this.type}`); + } - const sizes = [ + load () { + switch (this.kind) { + case 'video': return this.loadVideo(); + case 'image': return this.loadImage(); + default: + } + } + + async loadImage () { + + const { width, height } = await getImageDimensions(this.input); + + const ratioH = Math.round((height / width) * 100); + const ratioW = Math.round((width / height) * 100); + let orientation = 'wide'; + if (ratioH > 100) { + orientation = 'tall'; + } else if (ratioH === 100) { + orientation = 'square'; + } + + this.dimensions = { + width, + height, + ratioH, + ratioW, + orientation, + }; + + if (this.preprocessed) { + this.sizes = [ { + output: resolve(this.out), + url: this.url, + width, + height, + } ]; + } else { + this.sizes = [ { - url: path.join(siteDir, `${basename}.${filetype}`), - width: dimensions.width, - height: dimensions.height, + output: resolve(this.out), + url: this.url, + width, + height, }, ]; for (const w of RESOLUTIONS) { - if (w > dimensions.width) continue; - sizes.push({ - url: path.join(siteDir, `${basename}.${w}w.${filetype}`), + if (w > width) continue; + this.sizes.push({ + output: resolve(this.dest, `${this.name}.${w}w.${this.type}`), + url: path.join(this.dir, `${this.name}.${w}w.${this.type}`), width: w, - height: Math.ceil((w / dimensions.width) * dimensions.height), + height: Math.ceil((w / width) * height), }); } - sizes.reverse(); + this.sizes.reverse(); + } - return { - name: basename, - type: 'image', - sizes, - }; - }))).filter(Boolean); + return this; + } - const titlecard = (await glob('titlecard.{jpeg,jpg,png,gif}', { cwd }))[0]; + async loadVideo () { + const { width, height } = await getVideoDimensions(this.input); - return { - images: keyBy(images, 'name'), - titlecard: titlecard ? path.join(siteDir, titlecard) : '/images/titlecard.png', + const ratioH = Math.round((height / width) * 100); + const ratioW = Math.round((width / height) * 100); + let orientation = 'wide'; + if (ratioH > 100) { + orientation = 'tall'; + } else if (ratioH === 100) { + orientation = 'square'; + } + + this.dimensions = { + width, + height, + ratioH, + ratioW, + orientation, }; - }); -}; + + this.sizes = [ { + output: resolve(this.dest, this.basename), + url: path.join(this.dir, this.basename), + width, + height, + } ]; + + return this; + } + + toJson () { + return pick(this, [ + 'preprocessed', + 'type', + 'kind', + 'input', + 'cwd', + 'base', + 'dir', + 'name', + 'basename', + 'dest', + 'ext', + 'dimensions', + ]); + } + + webready () { + const { kind, name } = this; + return { + kind, + name, + sizes: this.sizes.map((s) => pick(s, [ 'url', 'width', 'height' ])), + }; + } + + tasks () { + return this.sizes.map(({ output, width }) => ({ + input: this.input, + output, + format: this.preprocessed ? undefined : this.type, + width: this.preprocessed ? undefined : width, + action: this.preprocessed ? actions.copy : actions.image, + })); + } + +} + +exports.Asset = Asset; diff --git a/gulp/content/index.js b/gulp/content/index.js index af5d9fe..753269a 100644 --- a/gulp/content/index.js +++ b/gulp/content/index.js @@ -8,28 +8,28 @@ const tweetparse = require('../lib/tweetparse'); const getEngines = require('./renderers'); const Twitter = require('twitter-lite'); const Page = require('./page'); -const createFileLoader = require('./files'); +const createAssetLoader = require('./files'); const ROOT = path.resolve(__dirname, '../..'); exports.parse = async function parsePageContent () { - const [ files, twitter, twitterBackup, twitterCache ] = await Promise.all([ + const [ files, twitter, twitterBackup, twitterCache, Assets ] = await Promise.all([ glob('pages/**/*.{md,hbs,html,xml}', { cwd: ROOT }), fs.readJson(resolve('twitter-config.json')).catch(() => null) .then(getTwitterClient), fs.readJson(resolve('twitter-backup.json')).catch(() => ({})), fs.readJson(resolve('twitter-cache.json')).catch(() => ({})), + createAssetLoader(), ]); let tweetsNeeded = []; const tweetsPresent = Object.keys(twitterCache); - const artifactLoader = createFileLoader(); let pages = await Promise.map(files, async (filepath) => { const page = new Page(filepath); if (!page.input) return; - await page.load({ artifactLoader }); + await page.load({ Assets }); if (page.tweets.length) { const missing = difference(page.tweets, tweetsPresent); diff --git a/gulp/content/page.js b/gulp/content/page.js index cc596fe..afa7973 100644 --- a/gulp/content/page.js +++ b/gulp/content/page.js @@ -57,13 +57,20 @@ module.exports = exports = class Page { // this is not a page file if (![ MD, HBS, HTML, XML ].includes(ext)) return false; - this.input = resolve(filepath); // /local/path/to/pages/folder/file.ext - this.cwd = resolve(file.dir); // /local/path/to/pages/, pages/folder, pages/folder/subfolder - this.base = file.dir.replace(/^pages\/?/, ''); // '', 'folder', 'folder/subfolder' - this.dir = file.dir.replace(/^pages\/?/, '/'); // /, /folder, /folder/subfolder - this.name = name; // index, fileA, fileB - this.basename = basename; // index.ext, fileA.ext, fileB.ext - this.dest = file.dir.replace(/^pages\/?/, 'dist/'); // dist/, dist/folder, dist/folder/subfolder + // remove the pages root and any _images segment from the dir + const dir = file.dir.split('/'); + if (dir[0] === 'pages') dir.shift(); + const i = dir.indexOf('_images'); + if (i > -1) dir.splice(i, 1); + + this.input = resolve(filepath); // /local/path/to/pages/file.ext + this.cwd = resolve(file.dir); // /local/path/to/pages/, pages/folder, pages/folder/subfolder + this.base = path.join(...dir); // '', 'folder', 'folder/subfolder' + this.dir = path.join('/', ...dir); // /, /folder, /folder/subfolder + this.name = name; // index, fileA, fileB + this.basename = basename; // index.ext, fileA.ext, fileB.ext + this.dest = path.join('dist/', ...dir); // dist/, dist/folder, dist/folder/subfolder + this.ext = file.ext; var isIndexPage = (name === 'index'); var isCleanUrl = [ HBS, MD ].includes(ext); @@ -102,13 +109,14 @@ module.exports = exports = class Page { } - async load ({ artifactLoader }) { - const [ raw, { ctime, mtime }, { images, titlecard } ] = await Promise.all([ + async load ({ Assets }) { + const [ raw, { ctime, mtime } ] = await Promise.all([ fs.readFile(this.input).catch(() => null), fs.stat(this.input).catch(() => {}), - artifactLoader(this.cwd, this.dir), ]); + const { titlecard, assets } = Assets.for(this.dir); + // empty file if (!raw || !ctime) { log.error('Could not load page: ' + this.filepath); @@ -124,7 +132,7 @@ module.exports = exports = class Page { this.source = body; this.meta = meta; - this.images = images; + this.images = assets; this.titlecard = titlecard; this.tweets = (meta.tweets || []).map(parseTweetId); this.dateCreated = meta.date && new Date(meta.date) || ctime; @@ -151,6 +159,7 @@ module.exports = exports = class Page { 'base', 'dir', 'name', + 'ext', 'basename', 'dest', 'out', @@ -159,6 +168,7 @@ module.exports = exports = class Page { 'engine', 'source', 'images', + 'assets', 'titlecard', 'tweets', 'classes', diff --git a/gulp/imgflow/index.js b/gulp/imgflow/index.js index 6672e11..42af7d6 100644 --- a/gulp/imgflow/index.js +++ b/gulp/imgflow/index.js @@ -1,130 +1,47 @@ const path = require('path'); -const glob = require('../lib/glob'); -const { sortBy, uniqBy } = require('lodash'); +const { uniqBy } = require('lodash'); const Promise = require('bluebird'); const fs = require('fs-extra'); -const log = require('fancy-log'); const actions = require('./actions'); -const getDimensions = require('../lib/dimensions'); +const createAssetLoader = require('../content/files'); -const CWD = path.resolve(__dirname, '../..'); -const PAGES = path.join(CWD, 'pages'); -const SOURCE = path.resolve(CWD, 'pages/**/*.{jpeg,jpg,png,gif,mp4}'); -const MANIFEST_PATH = path.resolve(CWD, 'if-manifest.json'); -const REV_MANIFEST_PATH = path.resolve(CWD, 'rev-manifest.json'); -const MEDIA_INDEX = path.resolve(CWD, 'twitter-media.json'); + +const ROOT = path.resolve(__dirname, '../..'); const CACHE = 'if-cache'; -const revHash = require('rev-hash'); -const revPath = require('rev-path'); +const { changed, execute } = require('./pipeline'); -const LOG = { - new: true, - update: true, - skip: true, - rebuild: true, - cached: false, - copy: false, -}; +function resolve (...args) { + args = args.filter(Boolean); + let fpath = args.shift(); + if (!fpath) return ROOT; + if (fpath[0] === '/') fpath = fpath.slice(1); + return path.resolve(ROOT, fpath, ...args); +} module.exports = exports = async function postImages ({ rev = false }) { - var manifest; - try { - manifest = JSON.parse(await fs.readFile(MANIFEST_PATH)); - } catch (e) { - manifest = {}; - } + const [ manifest, { tasks } ] = await Promise.all([ + fs.readJson(resolve('if-manifest.json')).catch(() => ({})), + createAssetLoader(), + fs.ensureDir(resolve(CACHE)), + ]); - await fs.ensureDir(path.resolve(CWD, CACHE)); - - const allfiles = (await glob(SOURCE)); - const tasks = []; - - for (const filepath of allfiles) { - const input = path.relative(CWD, filepath); - const output = path.relative(PAGES, filepath).replace('/_images', ''); - const file = path.parse(output); - // console.log(input, output); - - // is a titlecard image or a video - if (file.name === 'titlecard' || file.ext === '.mp4') { - tasks.push({ - input, - output, - action: actions.copy, - }); - continue; - } - - // is a file we've pre-sized and do not want processed - if (file.name[0] === '_') { - tasks.push({ - input, - output: path.format({ ...file, base: file.base.substring(1) }), - action: actions.copy, - }); - continue; - } - - const format = { - '.jpeg': 'jpeg', - '.jpg': 'jpeg', - '.png': 'png', - '.gif': 'gif', - }[file.ext]; - - if (!format) throw new Error('Got an unexpected format: ' + file.ext); - - const dimensions = await getDimensions(filepath); - - tasks.push({ - input: filepath, - output: `${file.dir}/${file.name}.${format}`, - format, - action: actions.image, - }); - - for (const w of [ 2048, 1024, 768, 576, 300, 100 ]) { - if (w > dimensions.width) continue; - tasks.push({ - input: filepath, - output: `${file.dir}/${file.name}.${w}w.${format}`, - format, - width: w, - action: actions.image, - }); - } - - } - - const filtered = await filter(manifest, tasks); + const filtered = await changed(manifest, tasks); await execute(manifest, filtered, rev); }; exports.prod = function imagesProd () { return exports({ rev: true }); }; exports.twitter = async function twitterImages ({ rev = false }) { - await fs.ensureDir(path.resolve(CWD, CACHE)); + const [ manifest, media ] = await Promise.all([ + fs.readJson(resolve('if-manifest.json')).catch(() => ({})), + fs.readJson(resolve('twitter-media.json')).catch(() => ([])), + fs.ensureDir(resolve(CACHE)), + ]); - var manifest; - try { - manifest = JSON.parse(await fs.readFile(MANIFEST_PATH)); - } catch (e) { - manifest = {}; - } - - var media; - try { - media = JSON.parse(await fs.readFile(MEDIA_INDEX)); - } catch (e) { - media = []; - } - - media = uniqBy(media, 'output'); - - const tasks = media.map((m) => ({ ...m, action: actions.fetch })); - const filtered = await filter(manifest, tasks); + const tasks = uniqBy(media, 'output').map((m) => ({ ...m, action: actions.fetch })); + const filtered = await changed(manifest, tasks); await execute(manifest, filtered, rev); }; @@ -132,16 +49,11 @@ exports.twitter.prod = function imagesProd () { return exports.twitter({ rev: tr exports.favicon = async function favicon ({ rev = false }) { - await fs.ensureDir(path.resolve(CWD, CACHE)); - - const input = path.resolve(CWD, 'favicon.png'); - - var manifest; - try { - manifest = JSON.parse(await fs.readFile(MANIFEST_PATH)); - } catch (e) { - manifest = {}; - } + const input = resolve('favicon.png'); + const [ manifest ] = await Promise.all([ + fs.readJson(resolve('if-manifest.json')).catch(() => ({})), + fs.ensureDir(resolve(CACHE)), + ]); const tasks = [ 32, 57, 64, 76, 96, 114, 120, 128, 144, 152, 180, 192, 196, 228 ].map((width) => ({ input, @@ -158,169 +70,9 @@ exports.favicon = async function favicon ({ rev = false }) { action: actions.image, }); - const filtered = await filter(manifest, tasks); + const filtered = await changed(manifest, tasks); await execute(manifest, filtered, rev); }; exports.favicon.prod = function imagesProd () { return exports.favicon({ rev: true }); }; - -async function filter (manifest, tasks) { - const statMap = new Map(); - async function stat (f) { - if (statMap.has(f)) return statMap.get(f); - - const p = fs.stat(path.resolve(CWD, f)) - .catch(() => null) - .then((stats) => (stats && Math.floor(stats.mtimeMs / 1000))); - - statMap.set(f, p); - return p; - } - - return Promise.filter(tasks, async (task) => { - - const local = task.input.slice(0, 4) !== 'http'; - const hash = task.action.name + '.' + revHash(task.input) + '|' + revHash(task.output); - const cachePath = path.join(CACHE, `${hash}${path.extname(task.output)}`); - const [ inTime, outTime, cachedTime ] = await Promise.all([ - local && stat(path.resolve(CWD, task.input)), - stat(path.resolve(CWD, 'dist', task.output)), - stat(path.resolve(CWD, cachePath)), - ]); - - task.manifest = manifest[hash]; - task.hash = hash; - task.cache = cachePath; - - // how did this happen? - if (local && !inTime) { - log.error('Input file could not be found?', task.input); - return false; - } - - // never seen this file before - if (!task.manifest) { - task.apply = { - hash, - input: task.input, - output: task.output, - mtime: inTime, - }; - task.log = [ 'new', task.input, task.output, hash ]; - return true; - } - - // file modification time does not match last read, rebuild - if (local && inTime > task.manifest.mtime) { - task.log = [ 'update', task.input, task.output ]; - task.apply = { - mtime: inTime, - }; - return true; - } - - task.apply = { - mtime: local ? inTime : Math.floor(Date.now() / 1000), - }; - - // target file exists, nothing to do - if (outTime) { - return false; - // task.log = [ 'skip', task.input, task.output, inTime, task.manifest.mtime ]; - // task.action = null; - // return true; - } - - // file exists in the cache, change the task to a copy action - if (cachedTime) { - task.log = [ 'cached', task.input, task.output ]; - task.action = actions.copy; - task.input = cachePath; - return true; - } - - // task is a file copy - if (task.action === actions.copy) { - task.log = [ 'copy', task.input, task.output ]; - return true; - } - - // file does not exist in cache, build it - task.log = [ 'rebuild', task.input, task.output ]; - return true; - }); -} - - -async function execute (manifest, tasks, rev) { - const lastSeen = Math.floor(Date.now() / 1000); - const revManifest = {}; - - let writeCounter = 0; - let lastWriteTime = 0; - async function writeManifest (force) { - if (!force && rev) return; // disable interim writes during prod builds. - if (!force && ++writeCounter % 100) return; - const now = Date.now(); - if (!force && now - lastWriteTime < 10000) return; - lastWriteTime = now; - await fs.writeFile(MANIFEST_PATH, JSON.stringify(manifest, null, 2)); - } - - await Promise.map(sortBy(tasks, [ 'input', 'output' ]), async (task) => { - const output = path.resolve(CWD, 'dist', task.output); - - const result = task.action && await task.action({ ...task, output }); - const apply = task.apply || {}; - if (task.log && LOG[task.log[0]]) log.info(...task.log); - apply.lastSeen = lastSeen; - apply.lastSeenHuman = new Date(); - - if (!result) log('Nothing happened?', task); - - const rhash = result && revHash(result); - const hashedPath = revPath(task.output, rhash); - apply.revHash = rhash; - apply.revPath = hashedPath; - - if (rev && rhash) { - const rOutPath = task.output; - const rNewPath = hashedPath; - - revManifest[rOutPath] = rNewPath; - - await fs.copy(output, path.resolve(CWD, 'dist', hashedPath)); - } - - manifest[task.hash] = { ...manifest[task.hash], ...apply }; - await writeManifest(); - - }, { concurrency: rev ? 20 : 10 }); - - // filter unseen files from history - // manifest = omitBy(manifest, (m) => m.lastSeen !== lastSeen); - - await writeManifest(true); - - if (rev) { - let originalManifest = {}; - try { - if (await fs.exists(REV_MANIFEST_PATH)) { - originalManifest = JSON.parse(await fs.readFile(REV_MANIFEST_PATH)); - } - } catch (e) { - // do nothing - } - - Object.assign(originalManifest, revManifest); - - await fs.writeFile(REV_MANIFEST_PATH, JSON.stringify(originalManifest, null, 2)); - } - -} - -if (require.main === module) { - exports().catch(console.error).then(() => process.exit()); // eslint-disable-line -} - diff --git a/gulp/imgflow/pipeline.js b/gulp/imgflow/pipeline.js new file mode 100644 index 0000000..23d1fe1 --- /dev/null +++ b/gulp/imgflow/pipeline.js @@ -0,0 +1,179 @@ +const path = require('path'); +const { sortBy } = require('lodash'); +const Promise = require('bluebird'); +const fs = require('fs-extra'); +const log = require('fancy-log'); +const actions = require('./actions'); +const revHash = require('rev-hash'); +const revPath = require('rev-path'); + +const CWD = path.resolve(__dirname, '../..'); +const PAGES = path.join(CWD, 'pages'); +const SOURCE = path.resolve(PAGES, '**/*.{jpeg,jpg,png,gif,mp4}'); +const MANIFEST_PATH = path.resolve(CWD, 'if-manifest.json'); +const REV_MANIFEST_PATH = path.resolve(CWD, 'rev-manifest.json'); +const MEDIA_INDEX = path.resolve(CWD, 'twitter-media.json'); +const CACHE = 'if-cache'; + +const LOG = { + new: true, + update: true, + skip: true, + rebuild: true, + cached: false, + copy: false, +}; + +exports.changed = async function changed (manifest, tasks) { + const statMap = new Map(); + async function stat (f) { + if (statMap.has(f)) return statMap.get(f); + + const p = fs.stat(path.resolve(CWD, f)) + .catch(() => null) + .then((stats) => (stats && Math.floor(stats.mtimeMs / 1000))); + + statMap.set(f, p); + return p; + } + + return Promise.filter(tasks, async (task) => { + + const local = task.input.slice(0, 4) !== 'http'; + const hash = task.action.name + '.' + revHash(task.input) + '|' + revHash(task.output); + const cachePath = path.join(CACHE, `${hash}${path.extname(task.output)}`); + const [ inTime, outTime, cachedTime ] = await Promise.all([ + local && stat(path.resolve(CWD, task.input)), + stat(path.resolve(CWD, 'dist', task.output)), + stat(path.resolve(CWD, cachePath)), + ]); + + task.manifest = manifest[hash]; + task.hash = hash; + task.cache = cachePath; + + // how did this happen? + if (local && !inTime) { + log.error('Input file could not be found?', task.input); + return false; + } + + // never seen this file before + if (!task.manifest) { + task.apply = { + hash, + input: task.input, + output: task.output, + mtime: inTime, + }; + task.log = [ 'new', task.input, task.output, hash ]; + return true; + } + + // file modification time does not match last read, rebuild + if (local && inTime > task.manifest.mtime) { + task.log = [ 'update', task.input, task.output ]; + task.apply = { + mtime: inTime, + }; + return true; + } + + task.apply = { + mtime: local ? inTime : Math.floor(Date.now() / 1000), + }; + + // target file exists, nothing to do + if (outTime) { + return false; + // task.log = [ 'skip', task.input, task.output, inTime, task.manifest.mtime ]; + // task.action = null; + // return true; + } + + // file exists in the cache, change the task to a copy action + if (cachedTime) { + task.log = [ 'cached', task.input, task.output ]; + task.action = actions.copy; + task.input = cachePath; + return true; + } + + // task is a file copy + if (task.action === actions.copy) { + task.log = [ 'copy', task.input, task.output ]; + return true; + } + + // file does not exist in cache, build it + task.log = [ 'rebuild', task.input, task.output ]; + return true; + }); +}; + + +exports.execute = async function execute (manifest, tasks, rev) { + const lastSeen = Math.floor(Date.now() / 1000); + const revManifest = {}; + + let writeCounter = 0; + let lastWriteTime = 0; + async function writeManifest (force) { + if (!force && rev) return; // disable interim writes during prod builds. + if (!force && ++writeCounter % 100) return; + const now = Date.now(); + if (!force && now - lastWriteTime < 10000) return; + lastWriteTime = now; + await fs.writeFile(MANIFEST_PATH, JSON.stringify(manifest, null, 2)); + } + + await Promise.map(sortBy(tasks, [ 'input', 'output' ]), async (task) => { + const output = path.resolve(CWD, 'dist', task.output); + + const result = task.action && await task.action({ ...task, output }); + const apply = task.apply || {}; + if (task.log && LOG[task.log[0]]) log.info(...task.log); + apply.lastSeen = lastSeen; + apply.lastSeenHuman = new Date(); + + if (!result) log('Nothing happened?', task); + + const rhash = result && revHash(result); + const hashedPath = revPath(task.output, rhash); + apply.revHash = rhash; + apply.revPath = hashedPath; + + if (rev && rhash) { + const rOutPath = task.output; + const rNewPath = hashedPath; + + revManifest[rOutPath] = rNewPath; + + await fs.copy(output, path.resolve(CWD, 'dist', hashedPath)); + } + + manifest[task.hash] = { ...manifest[task.hash], ...apply }; + await writeManifest(); + + }, { concurrency: rev ? 20 : 10 }); + + // filter unseen files from history + // manifest = omitBy(manifest, (m) => m.lastSeen !== lastSeen); + + await writeManifest(true); + + if (rev) { + let originalManifest = {}; + try { + if (await fs.exists(REV_MANIFEST_PATH)) { + originalManifest = JSON.parse(await fs.readFile(REV_MANIFEST_PATH)); + } + } catch (e) { + // do nothing + } + + Object.assign(originalManifest, revManifest); + + await fs.writeFile(REV_MANIFEST_PATH, JSON.stringify(originalManifest, null, 2)); + } +}; diff --git a/package-lock.json b/package-lock.json index 90bf648..13d00a0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -522,6 +522,12 @@ "integrity": "sha512-uMgjozySS8adZZYePpaWs8cxB9/kdzmpX6SgJZ+wbz1K5eYk5QMYDVJaZKhxyIHUdnnJkfR7SVgStgH7LkGUyg==", "dev": true }, + "any-promise": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz", + "integrity": "sha1-q8av7tzqUugJzcA3au0845Y10X8=", + "dev": true + }, "anymatch": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-2.0.0.tgz", @@ -4005,6 +4011,15 @@ "integrity": "sha1-3BXKHGcjh8p2vTesCjlbogQqLCg=", "dev": true }, + "get-video-dimensions": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/get-video-dimensions/-/get-video-dimensions-1.0.0.tgz", + "integrity": "sha1-/H5ayBw5JEH1uG1Q3XeDiptTFHo=", + "dev": true, + "requires": { + "mz": "^1.2.1" + } + }, "getpass": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", @@ -6411,6 +6426,17 @@ "integrity": "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==", "dev": true }, + "mz": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/mz/-/mz-1.3.0.tgz", + "integrity": "sha1-BvCT/dmVagbTfhsegTROJ0eMQvA=", + "dev": true, + "requires": { + "native-or-bluebird": "1", + "thenify": "3", + "thenify-all": "1" + } + }, "nan": { "version": "2.14.0", "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.0.tgz", @@ -6436,6 +6462,12 @@ "to-regex": "^3.0.1" } }, + "native-or-bluebird": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/native-or-bluebird/-/native-or-bluebird-1.2.0.tgz", + "integrity": "sha1-OcR7/Xgl0fuf+tMiEK4l2q3xAck=", + "dev": true + }, "natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -9033,6 +9065,24 @@ "integrity": "sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=", "dev": true }, + "thenify": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/thenify/-/thenify-3.3.0.tgz", + "integrity": "sha1-5p44obq+lpsBCCB5eLn2K4hgSDk=", + "dev": true, + "requires": { + "any-promise": "^1.0.0" + } + }, + "thenify-all": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/thenify-all/-/thenify-all-1.6.0.tgz", + "integrity": "sha1-GhkY1ALY/D+Y+/I02wvMjMEOlyY=", + "dev": true, + "requires": { + "thenify": ">= 3.1.0 < 4" + } + }, "through": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", diff --git a/package.json b/package.json index 82ebae6..76cc35a 100644 --- a/package.json +++ b/package.json @@ -40,6 +40,7 @@ "forever": "~2.0.0", "front-matter": "~3.1.0", "fs-extra": "~8.1.0", + "get-video-dimensions": "~1.0.0", "glob": "~7.1.6", "gm": "~1.23.1", "gulp": "~4.0.2",