mirror of
https://github.com/GenderDysphoria/GenderDysphoria.fyi.git
synced 2025-01-31 07:16:17 +00:00
More overhaul, image tasks and content assets now come from the same code.
This commit is contained in:
Jocelyn Badgley (Twipped)
parent
24dab66898
commit
9c483c9dd7
@ -1,95 +1,249 @@
|
||||
const path = require('path');
|
||||
const glob = require('../lib/glob');
|
||||
const memoize = require('memoizepromise');
|
||||
const getDimensions = require('../lib/dimensions');
|
||||
const { keyBy } = require('lodash');
|
||||
const getImageDimensions = require('../lib/dimensions');
|
||||
const getVideoDimensions = require('get-video-dimensions');
|
||||
const { keyBy, pick, filter, get, set, memoize } = require('lodash');
|
||||
const actions = require('../imgflow/actions');
|
||||
|
||||
const ROOT = path.resolve(__dirname, '../..');
|
||||
|
||||
function resolve (...args) {
|
||||
args = args.filter(Boolean);
|
||||
let fpath = args.shift();
|
||||
if (!fpath) return ROOT;
|
||||
if (fpath[0] === '/') fpath = fpath.slice(1);
|
||||
return path.resolve(ROOT, fpath, ...args);
|
||||
}
|
||||
|
||||
|
||||
module.exports = exports = async function findAssets () {
|
||||
const files = await glob('pages/**/*.{jpeg,jpg,png,gif,mp4}', { cwd: ROOT });
|
||||
const map = {};
|
||||
const assets = (await Promise.all(files.map(async (filepath) => {
|
||||
const asset = new Asset(path.relative(ROOT, filepath));
|
||||
await asset.load();
|
||||
set(map, [ ...asset.base.split('/'), asset.name ], asset);
|
||||
return asset;
|
||||
}))).filter(Boolean);
|
||||
|
||||
Object.freeze(map);
|
||||
|
||||
function within (dir) {
|
||||
const subset = filter(assets, { dir });
|
||||
return {
|
||||
get titlecard () {
|
||||
return get(filter(subset, { name: 'titlecard' }), [ 0, 'url' ]);
|
||||
},
|
||||
get assets () {
|
||||
return keyBy(subset.map((a) => a.webready()), 'name');
|
||||
},
|
||||
get all () {
|
||||
return [ ...subset ];
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
map,
|
||||
for: memoize(within),
|
||||
get tasks () {
|
||||
return assets.map((a) => a.tasks()).flat(1);
|
||||
},
|
||||
get all () {
|
||||
return [ ...assets ];
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
|
||||
const JPG = '.jpg';
|
||||
const JPEG = '.jpeg';
|
||||
const PNG = '.png';
|
||||
const GIF = '.gif';
|
||||
const MP4 = '.mp4';
|
||||
const M4V = '.m4v';
|
||||
|
||||
const FILETYPE = {
|
||||
[JPG]: 'jpeg',
|
||||
[JPEG]: 'jpeg',
|
||||
[PNG]: 'png',
|
||||
[GIF]: 'gif',
|
||||
[MP4]: 'mp4',
|
||||
[M4V]: 'mp4',
|
||||
};
|
||||
|
||||
const RESOLUTIONS = [ 2048, 1024, 768, 576, 300, 100 ];
|
||||
|
||||
module.exports = exports = function () {
|
||||
return memoize(async (cwd, siteDir) => {
|
||||
const imageFiles = (await glob('{*,_images/*}.{jpeg,jpg,png,gif,mp4}', { cwd }));
|
||||
|
||||
const images = (await Promise.all(imageFiles.map(async (imgpath) => {
|
||||
class Asset {
|
||||
|
||||
const ext = path.extname(imgpath);
|
||||
let basename = path.basename(imgpath, ext);
|
||||
constructor (filepath) {
|
||||
const file = path.parse(filepath);
|
||||
let { base: basename, name } = file;
|
||||
|
||||
if (basename === 'titlecard') return;
|
||||
|
||||
if (ext === '.mp4') {
|
||||
return {
|
||||
name: basename,
|
||||
type: 'movie',
|
||||
full: path.join(siteDir, `${basename}${ext}`),
|
||||
};
|
||||
this.preprocessed = false;
|
||||
if (name[0] === '_') {
|
||||
this.preprocessed = true;
|
||||
file.name = name = name.slice(1);
|
||||
file.basename = basename = basename.slice(1);
|
||||
}
|
||||
|
||||
const dimensions = await getDimensions(path.resolve(cwd, imgpath));
|
||||
const { width, height } = dimensions;
|
||||
dimensions.ratioH = Math.round((height / width) * 100);
|
||||
dimensions.ratioW = Math.round((width / height) * 100);
|
||||
if (dimensions.ratioH > 100) {
|
||||
dimensions.orientation = 'tall';
|
||||
} else if (dimensions.ratioH === 100) {
|
||||
dimensions.orientation = 'square';
|
||||
this.type = FILETYPE[file.ext] || file.ext.slice(1);
|
||||
if ([ JPG, JPEG, PNG, GIF ].includes(file.ext)) {
|
||||
this.kind = 'image';
|
||||
} else if ([ MP4, M4V ].includes(file.ext)) {
|
||||
this.kind = 'video';
|
||||
} else {
|
||||
dimensions.orientation = 'wide';
|
||||
this.kind = 'raw';
|
||||
}
|
||||
|
||||
const filetype = {
|
||||
'.jpeg': 'jpeg',
|
||||
'.jpg': 'jpeg',
|
||||
'.png': 'png',
|
||||
'.gif': 'gif',
|
||||
}[ext];
|
||||
// remove the pages root and any _images segment from the dir
|
||||
const dir = file.dir.split('/');
|
||||
if (dir[0] === 'pages') dir.shift();
|
||||
const i = dir.indexOf('_images');
|
||||
if (i > -1) dir.splice(i, 1);
|
||||
|
||||
if (basename[0] === '_') {
|
||||
basename = basename.slice(1);
|
||||
return {
|
||||
name: basename,
|
||||
type: 'image',
|
||||
sizes: [
|
||||
{
|
||||
url: path.join(siteDir, `${basename}${ext}`),
|
||||
width: dimensions.width,
|
||||
height: dimensions.height,
|
||||
},
|
||||
],
|
||||
this.input = resolve(filepath); // /local/path/to/pages/file.ext
|
||||
this.cwd = resolve(file.dir); // /local/path/to/pages/, pages/folder, pages/folder/subfolder
|
||||
this.base = path.join(...dir); // '', 'folder', 'folder/subfolder'
|
||||
this.dir = path.join('/', ...dir); // /, /folder, /folder/subfolder
|
||||
this.name = name; // index, fileA, fileB
|
||||
this.basename = basename; // index.ext, fileA.ext, fileB.ext
|
||||
this.dest = path.join('dist/', ...dir); // dist/, dist/folder, dist/folder/subfolder
|
||||
this.ext = file.ext;
|
||||
|
||||
this.out = path.join(this.dest, `${this.name}${this.preprocessed ? this.ext : '.' + this.type}`);
|
||||
this.url = path.join(this.dir, `${this.name}${this.preprocessed ? this.ext : '.' + this.type}`);
|
||||
}
|
||||
|
||||
load () {
|
||||
switch (this.kind) {
|
||||
case 'video': return this.loadVideo();
|
||||
case 'image': return this.loadImage();
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
async loadImage () {
|
||||
|
||||
const { width, height } = await getImageDimensions(this.input);
|
||||
|
||||
const ratioH = Math.round((height / width) * 100);
|
||||
const ratioW = Math.round((width / height) * 100);
|
||||
let orientation = 'wide';
|
||||
if (ratioH > 100) {
|
||||
orientation = 'tall';
|
||||
} else if (ratioH === 100) {
|
||||
orientation = 'square';
|
||||
}
|
||||
|
||||
this.dimensions = {
|
||||
width,
|
||||
height,
|
||||
ratioH,
|
||||
ratioW,
|
||||
orientation,
|
||||
};
|
||||
}
|
||||
|
||||
const sizes = [
|
||||
if (this.preprocessed) {
|
||||
this.sizes = [ {
|
||||
output: resolve(this.out),
|
||||
url: this.url,
|
||||
width,
|
||||
height,
|
||||
} ];
|
||||
} else {
|
||||
this.sizes = [
|
||||
{
|
||||
url: path.join(siteDir, `${basename}.${filetype}`),
|
||||
width: dimensions.width,
|
||||
height: dimensions.height,
|
||||
output: resolve(this.out),
|
||||
url: this.url,
|
||||
width,
|
||||
height,
|
||||
},
|
||||
];
|
||||
|
||||
for (const w of RESOLUTIONS) {
|
||||
if (w > dimensions.width) continue;
|
||||
sizes.push({
|
||||
url: path.join(siteDir, `${basename}.${w}w.${filetype}`),
|
||||
if (w > width) continue;
|
||||
this.sizes.push({
|
||||
output: resolve(this.dest, `${this.name}.${w}w.${this.type}`),
|
||||
url: path.join(this.dir, `${this.name}.${w}w.${this.type}`),
|
||||
width: w,
|
||||
height: Math.ceil((w / dimensions.width) * dimensions.height),
|
||||
height: Math.ceil((w / width) * height),
|
||||
});
|
||||
}
|
||||
|
||||
sizes.reverse();
|
||||
this.sizes.reverse();
|
||||
}
|
||||
|
||||
return {
|
||||
name: basename,
|
||||
type: 'image',
|
||||
sizes,
|
||||
return this;
|
||||
}
|
||||
|
||||
async loadVideo () {
|
||||
const { width, height } = await getVideoDimensions(this.input);
|
||||
|
||||
const ratioH = Math.round((height / width) * 100);
|
||||
const ratioW = Math.round((width / height) * 100);
|
||||
let orientation = 'wide';
|
||||
if (ratioH > 100) {
|
||||
orientation = 'tall';
|
||||
} else if (ratioH === 100) {
|
||||
orientation = 'square';
|
||||
}
|
||||
|
||||
this.dimensions = {
|
||||
width,
|
||||
height,
|
||||
ratioH,
|
||||
ratioW,
|
||||
orientation,
|
||||
};
|
||||
}))).filter(Boolean);
|
||||
|
||||
const titlecard = (await glob('titlecard.{jpeg,jpg,png,gif}', { cwd }))[0];
|
||||
this.sizes = [ {
|
||||
output: resolve(this.dest, this.basename),
|
||||
url: path.join(this.dir, this.basename),
|
||||
width,
|
||||
height,
|
||||
} ];
|
||||
|
||||
return this;
|
||||
}
|
||||
|
||||
toJson () {
|
||||
return pick(this, [
|
||||
'preprocessed',
|
||||
'type',
|
||||
'kind',
|
||||
'input',
|
||||
'cwd',
|
||||
'base',
|
||||
'dir',
|
||||
'name',
|
||||
'basename',
|
||||
'dest',
|
||||
'ext',
|
||||
'dimensions',
|
||||
]);
|
||||
}
|
||||
|
||||
webready () {
|
||||
const { kind, name } = this;
|
||||
return {
|
||||
images: keyBy(images, 'name'),
|
||||
titlecard: titlecard ? path.join(siteDir, titlecard) : '/images/titlecard.png',
|
||||
kind,
|
||||
name,
|
||||
sizes: this.sizes.map((s) => pick(s, [ 'url', 'width', 'height' ])),
|
||||
};
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
tasks () {
|
||||
return this.sizes.map(({ output, width }) => ({
|
||||
input: this.input,
|
||||
output,
|
||||
format: this.preprocessed ? undefined : this.type,
|
||||
width: this.preprocessed ? undefined : width,
|
||||
action: this.preprocessed ? actions.copy : actions.image,
|
||||
}));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
exports.Asset = Asset;
|
||||
|
||||
@ -8,28 +8,28 @@ const tweetparse = require('../lib/tweetparse');
|
||||
const getEngines = require('./renderers');
|
||||
const Twitter = require('twitter-lite');
|
||||
const Page = require('./page');
|
||||
const createFileLoader = require('./files');
|
||||
const createAssetLoader = require('./files');
|
||||
|
||||
const ROOT = path.resolve(__dirname, '../..');
|
||||
|
||||
exports.parse = async function parsePageContent () {
|
||||
const [ files, twitter, twitterBackup, twitterCache ] = await Promise.all([
|
||||
const [ files, twitter, twitterBackup, twitterCache, Assets ] = await Promise.all([
|
||||
glob('pages/**/*.{md,hbs,html,xml}', { cwd: ROOT }),
|
||||
fs.readJson(resolve('twitter-config.json')).catch(() => null)
|
||||
.then(getTwitterClient),
|
||||
fs.readJson(resolve('twitter-backup.json')).catch(() => ({})),
|
||||
fs.readJson(resolve('twitter-cache.json')).catch(() => ({})),
|
||||
createAssetLoader(),
|
||||
]);
|
||||
|
||||
|
||||
let tweetsNeeded = [];
|
||||
const tweetsPresent = Object.keys(twitterCache);
|
||||
const artifactLoader = createFileLoader();
|
||||
|
||||
let pages = await Promise.map(files, async (filepath) => {
|
||||
const page = new Page(filepath);
|
||||
if (!page.input) return;
|
||||
await page.load({ artifactLoader });
|
||||
await page.load({ Assets });
|
||||
|
||||
if (page.tweets.length) {
|
||||
const missing = difference(page.tweets, tweetsPresent);
|
||||
|
||||
@ -57,13 +57,20 @@ module.exports = exports = class Page {
|
||||
// this is not a page file
|
||||
if (![ MD, HBS, HTML, XML ].includes(ext)) return false;
|
||||
|
||||
this.input = resolve(filepath); // /local/path/to/pages/folder/file.ext
|
||||
// remove the pages root and any _images segment from the dir
|
||||
const dir = file.dir.split('/');
|
||||
if (dir[0] === 'pages') dir.shift();
|
||||
const i = dir.indexOf('_images');
|
||||
if (i > -1) dir.splice(i, 1);
|
||||
|
||||
this.input = resolve(filepath); // /local/path/to/pages/file.ext
|
||||
this.cwd = resolve(file.dir); // /local/path/to/pages/, pages/folder, pages/folder/subfolder
|
||||
this.base = file.dir.replace(/^pages\/?/, ''); // '', 'folder', 'folder/subfolder'
|
||||
this.dir = file.dir.replace(/^pages\/?/, '/'); // /, /folder, /folder/subfolder
|
||||
this.base = path.join(...dir); // '', 'folder', 'folder/subfolder'
|
||||
this.dir = path.join('/', ...dir); // /, /folder, /folder/subfolder
|
||||
this.name = name; // index, fileA, fileB
|
||||
this.basename = basename; // index.ext, fileA.ext, fileB.ext
|
||||
this.dest = file.dir.replace(/^pages\/?/, 'dist/'); // dist/, dist/folder, dist/folder/subfolder
|
||||
this.dest = path.join('dist/', ...dir); // dist/, dist/folder, dist/folder/subfolder
|
||||
this.ext = file.ext;
|
||||
|
||||
var isIndexPage = (name === 'index');
|
||||
var isCleanUrl = [ HBS, MD ].includes(ext);
|
||||
@ -102,13 +109,14 @@ module.exports = exports = class Page {
|
||||
|
||||
}
|
||||
|
||||
async load ({ artifactLoader }) {
|
||||
const [ raw, { ctime, mtime }, { images, titlecard } ] = await Promise.all([
|
||||
async load ({ Assets }) {
|
||||
const [ raw, { ctime, mtime } ] = await Promise.all([
|
||||
fs.readFile(this.input).catch(() => null),
|
||||
fs.stat(this.input).catch(() => {}),
|
||||
artifactLoader(this.cwd, this.dir),
|
||||
]);
|
||||
|
||||
const { titlecard, assets } = Assets.for(this.dir);
|
||||
|
||||
// empty file
|
||||
if (!raw || !ctime) {
|
||||
log.error('Could not load page: ' + this.filepath);
|
||||
@ -124,7 +132,7 @@ module.exports = exports = class Page {
|
||||
|
||||
this.source = body;
|
||||
this.meta = meta;
|
||||
this.images = images;
|
||||
this.images = assets;
|
||||
this.titlecard = titlecard;
|
||||
this.tweets = (meta.tweets || []).map(parseTweetId);
|
||||
this.dateCreated = meta.date && new Date(meta.date) || ctime;
|
||||
@ -151,6 +159,7 @@ module.exports = exports = class Page {
|
||||
'base',
|
||||
'dir',
|
||||
'name',
|
||||
'ext',
|
||||
'basename',
|
||||
'dest',
|
||||
'out',
|
||||
@ -159,6 +168,7 @@ module.exports = exports = class Page {
|
||||
'engine',
|
||||
'source',
|
||||
'images',
|
||||
'assets',
|
||||
'titlecard',
|
||||
'tweets',
|
||||
'classes',
|
||||
|
||||
@ -1,130 +1,47 @@
|
||||
const path = require('path');
|
||||
const glob = require('../lib/glob');
|
||||
const { sortBy, uniqBy } = require('lodash');
|
||||
const { uniqBy } = require('lodash');
|
||||
const Promise = require('bluebird');
|
||||
const fs = require('fs-extra');
|
||||
const log = require('fancy-log');
|
||||
const actions = require('./actions');
|
||||
const getDimensions = require('../lib/dimensions');
|
||||
const createAssetLoader = require('../content/files');
|
||||
|
||||
const CWD = path.resolve(__dirname, '../..');
|
||||
const PAGES = path.join(CWD, 'pages');
|
||||
const SOURCE = path.resolve(CWD, 'pages/**/*.{jpeg,jpg,png,gif,mp4}');
|
||||
const MANIFEST_PATH = path.resolve(CWD, 'if-manifest.json');
|
||||
const REV_MANIFEST_PATH = path.resolve(CWD, 'rev-manifest.json');
|
||||
const MEDIA_INDEX = path.resolve(CWD, 'twitter-media.json');
|
||||
|
||||
const ROOT = path.resolve(__dirname, '../..');
|
||||
const CACHE = 'if-cache';
|
||||
const revHash = require('rev-hash');
|
||||
const revPath = require('rev-path');
|
||||
|
||||
const { changed, execute } = require('./pipeline');
|
||||
|
||||
const LOG = {
|
||||
new: true,
|
||||
update: true,
|
||||
skip: true,
|
||||
rebuild: true,
|
||||
cached: false,
|
||||
copy: false,
|
||||
};
|
||||
function resolve (...args) {
|
||||
args = args.filter(Boolean);
|
||||
let fpath = args.shift();
|
||||
if (!fpath) return ROOT;
|
||||
if (fpath[0] === '/') fpath = fpath.slice(1);
|
||||
return path.resolve(ROOT, fpath, ...args);
|
||||
}
|
||||
|
||||
module.exports = exports = async function postImages ({ rev = false }) {
|
||||
|
||||
var manifest;
|
||||
try {
|
||||
manifest = JSON.parse(await fs.readFile(MANIFEST_PATH));
|
||||
} catch (e) {
|
||||
manifest = {};
|
||||
}
|
||||
const [ manifest, { tasks } ] = await Promise.all([
|
||||
fs.readJson(resolve('if-manifest.json')).catch(() => ({})),
|
||||
createAssetLoader(),
|
||||
fs.ensureDir(resolve(CACHE)),
|
||||
]);
|
||||
|
||||
await fs.ensureDir(path.resolve(CWD, CACHE));
|
||||
|
||||
const allfiles = (await glob(SOURCE));
|
||||
const tasks = [];
|
||||
|
||||
for (const filepath of allfiles) {
|
||||
const input = path.relative(CWD, filepath);
|
||||
const output = path.relative(PAGES, filepath).replace('/_images', '');
|
||||
const file = path.parse(output);
|
||||
// console.log(input, output);
|
||||
|
||||
// is a titlecard image or a video
|
||||
if (file.name === 'titlecard' || file.ext === '.mp4') {
|
||||
tasks.push({
|
||||
input,
|
||||
output,
|
||||
action: actions.copy,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
// is a file we've pre-sized and do not want processed
|
||||
if (file.name[0] === '_') {
|
||||
tasks.push({
|
||||
input,
|
||||
output: path.format({ ...file, base: file.base.substring(1) }),
|
||||
action: actions.copy,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
const format = {
|
||||
'.jpeg': 'jpeg',
|
||||
'.jpg': 'jpeg',
|
||||
'.png': 'png',
|
||||
'.gif': 'gif',
|
||||
}[file.ext];
|
||||
|
||||
if (!format) throw new Error('Got an unexpected format: ' + file.ext);
|
||||
|
||||
const dimensions = await getDimensions(filepath);
|
||||
|
||||
tasks.push({
|
||||
input: filepath,
|
||||
output: `${file.dir}/${file.name}.${format}`,
|
||||
format,
|
||||
action: actions.image,
|
||||
});
|
||||
|
||||
for (const w of [ 2048, 1024, 768, 576, 300, 100 ]) {
|
||||
if (w > dimensions.width) continue;
|
||||
tasks.push({
|
||||
input: filepath,
|
||||
output: `${file.dir}/${file.name}.${w}w.${format}`,
|
||||
format,
|
||||
width: w,
|
||||
action: actions.image,
|
||||
});
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
const filtered = await filter(manifest, tasks);
|
||||
const filtered = await changed(manifest, tasks);
|
||||
await execute(manifest, filtered, rev);
|
||||
};
|
||||
|
||||
exports.prod = function imagesProd () { return exports({ rev: true }); };
|
||||
|
||||
exports.twitter = async function twitterImages ({ rev = false }) {
|
||||
await fs.ensureDir(path.resolve(CWD, CACHE));
|
||||
const [ manifest, media ] = await Promise.all([
|
||||
fs.readJson(resolve('if-manifest.json')).catch(() => ({})),
|
||||
fs.readJson(resolve('twitter-media.json')).catch(() => ([])),
|
||||
fs.ensureDir(resolve(CACHE)),
|
||||
]);
|
||||
|
||||
var manifest;
|
||||
try {
|
||||
manifest = JSON.parse(await fs.readFile(MANIFEST_PATH));
|
||||
} catch (e) {
|
||||
manifest = {};
|
||||
}
|
||||
|
||||
var media;
|
||||
try {
|
||||
media = JSON.parse(await fs.readFile(MEDIA_INDEX));
|
||||
} catch (e) {
|
||||
media = [];
|
||||
}
|
||||
|
||||
media = uniqBy(media, 'output');
|
||||
|
||||
const tasks = media.map((m) => ({ ...m, action: actions.fetch }));
|
||||
const filtered = await filter(manifest, tasks);
|
||||
const tasks = uniqBy(media, 'output').map((m) => ({ ...m, action: actions.fetch }));
|
||||
const filtered = await changed(manifest, tasks);
|
||||
await execute(manifest, filtered, rev);
|
||||
};
|
||||
|
||||
@ -132,16 +49,11 @@ exports.twitter.prod = function imagesProd () { return exports.twitter({ rev: tr
|
||||
|
||||
|
||||
exports.favicon = async function favicon ({ rev = false }) {
|
||||
await fs.ensureDir(path.resolve(CWD, CACHE));
|
||||
|
||||
const input = path.resolve(CWD, 'favicon.png');
|
||||
|
||||
var manifest;
|
||||
try {
|
||||
manifest = JSON.parse(await fs.readFile(MANIFEST_PATH));
|
||||
} catch (e) {
|
||||
manifest = {};
|
||||
}
|
||||
const input = resolve('favicon.png');
|
||||
const [ manifest ] = await Promise.all([
|
||||
fs.readJson(resolve('if-manifest.json')).catch(() => ({})),
|
||||
fs.ensureDir(resolve(CACHE)),
|
||||
]);
|
||||
|
||||
const tasks = [ 32, 57, 64, 76, 96, 114, 120, 128, 144, 152, 180, 192, 196, 228 ].map((width) => ({
|
||||
input,
|
||||
@ -158,169 +70,9 @@ exports.favicon = async function favicon ({ rev = false }) {
|
||||
action: actions.image,
|
||||
});
|
||||
|
||||
const filtered = await filter(manifest, tasks);
|
||||
const filtered = await changed(manifest, tasks);
|
||||
await execute(manifest, filtered, rev);
|
||||
};
|
||||
|
||||
exports.favicon.prod = function imagesProd () { return exports.favicon({ rev: true }); };
|
||||
|
||||
|
||||
async function filter (manifest, tasks) {
|
||||
const statMap = new Map();
|
||||
async function stat (f) {
|
||||
if (statMap.has(f)) return statMap.get(f);
|
||||
|
||||
const p = fs.stat(path.resolve(CWD, f))
|
||||
.catch(() => null)
|
||||
.then((stats) => (stats && Math.floor(stats.mtimeMs / 1000)));
|
||||
|
||||
statMap.set(f, p);
|
||||
return p;
|
||||
}
|
||||
|
||||
return Promise.filter(tasks, async (task) => {
|
||||
|
||||
const local = task.input.slice(0, 4) !== 'http';
|
||||
const hash = task.action.name + '.' + revHash(task.input) + '|' + revHash(task.output);
|
||||
const cachePath = path.join(CACHE, `${hash}${path.extname(task.output)}`);
|
||||
const [ inTime, outTime, cachedTime ] = await Promise.all([
|
||||
local && stat(path.resolve(CWD, task.input)),
|
||||
stat(path.resolve(CWD, 'dist', task.output)),
|
||||
stat(path.resolve(CWD, cachePath)),
|
||||
]);
|
||||
|
||||
task.manifest = manifest[hash];
|
||||
task.hash = hash;
|
||||
task.cache = cachePath;
|
||||
|
||||
// how did this happen?
|
||||
if (local && !inTime) {
|
||||
log.error('Input file could not be found?', task.input);
|
||||
return false;
|
||||
}
|
||||
|
||||
// never seen this file before
|
||||
if (!task.manifest) {
|
||||
task.apply = {
|
||||
hash,
|
||||
input: task.input,
|
||||
output: task.output,
|
||||
mtime: inTime,
|
||||
};
|
||||
task.log = [ 'new', task.input, task.output, hash ];
|
||||
return true;
|
||||
}
|
||||
|
||||
// file modification time does not match last read, rebuild
|
||||
if (local && inTime > task.manifest.mtime) {
|
||||
task.log = [ 'update', task.input, task.output ];
|
||||
task.apply = {
|
||||
mtime: inTime,
|
||||
};
|
||||
return true;
|
||||
}
|
||||
|
||||
task.apply = {
|
||||
mtime: local ? inTime : Math.floor(Date.now() / 1000),
|
||||
};
|
||||
|
||||
// target file exists, nothing to do
|
||||
if (outTime) {
|
||||
return false;
|
||||
// task.log = [ 'skip', task.input, task.output, inTime, task.manifest.mtime ];
|
||||
// task.action = null;
|
||||
// return true;
|
||||
}
|
||||
|
||||
// file exists in the cache, change the task to a copy action
|
||||
if (cachedTime) {
|
||||
task.log = [ 'cached', task.input, task.output ];
|
||||
task.action = actions.copy;
|
||||
task.input = cachePath;
|
||||
return true;
|
||||
}
|
||||
|
||||
// task is a file copy
|
||||
if (task.action === actions.copy) {
|
||||
task.log = [ 'copy', task.input, task.output ];
|
||||
return true;
|
||||
}
|
||||
|
||||
// file does not exist in cache, build it
|
||||
task.log = [ 'rebuild', task.input, task.output ];
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
async function execute (manifest, tasks, rev) {
|
||||
const lastSeen = Math.floor(Date.now() / 1000);
|
||||
const revManifest = {};
|
||||
|
||||
let writeCounter = 0;
|
||||
let lastWriteTime = 0;
|
||||
async function writeManifest (force) {
|
||||
if (!force && rev) return; // disable interim writes during prod builds.
|
||||
if (!force && ++writeCounter % 100) return;
|
||||
const now = Date.now();
|
||||
if (!force && now - lastWriteTime < 10000) return;
|
||||
lastWriteTime = now;
|
||||
await fs.writeFile(MANIFEST_PATH, JSON.stringify(manifest, null, 2));
|
||||
}
|
||||
|
||||
await Promise.map(sortBy(tasks, [ 'input', 'output' ]), async (task) => {
|
||||
const output = path.resolve(CWD, 'dist', task.output);
|
||||
|
||||
const result = task.action && await task.action({ ...task, output });
|
||||
const apply = task.apply || {};
|
||||
if (task.log && LOG[task.log[0]]) log.info(...task.log);
|
||||
apply.lastSeen = lastSeen;
|
||||
apply.lastSeenHuman = new Date();
|
||||
|
||||
if (!result) log('Nothing happened?', task);
|
||||
|
||||
const rhash = result && revHash(result);
|
||||
const hashedPath = revPath(task.output, rhash);
|
||||
apply.revHash = rhash;
|
||||
apply.revPath = hashedPath;
|
||||
|
||||
if (rev && rhash) {
|
||||
const rOutPath = task.output;
|
||||
const rNewPath = hashedPath;
|
||||
|
||||
revManifest[rOutPath] = rNewPath;
|
||||
|
||||
await fs.copy(output, path.resolve(CWD, 'dist', hashedPath));
|
||||
}
|
||||
|
||||
manifest[task.hash] = { ...manifest[task.hash], ...apply };
|
||||
await writeManifest();
|
||||
|
||||
}, { concurrency: rev ? 20 : 10 });
|
||||
|
||||
// filter unseen files from history
|
||||
// manifest = omitBy(manifest, (m) => m.lastSeen !== lastSeen);
|
||||
|
||||
await writeManifest(true);
|
||||
|
||||
if (rev) {
|
||||
let originalManifest = {};
|
||||
try {
|
||||
if (await fs.exists(REV_MANIFEST_PATH)) {
|
||||
originalManifest = JSON.parse(await fs.readFile(REV_MANIFEST_PATH));
|
||||
}
|
||||
} catch (e) {
|
||||
// do nothing
|
||||
}
|
||||
|
||||
Object.assign(originalManifest, revManifest);
|
||||
|
||||
await fs.writeFile(REV_MANIFEST_PATH, JSON.stringify(originalManifest, null, 2));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
exports().catch(console.error).then(() => process.exit()); // eslint-disable-line
|
||||
}
|
||||
|
||||
|
||||
179
gulp/imgflow/pipeline.js
Normal file
179
gulp/imgflow/pipeline.js
Normal file
@ -0,0 +1,179 @@
|
||||
const path = require('path');
|
||||
const { sortBy } = require('lodash');
|
||||
const Promise = require('bluebird');
|
||||
const fs = require('fs-extra');
|
||||
const log = require('fancy-log');
|
||||
const actions = require('./actions');
|
||||
const revHash = require('rev-hash');
|
||||
const revPath = require('rev-path');
|
||||
|
||||
const CWD = path.resolve(__dirname, '../..');
|
||||
const PAGES = path.join(CWD, 'pages');
|
||||
const SOURCE = path.resolve(PAGES, '**/*.{jpeg,jpg,png,gif,mp4}');
|
||||
const MANIFEST_PATH = path.resolve(CWD, 'if-manifest.json');
|
||||
const REV_MANIFEST_PATH = path.resolve(CWD, 'rev-manifest.json');
|
||||
const MEDIA_INDEX = path.resolve(CWD, 'twitter-media.json');
|
||||
const CACHE = 'if-cache';
|
||||
|
||||
const LOG = {
|
||||
new: true,
|
||||
update: true,
|
||||
skip: true,
|
||||
rebuild: true,
|
||||
cached: false,
|
||||
copy: false,
|
||||
};
|
||||
|
||||
exports.changed = async function changed (manifest, tasks) {
|
||||
const statMap = new Map();
|
||||
async function stat (f) {
|
||||
if (statMap.has(f)) return statMap.get(f);
|
||||
|
||||
const p = fs.stat(path.resolve(CWD, f))
|
||||
.catch(() => null)
|
||||
.then((stats) => (stats && Math.floor(stats.mtimeMs / 1000)));
|
||||
|
||||
statMap.set(f, p);
|
||||
return p;
|
||||
}
|
||||
|
||||
return Promise.filter(tasks, async (task) => {
|
||||
|
||||
const local = task.input.slice(0, 4) !== 'http';
|
||||
const hash = task.action.name + '.' + revHash(task.input) + '|' + revHash(task.output);
|
||||
const cachePath = path.join(CACHE, `${hash}${path.extname(task.output)}`);
|
||||
const [ inTime, outTime, cachedTime ] = await Promise.all([
|
||||
local && stat(path.resolve(CWD, task.input)),
|
||||
stat(path.resolve(CWD, 'dist', task.output)),
|
||||
stat(path.resolve(CWD, cachePath)),
|
||||
]);
|
||||
|
||||
task.manifest = manifest[hash];
|
||||
task.hash = hash;
|
||||
task.cache = cachePath;
|
||||
|
||||
// how did this happen?
|
||||
if (local && !inTime) {
|
||||
log.error('Input file could not be found?', task.input);
|
||||
return false;
|
||||
}
|
||||
|
||||
// never seen this file before
|
||||
if (!task.manifest) {
|
||||
task.apply = {
|
||||
hash,
|
||||
input: task.input,
|
||||
output: task.output,
|
||||
mtime: inTime,
|
||||
};
|
||||
task.log = [ 'new', task.input, task.output, hash ];
|
||||
return true;
|
||||
}
|
||||
|
||||
// file modification time does not match last read, rebuild
|
||||
if (local && inTime > task.manifest.mtime) {
|
||||
task.log = [ 'update', task.input, task.output ];
|
||||
task.apply = {
|
||||
mtime: inTime,
|
||||
};
|
||||
return true;
|
||||
}
|
||||
|
||||
task.apply = {
|
||||
mtime: local ? inTime : Math.floor(Date.now() / 1000),
|
||||
};
|
||||
|
||||
// target file exists, nothing to do
|
||||
if (outTime) {
|
||||
return false;
|
||||
// task.log = [ 'skip', task.input, task.output, inTime, task.manifest.mtime ];
|
||||
// task.action = null;
|
||||
// return true;
|
||||
}
|
||||
|
||||
// file exists in the cache, change the task to a copy action
|
||||
if (cachedTime) {
|
||||
task.log = [ 'cached', task.input, task.output ];
|
||||
task.action = actions.copy;
|
||||
task.input = cachePath;
|
||||
return true;
|
||||
}
|
||||
|
||||
// task is a file copy
|
||||
if (task.action === actions.copy) {
|
||||
task.log = [ 'copy', task.input, task.output ];
|
||||
return true;
|
||||
}
|
||||
|
||||
// file does not exist in cache, build it
|
||||
task.log = [ 'rebuild', task.input, task.output ];
|
||||
return true;
|
||||
});
|
||||
};
|
||||
|
||||
|
||||
exports.execute = async function execute (manifest, tasks, rev) {
|
||||
const lastSeen = Math.floor(Date.now() / 1000);
|
||||
const revManifest = {};
|
||||
|
||||
let writeCounter = 0;
|
||||
let lastWriteTime = 0;
|
||||
async function writeManifest (force) {
|
||||
if (!force && rev) return; // disable interim writes during prod builds.
|
||||
if (!force && ++writeCounter % 100) return;
|
||||
const now = Date.now();
|
||||
if (!force && now - lastWriteTime < 10000) return;
|
||||
lastWriteTime = now;
|
||||
await fs.writeFile(MANIFEST_PATH, JSON.stringify(manifest, null, 2));
|
||||
}
|
||||
|
||||
await Promise.map(sortBy(tasks, [ 'input', 'output' ]), async (task) => {
|
||||
const output = path.resolve(CWD, 'dist', task.output);
|
||||
|
||||
const result = task.action && await task.action({ ...task, output });
|
||||
const apply = task.apply || {};
|
||||
if (task.log && LOG[task.log[0]]) log.info(...task.log);
|
||||
apply.lastSeen = lastSeen;
|
||||
apply.lastSeenHuman = new Date();
|
||||
|
||||
if (!result) log('Nothing happened?', task);
|
||||
|
||||
const rhash = result && revHash(result);
|
||||
const hashedPath = revPath(task.output, rhash);
|
||||
apply.revHash = rhash;
|
||||
apply.revPath = hashedPath;
|
||||
|
||||
if (rev && rhash) {
|
||||
const rOutPath = task.output;
|
||||
const rNewPath = hashedPath;
|
||||
|
||||
revManifest[rOutPath] = rNewPath;
|
||||
|
||||
await fs.copy(output, path.resolve(CWD, 'dist', hashedPath));
|
||||
}
|
||||
|
||||
manifest[task.hash] = { ...manifest[task.hash], ...apply };
|
||||
await writeManifest();
|
||||
|
||||
}, { concurrency: rev ? 20 : 10 });
|
||||
|
||||
// filter unseen files from history
|
||||
// manifest = omitBy(manifest, (m) => m.lastSeen !== lastSeen);
|
||||
|
||||
await writeManifest(true);
|
||||
|
||||
if (rev) {
|
||||
let originalManifest = {};
|
||||
try {
|
||||
if (await fs.exists(REV_MANIFEST_PATH)) {
|
||||
originalManifest = JSON.parse(await fs.readFile(REV_MANIFEST_PATH));
|
||||
}
|
||||
} catch (e) {
|
||||
// do nothing
|
||||
}
|
||||
|
||||
Object.assign(originalManifest, revManifest);
|
||||
|
||||
await fs.writeFile(REV_MANIFEST_PATH, JSON.stringify(originalManifest, null, 2));
|
||||
}
|
||||
};
|
||||
50
package-lock.json
generated
50
package-lock.json
generated
@ -522,6 +522,12 @@
|
||||
"integrity": "sha512-uMgjozySS8adZZYePpaWs8cxB9/kdzmpX6SgJZ+wbz1K5eYk5QMYDVJaZKhxyIHUdnnJkfR7SVgStgH7LkGUyg==",
|
||||
"dev": true
|
||||
},
|
||||
"any-promise": {
|
||||
"version": "1.3.0",
|
||||
"resolved": "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz",
|
||||
"integrity": "sha1-q8av7tzqUugJzcA3au0845Y10X8=",
|
||||
"dev": true
|
||||
},
|
||||
"anymatch": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/anymatch/-/anymatch-2.0.0.tgz",
|
||||
@ -4005,6 +4011,15 @@
|
||||
"integrity": "sha1-3BXKHGcjh8p2vTesCjlbogQqLCg=",
|
||||
"dev": true
|
||||
},
|
||||
"get-video-dimensions": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/get-video-dimensions/-/get-video-dimensions-1.0.0.tgz",
|
||||
"integrity": "sha1-/H5ayBw5JEH1uG1Q3XeDiptTFHo=",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"mz": "^1.2.1"
|
||||
}
|
||||
},
|
||||
"getpass": {
|
||||
"version": "0.1.7",
|
||||
"resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz",
|
||||
@ -6411,6 +6426,17 @@
|
||||
"integrity": "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==",
|
||||
"dev": true
|
||||
},
|
||||
"mz": {
|
||||
"version": "1.3.0",
|
||||
"resolved": "https://registry.npmjs.org/mz/-/mz-1.3.0.tgz",
|
||||
"integrity": "sha1-BvCT/dmVagbTfhsegTROJ0eMQvA=",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"native-or-bluebird": "1",
|
||||
"thenify": "3",
|
||||
"thenify-all": "1"
|
||||
}
|
||||
},
|
||||
"nan": {
|
||||
"version": "2.14.0",
|
||||
"resolved": "https://registry.npmjs.org/nan/-/nan-2.14.0.tgz",
|
||||
@ -6436,6 +6462,12 @@
|
||||
"to-regex": "^3.0.1"
|
||||
}
|
||||
},
|
||||
"native-or-bluebird": {
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/native-or-bluebird/-/native-or-bluebird-1.2.0.tgz",
|
||||
"integrity": "sha1-OcR7/Xgl0fuf+tMiEK4l2q3xAck=",
|
||||
"dev": true
|
||||
},
|
||||
"natural-compare": {
|
||||
"version": "1.4.0",
|
||||
"resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz",
|
||||
@ -9033,6 +9065,24 @@
|
||||
"integrity": "sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=",
|
||||
"dev": true
|
||||
},
|
||||
"thenify": {
|
||||
"version": "3.3.0",
|
||||
"resolved": "https://registry.npmjs.org/thenify/-/thenify-3.3.0.tgz",
|
||||
"integrity": "sha1-5p44obq+lpsBCCB5eLn2K4hgSDk=",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"any-promise": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"thenify-all": {
|
||||
"version": "1.6.0",
|
||||
"resolved": "https://registry.npmjs.org/thenify-all/-/thenify-all-1.6.0.tgz",
|
||||
"integrity": "sha1-GhkY1ALY/D+Y+/I02wvMjMEOlyY=",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"thenify": ">= 3.1.0 < 4"
|
||||
}
|
||||
},
|
||||
"through": {
|
||||
"version": "2.3.8",
|
||||
"resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz",
|
||||
|
||||
@ -40,6 +40,7 @@
|
||||
"forever": "~2.0.0",
|
||||
"front-matter": "~3.1.0",
|
||||
"fs-extra": "~8.1.0",
|
||||
"get-video-dimensions": "~1.0.0",
|
||||
"glob": "~7.1.6",
|
||||
"gm": "~1.23.1",
|
||||
"gulp": "~4.0.2",
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user