Implement pipelines to compose multiple asset types together (#1065)

Author: devongovett

src/Asset.js

@@ -26,7 +26,7 @@ class Asset {
     this.type = path.extname(this.name).slice(1);
 
     this.processed = false;
-    this.contents = null;
+    this.contents = options.rendition ? options.rendition.value : null;
     this.ast = null;
     this.generated = null;
     this.hash = null;
@@ -58,6 +58,13 @@ class Asset {
   }
 
   async getDependencies() {
+    if (
+      this.options.rendition &&
+      this.options.rendition.hasDependencies === false
+    ) {
+      return;
+    }
+
     await this.loadIfNeeded();
 
     if (this.contents && this.mightHaveDependencies()) {
@@ -154,6 +161,10 @@ class Asset {
     return this.generated;
   }
 
+  async postProcess(generated) {
+    return generated;
+  }
+
   generateHash() {
     return objectHash(this.generated);
   }

src/Pipeline.js

@@ -0,0 +1,102 @@
+const Parser = require('./Parser');
+const path = require('path');
+const md5 = require('./utils/md5');
+
+/**
+ * A Pipeline composes multiple Asset types together.
+ */
+class Pipeline {
+  constructor(options) {
+    this.options = options;
+    this.parser = new Parser(options);
+  }
+
+  async process(path, pkg, options) {
+    let asset = this.parser.getAsset(path, pkg, options);
+    let generated = await this.processAsset(asset);
+    let generatedMap = {};
+    for (let rendition of generated) {
+      generatedMap[rendition.type] = rendition.value;
+    }
+
+    return {
+      dependencies: Array.from(asset.dependencies.values()),
+      generated: generatedMap,
+      hash: asset.hash,
+      cacheData: asset.cacheData
+    };
+  }
+
+  async processAsset(asset) {
+    try {
+      await asset.process();
+    } catch (err) {
+      throw asset.generateErrorMessage(err);
+    }
+
+    let inputType = path.extname(asset.name).slice(1);
+    let generated = [];
+
+    for (let rendition of this.iterateRenditions(asset)) {
+      let {type, value} = rendition;
+      if (typeof value !== 'string' || rendition.final) {
+        generated.push(rendition);
+        continue;
+      }
+
+      // Find an asset type for the rendition type.
+      // If the asset is not already an instance of this asset type, process it.
+      let AssetType = this.parser.findParser(
+        asset.name.slice(0, -inputType.length) + type
+      );
+      if (!(asset instanceof AssetType)) {
+        let opts = Object.assign({rendition}, asset.options);
+        let subAsset = new AssetType(asset.name, asset.package, opts);
+        subAsset.contents = value;
+        subAsset.dependencies = asset.dependencies;
+
+        let processed = await this.processAsset(subAsset);
+        generated = generated.concat(processed);
+        Object.assign(asset.cacheData, subAsset.cacheData);
+        asset.hash = md5(asset.hash + subAsset.hash);
+      } else {
+        generated.push(rendition);
+      }
+    }
+
+    // Post process. This allows assets a chance to modify the output produced by sub-asset types.
+    asset.generated = generated;
+    try {
+      generated = await asset.postProcess(generated);
+    } catch (err) {
+      throw asset.generateErrorMessage(err);
+    }
+
+    return generated;
+  }
+
+  *iterateRenditions(asset) {
+    if (Array.isArray(asset.generated)) {
+      return yield* asset.generated;
+    }
+
+    if (typeof asset.generated === 'string') {
+      return yield {
+        type: asset.type,
+        value: asset.generated
+      };
+    }
+
+    // Backward compatibility support for the old API.
+    // Assume all renditions are final - don't compose asset types together.
+    for (let type in asset.generated) {
+      yield {
+        type,
+        value: asset.generated[type],
+        final: true
+      };
+    }
+  }
+}
+
+module.exports = Pipeline;

src/assets/CSSAsset.js

@@ -116,7 +116,17 @@ class CSSAsset extends Asset {
         'module.exports = ' + JSON.stringify(this.cssModules, false, 2) + ';';
     }
 
-    return {css, js};
+    return [
+      {
+        type: 'css',
+        value: css
+      },
+      {
+        type: 'js',
+        value: js,
+        final: true
+      }
+    ];
   }
 
   generateErrorMessage(err) {

src/assets/CoffeeScriptAsset.js

@@ -1,24 +1,35 @@
-const JSAsset = require('./JSAsset');
+const Asset = require('../Asset');
 const localRequire = require('../utils/localRequire');
 
-class CoffeeScriptAsset extends JSAsset {
-  async parse(code) {
+class CoffeeScriptAsset extends Asset {
+  constructor(name, pkg, options) {
+    super(name, pkg, options);
+    this.type = 'js';
+  }
+
+  async generate() {
     // require coffeescript, installed locally in the app
     let coffee = await localRequire('coffeescript', this.name);
 
     // Transpile Module using CoffeeScript and parse result as ast format through babylon
-    let transpiled = coffee.compile(code, {
+    let transpiled = coffee.compile(this.contents, {
       sourceMap: this.options.sourceMaps
     });
 
+    let sourceMap;
     if (transpiled.sourceMap) {
-      this.sourceMap = transpiled.sourceMap.generate();
-      this.sourceMap.sources = [this.relativeName];
-      this.sourceMap.sourcesContent = [this.contents];
+      sourceMap = transpiled.sourceMap.generate();
+      sourceMap.sources = [this.relativeName];
+      sourceMap.sourcesContent = [this.contents];
     }
 
-    this.contents = this.options.sourceMaps ? transpiled.js : transpiled;
-    return await super.parse(this.contents);
+    return [
+      {
+        type: 'js',
+        value: this.options.sourceMaps ? transpiled.js : transpiled,
+        sourceMap
+      }
+    ];
   }
 }
 

src/assets/HTMLAsset.js

@@ -160,8 +160,7 @@ class HTMLAsset extends Asset {
   }
 
   generate() {
-    let html = this.isAstDirty ? render(this.ast) : this.contents;
-    return {html};
+    return this.isAstDirty ? render(this.ast) : this.contents;
   }
 }
 

src/assets/JSAsset.js

@@ -27,6 +27,7 @@ class JSAsset extends Asset {
     this.isES6Module = false;
     this.outputCode = null;
     this.cacheData.env = {};
+    this.sourceMap = options.rendition ? options.rendition.sourceMap : null;
   }
 
   shouldInvalidate(cacheData) {

src/assets/LESSAsset.js

@@ -1,8 +1,13 @@
-const CSSAsset = require('./CSSAsset');
+const Asset = require('../Asset');
 const localRequire = require('../utils/localRequire');
 const promisify = require('../utils/promisify');
 
-class LESSAsset extends CSSAsset {
+class LESSAsset extends Asset {
+  constructor(name, pkg, options) {
+    super(name, pkg, options);
+    this.type = 'css';
+  }
+
   async parse(code) {
     // less should be installed locally in the module that's being required
     let less = await localRequire('less', this.name);
@@ -15,16 +20,24 @@ class LESSAsset extends CSSAsset {
     opts.filename = this.name;
     opts.plugins = (opts.plugins || []).concat(urlPlugin(this));
 
-    let res = await render(code, opts);
-    res.render = () => res.css;
-    return res;
+    return await render(code, opts);
   }
 
   collectDependencies() {
     for (let dep of this.ast.imports) {
       this.addDependency(dep, {includedInParent: true});
     }
   }
+
+  generate() {
+    return [
+      {
+        type: 'css',
+        value: this.ast.css,
+        hasDependencies: false
+      }
+    ];
+  }
 }
 
 function urlPlugin(asset) {

src/assets/ReasonAsset.js

@@ -1,9 +1,14 @@
-const JSAsset = require('./JSAsset');
+const Asset = require('../Asset');
 const fs = require('../utils/fs');
 const localRequire = require('../utils/localRequire');
 
-class ReasonAsset extends JSAsset {
-  async parse() {
+class ReasonAsset extends Asset {
+  constructor(name, pkg, options) {
+    super(name, pkg, options);
+    this.type = 'js';
+  }
+
+  async generate() {
     const bsb = await localRequire('bsb-js', this.name);
 
     // This runs BuckleScript - the Reason to JS compiler.
@@ -18,10 +23,7 @@ class ReasonAsset extends JSAsset {
     // BuckleScript configuration to simplify the file processing.
     const outputFile = this.name.replace(/\.(re|ml)$/, '.bs.js');
     const outputContent = await fs.readFile(outputFile);
-    this.contents = outputContent.toString();
-
-    // After loading the compiled JS source, use the normal JS behavior.
-    return await super.parse(this.contents);
+    return outputContent.toString();
   }
 }
 

src/assets/SASSAsset.js

@@ -1,10 +1,15 @@
-const CSSAsset = require('./CSSAsset');
+const Asset = require('../Asset');
 const localRequire = require('../utils/localRequire');
 const promisify = require('../utils/promisify');
 const path = require('path');
 const os = require('os');
 
-class SASSAsset extends CSSAsset {
+class SASSAsset extends Asset {
+  constructor(name, pkg, options) {
+    super(name, pkg, options);
+    this.type = 'css';
+  }
+
   async parse(code) {
     // node-sass should be installed locally in the module that's being required
     let sass = await localRequire('node-sass', this.name);
@@ -17,7 +22,7 @@ class SASSAsset extends CSSAsset {
     opts.includePaths = (opts.includePaths || []).concat(
       path.dirname(this.name)
     );
-    opts.data = opts.data ? (opts.data + os.EOL + code) : code;
+    opts.data = opts.data ? opts.data + os.EOL + code : code;
     opts.indentedSyntax =
       typeof opts.indentedSyntax === 'boolean'
         ? opts.indentedSyntax
@@ -30,16 +35,24 @@ class SASSAsset extends CSSAsset {
       }
     });
 
-    let res = await render(opts);
-    res.render = () => res.css.toString();
-    return res;
+    return await render(opts);
   }
 
   collectDependencies() {
     for (let dep of this.ast.stats.includedFiles) {
       this.addDependency(dep, {includedInParent: true});
     }
   }
+
+  generate() {
+    return [
+      {
+        type: 'css',
+        value: this.ast.css.toString(),
+        hasDependencies: false
+      }
+    ];
+  }
 }
 
 module.exports = SASSAsset;

src/assets/StylusAsset.js

@@ -1,11 +1,17 @@
-const CSSAsset = require('./CSSAsset');
+// const CSSAsset = require('./CSSAsset');
+const Asset = require('../Asset');
 const localRequire = require('../utils/localRequire');
 const Resolver = require('../Resolver');
 const syncPromise = require('../utils/syncPromise');
 
 const URL_RE = /^(?:url\s*\(\s*)?['"]?(?:[#/]|(?:https?:)?\/\/)/i;
 
-class StylusAsset extends CSSAsset {
+class StylusAsset extends Asset {
+  constructor(name, pkg, options) {
+    super(name, pkg, options);
+    this.type = 'css';
+  }
+
   async parse(code) {
     // stylus should be installed locally in the module that's being required
     let stylus = await localRequire('stylus', this.name);
@@ -26,8 +32,14 @@ class StylusAsset extends CSSAsset {
     return style;
   }
 
-  collectDependencies() {
-    // Do nothing. Dependencies are collected by our custom evaluator.
+  generate() {
+    return [
+      {
+        type: 'css',
+        value: this.ast.render(),
+        hasDependencies: false
+      }
+    ];
   }
 
   generateErrorMessage(err) {