chore: use new parallel build feature of jsii

Improve build times by using the new "build-all-at-once" feature of
jsii-pacmak.

Goes together with aws/jsii#849.

Author: rix0rrr

pack.sh

@@ -2,7 +2,7 @@
 # Runs "npm package" in all modules. This will produce a "dist/" directory in each module.
 # Then, calls pack-collect.sh to merge all outputs into a root ./pack directory, which is
 # later read by bundle-beta.sh.
-set -e
+set -eu
 export PATH=$PWD/node_modules/.bin:$PATH
 export NODE_OPTIONS="--max-old-space-size=4096 ${NODE_OPTIONS:-}"
 root=$PWD
@@ -11,15 +11,33 @@ distdir="$PWD/dist"
 rm -fr ${distdir}
 mkdir -p ${distdir}
 
-scopes=$(lerna ls 2>/dev/null | grep -v "(private)" | cut -d" " -f1 | xargs -n1 -I{} echo "--scope {}" | tr "\n" " ")
+# Split out jsii and non-jsii packages. Jsii packages will be built all at once.
+# Non-jsii packages will be run individually.
+echo "Collecting package list..." >&2
+scripts/list-packages $TMPDIR/jsii.txt $TMPDIR/nonjsii.txt
 
-# Run the "cdk-package" script in all modules. For jsii modules, this invokes jsii-pacmak which generates and builds multi-language
-# outputs. For non-jsii module, it will just run "npm pack" and place the output in dist/npm
-# (which is similar to how pacmak outputs it).
-lerna run ${scopes} --sort --concurrency=1 --stream package
+# Return lerna scopes from a package list
+function lerna_scopes() {
+  while [[ "${1:-}" != "" ]]; do
+    echo "--scope $1 "
+    shift
+  done
+}
+
+echo "Packaging jsii modules" >&2
+
+# Jsii packaging (all at once using jsii-pacmak)
+jsii-pacmak \
+  --verbose \
+  --outdir $distdir/ \
+  $(cat $TMPDIR/jsii.txt)
+
+# Non-jsii packaging, which means running 'package' in every individual
+# module and rsync'ing the result to the shared dist directory.
+echo "Packaging non-jsii modules" >&2
+lerna run $(lerna_scopes $(cat $TMPDIR/nonjsii.txt)) --sort --concurrency=1 --stream package
 
-# Collect dist/ from all modules into the root dist/
-for dir in $(find packages -name dist | grep -v node_modules); do
+for dir in $(find packages -name dist | grep -v node_modules | grep -v run-wrappers); do
   echo "Merging ${dir} into ${distdir}"
   rsync -av $dir/ ${distdir}/
 done

package.json

@@ -15,6 +15,7 @@
     "fs-extra": "^8.1.0",
     "jest": "^24.9.0",
     "jsii-diff": "^0.17.1",
+    "jsii-pacmak": "^0.17.1",
     "lerna": "^3.16.4",
     "nodeunit": "^0.11.3",
     "nyc": "^14.1.1",

scripts/list-packages

@@ -0,0 +1,36 @@
+#!/usr/bin/env node
+/**
+ * Collects all packages in the repository in a way that makes it
+ * easy to process for packaging.
+ */
+const child_process = require('child_process');
+const fs = require('fs-extra');
+const path = require('path');
+
+if (process.argv.length < 4) {
+  process.stderr.write('Usage: list-packages <jsii file> <nonjsii file>\n');
+  process.exit(1);
+}
+
+child_process.exec('lerna ls --json', { shell: true }, (error, stdout) => {
+  if (error) {
+    console.error('Error: ', error);
+    process.exit(-1);
+  }
+  const modules = JSON.parse(stdout.toString('utf8'));
+
+  const jsiiDirectories = [];
+  const nonJsiiNames = [];
+
+  for (const module of modules) {
+    const pkgJson = require(path.join(module.location, 'package.json'));
+    if (pkgJson.jsii) {
+      jsiiDirectories.push(module.location);
+    } else {
+      nonJsiiNames.push(pkgJson.name);
+    }
+  }
+
+  fs.writeFileSync(process.argv[2], jsiiDirectories.join('\n'));
+  fs.writeFileSync(process.argv[3], nonJsiiNames.join('\n'));
+});