Use make -j for parallel build (#10506)

* build: add build-babel-types task

* chore: revamp Makefile to for parallel builds

* chore: style fix

* chore: skip install step as we handle yarn in makefile

* test job execution order

.travis.yml

@@ -6,48 +6,40 @@ cache:
 
 os: linux
 
-node_js:
-  # We test the latest version on circleci
-  - "11"
-  - "10"
-  - "8"
-  - "6"
-
 env:
   global:
     - PATH=$HOME/.yarn/bin:$PATH
     - JOB=test
 
-before_install:
+install: skip
-  - curl -o- -L https://yarnpkg.com/install.sh | bash
-
-install:
-  # the `make test-ci` script runs this command already
-  - if [ "$JOB" != "test" ] && [ "$JOB" != "lint" ]; then yarn install; fi
-  - if [ "$JOB" = "lint" ]; then make bootstrap; fi
-
 before_script:
+  - curl -o- -L https://yarnpkg.com/install.sh | bash
   - if [ "$TRAVIS_OS_NAME" = "windows" ]; then choco install make; fi
-  - if [ "$JOB" = "babel-parser-flow-tests" ]; then make bootstrap-flow; fi
-  - if [ "$JOB" = "babel-parser-test262-tests" ]; then make bootstrap-test262; fi
 
 script:
-  - if [ "$JOB" = "test" ]; then make test-ci; fi
+  - if [ "$JOB" = "test" ]; then make -j test-ci; fi
-  - if [ "$JOB" = "lint" ]; then make lint && make flow; fi
+  - if [ "$JOB" = "lint" ]; then make -j code-quality-ci; fi
-  - if [ "$JOB" = "babel-parser-flow-tests" ]; then make test-flow-ci; fi
+  - if [ "$JOB" = "babel-parser-flow-tests" ]; then make -j test-flow-ci; fi
-  - if [ "$JOB" = "babel-parser-test262-tests" ]; then make test-test262-ci; fi
+  - if [ "$JOB" = "babel-parser-test262-tests" ]; then make -j test-test262-ci; fi
 
 matrix:
   fast_finish: true
   include:
+    - node_js: "node"
+      env: JOB=lint
+  # We test the latest version on circleci
+    - node_js: "11"
+  # Move `windows` build to be the third since it is slow
     - os: windows
       node_js: "node"
       env:
       - JOB=test
       # https://travis-ci.community/t/build-doesnt-finish-after-completing-tests/288/9
       - YARN_GPG=no
-    - node_js: "node"
+  # Continue node_js matrix
-      env: JOB=lint
+    - node_js: "6"
+    - node_js: "10"
+    - node_js: "8"
     - node_js: "node"
       env: JOB=babel-parser-flow-tests
     - node_js: "node"

Gulpfile.js

@@ -11,25 +11,20 @@ const filter = require("gulp-filter");
 const gulp = require("gulp");
 const path = require("path");
 const webpack = require("webpack");
-const merge = require("merge-stream");
 const rollup = require("rollup");
 const rollupBabel = require("rollup-plugin-babel");
 const rollupNodeResolve = require("rollup-plugin-node-resolve");
 const rollupReplace = require("rollup-plugin-replace");
 const { registerStandalonePackageTask } = require("./scripts/gulp-tasks");
 
-const sources = ["codemods", "packages"];
+const defaultSourcesGlob = "./@(codemods|packages)/*/src/**/*.js";
 
 function swapSrcWithLib(srcPath) {
   const parts = srcPath.split(path.sep);
-  parts[1] = "lib";
+  parts[2] = "lib";
   return parts.join(path.sep);
 }
 
-function getGlobFromSource(source) {
-  return `./${source}/*/src/**/*.js`;
-}
-
 function getIndexFromPackage(name) {
   return `${name}/src/index.js`;
 }
@@ -56,32 +51,28 @@ function rename(fn) {
   });
 }
 
-function buildBabel(exclude) {
+function buildBabel(exclude, sourcesGlob = defaultSourcesGlob) {
-  return merge(
+  const base = __dirname;
-    sources.map(source => {
-      const base = path.join(__dirname, source);
 
-      let stream = gulp.src(getGlobFromSource(source), { base: base });
+  let stream = gulp.src(sourcesGlob, { base: __dirname });
 
-      if (exclude) {
+  if (exclude) {
-        const filters = exclude.map(p => `!**/${p}/**`);
+    const filters = exclude.map(p => `!**/${p}/**`);
-        filters.unshift("**");
+    filters.unshift("**");
-        stream = stream.pipe(filter(filters));
+    stream = stream.pipe(filter(filters));
-      }
+  }
 
-      return stream
+  return stream
-        .pipe(errorsLogger())
+    .pipe(errorsLogger())
-        .pipe(newer({ dest: base, map: swapSrcWithLib }))
+    .pipe(newer({ dest: base, map: swapSrcWithLib }))
-        .pipe(compilationLogger())
+    .pipe(compilationLogger())
-        .pipe(babel())
+    .pipe(babel())
-        .pipe(
+    .pipe(
-          // Passing 'file.relative' because newer() above uses a relative
+      // Passing 'file.relative' because newer() above uses a relative
-          // path and this keeps it consistent.
+      // path and this keeps it consistent.
-          rename(file => path.resolve(file.base, swapSrcWithLib(file.relative)))
+      rename(file => path.resolve(file.base, swapSrcWithLib(file.relative)))
-        )
+    )
-        .pipe(gulp.dest(base));
+    .pipe(gulp.dest(base));
-    })
-  );
 }
 
 function buildRollup(packages) {
@@ -118,6 +109,9 @@ const bundles = ["packages/babel-parser"];
 
 gulp.task("build-rollup", () => buildRollup(bundles));
 gulp.task("build-babel", () => buildBabel(/* exclude */ bundles));
+gulp.task("build-babel-types", () =>
+  buildBabel(/* exclude */ bundles, "packages/babel-types/src/**/*.js")
+);
 gulp.task("build", gulp.parallel("build-rollup", "build-babel"));
 
 gulp.task("default", gulp.series("build"));
@@ -128,7 +122,7 @@ gulp.task(
   "watch",
   gulp.series("build-no-bundle", function watch() {
     gulpWatch(
-      sources.map(getGlobFromSource),
+      defaultSourcesGlob,
       { debounceDelay: 200 },
       gulp.task("build-no-bundle")
     );

Makefile

@@ -1,4 +1,3 @@
-MAKEFLAGS = -j1
 FLOW_COMMIT = 09669846b7a7ca5a6c23c12d56bb3bebdafd67e9
 TEST262_COMMIT = a1acc23cd85c552cde9c06cfed300c58d4b5b04c
 
@@ -9,21 +8,42 @@ SOURCES = packages codemods
 
 .PHONY: build build-dist watch lint fix clean test-clean test-only test test-ci publish bootstrap
 
-build: clean clean-lib
+build: build-bundle
-	yarn gulp build
-	node packages/babel-standalone/scripts/generate.js
-	node packages/babel-types/scripts/generateTypeHelpers.js
-	# call build again as the generated files might need to be compiled again.
-	yarn gulp build
-	# generate flow and typescript typings
-	node packages/babel-types/scripts/generators/flow.js > packages/babel-types/lib/index.js.flow
-	node packages/babel-types/scripts/generators/typescript.js > packages/babel-types/lib/index.d.ts
 ifneq ("$(BABEL_COVERAGE)", "true")
-	make build-standalone
+	$(MAKE) build-standalone
-	make build-preset-env-standalone
 endif
 
-build-standalone:
+build-bundle: clean clean-lib
+	yarn gulp build
+	$(MAKE) generate-standalone generate-type-helpers
+	# call build again as the generated files might need to be compiled again.
+	yarn gulp build
+	$(MAKE) build-typings
+	$(MAKE) build-dist
+
+build-bundle-ci: bootstrap-only
+	$(MAKE) build-bundle
+
+generate-standalone:
+	node packages/babel-standalone/scripts/generate.js
+
+generate-type-helpers:
+	node packages/babel-types/scripts/generateTypeHelpers.js
+
+build-typings: build-flow-typings build-typescript-typings
+
+build-flow-typings:
+	node packages/babel-types/scripts/generators/flow.js > packages/babel-types/lib/index.js.flow
+
+build-typescript-typings:
+	node packages/babel-types/scripts/generators/typescript.js > packages/babel-types/lib/index.d.ts
+
+build-standalone: build-babel-standalone build-preset-env-standalone
+
+build-standalone-ci: build-bundle-ci
+	$(MAKE) build-standalone
+
+build-babel-standalone:
 	yarn gulp build-babel-standalone
 
 build-preset-env-standalone:
@@ -35,25 +55,48 @@ prepublish-build-standalone:
 prepublish-build-preset-env-standalone:
 	BABEL_ENV=production IS_PUBLISH=true yarn gulp build-babel-preset-env-standalone
 
-build-dist: build
+build-dist: build-polyfill-dist build-plugin-transform-runtime-dist
+
+build-polyfill-dist:
 	cd packages/babel-polyfill; \
 	scripts/build-dist.sh
+
+build-plugin-transform-runtime-dist:
 	cd packages/babel-plugin-transform-runtime; \
 	node scripts/build-dist.js
 
-watch: clean clean-lib
+build-no-bundle: clean clean-lib
-
+	BABEL_ENV=development yarn gulp build-no-bundle
 	# Ensure that build artifacts for types are created during local
 	# development too.
-	BABEL_ENV=development yarn gulp build-no-bundle
+	$(MAKE) generate-type-helpers
-	node packages/babel-types/scripts/generateTypeHelpers.js
+	$(MAKE) build-typings
-	node packages/babel-types/scripts/generators/flow.js > packages/babel-types/lib/index.js.flow
+
-	node packages/babel-types/scripts/generators/typescript.js > packages/babel-types/lib/index.d.ts
+watch: build-no-bundle
 	BABEL_ENV=development yarn gulp watch
 
+code-quality-ci: flowcheck-ci lint-ci
+
+flowcheck-ci: bootstrap-flowcheck
+	$(MAKE) flow
+
+code-quality: flow lint
+
 flow:
 	yarn flow check --strip-root
 
+bootstrap-flowcheck: bootstrap-only
+	yarn gulp build-babel-types
+	$(MAKE) build-typings
+
+lint-ci: lint-js-ci lint-ts-ci
+
+lint-js-ci: bootstrap-only
+	$(MAKE) lint-js
+
+lint-ts-ci: bootstrap-flowcheck
+	$(MAKE) lint-ts
+
 lint: lint-js lint-ts
 
 lint-js:
@@ -62,7 +105,9 @@ lint-js:
 lint-ts:
 	scripts/tests/typescript/lint.sh
 
-fix: fix-json
+fix: fix-json fix-js
+
+fix-js:
 	yarn eslint scripts $(SOURCES) '*.js' --format=codeframe --fix
 
 fix-json:
@@ -82,18 +127,20 @@ test-clean:
 # Does not work on Windows; use "yarn jest" instead
 test-only:
 	BABEL_ENV=test ./scripts/test.sh
-	make test-clean
+	$(MAKE) test-clean
 
 test: lint test-only
 
-test-ci: bootstrap
+test-ci: jest-ci
+
+jest-ci: build-standalone-ci
 	BABEL_ENV=test yarn jest --maxWorkers=4 --ci
-	make test-clean
+	$(MAKE) test-clean
 
 # Does not work on Windows
 test-ci-coverage: SHELL:=/bin/bash
 test-ci-coverage:
-	BABEL_COVERAGE=true BABEL_ENV=test make bootstrap
+	BABEL_COVERAGE=true BABEL_ENV=test $(MAKE) bootstrap
 	BABEL_ENV=test TEST_TYPE=cov ./scripts/test-cov.sh
 	bash <(curl -s https://codecov.io/bash) -f coverage/coverage-final.json
 
@@ -106,7 +153,8 @@ bootstrap-flow:
 test-flow:
 	node scripts/tests/flow/run_babel_parser_flow_tests.js
 
-test-flow-ci: bootstrap test-flow
+test-flow-ci: build-bundle-ci bootstrap-flow
+	$(MAKE) test-flow
 
 test-flow-update-whitelist:
 	node scripts/tests/flow/run_babel_parser_flow_tests.js --update-whitelist
@@ -120,7 +168,8 @@ bootstrap-test262:
 test-test262:
 	node scripts/tests/test262/run_babel_parser_test262.js
 
-test-test262-ci: bootstrap test-test262
+test-test262-ci: build-bundle-ci bootstrap-test262
+	$(MAKE) test-test262
 
 test-test262-update-whitelist:
 	node scripts/tests/test262/run_babel_parser_test262.js --update-whitelist
@@ -129,18 +178,14 @@ test-test262-update-whitelist:
 clone-license:
 	./scripts/clone-license.sh
 
-prepublish-build:
+prepublish-build: clean-lib clean-runtime-helpers
-	make clean-lib
+	NODE_ENV=production BABEL_ENV=production $(MAKE) build-dist
-	rm -rf packages/babel-runtime/helpers
+	$(MAKE) clone-license
-	rm -rf packages/babel-runtime-corejs2/helpers
-	rm -rf packages/babel-runtime-corejs2/core-js
-	NODE_ENV=production BABEL_ENV=production make build-dist
-	make clone-license
 
 prepublish:
-	make bootstrap-only
+	$(MAKE) bootstrap-only
-	make prepublish-build
+	$(MAKE) prepublish-build
-	make test
+	$(MAKE) test
 
 new-version:
 	git pull --rebase
@@ -149,7 +194,7 @@ new-version:
 # NOTE: Run make new-version first
 publish: prepublish
 	yarn lerna publish from-git --require-scripts
-	make clean
+	$(MAKE) clean
 
 publish-ci: prepublish
 ifneq ("$(NPM_TOKEN)", "")
@@ -160,21 +205,28 @@ else
 endif
 	yarn lerna publish from-git --require-scripts --yes
 	rm -f .npmrc
-	make clean
+	$(MAKE) clean
 
-bootstrap-only: clean-all
+bootstrap-only: lerna-bootstrap
+
+yarn-install: clean-all
 	yarn --ignore-engines
+
+lerna-bootstrap: yarn-install
 	yarn lerna bootstrap -- --ignore-engines
 
 bootstrap: bootstrap-only
-	make build
+	$(MAKE) build
-	cd packages/babel-plugin-transform-runtime; \
-	node scripts/build-dist.js
 
 clean-lib:
 	$(foreach source, $(SOURCES), \
 		$(call clean-source-lib, $(source)))
 
+clean-runtime-helpers:
+	rm -rf packages/babel-runtime/helpers
+	rm -rf packages/babel-runtime-corejs2/helpers
+	rm -rf packages/babel-runtime-corejs2/core-js
+
 clean-all:
 	rm -rf node_modules
 	rm -rf package-lock.json
@@ -183,7 +235,7 @@ clean-all:
 	$(foreach source, $(SOURCES), \
 		$(call clean-source-all, $(source)))
 
-	make clean
+	$(MAKE) clean
 
 define clean-source-lib
 	rm -rf $(1)/*/lib

package.json

@@ -56,7 +56,6 @@
     "lerna-changelog": "^0.5.0",
     "lint-staged": "^9.2.0",
     "lodash": "^4.17.13",
-    "merge-stream": "^1.0.1",
     "output-file-sync": "^2.0.0",
     "prettier": "^1.17.1",
     "pump": "^3.0.0",