Move some build steps to gulp (#12127)

* chore: move generate-standalone to gulp tasks

* move build-flow-typings to gulp tasks

* chore: move build-typescript-typings to gulp tasks

* move generate-type-helpers to gulp tasks

* chore: include build-typings and generate-type-helpers in gulp build

* update generate-standalone comments

* address review comments

* chore: revert build-typings refactor

* fix: run build-typings in build-no-bundle

Gulpfile.js

@@ -5,6 +5,7 @@ const through = require("through2");
 const chalk = require("chalk");
 const newer = require("gulp-newer");
 const babel = require("gulp-babel");
+const camelCase = require("lodash/camelCase");
 const fancyLog = require("fancy-log");
 const filter = require("gulp-filter");
 const gulp = require("gulp");
@@ -21,6 +22,12 @@ const rollupReplace = require("@rollup/plugin-replace");
 const { terser: rollupTerser } = require("rollup-plugin-terser");
 
 const defaultSourcesGlob = "./@(codemods|packages|eslint)/*/src/**/*.{js,ts}";
+const babelStandalonePluginConfigGlob =
+  "./packages/babel-standalone/scripts/pluginConfig.json";
+const buildTypingsWatchGlob = [
+  "./packages/babel-types/lib/definitions/**/*.js",
+  "./packages/babel-types/scripts/generators/*.js",
+];
 
 /**
  * map source code path to the generated artifacts path
@@ -70,6 +77,68 @@ function rename(fn) {
   });
 }
 
+/**
+ *
+ * @typedef {("asserts" | "builders" | "constants" | "validators")} HelperKind
+ * @param {HelperKind} helperKind
+ */
+function generateTypeHelpers(helperKind) {
+  const dest = `./packages/babel-types/src/${helperKind}/generated/`;
+  const formatCode = require("./scripts/utils/formatCode");
+  return gulp
+    .src(".", { base: __dirname })
+    .pipe(errorsLogger())
+    .pipe(
+      through.obj(function (file, enc, callback) {
+        file.path = "index.js";
+        file.contents = Buffer.from(
+          formatCode(
+            require(`./packages/babel-types/scripts/generators/${helperKind}`)(),
+            dest + file.path
+          )
+        );
+        fancyLog(`${chalk.green("✔")} Generated ${helperKind}`);
+        callback(null, file);
+      })
+    )
+    .pipe(gulp.dest(dest));
+}
+
+function generateStandalone() {
+  const dest = "./packages/babel-standalone/src/generated/";
+  const formatCode = require("./scripts/utils/formatCode");
+  return gulp
+    .src(babelStandalonePluginConfigGlob, { base: __dirname })
+    .pipe(
+      through.obj((file, enc, callback) => {
+        fancyLog("Generating @babel/standalone files");
+        const pluginConfig = JSON.parse(file.contents);
+        let imports = "";
+        let list = "";
+        let allList = "";
+
+        for (const plugin of pluginConfig) {
+          const camelPlugin = camelCase(plugin);
+          imports += `import ${camelPlugin} from "@babel/plugin-${plugin}";`;
+          list += `${camelPlugin},`;
+          allList += `"${plugin}": ${camelPlugin},`;
+        }
+
+        const fileContents = `/*
+ * This file is auto-generated! Do not modify it directly.
+ * To re-generate run 'yarn gulp generate-standalone'
+ */
+${imports}
+export {${list}};
+export const all = {${allList}};`;
+        file.path = "plugins.js";
+        file.contents = Buffer.from(formatCode(fileContents, dest));
+        callback(null, file);
+      })
+    )
+    .pipe(gulp.dest(dest));
+}
+
 function buildBabel(exclude, sourcesGlob = defaultSourcesGlob) {
   const base = __dirname;
 
@@ -235,19 +304,68 @@ const standaloneBundle = [
   },
 ];
 
+gulp.task("generate-type-helpers", () => {
+  fancyLog("Generating @babel/types dynamic functions");
+  return Promise.all(
+    ["asserts", "builders", "constants", "validators"].map(helperKind =>
+      generateTypeHelpers(helperKind)
+    )
+  );
+});
+
+gulp.task("generate-standalone", () => generateStandalone());
+
 gulp.task("build-rollup", () => buildRollup(libBundles));
-gulp.task("build-babel-standalone", () => buildRollup(standaloneBundle, true));
+gulp.task("rollup-babel-standalone", () => buildRollup(standaloneBundle, true));
+gulp.task(
+  "build-babel-standalone",
+  gulp.series("generate-standalone", "rollup-babel-standalone")
+);
 
 gulp.task("build-babel", () => buildBabel(/* exclude */ libBundles));
-gulp.task("build", gulp.parallel("build-rollup", "build-babel"));
+
+gulp.task(
+  "build",
+  gulp.series(
+    gulp.parallel("build-rollup", "build-babel"),
+    gulp.parallel(
+      "generate-standalone",
+      gulp.series(
+        "generate-type-helpers",
+        // rebuild @babel/types since type-helpers may be changed
+        "build-babel"
+      )
+    )
+  )
+);
 
 gulp.task("default", gulp.series("build"));
 
 gulp.task("build-no-bundle", () => buildBabel());
 
+gulp.task(
+  "build-dev",
+  gulp.series(
+    "build-no-bundle",
+    gulp.parallel(
+      "generate-standalone",
+      gulp.series(
+        "generate-type-helpers",
+        // rebuild @babel/types since type-helpers may be changed
+        "build-no-bundle"
+      )
+    )
+  )
+);
+
 gulp.task(
   "watch",
-  gulp.series("build-no-bundle", function watch() {
+  gulp.series("build-dev", function watch() {
     gulp.watch(defaultSourcesGlob, gulp.task("build-no-bundle"));
+    gulp.watch(
+      babelStandalonePluginConfigGlob,
+      gulp.task("generate-standalone")
+    );
+    gulp.watch(buildTypingsWatchGlob, gulp.task("generate-type-helpers"));
   })
 );

Makefile

@@ -26,9 +26,6 @@ ifneq ("$(BABEL_COVERAGE)", "true")
 endif
 
 build-bundle: clean clean-lib
-	$(YARN) gulp build
-	$(MAKE) generate-standalone generate-type-helpers
-	# call build again as the generated files might need to be compiled again.
 	$(YARN) gulp build
 	$(MAKE) build-typings
 	$(MAKE) build-dist
@@ -39,11 +36,8 @@ build-bundle-ci: bootstrap-only
 generate-tsconfig:
 	$(NODE) scripts/generators/tsconfig.js
 
-generate-standalone:
-	$(NODE) packages/babel-standalone/scripts/generate.js
-
 generate-type-helpers:
-	$(NODE) packages/babel-types/scripts/generateTypeHelpers.js
+	$(YARN) gulp generate-type-helpers
 
 build-typings: build-flow-typings build-typescript-typings
 
@@ -76,11 +70,11 @@ build-plugin-transform-runtime-dist:
 	$(NODE) scripts/build-dist.js
 
 build-no-bundle: clean clean-lib
-	BABEL_ENV=development $(YARN) gulp build-no-bundle
+	BABEL_ENV=development $(YARN) gulp build-dev
 	# Ensure that build artifacts for types are created during local
 	# development too.
 	# Babel-transform-fixture-test-runner requires minified polyfill for performance
-	$(MAKE) generate-type-helpers build-typings build-polyfill-dist
+	$(MAKE) build-typings build-polyfill-dist
 
 watch: build-no-bundle
 	BABEL_ENV=development $(YARN) gulp watch

packages/babel-standalone/package.json

@@ -105,9 +105,7 @@
     "@babel/preset-env": "workspace:*",
     "@babel/preset-flow": "workspace:*",
     "@babel/preset-react": "workspace:*",
-    "@babel/preset-typescript": "workspace:*",
+    "@babel/preset-typescript": "workspace:*"
-    "chalk": "^4.0.0",
-    "lodash": "^4.17.20"
   },
   "keywords": [
     "babel",

packages/babel-standalone/scripts/generate.js

@@ -1,37 +0,0 @@
-// @flow
-const pluginConfig = require("./pluginConfig.json");
-const path = require("path");
-const chalk = require("chalk");
-const camelCase = require("lodash/camelCase");
-const format = require("../../../scripts/utils/formatCode");
-const writeFile = require("../../../scripts/utils/writeFileAndMkDir");
-
-const outputFile = path.join(__dirname, "../src/generated/plugins.js");
-
-console.log("Generating @babel/standalone files");
-
-let imports = "";
-let list = "";
-let allList = "";
-
-for (const plugin of pluginConfig) {
-  const camelPlugin = camelCase(plugin);
-  imports += `import ${camelPlugin} from "@babel/plugin-${plugin}";`;
-  list += `${camelPlugin},`;
-  allList += `"${plugin}": ${camelPlugin},`;
-}
-
-const fileContent = `
-/*
- * This file is auto-generated! Do not modify it directly.
- * To re-generate run 'make build'
- */
-${imports}
-
-export {${list}};
-
-export const all = {${allList}};
-`;
-
-writeFile(outputFile, format(fileContent, outputFile));
-console.log(`  ${chalk.green("✔")} Generated plugin list`);

packages/babel-standalone/src/generated/plugins.js

@@ -1,6 +1,6 @@
 /*
  * This file is auto-generated! Do not modify it directly.
- * To re-generate run 'make build'
+ * To re-generate run 'yarn gulp generate-standalone'
  */
 import externalHelpers from "@babel/plugin-external-helpers";
 import syntaxAsyncGenerators from "@babel/plugin-syntax-async-generators";

packages/babel-types/package.json

@@ -29,7 +29,6 @@
   },
   "devDependencies": {
     "@babel/generator": "workspace:*",
-    "@babel/parser": "workspace:*",
+    "@babel/parser": "workspace:*"
-    "chalk": "^4.1.0"
   }
 }

packages/babel-types/scripts/generateTypeHelpers.js

@@ -1,29 +0,0 @@
-"use strict";
-const path = require("path");
-const chalk = require("chalk");
-const generateBuilders = require("./generators/generateBuilders");
-const generateValidators = require("./generators/generateValidators");
-const generateAsserts = require("./generators/generateAsserts");
-const generateConstants = require("./generators/generateConstants");
-const format = require("../../../scripts/utils/formatCode");
-const writeFile = require("../../../scripts/utils/writeFileAndMkDir");
-
-const baseDir = path.join(__dirname, "../src");
-
-console.log("Generating @babel/types dynamic functions");
-
-const buildersFile = path.join(baseDir, "builders/generated/index.js");
-writeFile(buildersFile, format(generateBuilders(), buildersFile));
-console.log(`  ${chalk.green("✔")} Generated builders`);
-
-const validatorsFile = path.join(baseDir, "validators/generated/index.js");
-writeFile(validatorsFile, format(generateValidators(), validatorsFile));
-console.log(`  ${chalk.green("✔")} Generated validators`);
-
-const assertsFile = path.join(baseDir, "asserts/generated/index.js");
-writeFile(assertsFile, format(generateAsserts(), assertsFile));
-console.log(`  ${chalk.green("✔")} Generated asserts`);
-
-const constantsFile = path.join(baseDir, "constants/generated/index.js");
-writeFile(constantsFile, format(generateConstants(), constantsFile));
-console.log(`  ${chalk.green("✔")} Generated constants`);

scripts/utils/writeFileAndMkDir.js

@@ -1,14 +0,0 @@
-const fs = require("fs");
-const path = require("path");
-
-module.exports = function writeFileAndMkDir(file, content) {
-  try {
-    fs.mkdirSync(path.dirname(file));
-  } catch (error) {
-    if (error.code !== "EEXIST") {
-      throw error;
-    }
-  }
-
-  fs.writeFileSync(file, content);
-};

yarn.lock

@@ -3295,8 +3295,6 @@ __metadata:
     "@babel/preset-flow": "workspace:*"
     "@babel/preset-react": "workspace:*"
     "@babel/preset-typescript": "workspace:*"
-    chalk: ^4.0.0
-    lodash: ^4.17.20
   languageName: unknown
   linkType: soft
 
@@ -3383,7 +3381,6 @@ __metadata:
     "@babel/generator": "workspace:*"
     "@babel/helper-validator-identifier": "workspace:^7.10.4"
     "@babel/parser": "workspace:*"
-    chalk: ^4.1.0
     lodash: ^4.17.19
     to-fast-properties: ^2.0.0
   languageName: unknown
@@ -9676,7 +9673,7 @@ fsevents@^1.2.7:
   languageName: node
   linkType: hard
 
-"lodash@npm:^4.17.14, lodash@npm:^4.17.15, lodash@npm:^4.17.19, lodash@npm:^4.17.20":
+"lodash@npm:^4.17.14, lodash@npm:^4.17.15, lodash@npm:^4.17.19":
   version: 4.17.20
   resolution: "lodash@npm:4.17.20"
   checksum: c62101d2500c383b5f174a7e9e6fe8098149ddd6e9ccfa85f36d4789446195f5c4afd3cfba433026bcaf3da271256566b04a2bf2618e5a39f6e67f8c12030cb6