Integrate Test262 (#654)

* Integrate Test262

Introduce a GNU Make target for retrieving TC-39's Test262 suite and
validating parsing of the files it contains. Interpret each file as a
parser test in accordance with that project's `INTERPRETING.md`
document. Allow for the specification of allowed failures via a
"whitelist" file so that the test suite may help prevent regressions in
this project in situations where this project has known bugs. Initialize
the "whitelist" file with a listing of all tests that are currently
failing. Extend the continuous integration environment's configuration
to automatically run these tests.

* use graceful-fs and latest yarn on travis

scripts/run_test262.js

@@ -0,0 +1,113 @@
+"use strict";
+
+const path = require("path");
+const chalk = require("chalk");
+const utils = require("./run_test262_utils");
+
+const testDir = path.join(__dirname, "..", "build", "test262", "test");
+const whitelistFile = path.join(__dirname, "test262_whitelist.txt");
+const plugins = ["asyncGenerators", "objectRestSpread"];
+const shouldUpdate = process.argv.indexOf("--update-whitelist") > -1;
+
+Promise.all([utils.getTests(testDir), utils.getWhitelist(whitelistFile)])
+  .then(function([tests, whitelist]) {
+    const total = tests.length;
+    const reportInc = Math.floor(total / 20);
+
+    console.log(`Now running ${total} tests...`);
+
+    const results = tests.map(function(test, idx) {
+      if (idx % reportInc === 0) {
+        console.log(`> ${Math.round(100 * idx / total)}% complete`);
+      }
+
+      return utils.runTest(test, plugins);
+    });
+
+    return utils.interpret(results, whitelist);
+  })
+  .then(function(summary) {
+    const goodnews = [
+      summary.allowed.success.length + " valid programs parsed without error",
+      summary.allowed.failure.length +
+        " invalid programs produced a parsing error",
+      summary.allowed.falsePositive.length +
+        " invalid programs did not produce a parsing error" +
+        " (and allowed by the whitelist file)",
+      summary.allowed.falseNegative.length +
+        " valid programs produced a parsing error" +
+        " (and allowed by the whitelist file)",
+    ];
+    const badnews = [];
+    const badnewsDetails = [];
+
+    void [
+      {
+        tests: summary.disallowed.success,
+        label:
+          "valid programs parsed without error" +
+          " (in violation of the whitelist file)",
+      },
+      {
+        tests: summary.disallowed.failure,
+        label:
+          "invalid programs produced a parsing error" +
+          " (in violation of the whitelist file)",
+      },
+      {
+        tests: summary.disallowed.falsePositive,
+        label:
+          "invalid programs did not produce a parsing error" +
+          " (without a corresponding entry in the whitelist file)",
+      },
+      {
+        tests: summary.disallowed.falseNegative,
+        label:
+          "valid programs produced a parsing error" +
+          " (without a corresponding entry in the whitelist file)",
+      },
+      {
+        tests: summary.unrecognized,
+        label: "non-existent programs specified in the whitelist file",
+      },
+    ].forEach(function({ tests, label }) {
+      if (!tests.length) {
+        return;
+      }
+
+      const desc = tests.length + " " + label;
+
+      badnews.push(desc);
+      badnewsDetails.push(desc + ":");
+      badnewsDetails.push(
+        ...tests.map(function(test) {
+          return test.id;
+        })
+      );
+    });
+
+    console.log("Testing complete.");
+    console.log("Summary:");
+    console.log(chalk.green(goodnews.join("\n").replace(/^/gm, " ✔ ")));
+
+    if (!summary.passed) {
+      console.log("");
+      console.log(chalk.red(badnews.join("\n").replace(/^/gm, " ✘ ")));
+      console.log("");
+      console.log("Details:");
+      console.log(badnewsDetails.join("\n").replace(/^/gm, "   "));
+    }
+
+    if (shouldUpdate) {
+      return utils.updateWhitelist(whitelistFile, summary).then(function() {
+        console.log("");
+        console.log("Whitelist file updated.");
+      });
+    } else {
+      process.exitCode = summary.passed ? 0 : 1;
+    }
+  })
+  .catch(function(err) {
+    console.error(err);
+    process.exitCode = 1;
+  });

scripts/run_test262_utils.js

@@ -0,0 +1,233 @@
+"use strict";
+
+const fs = require("graceful-fs");
+const path = require("path");
+const promisify = require("util.promisify");
+const pfs = {
+  readFile: promisify(fs.readFile),
+  writeFile: promisify(fs.writeFile),
+  readdir: promisify(fs.readdir),
+  stat: promisify(fs.stat)
+};
+
+const parse = require("..").parse;
+
+const modulePattern = /^\s*-\s*module\s*$|^\s*flags\s*:.*\bmodule\b/m;
+const noStrictPattern = /^\s*-\s*noStrict\s*$|^\s*flags\s*:.*\bnoStrict\b/m;
+const onlyStrictPattern = /^\s*-\s*onlyStrict\s*$|^\s*flags\s*:.*\bonlyStrict\b/m;
+const rawPattern = /^\s*-\s*raw\s*$|^\s*flags\s*:.*\braw\b/m;
+const testNamePattern = /^(?!.*_FIXTURE).*\.[jJ][sS]$/;
+
+function flatten(array) {
+  const flattened = [];
+  array.forEach(function(element) {
+    if (Array.isArray(element)) {
+      flattened.push.apply(flattened, element);
+    } else {
+      flattened.push(element);
+    }
+  });
+  return flattened;
+}
+
+function hasEarlyError(src) {
+  return !!(
+    src.match(/^\s*negative:\s*$/m) && src.match(/^\s+phase:\s*early\s*$/m)
+  );
+}
+
+function readDirDeep(dirName) {
+  return pfs.readdir(dirName).then(function(contents) {
+    return Promise.all(
+      contents.map(function(name) {
+        return findTests(path.join(dirName, name));
+      })
+    ).then(flatten)
+  });
+}
+
+function findTests(name) {
+  return pfs.stat(name).then(function(stat) {
+    if (stat.isDirectory()) {
+      return readDirDeep(name);
+    }
+
+    return name;
+  });
+}
+
+function readTest(fileName, testDir) {
+  if (!testNamePattern.test(fileName)) {
+    return Promise.resolve([]);
+  }
+
+  return pfs.readFile(fileName, "utf-8").then(function(contents) {
+    return makeScenarios(path.relative(testDir, fileName), contents);
+  });
+}
+
+function makeScenarios(fileName, testContent) {
+  const scenarios = [];
+  const base = {
+    fileName: fileName,
+    isModule: modulePattern.test(testContent),
+    expectedError: hasEarlyError(testContent),
+  };
+  const isNoStrict = noStrictPattern.test(testContent);
+  const isOnlyStrict = onlyStrictPattern.test(testContent);
+  const isRaw = rawPattern.test(testContent);
+
+  if (!isOnlyStrict) {
+    scenarios.push(
+      Object.assign(
+        {
+          id: fileName + "(default)",
+          content: testContent,
+        },
+        base
+      )
+    );
+  }
+
+  if (!isNoStrict && !isRaw) {
+    scenarios.push(
+      Object.assign(
+        {
+          id: fileName + "(strict mode)",
+          content: "'use strict';\n" + testContent,
+        },
+        base
+      )
+    );
+  }
+
+  return scenarios;
+}
+
+exports.getTests = function(testDir) {
+  return findTests(testDir)
+    .then(function(testPaths) {
+      return Promise.all(
+        testPaths.map(function(path) {
+          return readTest(path, testDir);
+        })
+      );
+    })
+    .then(flatten);
+};
+
+exports.runTest = function(test, plugins) {
+  const sourceType = test.isModule ? "module" : "script";
+
+  try {
+    parse(test.content, { sourceType: sourceType, plugins: plugins });
+    test.actualError = false;
+  } catch (err) {
+    test.actualError = true;
+  }
+
+  test.result = test.expectedError !== test.actualError ? "fail" : "pass";
+
+  return test;
+};
+
+exports.getWhitelist = function(filename) {
+  return pfs.readFile(filename, "utf-8").then(function(contents) {
+    return contents
+      .split("\n")
+      .map(function(line) {
+        return line.replace(/#.*$/, "").trim();
+      })
+      .filter(function(line) {
+        return line.length > 0;
+      })
+      .reduce(function(table, filename) {
+        table[filename] = true;
+        return table;
+      }, Object.create(null));
+  });
+};
+
+exports.updateWhitelist = function(filename, summary) {
+  return pfs.readFile(filename, "utf-8").then(function(contents) {
+    const toRemove = summary.disallowed.success
+      .concat(summary.disallowed.failure)
+      .map(function(test) {
+        return test.id;
+      });
+    const toAdd = summary.disallowed.falsePositive
+      .concat(summary.disallowed.falseNegative)
+      .map(function(test) {
+        return test.id;
+      });
+    const newContents = contents
+      .split("\n")
+      .map(function(line) {
+        const testId = line.replace(/#.*$/, "").trim();
+
+        if (toRemove.indexOf(testId) > -1) {
+          return null;
+        }
+
+        return line;
+      })
+      .filter(function(line) {
+        return line !== null;
+      })
+      .concat(toAdd)
+      .join("\n");
+
+    return pfs.writeFile(filename, newContents, "utf-8");
+  });
+};
+
+exports.interpret = function(results, whitelist) {
+  const summary = {
+    passed: true,
+    allowed: {
+      success: [],
+      failure: [],
+      falsePositive: [],
+      falseNegative: [],
+    },
+    disallowed: {
+      success: [],
+      failure: [],
+      falsePositive: [],
+      falseNegative: [],
+    },
+    unrecognized: null,
+  };
+
+  results.forEach(function(result) {
+    let classification, isAllowed;
+    const inWhitelist = result.id in whitelist;
+    delete whitelist[result.id];
+
+    if (!result.expectedError) {
+      if (!result.actualError) {
+        classification = "success";
+        isAllowed = !inWhitelist;
+      } else {
+        classification = "falseNegative";
+        isAllowed = inWhitelist;
+      }
+    } else {
+      if (!result.actualError) {
+        classification = "falsePositive";
+        isAllowed = inWhitelist;
+      } else {
+        classification = "failure";
+        isAllowed = !inWhitelist;
+      }
+    }
+
+    summary.passed &= isAllowed;
+    summary[isAllowed ? "allowed" : "disallowed"][classification].push(result);
+  });
+
+  summary.unrecognized = Object.keys(whitelist);
+  summary.passed = !!summary.passed && summary.unrecognized.length === 0;
+
+  return summary;
+};