fix(core): optimize task hasher speed when dealing with external nodes

packages/nx/src/hasher/task-hasher.spec.ts

@@ -146,7 +146,7 @@ describe('TaskHasher', () => {
     expect(hash.details.nodes).toEqual({
       'parent:{projectRoot}/**/*':
         '/file|file.hash|{"root":"libs/parent","targets":{"build":{"executor":"nx:run-commands","inputs":["default","^default",{"runtime":"echo runtime123"},{"env":"TESTENV"},{"env":"NONEXISTENTENV"},{"input":"default","projects":["unrelated"]}]}}}|{"compilerOptions":{"paths":{"@nx/parent":["libs/parent/src/index.ts"],"@nx/child":["libs/child/src/index.ts"]}}}',
-      parent: 'nx:run-commands',
+      target: 'nx:run-commands',
       'unrelated:{projectRoot}/**/*':
         'libs/unrelated/filec.ts|filec.hash|{"root":"libs/unrelated","targets":{"build":{}}}|{"compilerOptions":{"paths":{"@nx/parent":["libs/parent/src/index.ts"],"@nx/child":["libs/child/src/index.ts"]}}}',
       '{workspaceRoot}/nx.json': 'nx.json.hash',
@@ -854,8 +854,10 @@ describe('TaskHasher', () => {
       });
 
       assertFilesets(hash, {
-        'npm:@nx/webpack': { contains: '16.0.0' },
+        target: { contains: '@nx/webpack:webpack' },
       });
+
+      expect(hash.value).toContain('|16.0.0|');
     });
 
     it('should hash entire subtree of dependencies', async () => {
@@ -949,11 +951,13 @@ describe('TaskHasher', () => {
       });
 
       assertFilesets(hash, {
-        'npm:@nx/webpack': { contains: '$nx/webpack16$' },
+        target: { contains: '@nx/webpack:webpack' },
-        'npm:@nx/devkit': { contains: '$nx/devkit16$' },
-        'npm:nx': { contains: '$nx16$' },
-        'npm:webpack': { contains: '5.0.0' },
       });
+
+      expect(hash.value).toContain('|$nx/webpack16$|');
+      expect(hash.value).toContain('|$nx/devkit16$|');
+      expect(hash.value).toContain('|$nx16$|');
+      expect(hash.value).toContain('|5.0.0|');
     });
 
     it('should not hash when nx:run-commands executor', async () => {
@@ -994,8 +998,8 @@ describe('TaskHasher', () => {
         overrides: { prop: 'prop-value' },
       });
 
-      expect(hash.details.nodes['npm:nx']).not.toBeDefined();
+      expect(hash.value).not.toContain('|16.0.0|');
-      expect(hash.details.nodes['app']).toEqual('nx:run-commands');
+      expect(hash.details.nodes['target']).toEqual('nx:run-commands');
     });
 
     it('should use externalDependencies to override nx:run-commands', async () => {
@@ -1060,10 +1064,10 @@ describe('TaskHasher', () => {
         overrides: { prop: 'prop-value' },
       });
 
-      expect(hash.details.nodes['npm:nx']).not.toBeDefined();
+      expect(hash.value).not.toContain('|16.0.0|');
-      expect(hash.details.nodes['app']).not.toBeDefined();
+      expect(hash.value).toContain('|17.0.0|');
-      expect(hash.details.nodes['npm:webpack']).toEqual('5.0.0');
+      expect(hash.value).toContain('|5.0.0|');
-      expect(hash.details.nodes['npm:react']).toEqual('17.0.0');
+      expect(hash.details.nodes['target']).toEqual('nx:run-commands');
     });
 
     it('should use externalDependencies with empty array to ignore all deps', async () => {

packages/nx/src/hasher/task-hasher.ts

@@ -161,7 +161,7 @@ class TaskHasherImpl {
   private runtimeHashes: {
     [runtime: string]: Promise<PartialHash>;
   } = {};
-  private externalDepsHashCache: { [packageName: string]: PartialHash } = {};
+  private externalDepsHashCache: { [packageName: string]: string } = {};
   private projectRootMappings = createProjectRootMappings(
     this.projectGraph.nodes
   );
@@ -181,7 +181,13 @@ class TaskHasherImpl {
     return Promise.resolve().then(async () => {
       const projectNode = this.projectGraph.nodes[task.target.project];
       if (!projectNode) {
-        return this.hashExternalDependency(task.target.project);
+        const hash = this.hashExternalDependency(task.target.project);
+        return {
+          value: hash,
+          details: {
+            [task.target.project]: hash,
+          },
+        };
       }
       const namedInputs = getNamedInputs(this.nxJson, projectNode);
       const targetData = projectNode.data.targets[task.target.target];
@@ -226,7 +232,13 @@ class TaskHasherImpl {
   ): Promise<PartialHash> {
     const projectNode = this.projectGraph.nodes[projectName];
     if (!projectNode) {
-      return this.hashExternalDependency(projectName);
+      const hash = this.hashExternalDependency(projectName);
+      return {
+        value: hash,
+        details: {
+          [projectName]: hash,
+        },
+      };
     }
     const namedInputs = {
       default: [{ fileset: '{projectRoot}/**/*' }],
@@ -316,25 +328,24 @@ class TaskHasherImpl {
   private hashExternalDependency(
     projectName: string,
     visited = new Set<string>()
-  ): PartialHash {
+  ): string {
     // try to retrieve the hash from cache
     if (this.externalDepsHashCache[projectName]) {
       return this.externalDepsHashCache[projectName];
     }
     visited.add(projectName);
     const node = this.projectGraph.externalNodes[projectName];
-    let partialHash;
+    let partialHash: string;
     if (node) {
-      let hash;
+      const partialHashes: string[] = [];
       if (node.data.hash) {
         // we already know the hash of this dependency
-        hash = node.data.hash;
+        partialHashes.push(node.data.hash);
       } else {
         // we take version as a hash
-        hash = node.data.version;
+        partialHashes.push(node.data.version);
       }
       // we want to calculate the hash of the entire dependency tree
-      const partialHashes: PartialHash[] = [];
       if (this.projectGraph.dependencies[projectName]) {
         this.projectGraph.dependencies[projectName].forEach((d) => {
           if (!visited.has(d.target)) {
@@ -342,24 +353,13 @@ class TaskHasherImpl {
           }
         });
       }
-      partialHash = {
+      partialHash = hashArray(partialHashes);
-        value: hashArray([hash, ...partialHashes.map((p) => p.value)]),
-        details: {
-          [projectName]: hash,
-          ...partialHashes.reduce((m, c) => ({ ...m, ...c.details }), {}),
-        },
-      };
     } else {
       // unknown dependency
       // this may occur if dependency is not an npm package
       // but rather symlinked in node_modules or it's pointing to a remote git repo
       // in this case we have no information about the versioning of the given package
-      partialHash = {
+      partialHash = `__${projectName}__`;
-        value: `__${projectName}__`,
-        details: {
-          [projectName]: `__${projectName}__`,
-        },
-      };
     }
     this.externalDepsHashCache[projectName] = partialHash;
     return partialHash;
@@ -377,6 +377,7 @@ class TaskHasherImpl {
       return;
     }
 
+    let hash;
     // we can only vouch for @nx packages's executor dependencies
     // if it's "run commands" or third-party we skip traversing since we have no info what this command depends on
     if (
@@ -386,38 +387,39 @@ class TaskHasherImpl {
       const executorPackage = target.executor.split(':')[0];
       const executorNodeName =
         this.findExternalDependencyNodeName(executorPackage);
-      return this.hashExternalDependency(executorNodeName);
+      hash = this.hashExternalDependency(executorNodeName);
-    }
+    } else {
-
+      // use command external dependencies if available to construct the hash
-    // use command external dependencies if available to construct the hash
+      const partialHashes: string[] = [];
-    const partialHashes: PartialHash[] = [];
+      let hasCommandExternalDependencies = false;
-    let hasCommandExternalDependencies = false;
+      for (const input of selfInputs) {
-    for (const input of selfInputs) {
+        if (input['externalDependencies']) {
-      if (input['externalDependencies']) {
+          // if we have externalDependencies with empty array we still want to override the default hash
-        // if we have externalDependencies with empty array we still want to override the default hash
+          hasCommandExternalDependencies = true;
-        hasCommandExternalDependencies = true;
+          const externalDependencies = input['externalDependencies'];
-        const externalDependencies = input['externalDependencies'];
+          for (let dep of externalDependencies) {
-        for (let dep of externalDependencies) {
+            dep = this.findExternalDependencyNodeName(dep);
-          dep = this.findExternalDependencyNodeName(dep);
+            partialHashes.push(this.hashExternalDependency(dep));
-          partialHashes.push(this.hashExternalDependency(dep));
+          }
+        }
+      }
+      if (hasCommandExternalDependencies) {
+        hash = hashArray(partialHashes);
+      } else {
+        // cache the hash of the entire external dependencies tree
+        if (this.externalDepsHashCache['']) {
+          hash = this.externalDepsHashCache[''];
+        } else {
+          hash = hashArray([JSON.stringify(this.projectGraph.externalNodes)]);
+          this.externalDepsHashCache[''] = hash;
         }
       }
     }
-    if (hasCommandExternalDependencies) {
-      return {
-        value: hashArray(partialHashes.map((h) => h.value)),
-        details: partialHashes.reduce(
-          (acc, c) => ({ ...acc, ...c.details }),
-          {}
-        ),
-      };
-    }
 
-    const hash = hashArray([JSON.stringify(this.projectGraph.externalNodes)]);
     return {
       value: hash,
       details: {
-        [projectNode.name]: target.executor,
+        target: target.executor,
       },
     };
   }