fix(core): optimize task hasher speed when dealing with external nodes
This commit is contained in:
Miroslav Jonas
Victor Savkin
parent
0dfe6fc48a
commit
42ffa7c882
@ -146,7 +146,7 @@ describe('TaskHasher', () => {
|
||||
expect(hash.details.nodes).toEqual({
|
||||
'parent:{projectRoot}/**/*':
|
||||
'/file|file.hash|{"root":"libs/parent","targets":{"build":{"executor":"nx:run-commands","inputs":["default","^default",{"runtime":"echo runtime123"},{"env":"TESTENV"},{"env":"NONEXISTENTENV"},{"input":"default","projects":["unrelated"]}]}}}|{"compilerOptions":{"paths":{"@nx/parent":["libs/parent/src/index.ts"],"@nx/child":["libs/child/src/index.ts"]}}}',
|
||||
parent: 'nx:run-commands',
|
||||
target: 'nx:run-commands',
|
||||
'unrelated:{projectRoot}/**/*':
|
||||
'libs/unrelated/filec.ts|filec.hash|{"root":"libs/unrelated","targets":{"build":{}}}|{"compilerOptions":{"paths":{"@nx/parent":["libs/parent/src/index.ts"],"@nx/child":["libs/child/src/index.ts"]}}}',
|
||||
'{workspaceRoot}/nx.json': 'nx.json.hash',
|
||||
@ -854,8 +854,10 @@ describe('TaskHasher', () => {
|
||||
});
|
||||
|
||||
assertFilesets(hash, {
|
||||
'npm:@nx/webpack': { contains: '16.0.0' },
|
||||
target: { contains: '@nx/webpack:webpack' },
|
||||
});
|
||||
|
||||
expect(hash.value).toContain('|16.0.0|');
|
||||
});
|
||||
|
||||
it('should hash entire subtree of dependencies', async () => {
|
||||
@ -949,11 +951,13 @@ describe('TaskHasher', () => {
|
||||
});
|
||||
|
||||
assertFilesets(hash, {
|
||||
'npm:@nx/webpack': { contains: '$nx/webpack16$' },
|
||||
'npm:@nx/devkit': { contains: '$nx/devkit16$' },
|
||||
'npm:nx': { contains: '$nx16$' },
|
||||
'npm:webpack': { contains: '5.0.0' },
|
||||
target: { contains: '@nx/webpack:webpack' },
|
||||
});
|
||||
|
||||
expect(hash.value).toContain('|$nx/webpack16$|');
|
||||
expect(hash.value).toContain('|$nx/devkit16$|');
|
||||
expect(hash.value).toContain('|$nx16$|');
|
||||
expect(hash.value).toContain('|5.0.0|');
|
||||
});
|
||||
|
||||
it('should not hash when nx:run-commands executor', async () => {
|
||||
@ -994,8 +998,8 @@ describe('TaskHasher', () => {
|
||||
overrides: { prop: 'prop-value' },
|
||||
});
|
||||
|
||||
expect(hash.details.nodes['npm:nx']).not.toBeDefined();
|
||||
expect(hash.details.nodes['app']).toEqual('nx:run-commands');
|
||||
expect(hash.value).not.toContain('|16.0.0|');
|
||||
expect(hash.details.nodes['target']).toEqual('nx:run-commands');
|
||||
});
|
||||
|
||||
it('should use externalDependencies to override nx:run-commands', async () => {
|
||||
@ -1060,10 +1064,10 @@ describe('TaskHasher', () => {
|
||||
overrides: { prop: 'prop-value' },
|
||||
});
|
||||
|
||||
expect(hash.details.nodes['npm:nx']).not.toBeDefined();
|
||||
expect(hash.details.nodes['app']).not.toBeDefined();
|
||||
expect(hash.details.nodes['npm:webpack']).toEqual('5.0.0');
|
||||
expect(hash.details.nodes['npm:react']).toEqual('17.0.0');
|
||||
expect(hash.value).not.toContain('|16.0.0|');
|
||||
expect(hash.value).toContain('|17.0.0|');
|
||||
expect(hash.value).toContain('|5.0.0|');
|
||||
expect(hash.details.nodes['target']).toEqual('nx:run-commands');
|
||||
});
|
||||
|
||||
it('should use externalDependencies with empty array to ignore all deps', async () => {
|
||||
|
||||
@ -161,7 +161,7 @@ class TaskHasherImpl {
|
||||
private runtimeHashes: {
|
||||
[runtime: string]: Promise<PartialHash>;
|
||||
} = {};
|
||||
private externalDepsHashCache: { [packageName: string]: PartialHash } = {};
|
||||
private externalDepsHashCache: { [packageName: string]: string } = {};
|
||||
private projectRootMappings = createProjectRootMappings(
|
||||
this.projectGraph.nodes
|
||||
);
|
||||
@ -181,7 +181,13 @@ class TaskHasherImpl {
|
||||
return Promise.resolve().then(async () => {
|
||||
const projectNode = this.projectGraph.nodes[task.target.project];
|
||||
if (!projectNode) {
|
||||
return this.hashExternalDependency(task.target.project);
|
||||
const hash = this.hashExternalDependency(task.target.project);
|
||||
return {
|
||||
value: hash,
|
||||
details: {
|
||||
[task.target.project]: hash,
|
||||
},
|
||||
};
|
||||
}
|
||||
const namedInputs = getNamedInputs(this.nxJson, projectNode);
|
||||
const targetData = projectNode.data.targets[task.target.target];
|
||||
@ -226,7 +232,13 @@ class TaskHasherImpl {
|
||||
): Promise<PartialHash> {
|
||||
const projectNode = this.projectGraph.nodes[projectName];
|
||||
if (!projectNode) {
|
||||
return this.hashExternalDependency(projectName);
|
||||
const hash = this.hashExternalDependency(projectName);
|
||||
return {
|
||||
value: hash,
|
||||
details: {
|
||||
[projectName]: hash,
|
||||
},
|
||||
};
|
||||
}
|
||||
const namedInputs = {
|
||||
default: [{ fileset: '{projectRoot}/**/*' }],
|
||||
@ -316,25 +328,24 @@ class TaskHasherImpl {
|
||||
private hashExternalDependency(
|
||||
projectName: string,
|
||||
visited = new Set<string>()
|
||||
): PartialHash {
|
||||
): string {
|
||||
// try to retrieve the hash from cache
|
||||
if (this.externalDepsHashCache[projectName]) {
|
||||
return this.externalDepsHashCache[projectName];
|
||||
}
|
||||
visited.add(projectName);
|
||||
const node = this.projectGraph.externalNodes[projectName];
|
||||
let partialHash;
|
||||
let partialHash: string;
|
||||
if (node) {
|
||||
let hash;
|
||||
const partialHashes: string[] = [];
|
||||
if (node.data.hash) {
|
||||
// we already know the hash of this dependency
|
||||
hash = node.data.hash;
|
||||
partialHashes.push(node.data.hash);
|
||||
} else {
|
||||
// we take version as a hash
|
||||
hash = node.data.version;
|
||||
partialHashes.push(node.data.version);
|
||||
}
|
||||
// we want to calculate the hash of the entire dependency tree
|
||||
const partialHashes: PartialHash[] = [];
|
||||
if (this.projectGraph.dependencies[projectName]) {
|
||||
this.projectGraph.dependencies[projectName].forEach((d) => {
|
||||
if (!visited.has(d.target)) {
|
||||
@ -342,24 +353,13 @@ class TaskHasherImpl {
|
||||
}
|
||||
});
|
||||
}
|
||||
partialHash = {
|
||||
value: hashArray([hash, ...partialHashes.map((p) => p.value)]),
|
||||
details: {
|
||||
[projectName]: hash,
|
||||
...partialHashes.reduce((m, c) => ({ ...m, ...c.details }), {}),
|
||||
},
|
||||
};
|
||||
partialHash = hashArray(partialHashes);
|
||||
} else {
|
||||
// unknown dependency
|
||||
// this may occur if dependency is not an npm package
|
||||
// but rather symlinked in node_modules or it's pointing to a remote git repo
|
||||
// in this case we have no information about the versioning of the given package
|
||||
partialHash = {
|
||||
value: `__${projectName}__`,
|
||||
details: {
|
||||
[projectName]: `__${projectName}__`,
|
||||
},
|
||||
};
|
||||
partialHash = `__${projectName}__`;
|
||||
}
|
||||
this.externalDepsHashCache[projectName] = partialHash;
|
||||
return partialHash;
|
||||
@ -377,6 +377,7 @@ class TaskHasherImpl {
|
||||
return;
|
||||
}
|
||||
|
||||
let hash;
|
||||
// we can only vouch for @nx packages's executor dependencies
|
||||
// if it's "run commands" or third-party we skip traversing since we have no info what this command depends on
|
||||
if (
|
||||
@ -386,38 +387,39 @@ class TaskHasherImpl {
|
||||
const executorPackage = target.executor.split(':')[0];
|
||||
const executorNodeName =
|
||||
this.findExternalDependencyNodeName(executorPackage);
|
||||
return this.hashExternalDependency(executorNodeName);
|
||||
}
|
||||
|
||||
// use command external dependencies if available to construct the hash
|
||||
const partialHashes: PartialHash[] = [];
|
||||
let hasCommandExternalDependencies = false;
|
||||
for (const input of selfInputs) {
|
||||
if (input['externalDependencies']) {
|
||||
// if we have externalDependencies with empty array we still want to override the default hash
|
||||
hasCommandExternalDependencies = true;
|
||||
const externalDependencies = input['externalDependencies'];
|
||||
for (let dep of externalDependencies) {
|
||||
dep = this.findExternalDependencyNodeName(dep);
|
||||
partialHashes.push(this.hashExternalDependency(dep));
|
||||
hash = this.hashExternalDependency(executorNodeName);
|
||||
} else {
|
||||
// use command external dependencies if available to construct the hash
|
||||
const partialHashes: string[] = [];
|
||||
let hasCommandExternalDependencies = false;
|
||||
for (const input of selfInputs) {
|
||||
if (input['externalDependencies']) {
|
||||
// if we have externalDependencies with empty array we still want to override the default hash
|
||||
hasCommandExternalDependencies = true;
|
||||
const externalDependencies = input['externalDependencies'];
|
||||
for (let dep of externalDependencies) {
|
||||
dep = this.findExternalDependencyNodeName(dep);
|
||||
partialHashes.push(this.hashExternalDependency(dep));
|
||||
}
|
||||
}
|
||||
}
|
||||
if (hasCommandExternalDependencies) {
|
||||
hash = hashArray(partialHashes);
|
||||
} else {
|
||||
// cache the hash of the entire external dependencies tree
|
||||
if (this.externalDepsHashCache['']) {
|
||||
hash = this.externalDepsHashCache[''];
|
||||
} else {
|
||||
hash = hashArray([JSON.stringify(this.projectGraph.externalNodes)]);
|
||||
this.externalDepsHashCache[''] = hash;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (hasCommandExternalDependencies) {
|
||||
return {
|
||||
value: hashArray(partialHashes.map((h) => h.value)),
|
||||
details: partialHashes.reduce(
|
||||
(acc, c) => ({ ...acc, ...c.details }),
|
||||
{}
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
const hash = hashArray([JSON.stringify(this.projectGraph.externalNodes)]);
|
||||
return {
|
||||
value: hash,
|
||||
details: {
|
||||
[projectNode.name]: target.executor,
|
||||
target: target.executor,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user