miel.truyen/nx
1
0
Fork 0
Code Issues Pull Requests Actions 4 Packages Projects Releases Wiki Activity

feat(core): provide an experimental hashing mode for jest and cyrpess

Browse Source
This commit is contained in:
Victor Savkin 2022-03-10 16:33:26 -05:00
parent 90abd6f101
commit a32d46c5a3
11 changed files with 285 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
packages/cypress/executors.json
View File

@ -10,6 +10,7 @@
"cypress": {
"implementation": "./src/executors/cypress/cypress.impl",
"schema": "./src/executors/cypress/schema.json",
"hasher": "./src/executors/cypress/hasher",
"description": "Run Cypress e2e tests"
}
}

27
packages/cypress/src/executors/cypress/hasher.ts Normal file
View File

@ -0,0 +1,27 @@
import {
NxJsonConfiguration,
ProjectGraph,
Task,
TaskGraph,
WorkspaceJsonConfiguration,
} from '@nrwl/devkit';
import { Hash, Hasher } from '@nrwl/workspace/src/core/hasher/hasher';
export default async function run(
task: Task,
context: {
hasher: Hasher;
projectGraph: ProjectGraph;
taskGraph: TaskGraph;
workspaceConfig: WorkspaceJsonConfiguration & NxJsonConfiguration;
}
): Promise<Hash> {
const cypressPluginConfig = context.workspaceConfig.pluginsConfig
? (context.workspaceConfig.pluginsConfig['@nrwl/cypress'] as any)
: undefined;
const filter =
cypressPluginConfig && cypressPluginConfig.hashingExcludesTestsOfDeps
? 'exclude-tests-of-deps'
: 'all-files';
return context.hasher.hashTaskWithDepsAndContext(task, filter);
}

1
packages/jest/executors.json
View File

@ -11,6 +11,7 @@
"implementation": "./src/executors/jest/jest.impl",
"batchImplementation": "./src/executors/jest/jest.impl#batchJest",
"schema": "./src/executors/jest/schema.json",
"hasher": "./src/executors/jest/hasher",
"description": "Run Jest unit tests"
}
}

27
packages/jest/src/executors/jest/hasher.ts Normal file
View File

@ -0,0 +1,27 @@
import {
NxJsonConfiguration,
ProjectGraph,
Task,
TaskGraph,
WorkspaceJsonConfiguration,
} from '@nrwl/devkit';
import { Hash, Hasher } from '@nrwl/workspace/src/core/hasher/hasher';
export default async function run(
task: Task,
context: {
hasher: Hasher;
projectGraph: ProjectGraph;
taskGraph: TaskGraph;
workspaceConfig: WorkspaceJsonConfiguration & NxJsonConfiguration;
}
): Promise<Hash> {
const jestPluginConfig = context.workspaceConfig.pluginsConfig
? (context.workspaceConfig.pluginsConfig['@nrwl/jest'] as any)
: undefined;
const filter =
jestPluginConfig && jestPluginConfig.hashingExcludesTestsOfDeps
? 'exclude-tests-of-deps'
: 'all-files';
return context.hasher.hashTaskWithDepsAndContext(task, filter);
}

58
packages/linter/src/executors/eslint/hasher.ts
View File

@ -1,48 +1,44 @@
import { ProjectGraph, Task, TaskGraph } from '@nrwl/devkit';
import {
ProjectGraph,
Task,
TaskGraph,
WorkspaceJsonConfiguration,
} from '@nrwl/devkit';
import { Hash, Hasher } from '@nrwl/workspace/src/core/hasher/hasher';
import { appRootPath } from '@nrwl/tao/src/utils/app-root';
import { Workspaces } from '@nrwl/tao/src/shared/workspace';
import { readCachedProjectGraph } from '@nrwl/workspace/src/core/project-graph';
export default async function run(
task: Task,
taskGraph: TaskGraph,
hasher: Hasher
context: {
hasher: Hasher;
projectGraph: ProjectGraph;
taskGraph: TaskGraph;
workspaceConfig: WorkspaceJsonConfiguration;
}
): Promise<Hash> {
if (task.overrides['hasTypeAwareRules'] === true) {
return hasher.hashTaskWithDepsAndContext(task);
return context.hasher.hashTaskWithDepsAndContext(task);
}
if (!(global as any).projectGraph) {
try {
(global as any).projectGraph = readCachedProjectGraph();
} catch {
// do nothing, if project graph is unavailable we fallback to using all projects
}
}
const projectGraph = (global as any).projectGraph;
const command = hasher.hashCommand(task);
const sources = await hasher.hashSource(task);
const workspace = new Workspaces(appRootPath).readWorkspaceConfiguration();
const deps = projectGraph
? allDeps(task.id, taskGraph, projectGraph)
: Object.keys(workspace.projects);
const tags = hasher.hashArray(
deps.map((d) => (workspace.projects[d].tags || []).join('|'))
const command = context.hasher.hashCommand(task);
const source = await context.hasher.hashSource(task);
const deps = allDeps(task.id, context.taskGraph, context.projectGraph);
const tags = context.hasher.hashArray(
deps.map((d) => (context.workspaceConfig.projects[d].tags || []).join('|'))
);
const context = await hasher.hashContext();
const taskContext = await context.hasher.hashContext();
return {
value: hasher.hashArray([
value: context.hasher.hashArray([
command,
sources,
source,
tags,
context.implicitDeps.value,
context.runtime.value,
taskContext.implicitDeps.value,
taskContext.runtime.value,
]),
details: {
command,
nodes: { [task.target.project]: sources, tags },
implicitDeps: context.implicitDeps.files,
runtime: context.runtime.runtime,
nodes: { [task.target.project]: source, tags },
implicitDeps: taskContext.implicitDeps.files,
runtime: taskContext.runtime.runtime,
},
};
}

7
packages/tao/src/shared/workspace.ts
View File

@ -263,6 +263,9 @@ export interface ExecutorContext {
}
export class Workspaces {
private cachedWorkspaceConfig: WorkspaceJsonConfiguration &
NxJsonConfiguration;
constructor(private root: string) {}
relativeCwd(cwd: string) {
@ -289,6 +292,7 @@ export class Workspaces {
readWorkspaceConfiguration(): WorkspaceJsonConfiguration &
NxJsonConfiguration {
if (this.cachedWorkspaceConfig) return this.cachedWorkspaceConfig;
const nxJsonPath = path.join(this.root, 'nx.json');
const nxJson = readNxJson(nxJsonPath);
const workspaceFile = workspaceConfigName(this.root);
@ -305,7 +309,8 @@ export class Workspaces {
);
assertValidWorkspaceConfiguration(nxJson);
return { ...workspace, ...nxJson };
this.cachedWorkspaceConfig = { ...workspace, ...nxJson };
return this.cachedWorkspaceConfig;
}
isNxExecutor(nodeModule: string, executor: string) {

130
packages/workspace/src/core/hasher/hasher.spec.ts
View File

@ -72,7 +72,6 @@ describe('Hasher', () => {
});
it('should create project hash', async () => {
hashes['/file'] = 'file.hash';
const hasher = new Hasher(
{
nodes: {
@ -128,7 +127,6 @@ describe('Hasher', () => {
});
it('should create project hash with tsconfig.base.json cache', async () => {
hashes['/file'] = 'file.hash';
const hasher = new Hasher(
{
nodes: {
@ -227,8 +225,6 @@ describe('Hasher', () => {
});
it('should hash projects with dependencies', async () => {
hashes['/filea'] = 'a.hash';
hashes['/fileb'] = 'b.hash';
const hasher = new Hasher(
{
nodes: {
@ -237,7 +233,10 @@ describe('Hasher', () => {
type: 'lib',
data: {
root: '',
files: [{ file: '/filea.ts', hash: 'a.hash' }],
files: [
{ file: '/filea.ts', hash: 'a.hash' },
{ file: '/filea.spec.ts', hash: 'a.spec.hash' },
],
},
},
child: {
@ -245,7 +244,10 @@ describe('Hasher', () => {
type: 'lib',
data: {
root: '',
files: [{ file: '/fileb.ts', hash: 'b.hash' }],
files: [
{ file: '/fileb.ts', hash: 'b.hash' },
{ file: '/fileb.spec.ts', hash: 'b.spec.hash' },
],
},
},
},
@ -264,6 +266,114 @@ describe('Hasher', () => {
overrides: { prop: 'prop-value' },
});
// note that the parent hash is based on parent source files only!
expect(hash.details.nodes).toEqual({
child:
'/fileb.ts|/fileb.spec.ts|b.hash|b.spec.hash|{"root":"libs/child"}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}',
parent:
'/filea.ts|/filea.spec.ts|a.hash|a.spec.hash|{"root":"libs/parent"}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}',
});
});
it('should hash projects with dependencies (exclude spec files of dependencies)', async () => {
const hasher = new Hasher(
{
nodes: {
parent: {
name: 'parent',
type: 'lib',
data: {
root: '',
files: [
{ file: '/filea.ts', hash: 'a.hash' },
{ file: '/filea.spec.ts', hash: 'a.spec.hash' },
],
},
},
child: {
name: 'child',
type: 'lib',
data: {
root: '',
files: [
{ file: '/fileb.ts', hash: 'b.hash' },
{ file: '/fileb.spec.ts', hash: 'b.spec.hash' },
],
},
},
},
dependencies: {
parent: [{ source: 'parent', target: 'child', type: 'static' }],
},
},
{} as any,
{},
createHashing()
);
const hash = await hasher.hashTaskWithDepsAndContext(
{
target: { project: 'parent', target: 'build' },
id: 'parent-build',
overrides: { prop: 'prop-value' },
},
'exclude-tests-of-deps'
);
// note that the parent hash is based on parent source files only!
expect(hash.details.nodes).toEqual({
child:
'/fileb.ts|b.hash|{"root":"libs/child"}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}',
parent:
'/filea.ts|/filea.spec.ts|a.hash|a.spec.hash|{"root":"libs/parent"}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}',
});
});
it('should hash projects with dependencies (exclude spec files of all projects)', async () => {
const hasher = new Hasher(
{
nodes: {
parent: {
name: 'parent',
type: 'lib',
data: {
root: '',
files: [
{ file: '/filea.ts', hash: 'a.hash' },
{ file: '/filea.spec.ts', hash: 'a.spec.hash' },
],
},
},
child: {
name: 'child',
type: 'lib',
data: {
root: '',
files: [
{ file: '/fileb.ts', hash: 'b.hash' },
{ file: '/fileb.spec.ts', hash: 'b.spec.hash' },
],
},
},
},
dependencies: {
parent: [{ source: 'parent', target: 'child', type: 'static' }],
},
},
{} as any,
{},
createHashing()
);
const hash = await hasher.hashTaskWithDepsAndContext(
{
target: { project: 'parent', target: 'build' },
id: 'parent-build',
overrides: { prop: 'prop-value' },
},
'exclude-tests-of-all'
);
// note that the parent hash is based on parent source files only!
expect(hash.details.nodes).toEqual({
child:
@ -274,8 +384,6 @@ describe('Hasher', () => {
});
it('should hash dependent npm project versions', async () => {
hashes['/filea'] = 'a.hash';
hashes['/fileb'] = 'b.hash';
const hasher = new Hasher(
{
nodes: {
@ -324,8 +432,6 @@ describe('Hasher', () => {
});
it('should hash when circular dependencies', async () => {
hashes['/filea'] = 'a.hash';
hashes['/fileb'] = 'b.hash';
const hasher = new Hasher(
{
nodes: {
@ -396,8 +502,6 @@ describe('Hasher', () => {
});
it('should hash implicit deps', async () => {
hashes['/filea'] = 'a.hash';
hashes['/fileb'] = 'b.hash';
const hasher = new Hasher(
{
nodes: {
@ -442,8 +546,6 @@ describe('Hasher', () => {
});
it('should hash missing dependent npm project versions', async () => {
hashes['/filea'] = 'a.hash';
hashes['/fileb'] = 'b.hash';
const hasher = new Hasher(
{
nodes: {

69
packages/workspace/src/core/hasher/hasher.ts
View File

@ -73,13 +73,21 @@ export class Hasher {
});
}
async hashTaskWithDepsAndContext(task: Task): Promise<Hash> {
async hashTaskWithDepsAndContext(
task: Task,
filter:
| 'all-files'
| 'exclude-tests-of-all'
| 'exclude-tests-of-deps' = 'all-files'
): Promise<Hash> {
const command = this.hashCommand(task);
const values = (await Promise.all([
this.projectHashes.hashProject(task.target.project, [
this.projectHashes.hashProject(
task.target.project,
]),
[task.target.project],
filter
),
this.implicitDepsHash(),
this.runtimeInputsHash(),
])) as [
@ -131,7 +139,10 @@ export class Hasher {
}
async hashSource(task: Task): Promise<string> {
return this.projectHashes.hashProjectNodeSource(task.target.project);
return this.projectHashes.hashProjectNodeSource(
task.target.project,
'all-files'
);
}
hashArray(values: string[]): string {
@ -306,7 +317,8 @@ class ProjectHasher {
async hashProject(
projectName: string,
visited: string[]
visited: string[],
filter: 'all-files' | 'exclude-tests-of-all' | 'exclude-tests-of-deps'
): Promise<ProjectHashResult> {
return Promise.resolve().then(async () => {
const deps = this.projectGraph.dependencies[projectName] ?? [];
@ -317,12 +329,21 @@ class ProjectHasher {
return null;
} else {
visited.push(d.target);
return await this.hashProject(d.target, visited);
return await this.hashProject(d.target, visited, filter);
}
})
)
).filter((r) => !!r);
const projectHash = await this.hashProjectNodeSource(projectName);
const filterForProject =
filter === 'all-files'
? 'all-files'
: filter === 'exclude-tests-of-deps' && visited[0] === projectName
? 'all-files'
: 'exclude-tests';
const projectHash = await this.hashProjectNodeSource(
projectName,
filterForProject
);
const nodes = depHashes.reduce(
(m, c) => {
return { ...m, ...c.nodes };
@ -337,9 +358,13 @@ class ProjectHasher {
});
}
async hashProjectNodeSource(projectName: string) {
if (!this.sourceHashes[projectName]) {
this.sourceHashes[projectName] = new Promise(async (res) => {
async hashProjectNodeSource(
projectName: string,
filter: 'all-files' | 'exclude-tests'
) {
const mapKey = `${projectName}-${filter}`;
if (!this.sourceHashes[mapKey]) {
this.sourceHashes[mapKey] = new Promise(async (res) => {
const p = this.projectGraph.nodes[projectName];
if (!p) {
@ -366,8 +391,13 @@ class ProjectHasher {
return;
}
const fileNames = p.data.files.map((f) => f.file);
const values = p.data.files.map((f) => f.hash);
const filteredFiles =
filter === 'all-files'
? p.data.files
: p.data.files.filter((f) => !this.isSpec(f.file));
const fileNames = filteredFiles.map((f) => f.file);
const values = filteredFiles.map((f) => f.hash);
const workspaceJson = JSON.stringify(
this.workspaceJson.projects[projectName] ?? ''
@ -391,7 +421,20 @@ class ProjectHasher {
);
});
}
return this.sourceHashes[projectName];
return this.sourceHashes[mapKey];
}
private isSpec(file: string) {
return (
file.endsWith('.spec.ts') ||
file.endsWith('.test.ts') ||
file.endsWith('-test.ts') ||
file.endsWith('-spec.ts') ||
file.endsWith('.spec.js') ||
file.endsWith('.test.js') ||
file.endsWith('-test.js') ||
file.endsWith('-spec.js')
);
}
private removeOtherProjectsPathRecords(projectName: string) {

1
packages/workspace/src/tasks-runner/task-orchestrator.ts
View File

@ -24,6 +24,7 @@ export class TaskOrchestrator {
private forkedProcessTaskRunner = new ForkedProcessTaskRunner(this.options);
private tasksSchedule = new TasksSchedule(
this.hasher,
this.projectGraph,
this.taskGraph,
this.workspace,
this.options

3
packages/workspace/src/tasks-runner/tasks-schedule.spec.ts
View File

@ -82,12 +82,15 @@ describe('TasksSchedule', () => {
},
};
const projectGraph = {} as any;
const hasher = {
hashTaskWithDepsAndContext: () => 'hash',
} as any;
taskSchedule = new TasksSchedule(
hasher,
projectGraph,
taskGraph,
workspace as Workspaces,
{

29
packages/workspace/src/tasks-runner/tasks-schedule.ts
View File

@ -1,4 +1,9 @@
import { Task, TaskGraph } from '@nrwl/devkit';
import {
ProjectGraph,
Task,
TaskGraph,
WorkspaceConfiguration,
} from '@nrwl/devkit';
import { Workspaces } from '@nrwl/tao/src/shared/workspace';
@ -29,9 +34,10 @@ export class TasksSchedule {
constructor(
private readonly hasher: Hasher,
private taskGraph: TaskGraph,
private workspace: Workspaces,
private options: DefaultTasksRunnerOptions
private readonly projectGraph: ProjectGraph,
private readonly taskGraph: TaskGraph,
private readonly workspaces: Workspaces,
private readonly options: DefaultTasksRunnerOptions
) {}
public async scheduleNextTasks() {
@ -97,7 +103,7 @@ export class TasksSchedule {
const batchMap: Record<string, TaskGraph> = {};
for (const root of this.notScheduledTaskGraph.roots) {
const rootTask = this.notScheduledTaskGraph.tasks[root];
const executorName = getExecutorNameForTask(rootTask, this.workspace);
const executorName = getExecutorNameForTask(rootTask, this.workspaces);
this.processTaskForBatches(batchMap, rootTask, executorName, true);
}
for (const [executorName, taskGraph] of Object.entries(batchMap)) {
@ -123,9 +129,9 @@ export class TasksSchedule {
) {
const { batchImplementationFactory } = getExecutorForTask(
task,
this.workspace
this.workspaces
);
const executorName = getExecutorNameForTask(task, this.workspace);
const executorName = getExecutorNameForTask(task, this.workspaces);
if (rootExecutorName !== executorName) {
return;
}
@ -161,9 +167,14 @@ export class TasksSchedule {
}
private async hashTask(task: Task) {
const customHasher = getCustomHasher(task, this.workspace);
const customHasher = getCustomHasher(task, this.workspaces);
const { value, details } = await (customHasher
? customHasher(task, this.taskGraph, this.hasher)
? customHasher(task, {
hasher: this.hasher,
projectGraph: this.projectGraph,
taskGraph: this.taskGraph,
workspaceConfig: this.workspaces.readWorkspaceConfiguration(),
})
: this.hasher.hashTaskWithDepsAndContext(task));
task.hash = value;
task.hashDetails = details;