miel.truyen/nx
1
0
Fork 0
Code Issues Pull Requests Actions 6 Packages Projects Releases Wiki Activity

cleanup(core): copy from cache only when needed

Browse Source
This commit is contained in:
Leosvel Pérez Espinosa 2021-04-16 15:35:08 +01:00 committed by Victor Savkin
parent ea2a413292
commit c4967fa462
4 changed files with 151 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
e2e/workspace/src/workspace.test.ts
View File

@ -15,6 +15,7 @@ import {
updateFile,
workspaceConfigName,
} from '@nrwl/e2e/utils';
import { TaskCacheStatus } from '@nrwl/workspace/src/utilities/output';
describe('run-one', () => {
let proj: string;
@ -637,7 +638,7 @@ describe('cache', () => {
});
const outputWithBuildApp2Cached = runCLI(`affected:build ${files}`);
expect(outputWithBuildApp2Cached).toContain('read the output from cache');
expectCached(outputWithBuildApp2Cached, [myapp2]);
expectMatchedOutput(outputWithBuildApp2Cached, [myapp2]);
// touch package.json
// --------------------------------------------
@ -651,13 +652,17 @@ describe('cache', () => {
// build individual project with caching
const individualBuildWithCache = runCLI(`build ${myapp1}`);
expect(individualBuildWithCache).toContain('from cache');
expect(individualBuildWithCache).toContain(
TaskCacheStatus.MatchedExistingOutput
);
// skip caching when building individual projects
const individualBuildWithSkippedCache = runCLI(
`build ${myapp1} --skip-nx-cache`
);
expect(individualBuildWithSkippedCache).not.toContain('from cache');
expect(individualBuildWithSkippedCache).not.toContain(
TaskCacheStatus.MatchedExistingOutput
);
// run lint with caching
// --------------------------------------------
@ -668,7 +673,7 @@ describe('cache', () => {
expect(outputWithBothLintTasksCached).toContain(
'read the output from cache'
);
expectCached(outputWithBothLintTasksCached, [
expectMatchedOutput(outputWithBothLintTasksCached, [
myapp1,
myapp2,
`${myapp1}-e2e`,
@ -691,13 +696,13 @@ describe('cache', () => {
silenceError: true,
env: { ...process.env, NX_CACHE_FAILURES: 'true' },
});
expect(failingRun).not.toContain('[retrieved from cache]');
expect(failingRun).not.toContain(TaskCacheStatus.RetrievedFromCache);
const cachedFailingRun = runCLI(`lint ${myapp1}`, {
silenceError: true,
env: { ...process.env, NX_CACHE_FAILURES: 'true' },
});
expect(cachedFailingRun).toContain('[retrieved from cache]');
expect(cachedFailingRun).toContain(TaskCacheStatus.MatchedExistingOutput);
// run without caching
// --------------------------------------------
@ -771,19 +776,38 @@ describe('cache', () => {
actualOutput: string,
expectedCachedProjects: string[]
) {
const cachedProjects = [];
expectProjectMatchTaskCacheStatus(actualOutput, expectedCachedProjects);
}
function expectMatchedOutput(
actualOutput: string,
expectedMatchedOutputProjects: string[]
) {
expectProjectMatchTaskCacheStatus(
actualOutput,
expectedMatchedOutputProjects,
TaskCacheStatus.MatchedExistingOutput
);
}
function expectProjectMatchTaskCacheStatus(
actualOutput: string,
expectedProjects: string[],
cacheStatus: TaskCacheStatus = TaskCacheStatus.RetrievedFromCache
) {
const matchingProjects = [];
const lines = actualOutput.split('\n');
lines.forEach((s, i) => {
lines.forEach((s) => {
if (s.startsWith(`> nx run`)) {
const projectName = s.split(`> nx run `)[1].split(':')[0].trim();
if (s.indexOf('from cache') > -1) {
cachedProjects.push(projectName);
if (s.indexOf(cacheStatus) > -1) {
matchingProjects.push(projectName);
}
}
});
cachedProjects.sort((a, b) => a.localeCompare(b));
expectedCachedProjects.sort((a, b) => a.localeCompare(b));
expect(cachedProjects).toEqual(expectedCachedProjects);
matchingProjects.sort((a, b) => a.localeCompare(b));
expectedProjects.sort((a, b) => a.localeCompare(b));
expect(matchingProjects).toEqual(expectedProjects);
}
});

82
packages/workspace/src/tasks-runner/cache.ts
View File

@ -6,12 +6,14 @@ import {
readFileSync,
writeFileSync,
lstatSync,
unlinkSync,
} from 'fs';
import { join, resolve } from 'path';
import * as fsExtra from 'fs-extra';
import { DefaultTasksRunnerOptions } from './default-tasks-runner';
import { spawn } from 'child_process';
import { cacheDirectory } from '../utilities/cache-directory';
import { writeToFile } from '../utilities/fileutils';
export type CachedResult = {
terminalOutput: string;
@ -42,6 +44,7 @@ export class Cache {
root = appRootPath;
cachePath = this.createCacheDir();
terminalOutputsDir = this.createTerminalOutputsDir();
latestTasksHashesDir = this.ensureLatestTasksHashesDir();
cacheConfig = new CacheConfig(this.options);
constructor(private readonly options: DefaultTasksRunnerOptions) {}
@ -136,12 +139,12 @@ export class Cache {
}
copyFilesFromCache(
hash: string,
cachedResult: CachedResult,
taskWithCachedResult: TaskWithCachedResult,
outputs: string[]
) {
this.removeRecordedTaskHash(taskWithCachedResult.task, outputs);
outputs.forEach((f) => {
const cached = join(cachedResult.outputsPath, f);
const cached = join(taskWithCachedResult.cachedResult.outputsPath, f);
if (existsSync(cached)) {
const isFile = lstatSync(cached).isFile();
const src = join(this.root, f);
@ -154,6 +157,7 @@ export class Cache {
fsExtra.copySync(cached, src);
}
});
this.recordTaskHash(taskWithCachedResult.task, outputs);
}
temporaryOutputPath(task: Task) {
@ -164,6 +168,72 @@ export class Cache {
}
}
removeRecordedTaskHash(task: Task, outputs: string[]): void {
if (outputs.length === 0) {
return;
}
const hashFile = this.getFileNameWithLatestRecordedHashForTask(task);
try {
unlinkSync(hashFile);
} catch (e) {}
}
recordTaskHash(task: Task, outputs: string[]): void {
if (outputs.length === 0) {
return;
}
const hashFile = this.getFileNameWithLatestRecordedHashForTask(task);
writeToFile(hashFile, task.hash);
}
shouldCopyOutputsFromCache(
taskWithCachedResult: TaskWithCachedResult,
outputs: string[]
): boolean {
if (outputs.length === 0) {
return false;
}
if (
this.getLatestRecordedHashForTask(taskWithCachedResult.task) !==
taskWithCachedResult.task.hash
) {
return true;
}
return this.isAnyOutputMissing(taskWithCachedResult.cachedResult, outputs);
}
private getLatestRecordedHashForTask(task: Task): string | null {
try {
return readFileSync(
this.getFileNameWithLatestRecordedHashForTask(task)
).toString();
} catch (e) {
return null;
}
}
private isAnyOutputMissing(
cachedResult: CachedResult,
outputs: string[]
): boolean {
return outputs.some(
(output) =>
existsSync(join(cachedResult.outputsPath, output)) &&
!existsSync(join(this.root, output))
);
}
private getFileNameWithLatestRecordedHashForTask(task: Task): string {
return join(
this.latestTasksHashesDir,
`${task.target.project}-${task.target.target}.hash`
);
}
private getFromLocalDir(task: Task) {
const tdCommit = join(this.cachePath, `${task.hash}.commit`);
const td = join(this.cachePath, task.hash);
@ -199,4 +269,10 @@ export class Cache {
fsExtra.ensureDirSync(path);
return path;
}
private ensureLatestTasksHashesDir() {
const path = join(this.cachePath, 'latestTasksHashes');
fsExtra.ensureDirSync(path);
return path;
}
}

28
packages/workspace/src/tasks-runner/task-orchestrator.ts
View File

@ -4,7 +4,7 @@ import * as dotenv from 'dotenv';
import * as fs from 'fs';
import { ProjectGraph } from '../core/project-graph';
import { appRootPath } from '../utilities/app-root';
import { output } from '../utilities/output';
import { output, TaskCacheStatus } from '../utilities/output';
import { Cache, TaskWithCachedResult } from './cache';
import { DefaultTasksRunnerOptions } from './default-tasks-runner';
import { AffectedEventType, Task } from './tasks-runner';
@ -115,18 +115,29 @@ export class TaskOrchestrator {
tasks.forEach((t) => {
this.options.lifeCycle.startTask(t.task);
const outputs = getOutputs(this.projectGraph.nodes, t.task);
const shouldCopyOutputsFromCache = this.cache.shouldCopyOutputsFromCache(
t,
outputs
);
if (shouldCopyOutputsFromCache) {
this.cache.copyFilesFromCache(t, outputs);
}
if (
!this.initiatingProject ||
this.initiatingProject === t.task.target.project
) {
const args = this.getCommandArgs(t.task);
output.logCommand(`nx ${args.join(' ')}`, true);
output.logCommand(
`nx ${args.join(' ')}`,
shouldCopyOutputsFromCache
? TaskCacheStatus.RetrievedFromCache
: TaskCacheStatus.MatchedExistingOutput
);
process.stdout.write(t.cachedResult.terminalOutput);
}
const outputs = getOutputs(this.projectGraph.nodes, t.task);
this.cache.copyFilesFromCache(t.task.hash, t.cachedResult, outputs);
this.options.lifeCycle.endTask(t.task, t.cachedResult.code);
});
@ -175,6 +186,7 @@ export class TaskOrchestrator {
if (forwardOutput) {
output.logCommand(commandLine);
}
this.cache.removeRecordedTaskHash(task, taskOutputs);
const p = fork(this.getCommand(), args, {
stdio: ['inherit', 'pipe', 'pipe', 'ipc'],
env,
@ -211,6 +223,7 @@ export class TaskOrchestrator {
this.cache
.put(task, terminalOutput, taskOutputs, code)
.then(() => {
this.cache.recordTaskHash(task, taskOutputs);
this.options.lifeCycle.endTask(task, code);
res(code);
})
@ -218,10 +231,12 @@ export class TaskOrchestrator {
rej(e);
});
} else {
this.cache.recordTaskHash(task, taskOutputs);
this.options.lifeCycle.endTask(task, code);
res(code);
}
} else {
this.cache.recordTaskHash(task, taskOutputs);
this.options.lifeCycle.endTask(task, code);
res(code);
}
@ -252,6 +267,7 @@ export class TaskOrchestrator {
if (forwardOutput) {
output.logCommand(commandLine);
}
this.cache.removeRecordedTaskHash(task, taskOutputs);
const p = fork(this.getCommand(), args, {
stdio: ['inherit', 'inherit', 'inherit', 'ipc'],
env,
@ -277,6 +293,7 @@ export class TaskOrchestrator {
this.cache
.put(task, this.readTerminalOutput(outputPath), taskOutputs, code)
.then(() => {
this.cache.recordTaskHash(task, taskOutputs);
this.options.lifeCycle.endTask(task, code);
res(code);
})
@ -284,6 +301,7 @@ export class TaskOrchestrator {
rej(e);
});
} else {
this.cache.recordTaskHash(task, taskOutputs);
this.options.lifeCycle.endTask(task, code);
res(code);
}

15
packages/workspace/src/utilities/output.ts
View File

@ -22,6 +22,12 @@ export interface CLISuccessMessageConfig {
bodyLines?: string[];
}
export enum TaskCacheStatus {
NoCache = '[no cache]',
MatchedExistingOutput = '[existing outputs match the cache, left as is]',
RetrievedFromCache = '[retrieved from cache]',
}
/**
* Automatically disable styling applied by chalk if CI=true
*/
@ -177,13 +183,16 @@ class CLIOutput {
this.addNewline();
}
logCommand(message: string, isCached: boolean = false) {
logCommand(
message: string,
cacheStatus: TaskCacheStatus = TaskCacheStatus.NoCache
) {
this.addNewline();
this.writeToStdOut(chalk.bold(`> ${message} `));
if (isCached) {
this.writeToStdOut(chalk.bold.grey(`[retrieved from cache]`));
if (cacheStatus !== TaskCacheStatus.NoCache) {
this.writeToStdOut(chalk.bold.grey(cacheStatus));
}
this.addNewline();