fix(core): parse project configs only in js (#18009)

This commit is contained in:
Jason Jean 2023-07-07 16:06:44 -04:00 committed by GitHub
parent b1f19e3e91
commit c04053b4e9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 299 additions and 378 deletions

43
Cargo.lock generated
View File

@ -844,15 +844,6 @@ version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6"
[[package]]
name = "jsonc-parser"
version = "0.21.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b56a20e76235284255a09fcd1f45cf55d3c524ea657ebd3854735925c57743d"
dependencies = [
"serde_json",
]
[[package]] [[package]]
name = "kqueue" name = "kqueue"
version = "1.0.7" version = "1.0.7"
@ -1171,13 +1162,10 @@ dependencies = [
"ignore", "ignore",
"ignore-files", "ignore-files",
"itertools", "itertools",
"jsonc-parser",
"napi", "napi",
"napi-build", "napi-build",
"napi-derive", "napi-derive",
"rayon", "rayon",
"serde",
"serde_json",
"thiserror", "thiserror",
"tokio", "tokio",
"tracing", "tracing",
@ -1447,12 +1435,6 @@ dependencies = [
"windows-sys 0.45.0", "windows-sys 0.45.0",
] ]
[[package]]
name = "ryu"
version = "1.0.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041"
[[package]] [[package]]
name = "same-file" name = "same-file"
version = "1.0.6" version = "1.0.6"
@ -1479,31 +1461,6 @@ name = "serde"
version = "1.0.152" version = "1.0.152"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.152"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e"
dependencies = [
"proc-macro2",
"quote",
"syn 1.0.107",
]
[[package]]
name = "serde_json"
version = "1.0.96"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1"
dependencies = [
"itoa",
"ryu",
"serde",
]
[[package]] [[package]]
name = "sha1_smol" name = "sha1_smol"

View File

@ -13,12 +13,9 @@ hashbrown = { version = "0.14.0", features = ["rayon"] }
ignore = '0.4' ignore = '0.4'
ignore-files = "1.3.0" ignore-files = "1.3.0"
itertools = "0.10.5" itertools = "0.10.5"
jsonc-parser = { version = "0.21.1", features = ["serde"] }
napi = { version = '2.12.6', default-features = false, features = ['anyhow', 'napi4', 'tokio_rt'] } napi = { version = '2.12.6', default-features = false, features = ['anyhow', 'napi4', 'tokio_rt'] }
napi-derive = '2.9.3' napi-derive = '2.9.3'
rayon = "1.7.0" rayon = "1.7.0"
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
thiserror = "1.0.40" thiserror = "1.0.40"
tokio = { version = "1.28.2", features = ["fs"] } tokio = { version = "1.28.2", features = ["fs"] }
tracing = "0.1.37" tracing = "0.1.37"

View File

@ -95,7 +95,7 @@ export class Workspaces {
return this.cachedProjectsConfig; return this.cachedProjectsConfig;
} }
const nxJson = this.readNxJson(); const nxJson = this.readNxJson();
const projectsConfigurations = buildProjectsConfigurationsFromProjectPaths( let projectsConfigurations = buildProjectsConfigurationsFromProjectPaths(
nxJson, nxJson,
globForProjectFiles( globForProjectFiles(
this.root, this.root,
@ -116,15 +116,18 @@ export class Workspaces {
opts?._includeProjectsFromAngularJson opts?._includeProjectsFromAngularJson
) )
) { ) {
projectsConfigurations.projects = mergeAngularJsonAndProjects( projectsConfigurations = mergeAngularJsonAndProjects(
projectsConfigurations.projects, projectsConfigurations,
this.root this.root
); );
} }
this.cachedProjectsConfig = this.mergeTargetDefaultsIntoProjectDescriptions( this.cachedProjectsConfig = {
version: 2,
projects: this.mergeTargetDefaultsIntoProjectDescriptions(
projectsConfigurations, projectsConfigurations,
nxJson nxJson
); ),
};
return this.cachedProjectsConfig; return this.cachedProjectsConfig;
} }
@ -140,10 +143,10 @@ export class Workspaces {
} }
private mergeTargetDefaultsIntoProjectDescriptions( private mergeTargetDefaultsIntoProjectDescriptions(
config: ProjectsConfigurations, projects: Record<string, ProjectConfiguration>,
nxJson: NxJsonConfiguration nxJson: NxJsonConfiguration
) { ) {
for (const proj of Object.values(config.projects)) { for (const proj of Object.values(projects)) {
if (proj.targets) { if (proj.targets) {
for (const targetName of Object.keys(proj.targets)) { for (const targetName of Object.keys(proj.targets)) {
const projectTargetDefinition = proj.targets[targetName]; const projectTargetDefinition = proj.targets[targetName];
@ -163,7 +166,7 @@ export class Workspaces {
} }
} }
} }
return config; return projects;
} }
isNxExecutor(nodeModule: string, executor: string) { isNxExecutor(nodeModule: string, executor: string) {
@ -808,7 +811,7 @@ export function buildProjectsConfigurationsFromProjectPaths(
projectFiles: string[], // making this parameter allows devkit to pick up newly created projects projectFiles: string[], // making this parameter allows devkit to pick up newly created projects
readJson: <T extends Object>(string) => T = <T extends Object>(string) => readJson: <T extends Object>(string) => T = <T extends Object>(string) =>
readJsonFile<T>(string) // making this an arg allows us to reuse in devkit readJsonFile<T>(string) // making this an arg allows us to reuse in devkit
): ProjectsConfigurations { ): Record<string, ProjectConfiguration> {
const projects: Record<string, ProjectConfiguration> = {}; const projects: Record<string, ProjectConfiguration> = {};
for (const file of projectFiles) { for (const file of projectFiles) {
@ -868,10 +871,7 @@ export function buildProjectsConfigurationsFromProjectPaths(
} }
} }
return { return projects;
version: 2,
projects: projects,
};
} }
export function mergeTargetConfigurations( export function mergeTargetConfigurations(

View File

@ -198,7 +198,7 @@ function readAndCombineAllProjectConfigurations(tree: Tree): {
nxJson, nxJson,
projectFiles, projectFiles,
(file) => readJson(tree, file) (file) => readJson(tree, file)
).projects; );
} }
/** /**

View File

@ -37,14 +37,13 @@ export const enum WorkspaceErrors {
Generic = 'Generic' Generic = 'Generic'
} }
/** Get workspace config files based on provided globs */ /** Get workspace config files based on provided globs */
export function getConfigFiles(workspaceRoot: string, globs: Array<string>): Array<string> export function getProjectConfigurations(workspaceRoot: string, globs: Array<string>, parseConfigurations: (arg0: Array<string>) => Record<string, object>): Record<string, object>
export interface NxWorkspaceFiles { export interface NxWorkspaceFiles {
projectFileMap: Record<string, Array<FileData>> projectFileMap: Record<string, Array<FileData>>
globalFiles: Array<FileData> globalFiles: Array<FileData>
configFiles: Array<string> projectConfigurations: Record<string, object>
} }
/** Throws exceptions */ export function getWorkspaceFilesNative(workspaceRoot: string, globs: Array<string>, parseConfigurations: (arg0: Array<string>) => Record<string, object>): NxWorkspaceFiles
export function getWorkspaceFilesNative(workspaceRoot: string, globs: Array<string>): NxWorkspaceFiles
export class Watcher { export class Watcher {
origin: string origin: string
/** /**

View File

@ -246,7 +246,7 @@ if (!nativeBinding) {
throw new Error(`Failed to load native binding`) throw new Error(`Failed to load native binding`)
} }
const { expandOutputs, remove, copy, hashArray, hashFile, hashFiles, hashFilesMatchingGlobs, EventType, Watcher, WorkspaceErrors, getConfigFiles, getWorkspaceFilesNative } = nativeBinding const { expandOutputs, remove, copy, hashArray, hashFile, hashFiles, hashFilesMatchingGlobs, EventType, Watcher, WorkspaceErrors, getProjectConfigurations, getWorkspaceFilesNative } = nativeBinding
module.exports.expandOutputs = expandOutputs module.exports.expandOutputs = expandOutputs
module.exports.remove = remove module.exports.remove = remove
@ -258,5 +258,5 @@ module.exports.hashFilesMatchingGlobs = hashFilesMatchingGlobs
module.exports.EventType = EventType module.exports.EventType = EventType
module.exports.Watcher = Watcher module.exports.Watcher = Watcher
module.exports.WorkspaceErrors = WorkspaceErrors module.exports.WorkspaceErrors = WorkspaceErrors
module.exports.getConfigFiles = getConfigFiles module.exports.getProjectConfigurations = getProjectConfigurations
module.exports.getWorkspaceFilesNative = getWorkspaceFilesNative module.exports.getWorkspaceFilesNative = getWorkspaceFilesNative

View File

@ -1,12 +1,24 @@
import { import { getProjectConfigurations, getWorkspaceFilesNative } from '../index';
getConfigFiles,
getWorkspaceFilesNative,
WorkspaceErrors,
} from '../index';
import { TempFs } from '../../utils/testing/temp-fs'; import { TempFs } from '../../utils/testing/temp-fs';
import { NxJsonConfiguration } from '../../config/nx-json'; import { NxJsonConfiguration } from '../../config/nx-json';
import { dirname, join } from 'path';
import { readJsonFile } from '../../utils/fileutils';
describe('workspace files', () => { describe('workspace files', () => {
function createParseConfigurationsFunction(tempDir: string) {
return (filenames: string[]) => {
const res = {};
for (const filename of filenames) {
const json = readJsonFile(join(tempDir, filename));
res[json.name] = {
...json,
root: dirname(filename),
};
}
return res;
};
}
it('should gather workspace file information', async () => { it('should gather workspace file information', async () => {
const fs = new TempFs('workspace-files'); const fs = new TempFs('workspace-files');
const nxJson: NxJsonConfiguration = {}; const nxJson: NxJsonConfiguration = {};
@ -41,12 +53,16 @@ describe('workspace files', () => {
}); });
let globs = ['project.json', '**/project.json', 'libs/*/package.json']; let globs = ['project.json', '**/project.json', 'libs/*/package.json'];
let { projectFileMap, configFiles, globalFiles } = getWorkspaceFilesNative( let { projectFileMap, projectConfigurations, globalFiles } =
getWorkspaceFilesNative(
fs.tempDir, fs.tempDir,
globs globs,
createParseConfigurationsFunction(fs.tempDir)
); );
let sortedConfigs = configFiles.sort(); let sortedConfigs = Object.values(projectConfigurations).sort((a, b) =>
a['name'].localeCompare(b['name'])
);
expect(projectFileMap).toMatchInlineSnapshot(` expect(projectFileMap).toMatchInlineSnapshot(`
{ {
@ -104,11 +120,26 @@ describe('workspace files', () => {
`); `);
expect(sortedConfigs).toMatchInlineSnapshot(` expect(sortedConfigs).toMatchInlineSnapshot(`
[ [
"libs/nested/project/project.json", {
"libs/package-project/package.json", "name": "nested-project",
"libs/project1/project.json", "root": "libs/nested/project",
"libs/project2/project.json", },
"libs/project3/project.json", {
"name": "package-project",
"root": "libs/package-project",
},
{
"name": "project1",
"root": "libs/project1",
},
{
"name": "project2",
"root": "libs/project2",
},
{
"name": "project3",
"root": "libs/project3",
},
] ]
`); `);
expect(globalFiles).toMatchInlineSnapshot(` expect(globalFiles).toMatchInlineSnapshot(`
@ -148,7 +179,8 @@ describe('workspace files', () => {
const globs = ['project.json', '**/project.json', '**/package.json']; const globs = ['project.json', '**/project.json', '**/package.json'];
const { globalFiles, projectFileMap } = getWorkspaceFilesNative( const { globalFiles, projectFileMap } = getWorkspaceFilesNative(
fs.tempDir, fs.tempDir,
globs globs,
createParseConfigurationsFunction(fs.tempDir)
); );
expect(globalFiles).toEqual([]); expect(globalFiles).toEqual([]);
@ -201,140 +233,117 @@ describe('workspace files', () => {
}); });
let globs = ['project.json', '**/project.json', '**/package.json']; let globs = ['project.json', '**/project.json', '**/package.json'];
let { configFiles } = getWorkspaceFilesNative(fs.tempDir, globs);
configFiles = configFiles.sort(); let projectConfigurations = getProjectConfigurations(
fs.tempDir,
expect(configFiles).toMatchInlineSnapshot(` globs,
[ (filenames) => {
"libs/project1/project.json", const res = {};
"project.json", for (const filename of filenames) {
] const json = readJsonFile(join(fs.tempDir, filename));
`); res[json.name] = {
...json,
let configFiles2 = getConfigFiles(fs.tempDir, globs).sort(); root: dirname(filename),
expect(configFiles2).toMatchInlineSnapshot(` };
[ }
"libs/project1/project.json", return res;
"project.json",
]
`);
});
describe('errors', () => {
it('it should infer names of configuration files without a name', async () => {
const fs = new TempFs('workspace-files');
const nxJson: NxJsonConfiguration = {};
await fs.createFiles({
'./nx.json': JSON.stringify(nxJson),
'./package.json': JSON.stringify({
name: 'repo-name',
version: '0.0.0',
dependencies: {},
}),
'./libs/project1/project.json': JSON.stringify({
name: 'project1',
}),
'./libs/project1/index.js': '',
'./libs/project2/project.json': JSON.stringify({}),
});
let globs = ['project.json', '**/project.json', 'libs/*/package.json'];
expect(getWorkspaceFilesNative(fs.tempDir, globs).projectFileMap)
.toMatchInlineSnapshot(`
{
"project1": [
{
"file": "libs/project1/index.js",
"hash": "3244421341483603138",
},
{
"file": "libs/project1/project.json",
"hash": "13466615737813422520",
},
],
"project2": [
{
"file": "libs/project2/project.json",
"hash": "1389868326933519382",
},
],
} }
`);
});
it('handles comments', async () => {
const fs = new TempFs('workspace-files');
const nxJson: NxJsonConfiguration = {};
await fs.createFiles({
'./nx.json': JSON.stringify(nxJson),
'./package.json': JSON.stringify({
name: 'repo-name',
version: '0.0.0',
dependencies: {},
}),
'./libs/project1/project.json': `{
"name": "temp"
// this should not fail
}`,
'./libs/project1/index.js': '',
});
let globs = ['project.json', '**/project.json', 'libs/*/package.json'];
expect(() => getWorkspaceFilesNative(fs.tempDir, globs)).not.toThrow();
});
it('handles extra comma', async () => {
const fs = new TempFs('workspace-files');
const nxJson: NxJsonConfiguration = {};
await fs.createFiles({
'./nx.json': JSON.stringify(nxJson),
'./package.json': JSON.stringify({
name: 'repo-name',
version: '0.0.0',
dependencies: {},
}),
'./libs/project1/project.json': `{
"name": "temp",
}`,
'./libs/project1/index.js': '',
});
let globs = ['**/project.json'];
expect(() => getWorkspaceFilesNative(fs.tempDir, globs)).not.toThrow();
});
it('throws parsing errors: missing brackets', async () => {
const fs = new TempFs('workspace-files');
const nxJson: NxJsonConfiguration = {};
await fs.createFiles({
'./nx.json': JSON.stringify(nxJson),
'./package.json': JSON.stringify({
name: 'repo-name',
version: '0.0.0',
dependencies: {},
}),
'./libs/project1/project.json': `{
"name": "temp", "property": "child": 2 }
}`,
'./libs/project1/index.js': '',
});
let globs = ['**/project.json'];
const error = getError(() => getWorkspaceFilesNative(fs.tempDir, globs));
expect(error.message).toMatchInlineSnapshot(
`"libs/project1/project.json"`
); );
expect(error).toHaveProperty('code', WorkspaceErrors.ParseError); expect(projectConfigurations).toMatchInlineSnapshot(`
}); {
}); "project1": {
}); "name": "project1",
"root": "libs/project1",
const getError = (fn: () => unknown): Error => { },
try { "repo-name": {
fn(); "name": "repo-name",
} catch (error: unknown) { "root": ".",
return error as Error; },
} }
}; `);
});
// describe('errors', () => {
// it('it should infer names of configuration files without a name', async () => {
// const fs = new TempFs('workspace-files');
// const nxJson: NxJsonConfiguration = {};
// await fs.createFiles({
// './nx.json': JSON.stringify(nxJson),
// './package.json': JSON.stringify({
// name: 'repo-name',
// version: '0.0.0',
// dependencies: {},
// }),
// './libs/project1/project.json': JSON.stringify({
// name: 'project1',
// }),
// './libs/project1/index.js': '',
// './libs/project2/project.json': JSON.stringify({}),
// });
//
// let globs = ['project.json', '**/project.json', 'libs/*/package.json'];
// expect(getWorkspaceFilesNative(fs.tempDir, globs).projectFileMap)
// .toMatchInlineSnapshot(`
// {
// "libs/project1": [
// {
// "file": "libs/project1/index.js",
// "hash": "3244421341483603138",
// },
// {
// "file": "libs/project1/project.json",
// "hash": "13466615737813422520",
// },
// ],
// "libs/project2": [
// {
// "file": "libs/project2/project.json",
// "hash": "1389868326933519382",
// },
// ],
// }
// `);
// });
//
// it('handles comments', async () => {
// const fs = new TempFs('workspace-files');
// const nxJson: NxJsonConfiguration = {};
// await fs.createFiles({
// './nx.json': JSON.stringify(nxJson),
// './package.json': JSON.stringify({
// name: 'repo-name',
// version: '0.0.0',
// dependencies: {},
// }),
// './libs/project1/project.json': `{
// "name": "temp"
// // this should not fail
// }`,
// './libs/project1/index.js': '',
// });
//
// let globs = ['project.json', '**/project.json', 'libs/*/package.json'];
// expect(() => getWorkspaceFilesNative(fs.tempDir, globs)).not.toThrow();
// });
//
// it('handles extra comma', async () => {
// const fs = new TempFs('workspace-files');
// const nxJson: NxJsonConfiguration = {};
// await fs.createFiles({
// './nx.json': JSON.stringify(nxJson),
// './package.json': JSON.stringify({
// name: 'repo-name',
// version: '0.0.0',
// dependencies: {},
// }),
// './libs/project1/project.json': `{
// "name": "temp",
// }`,
// './libs/project1/index.js': '',
// });
//
// let globs = ['**/project.json'];
// expect(() => getWorkspaceFilesNative(fs.tempDir, globs)).not.toThrow();
// });
// });
});

View File

@ -20,6 +20,10 @@ fn normalize_path<P>(path: P) -> String
where where
P: AsRef<Path>, P: AsRef<Path>,
{ {
if path.as_ref() == Path::new("") {
return ".".into();
}
// convert back-slashes in Windows paths, since the js expects only forward-slash path separators // convert back-slashes in Windows paths, since the js expects only forward-slash path separators
if cfg!(windows) { if cfg!(windows) {
path.as_ref().display().to_string().replace('\\', "/") path.as_ref().display().to_string().replace('\\', "/")

View File

@ -2,29 +2,42 @@ use crate::native::utils::glob::build_glob_set;
use crate::native::utils::path::Normalize; use crate::native::utils::path::Normalize;
use crate::native::walker::nx_walker; use crate::native::walker::nx_walker;
use globset::GlobSet; use globset::GlobSet;
use napi::JsObject;
use std::collections::hash_map::Entry; use std::collections::hash_map::Entry;
use std::collections::HashMap; use std::collections::HashMap;
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
#[napi] #[napi]
/// Get workspace config files based on provided globs /// Get workspace config files based on provided globs
pub fn get_config_files(workspace_root: String, globs: Vec<String>) -> anyhow::Result<Vec<String>> { pub fn get_project_configurations<ConfigurationParser>(
workspace_root: String,
globs: Vec<String>,
parse_configurations: ConfigurationParser,
) -> napi::Result<HashMap<String, JsObject>>
where
ConfigurationParser: Fn(Vec<String>) -> napi::Result<HashMap<String, JsObject>>,
{
let globs = build_glob_set(globs)?; let globs = build_glob_set(globs)?;
Ok(nx_walker(workspace_root, move |rec| { let config_paths: Vec<String> = nx_walker(workspace_root, move |rec| {
let mut config_paths: HashMap<PathBuf, (PathBuf, Vec<u8>)> = HashMap::new(); let mut config_paths: HashMap<PathBuf, PathBuf> = HashMap::new();
for (path, content) in rec { for (path, _) in rec {
insert_config_file_into_map((path, content), &mut config_paths, &globs); insert_config_file_into_map(path, &mut config_paths, &globs);
} }
config_paths config_paths
.into_iter() .into_values()
.map(|(_, (val, _))| val.to_normalized_string()) .map(|p| p.to_normalized_string())
.collect() .collect()
})) });
parse_configurations(config_paths)
} }
pub fn insert_config_file_into_map( pub fn insert_config_file_into_map(
(path, content): (PathBuf, Vec<u8>), path: PathBuf,
config_paths: &mut HashMap<PathBuf, (PathBuf, Vec<u8>)>, config_paths: &mut HashMap<PathBuf, PathBuf>,
globs: &GlobSet, globs: &GlobSet,
) { ) {
if globs.is_match(&path) { if globs.is_match(&path) {
@ -34,25 +47,24 @@ pub fn insert_config_file_into_map(
.file_name() .file_name()
.expect("Config paths always have file names"); .expect("Config paths always have file names");
if file_name == "project.json" { if file_name == "project.json" {
config_paths.insert(parent, (path, content)); config_paths.insert(parent, path);
} else if file_name == "package.json" { } else if file_name == "package.json" {
match config_paths.entry(parent) { match config_paths.entry(parent) {
Entry::Occupied(mut o) => { Entry::Occupied(mut o) => {
if o.get() if o.get()
.0
.file_name() .file_name()
.expect("Config paths always have file names") .expect("Config paths always have file names")
!= "project.json" != "project.json"
{ {
o.insert((path, content)); o.insert(path);
} }
} }
Entry::Vacant(v) => { Entry::Vacant(v) => {
v.insert((path, content)); v.insert(path);
} }
} }
} else { } else {
config_paths.entry(parent).or_insert((path, content)); config_paths.entry(parent).or_insert(path);
} }
} }
} }
@ -65,34 +77,23 @@ mod test {
#[test] #[test]
fn should_insert_config_files_properly() { fn should_insert_config_files_properly() {
let mut config_paths: HashMap<PathBuf, (PathBuf, Vec<u8>)> = HashMap::new(); let mut config_paths: HashMap<PathBuf, PathBuf> = HashMap::new();
let globs = build_glob_set(vec!["**/*".into()]).unwrap(); let globs = build_glob_set(vec!["**/*".into()]).unwrap();
insert_config_file_into_map(PathBuf::from("project.json"), &mut config_paths, &globs);
insert_config_file_into_map(PathBuf::from("package.json"), &mut config_paths, &globs);
insert_config_file_into_map( insert_config_file_into_map(
(PathBuf::from("project.json"), vec![]), PathBuf::from("lib1/project.json"),
&mut config_paths, &mut config_paths,
&globs, &globs,
); );
insert_config_file_into_map( insert_config_file_into_map(
(PathBuf::from("package.json"), vec![]), PathBuf::from("lib2/package.json"),
&mut config_paths,
&globs,
);
insert_config_file_into_map(
(PathBuf::from("lib1/project.json"), vec![]),
&mut config_paths,
&globs,
);
insert_config_file_into_map(
(PathBuf::from("lib2/package.json"), vec![]),
&mut config_paths, &mut config_paths,
&globs, &globs,
); );
let config_files: Vec<PathBuf> = config_paths let config_files: Vec<PathBuf> = config_paths.into_values().collect();
.into_iter()
.map(|(_, (path, _))| path)
.collect();
assert!(config_files.contains(&PathBuf::from("project.json"))); assert!(config_files.contains(&PathBuf::from("project.json")));
assert!(config_files.contains(&PathBuf::from("lib1/project.json"))); assert!(config_files.contains(&PathBuf::from("lib1/project.json")));

View File

@ -1,5 +1,7 @@
use jsonc_parser::ParseOptions; use itertools::Itertools;
use std::collections::HashMap; use napi::threadsafe_function::{ErrorStrategy, ThreadsafeFunction, ThreadsafeFunctionCallMode};
use napi::{JsFunction, JsObject, JsUnknown, Status};
use std::collections::{HashMap, HashSet};
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use rayon::prelude::*; use rayon::prelude::*;
@ -13,21 +15,24 @@ use crate::native::utils::path::Normalize;
use crate::native::walker::nx_walker; use crate::native::walker::nx_walker;
use crate::native::workspace::errors::{InternalWorkspaceErrors, WorkspaceErrors}; use crate::native::workspace::errors::{InternalWorkspaceErrors, WorkspaceErrors};
use crate::native::workspace::get_config_files::insert_config_file_into_map; use crate::native::workspace::get_config_files::insert_config_file_into_map;
use crate::native::workspace::types::{FileLocation, ProjectConfiguration}; use crate::native::workspace::types::FileLocation;
#[napi(object)] #[napi(object)]
pub struct NxWorkspaceFiles { pub struct NxWorkspaceFiles {
pub project_file_map: HashMap<String, Vec<FileData>>, pub project_file_map: HashMap<String, Vec<FileData>>,
pub global_files: Vec<FileData>, pub global_files: Vec<FileData>,
pub config_files: Vec<String>, pub project_configurations: HashMap<String, JsObject>,
} }
#[napi] #[napi]
/// Throws exceptions pub fn get_workspace_files_native<ConfigurationParser>(
pub fn get_workspace_files_native(
workspace_root: String, workspace_root: String,
globs: Vec<String>, globs: Vec<String>,
) -> napi::Result<NxWorkspaceFiles, WorkspaceErrors> { parse_configurations: ConfigurationParser,
) -> napi::Result<NxWorkspaceFiles, WorkspaceErrors>
where
ConfigurationParser: Fn(Vec<String>) -> napi::Result<HashMap<String, JsObject>>,
{
enable_logger(); enable_logger();
trace!("{workspace_root}, {globs:?}"); trace!("{workspace_root}, {globs:?}");
@ -35,7 +40,12 @@ pub fn get_workspace_files_native(
let (projects, mut file_data) = get_file_data(&workspace_root, globs) let (projects, mut file_data) = get_file_data(&workspace_root, globs)
.map_err(|err| napi::Error::new(WorkspaceErrors::Generic, err.to_string()))?; .map_err(|err| napi::Error::new(WorkspaceErrors::Generic, err.to_string()))?;
let root_map = create_root_map(&projects)?; let projects_vec: Vec<String> = projects.iter().map(|p| p.to_normalized_string()).collect();
let project_configurations = parse_configurations(projects_vec)
.map_err(|e| napi::Error::new(WorkspaceErrors::ParseError, e.to_string()))?;
let root_map = create_root_map(&project_configurations);
trace!(?root_map); trace!(?root_map);
@ -46,14 +56,14 @@ pub fn get_workspace_files_native(
.into_par_iter() .into_par_iter()
.map(|file_data| { .map(|file_data| {
let file_path = Path::new(&file_data.file); let file_path = Path::new(&file_data.file);
let mut parent = file_path.parent().unwrap_or_else(|| Path::new("")); let mut parent = file_path.parent().unwrap_or_else(|| Path::new("."));
while root_map.get(parent).is_none() && parent != Path::new("") { while root_map.get(parent).is_none() && parent != Path::new(".") {
parent = parent.parent().unwrap_or_else(|| Path::new("")); parent = parent.parent().unwrap_or_else(|| Path::new("."));
} }
match root_map.get(parent) { match root_map.get(parent) {
Some(project_name) => (FileLocation::Project(project_name.clone()), file_data), Some(project_name) => (FileLocation::Project(project_name.into()), file_data),
None => (FileLocation::Global, file_data), None => (FileLocation::Global, file_data),
} }
}) })
@ -76,7 +86,7 @@ pub fn get_workspace_files_native(
FileLocation::Global => global_files.push(file_data), FileLocation::Global => global_files.push(file_data),
FileLocation::Project(project_name) => match project_file_map.get_mut(&project_name) { FileLocation::Project(project_name) => match project_file_map.get_mut(&project_name) {
None => { None => {
project_file_map.insert(project_name, vec![file_data]); project_file_map.insert(project_name.clone(), vec![file_data]);
} }
Some(project_files) => project_files.push(file_data), Some(project_files) => project_files.push(file_data),
}, },
@ -86,98 +96,34 @@ pub fn get_workspace_files_native(
Ok(NxWorkspaceFiles { Ok(NxWorkspaceFiles {
project_file_map, project_file_map,
global_files, global_files,
config_files: projects project_configurations,
.keys()
.map(|path| path.to_normalized_string())
.collect(),
}) })
} }
fn create_root_map( fn create_root_map(
projects: &HashMap<PathBuf, Vec<u8>>, project_configurations: &HashMap<String, JsObject>,
) -> Result<hashbrown::HashMap<&Path, String>, InternalWorkspaceErrors> { ) -> hashbrown::HashMap<PathBuf, String> {
projects project_configurations
.par_iter() .iter()
.map(|(path, content)| { .map(|(project_name, project_configuration)| {
let file_name = path let root: String = project_configuration.get("root").unwrap().unwrap();
.file_name() (PathBuf::from(root), project_name.clone())
.expect("path should always have a filename");
return if file_name == "project.json" || file_name == "package.json" {
// use serde_json to do the initial parse, if that fails fall back to jsonc_parser.
// If all those fail, expose the error from jsonc_parser
let project_configuration: ProjectConfiguration =
read_project_configuration(content, path)?;
let Some(parent_path) = path.parent() else {
return Err(InternalWorkspaceErrors::Generic {
msg: format!("{path:?} has no parent"),
})
};
let name: String = if let Some(name) = project_configuration.name {
Ok(name)
} else {
parent_path
.file_name()
.unwrap_or_default()
.to_os_string()
.into_string()
.map_err(|os_string| InternalWorkspaceErrors::Generic {
msg: format!("Cannot turn {os_string:?} into String"),
})
}?;
Ok((parent_path, name))
} else if let Some(parent_path) = path.parent() {
Ok((
parent_path,
parent_path
.file_name()
.unwrap_or_default()
.to_os_string()
.into_string()
.map_err(|os_string| InternalWorkspaceErrors::Generic {
msg: format!("Cannot turn {os_string:?} into String"),
})?,
))
} else {
Err(InternalWorkspaceErrors::Generic {
msg: format!("{path:?} has no parent"),
})
};
}) })
.collect() .collect()
} }
fn read_project_configuration( type WorkspaceData = (HashSet<PathBuf>, Vec<FileData>);
content: &[u8],
path: &Path,
) -> Result<ProjectConfiguration, InternalWorkspaceErrors> {
serde_json::from_slice(content).or_else(|_| {
let content_str = std::str::from_utf8(content).expect("content should be valid utf8");
let parser_value =
jsonc_parser::parse_to_serde_value(content_str, &ParseOptions::default()).map_err(
|_| InternalWorkspaceErrors::ParseError {
file: PathBuf::from(path),
},
)?;
serde_json::from_value(parser_value.into()).map_err(|_| InternalWorkspaceErrors::Generic {
msg: format!("Failed to parse {path:?}"),
})
})
}
type WorkspaceData = (HashMap<PathBuf, Vec<u8>>, Vec<FileData>);
fn get_file_data(workspace_root: &str, globs: Vec<String>) -> anyhow::Result<WorkspaceData> { fn get_file_data(workspace_root: &str, globs: Vec<String>) -> anyhow::Result<WorkspaceData> {
let globs = build_glob_set(globs)?; let globs = build_glob_set(globs)?;
let (projects, file_data) = nx_walker(workspace_root, move |rec| { let (projects, file_data) = nx_walker(workspace_root, move |rec| {
let mut projects: HashMap<PathBuf, (PathBuf, Vec<u8>)> = HashMap::new(); let mut projects: HashMap<PathBuf, PathBuf> = HashMap::new();
let mut file_hashes: Vec<FileData> = vec![]; let mut file_hashes: Vec<FileData> = vec![];
for (path, content) in rec { for (path, content) in rec {
file_hashes.push(FileData { file_hashes.push(FileData {
file: path.to_normalized_string(), file: path.to_normalized_string(),
hash: xxh3::xxh3_64(&content).to_string(), hash: xxh3::xxh3_64(&content).to_string(),
}); });
insert_config_file_into_map((path, content), &mut projects, &globs) insert_config_file_into_map(path, &mut projects, &globs)
} }
(projects, file_hashes) (projects, file_hashes)
}); });

View File

@ -1,10 +1,3 @@
use serde::Deserialize;
#[derive(Debug, Deserialize)]
pub(crate) struct ProjectConfiguration {
pub name: Option<String>,
}
#[derive(Debug, Eq, PartialEq)] #[derive(Debug, Eq, PartialEq)]
pub enum FileLocation { pub enum FileLocation {
Global, Global,

View File

@ -9,14 +9,17 @@ import {
import { getNxRequirePaths } from '../../utils/installation-directory'; import { getNxRequirePaths } from '../../utils/installation-directory';
import { readJsonFile } from '../../utils/fileutils'; import { readJsonFile } from '../../utils/fileutils';
import { join } from 'path'; import { join } from 'path';
import { ProjectsConfigurations } from '../../config/workspace-json-project-json'; import {
ProjectConfiguration,
ProjectsConfigurations,
} from '../../config/workspace-json-project-json';
import { import {
mergeAngularJsonAndProjects, mergeAngularJsonAndProjects,
shouldMergeAngularProjects, shouldMergeAngularProjects,
} from '../../adapter/angular-json'; } from '../../adapter/angular-json';
import { NxJsonConfiguration } from '../../config/nx-json'; import { NxJsonConfiguration } from '../../config/nx-json';
import { FileData, ProjectFileMap } from '../../config/project-graph'; import { FileData, ProjectFileMap } from '../../config/project-graph';
import { NxWorkspaceFiles, WorkspaceErrors } from '../../native'; import type { NxWorkspaceFiles } from '../../native';
/** /**
* Walks the workspace directory to create the `projectFileMap`, `ProjectConfigurations` and `allWorkspaceFiles` * Walks the workspace directory to create the `projectFileMap`, `ProjectConfigurations` and `allWorkspaceFiles`
@ -40,19 +43,21 @@ export async function retrieveWorkspaceFiles(
); );
performance.mark('get-workspace-files:start'); performance.mark('get-workspace-files:start');
let workspaceFiles: NxWorkspaceFiles;
try { const { projectConfigurations, projectFileMap, globalFiles } =
workspaceFiles = getWorkspaceFilesNative(workspaceRoot, globs); getWorkspaceFilesNative(
} catch (e) { workspaceRoot,
// If the error is a parse error from Rust, then use the JS readJsonFile function to write a pretty error message globs,
if (e.code === WorkspaceErrors.ParseError) { (configs: string[]): Record<string, ProjectConfiguration> => {
readJsonFile(join(workspaceRoot, e.message)); const projectConfigurations = createProjectConfigurations(
// readJsonFile should always fail, but if it doesn't, then throw the original error workspaceRoot,
throw e; nxJson,
} else { configs
throw e; );
}
return projectConfigurations.projects;
} }
) as NxWorkspaceFiles;
performance.mark('get-workspace-files:end'); performance.mark('get-workspace-files:end');
performance.measure( performance.measure(
'get-workspace-files', 'get-workspace-files',
@ -61,16 +66,12 @@ export async function retrieveWorkspaceFiles(
); );
return { return {
allWorkspaceFiles: buildAllWorkspaceFiles( allWorkspaceFiles: buildAllWorkspaceFiles(projectFileMap, globalFiles),
workspaceFiles.projectFileMap, projectFileMap,
workspaceFiles.globalFiles projectConfigurations: {
), version: 2,
projectFileMap: workspaceFiles.projectFileMap, projects: projectConfigurations,
projectConfigurations: createProjectConfigurations( } as ProjectsConfigurations,
workspaceRoot,
nxJson,
workspaceFiles.configFiles
),
}; };
} }
@ -84,10 +85,21 @@ export async function retrieveProjectConfigurations(
workspaceRoot: string, workspaceRoot: string,
nxJson: NxJsonConfiguration nxJson: NxJsonConfiguration
) { ) {
const { getConfigFiles } = require('../../native'); const { getProjectConfigurations } = require('../../native');
const globs = await configurationGlobs(workspaceRoot, nxJson); const globs = await configurationGlobs(workspaceRoot, nxJson);
const configPaths = getConfigFiles(workspaceRoot, globs); return getProjectConfigurations(
return createProjectConfigurations(workspaceRoot, nxJson, configPaths); workspaceRoot,
globs,
(configs: string[]): Record<string, ProjectConfiguration> => {
const projectConfigurations = createProjectConfigurations(
workspaceRoot,
nxJson,
configs
);
return projectConfigurations.projects;
}
);
} }
function buildAllWorkspaceFiles( function buildAllWorkspaceFiles(
@ -123,8 +135,8 @@ function createProjectConfigurations(
); );
if (shouldMergeAngularProjects(workspaceRoot, false)) { if (shouldMergeAngularProjects(workspaceRoot, false)) {
projectConfigurations.projects = mergeAngularJsonAndProjects( projectConfigurations = mergeAngularJsonAndProjects(
projectConfigurations.projects, projectConfigurations,
workspaceRoot workspaceRoot
); );
} }
@ -135,14 +147,17 @@ function createProjectConfigurations(
'build-project-configs:end' 'build-project-configs:end'
); );
return projectConfigurations; return {
version: 2,
projects: projectConfigurations,
};
} }
function mergeTargetDefaultsIntoProjectDescriptions( function mergeTargetDefaultsIntoProjectDescriptions(
config: ProjectsConfigurations, projects: Record<string, ProjectConfiguration>,
nxJson: NxJsonConfiguration nxJson: NxJsonConfiguration
) { ) {
for (const proj of Object.values(config.projects)) { for (const proj of Object.values(projects)) {
if (proj.targets) { if (proj.targets) {
for (const targetName of Object.keys(proj.targets)) { for (const targetName of Object.keys(proj.targets)) {
const projectTargetDefinition = proj.targets[targetName]; const projectTargetDefinition = proj.targets[targetName];
@ -162,7 +177,7 @@ function mergeTargetDefaultsIntoProjectDescriptions(
} }
} }
} }
return config; return projects;
} }
async function configurationGlobs( async function configurationGlobs(