miel.truyen/nx
1
0
Fork 0
Code Issues Pull Requests Actions 5 Packages Projects Releases Wiki Activity

fix(core): fix hashing of external dependencies (#22865)

Browse Source
This commit is contained in:
Jason Jean 2024-04-18 09:36:17 -04:00 committed by GitHub
parent 125c1d2786
commit 2d20ad819b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 162 additions and 188 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.prettierignore
View File

@ -17,7 +17,7 @@ packages/nx/src/plugins/js/lock-file/__fixtures__/**/*.*
packages/**/schematics/**/files/**/*.html
packages/**/generators/**/files/**/*.html
packages/nx/src/native/**/*.rs
packages/nx/src/native/index.js
packages/nx/src/native/native-bindings.js
packages/nx/src/native/index.d.ts
nx-dev/nx-dev/.next/
nx-dev/nx-dev/public/documentation

4
packages/nx/src/native/hasher.rs
View File

@ -26,8 +26,10 @@ pub fn hash_file_path<P: AsRef<Path>>(path: P) -> Option<String> {
trace!("Failed to read file: {:?}", path);
return None;
};
let hash = hash(&content);
trace!("Hashed file {:?} - {:?}", path, hash);
Some(hash(&content))
Some(hash)
}
#[cfg(test)]

5
packages/nx/src/native/index.d.ts vendored
View File

@ -29,11 +29,8 @@ export function findImports(projectFileMap: Record<string, Array<string>>): Arra
* This wont be needed once the project graph is created in Rust
*/
export function transferProjectGraph(projectGraph: ProjectGraph): ExternalObject<ProjectGraph>
export interface ExternalNodeData {
version: string
hash?: string
}
export interface ExternalNode {
packageName?: string
version: string
hash?: string
}

7
packages/nx/src/native/index.js
View File

@ -1,8 +1,8 @@
const { join, basename } = require('path');
const { copyFileSync, existsSync, mkdirSync } = require('fs');
const Module = require('module');
const { nxVersion} = require("../utils/versions")
const { cacheDir} = require("../utils/cache-directory")
const { nxVersion } = require('../utils/versions');
const { cacheDir } = require('../utils/cache-directory');
const nxPackages = new Set([
'@nx/nx-android-arm64',
@ -51,7 +51,7 @@ Module._load = function (request, parent, isMain) {
localNodeFiles.some((f) => modulePath.endsWith(f))
) {
const nativeLocation = require.resolve(modulePath);
const fileName = basename(nativeLocation)
const fileName = basename(nativeLocation);
// we copy the file to the cache directory (.nx/cache by default) and prefix with nxVersion to avoid stale files being loaded
const tmpFile = join(cacheDir, nxVersion + '-' + fileName);
if (existsSync(tmpFile)) {
@ -72,6 +72,5 @@ const indexModulePath = require.resolve('./native-bindings.js');
delete require.cache[indexModulePath];
const indexModule = require('./native-bindings.js');
module.exports = indexModule;
Module._load = originalLoad;

244
packages/nx/src/native/native-bindings.js
View File

@ -1,27 +1,24 @@
const { existsSync, readFileSync } = require('fs');
const { join } = require('path');
const { existsSync, readFileSync } = require('fs')
const { join } = require('path')
const { platform, arch } = process;
const { platform, arch } = process
let nativeBinding = null;
let localFileExisted = false;
let loadError = null;
let nativeBinding = null
let localFileExisted = false
let loadError = null
function isMusl() {
// For Node 10
if (!process.report || typeof process.report.getReport !== 'function') {
try {
const lddPath = require('child_process')
.execSync('which ldd')
.toString()
.trim();
return readFileSync(lddPath, 'utf8').includes('musl');
const lddPath = require('child_process').execSync('which ldd').toString().trim();
return readFileSync(lddPath, 'utf8').includes('musl')
} catch (e) {
return true;
return true
}
} else {
const { glibcVersionRuntime } = process.report.getReport().header;
return !glibcVersionRuntime;
const { glibcVersionRuntime } = process.report.getReport().header
return !glibcVersionRuntime
}
}
@ -29,262 +26,243 @@ switch (platform) {
case 'android':
switch (arch) {
case 'arm64':
localFileExisted = existsSync(join(__dirname, 'nx.android-arm64.node'));
localFileExisted = existsSync(join(__dirname, 'nx.android-arm64.node'))
try {
if (localFileExisted) {
nativeBinding = require('./nx.android-arm64.node');
nativeBinding = require('./nx.android-arm64.node')
} else {
nativeBinding = require('@nx/nx-android-arm64');
nativeBinding = require('@nx/nx-android-arm64')
}
} catch (e) {
loadError = e;
loadError = e
}
break;
break
case 'arm':
localFileExisted = existsSync(
join(__dirname, 'nx.android-arm-eabi.node')
);
localFileExisted = existsSync(join(__dirname, 'nx.android-arm-eabi.node'))
try {
if (localFileExisted) {
nativeBinding = require('./nx.android-arm-eabi.node');
nativeBinding = require('./nx.android-arm-eabi.node')
} else {
nativeBinding = require('@nx/nx-android-arm-eabi');
nativeBinding = require('@nx/nx-android-arm-eabi')
}
} catch (e) {
loadError = e;
loadError = e
}
break;
break
default:
throw new Error(`Unsupported architecture on Android ${arch}`);
throw new Error(`Unsupported architecture on Android ${arch}`)
}
break;
break
case 'win32':
switch (arch) {
case 'x64':
localFileExisted = existsSync(
join(__dirname, 'nx.win32-x64-msvc.node')
);
)
try {
if (localFileExisted) {
nativeBinding = require('./nx.win32-x64-msvc.node');
nativeBinding = require('./nx.win32-x64-msvc.node')
} else {
nativeBinding = require('@nx/nx-win32-x64-msvc');
nativeBinding = require('@nx/nx-win32-x64-msvc')
}
} catch (e) {
loadError = e;
loadError = e
}
break;
break
case 'ia32':
localFileExisted = existsSync(
join(__dirname, 'nx.win32-ia32-msvc.node')
);
)
try {
if (localFileExisted) {
nativeBinding = require('./nx.win32-ia32-msvc.node');
nativeBinding = require('./nx.win32-ia32-msvc.node')
} else {
nativeBinding = require('@nx/nx-win32-ia32-msvc');
nativeBinding = require('@nx/nx-win32-ia32-msvc')
}
} catch (e) {
loadError = e;
loadError = e
}
break;
break
case 'arm64':
localFileExisted = existsSync(
join(__dirname, 'nx.win32-arm64-msvc.node')
);
)
try {
if (localFileExisted) {
nativeBinding = require('./nx.win32-arm64-msvc.node');
nativeBinding = require('./nx.win32-arm64-msvc.node')
} else {
nativeBinding = require('@nx/nx-win32-arm64-msvc');
nativeBinding = require('@nx/nx-win32-arm64-msvc')
}
} catch (e) {
loadError = e;
loadError = e
}
break;
break
default:
throw new Error(`Unsupported architecture on Windows: ${arch}`);
throw new Error(`Unsupported architecture on Windows: ${arch}`)
}
break;
break
case 'darwin':
localFileExisted = existsSync(join(__dirname, 'nx.darwin-universal.node'));
localFileExisted = existsSync(join(__dirname, 'nx.darwin-universal.node'))
try {
if (localFileExisted) {
nativeBinding = require('./nx.darwin-universal.node');
nativeBinding = require('./nx.darwin-universal.node')
} else {
nativeBinding = require('@nx/nx-darwin-universal');
nativeBinding = require('@nx/nx-darwin-universal')
}
break;
break
} catch {}
switch (arch) {
case 'x64':
localFileExisted = existsSync(join(__dirname, 'nx.darwin-x64.node'));
localFileExisted = existsSync(join(__dirname, 'nx.darwin-x64.node'))
try {
if (localFileExisted) {
nativeBinding = require('./nx.darwin-x64.node');
nativeBinding = require('./nx.darwin-x64.node')
} else {
nativeBinding = require('@nx/nx-darwin-x64');
nativeBinding = require('@nx/nx-darwin-x64')
}
} catch (e) {
loadError = e;
loadError = e
}
break;
break
case 'arm64':
localFileExisted = existsSync(join(__dirname, 'nx.darwin-arm64.node'));
localFileExisted = existsSync(
join(__dirname, 'nx.darwin-arm64.node')
)
try {
if (localFileExisted) {
nativeBinding = require('./nx.darwin-arm64.node');
nativeBinding = require('./nx.darwin-arm64.node')
} else {
nativeBinding = require('@nx/nx-darwin-arm64');
nativeBinding = require('@nx/nx-darwin-arm64')
}
} catch (e) {
loadError = e;
loadError = e
}
break;
break
default:
throw new Error(`Unsupported architecture on macOS: ${arch}`);
throw new Error(`Unsupported architecture on macOS: ${arch}`)
}
break;
break
case 'freebsd':
if (arch !== 'x64') {
throw new Error(`Unsupported architecture on FreeBSD: ${arch}`);
throw new Error(`Unsupported architecture on FreeBSD: ${arch}`)
}
localFileExisted = existsSync(join(__dirname, 'nx.freebsd-x64.node'));
localFileExisted = existsSync(join(__dirname, 'nx.freebsd-x64.node'))
try {
if (localFileExisted) {
nativeBinding = require('./nx.freebsd-x64.node');
nativeBinding = require('./nx.freebsd-x64.node')
} else {
nativeBinding = require('@nx/nx-freebsd-x64');
nativeBinding = require('@nx/nx-freebsd-x64')
}
} catch (e) {
loadError = e;
loadError = e
}
break;
break
case 'linux':
switch (arch) {
case 'x64':
if (isMusl()) {
localFileExisted = existsSync(
join(__dirname, 'nx.linux-x64-musl.node')
);
)
try {
if (localFileExisted) {
nativeBinding = require('./nx.linux-x64-musl.node');
nativeBinding = require('./nx.linux-x64-musl.node')
} else {
nativeBinding = require('@nx/nx-linux-x64-musl');
nativeBinding = require('@nx/nx-linux-x64-musl')
}
} catch (e) {
loadError = e;
loadError = e
}
} else {
localFileExisted = existsSync(
join(__dirname, 'nx.linux-x64-gnu.node')
);
)
try {
if (localFileExisted) {
nativeBinding = require('./nx.linux-x64-gnu.node');
nativeBinding = require('./nx.linux-x64-gnu.node')
} else {
nativeBinding = require('@nx/nx-linux-x64-gnu');
nativeBinding = require('@nx/nx-linux-x64-gnu')
}
} catch (e) {
loadError = e;
loadError = e
}
}
break;
break
case 'arm64':
if (isMusl()) {
localFileExisted = existsSync(
join(__dirname, 'nx.linux-arm64-musl.node')
);
)
try {
if (localFileExisted) {
nativeBinding = require('./nx.linux-arm64-musl.node');
nativeBinding = require('./nx.linux-arm64-musl.node')
} else {
nativeBinding = require('@nx/nx-linux-arm64-musl');
nativeBinding = require('@nx/nx-linux-arm64-musl')
}
} catch (e) {
loadError = e;
loadError = e
}
} else {
localFileExisted = existsSync(
join(__dirname, 'nx.linux-arm64-gnu.node')
);
)
try {
if (localFileExisted) {
nativeBinding = require('./nx.linux-arm64-gnu.node');
nativeBinding = require('./nx.linux-arm64-gnu.node')
} else {
nativeBinding = require('@nx/nx-linux-arm64-gnu');
nativeBinding = require('@nx/nx-linux-arm64-gnu')
}
} catch (e) {
loadError = e;
loadError = e
}
}
break;
break
case 'arm':
localFileExisted = existsSync(
join(__dirname, 'nx.linux-arm-gnueabihf.node')
);
)
try {
if (localFileExisted) {
nativeBinding = require('./nx.linux-arm-gnueabihf.node');
nativeBinding = require('./nx.linux-arm-gnueabihf.node')
} else {
nativeBinding = require('@nx/nx-linux-arm-gnueabihf');
nativeBinding = require('@nx/nx-linux-arm-gnueabihf')
}
} catch (e) {
loadError = e;
loadError = e
}
break;
break
default:
throw new Error(`Unsupported architecture on Linux: ${arch}`);
throw new Error(`Unsupported architecture on Linux: ${arch}`)
}
break;
break
default:
throw new Error(`Unsupported OS: ${platform}, architecture: ${arch}`);
throw new Error(`Unsupported OS: ${platform}, architecture: ${arch}`)
}
if (!nativeBinding) {
if (loadError) {
throw loadError;
throw loadError
}
throw new Error(`Failed to load native binding`);
throw new Error(`Failed to load native binding`)
}
const {
expandOutputs,
getFilesForOutputs,
remove,
copy,
hashArray,
hashFile,
ImportResult,
findImports,
transferProjectGraph,
ChildProcess,
RustPseudoTerminal,
HashPlanner,
TaskHasher,
EventType,
Watcher,
WorkspaceContext,
WorkspaceErrors,
testOnlyTransferFileMap,
} = nativeBinding;
const { expandOutputs, getFilesForOutputs, remove, copy, hashArray, hashFile, ImportResult, findImports, transferProjectGraph, ChildProcess, RustPseudoTerminal, HashPlanner, TaskHasher, EventType, Watcher, WorkspaceContext, WorkspaceErrors, testOnlyTransferFileMap } = nativeBinding
module.exports.expandOutputs = expandOutputs;
module.exports.getFilesForOutputs = getFilesForOutputs;
module.exports.remove = remove;
module.exports.copy = copy;
module.exports.hashArray = hashArray;
module.exports.hashFile = hashFile;
module.exports.ImportResult = ImportResult;
module.exports.findImports = findImports;
module.exports.transferProjectGraph = transferProjectGraph;
module.exports.ChildProcess = ChildProcess;
module.exports.RustPseudoTerminal = RustPseudoTerminal;
module.exports.HashPlanner = HashPlanner;
module.exports.TaskHasher = TaskHasher;
module.exports.EventType = EventType;
module.exports.Watcher = Watcher;
module.exports.WorkspaceContext = WorkspaceContext;
module.exports.WorkspaceErrors = WorkspaceErrors;
module.exports.testOnlyTransferFileMap = testOnlyTransferFileMap;
module.exports.expandOutputs = expandOutputs
module.exports.getFilesForOutputs = getFilesForOutputs
module.exports.remove = remove
module.exports.copy = copy
module.exports.hashArray = hashArray
module.exports.hashFile = hashFile
module.exports.ImportResult = ImportResult
module.exports.findImports = findImports
module.exports.transferProjectGraph = transferProjectGraph
module.exports.ChildProcess = ChildProcess
module.exports.RustPseudoTerminal = RustPseudoTerminal
module.exports.HashPlanner = HashPlanner
module.exports.TaskHasher = TaskHasher
module.exports.EventType = EventType
module.exports.Watcher = Watcher
module.exports.WorkspaceContext = WorkspaceContext
module.exports.WorkspaceErrors = WorkspaceErrors
module.exports.testOnlyTransferFileMap = testOnlyTransferFileMap

7
packages/nx/src/native/project_graph/types.rs
View File

@ -1,14 +1,9 @@
use crate::native::types::JsInputs;
use std::collections::HashMap;
#[napi(object)]
pub struct ExternalNodeData {
pub version: String,
pub hash: Option<String>,
}
#[napi(object)]
pub struct ExternalNode {
pub package_name: Option<String>,
pub version: String,
pub hash: Option<String>,
}

59
packages/nx/src/native/tasks/hash_planner.rs
View File

@ -110,20 +110,13 @@ impl HashPlanner {
project_name: &str,
target_name: &str,
self_inputs: &[Input],
external_deps_map: &hashbrown::HashMap<&str, Vec<&'a str>>,
external_deps_map: &hashbrown::HashMap<&String, Vec<&'a String>>,
) -> anyhow::Result<Option<Vec<HashInstruction>>> {
let project = &self.project_graph.nodes[project_name];
let Some(target) = project.targets.get(target_name) else {
return Ok(None);
};
let external_nodes_keys: Vec<&str> = self
.project_graph
.external_nodes
.keys()
.map(|s| s.as_str())
.collect();
// we can only vouch for @nx packages's executor dependencies
// if it's "run commands" or third-party we skip traversing since we have no info what this command depends on
if target
@ -139,29 +132,29 @@ impl HashPlanner {
.next()
.expect("Executors should always have a ':'");
let Some(existing_package) =
find_external_dependency_node_name(executor_package, &external_nodes_keys)
find_external_dependency_node_name(executor_package, &self.project_graph)
else {
// this usually happens because the executor was a local plugin.
// todo)) @Cammisuli: we need to gather the project's inputs and its dep inputs similar to how we do it in `self_and_deps_inputs`
return Ok(None);
};
Ok(Some(vec![HashInstruction::External(
existing_package.to_string(),
existing_package.to_owned(),
)]))
} else {
let mut external_deps: Vec<&str> = vec![];
let mut external_deps: Vec<&'a String> = vec![];
for input in self_inputs {
match input {
Input::ExternalDependency(deps) => {
for dep in deps.iter() {
let external_node_name: Option<&str> =
find_external_dependency_node_name(dep, &external_nodes_keys);
let external_node_name =
find_external_dependency_node_name(dep, &self.project_graph);
let Some(external_node_name) = external_node_name else {
anyhow::bail!("The externalDependency '{dep}' for '{project_name}:{target_name}' could not be found")
};
external_deps.push(external_node_name);
external_deps.extend(&external_deps_map[external_node_name]);
external_deps.push(&external_node_name);
external_deps.extend(&external_deps_map[&external_node_name]);
}
}
_ => continue,
@ -186,12 +179,11 @@ impl HashPlanner {
task: &Task,
inputs: &SplitInputs,
task_graph: &TaskGraph,
external_deps_mapped: &hashbrown::HashMap<&str, Vec<&str>>,
external_deps_mapped: &hashbrown::HashMap<&String, Vec<&String>>,
visited: &mut Box<hashbrown::HashSet<String>>,
) -> anyhow::Result<Vec<HashInstruction>> {
let project_deps = &self.project_graph.dependencies[project_name]
.iter()
.map(|d| d.as_str())
.collect::<Vec<_>>();
let self_inputs = self.gather_self_inputs(project_name, &inputs.self_inputs);
let deps_inputs = self.gather_dependency_inputs(
@ -215,15 +207,13 @@ impl HashPlanner {
.collect())
}
fn setup_external_deps(&self) -> hashbrown::HashMap<&str, Vec<&str>> {
fn setup_external_deps(&self) -> hashbrown::HashMap<&String, Vec<&String>> {
self.project_graph
.external_nodes
.keys()
.collect::<Vec<_>>()
.par_iter()
.map(|external_node| {
(
external_node.as_str(),
external_node,
utils::find_all_project_node_dependencies(
external_node,
&self.project_graph,
@ -240,8 +230,8 @@ impl HashPlanner {
task: &Task,
inputs: &[Input],
task_graph: &TaskGraph,
project_deps: &[&'a str],
external_deps_mapped: &hashbrown::HashMap<&str, Vec<&'a str>>,
project_deps: &[&'a String],
external_deps_mapped: &hashbrown::HashMap<&String, Vec<&'a String>>,
visited: &mut Box<hashbrown::HashSet<String>>,
) -> anyhow::Result<Vec<HashInstruction>> {
let mut deps_inputs: Vec<HashInstruction> = vec![];
@ -377,12 +367,23 @@ impl HashPlanner {
fn find_external_dependency_node_name<'a>(
package_name: &str,
external_nodes: &[&'a str],
) -> Option<&'a str> {
external_nodes
.iter()
.find(|n| **n == package_name || n.ends_with(package_name))
.copied()
project_graph: &'a ProjectGraph,
) -> Option<&'a String> {
let npm_name = format!("npm:{}", &package_name);
if let Some((key, _)) = project_graph.external_nodes.get_key_value(package_name) {
Some(key)
} else if let Some((key, _)) = project_graph.external_nodes.get_key_value(&npm_name) {
Some(key)
} else {
for (node_name, node) in project_graph.external_nodes.iter() {
if let Some(pkg_name) = &node.package_name {
if pkg_name.as_str() == package_name {
return Some(node_name);
}
}
}
None
}
}
fn project_file_set_inputs(project_name: &str, file_sets: Vec<&str>) -> Vec<HashInstruction> {

2
packages/nx/src/native/tasks/hashers/hash_external.rs
View File

@ -54,6 +54,7 @@ mod test {
(
"my_external".to_string(),
ExternalNode {
package_name: Some("my_external".into()),
version: "0.0.1".into(),
hash: None,
},
@ -61,6 +62,7 @@ mod test {
(
"my_external_with_hash".to_string(),
ExternalNode {
package_name: Some("my_external_with_hash".into()),
version: "0.0.1".into(),
hash: Some("hashvalue".into()),
},

15
packages/nx/src/native/tasks/utils.rs
View File

@ -5,7 +5,7 @@ pub(super) fn find_all_project_node_dependencies<'a>(
parent_node_name: &str,
project_graph: &'a ProjectGraph,
exclude_external_dependencies: bool,
) -> Vec<&'a str> {
) -> Vec<&'a String> {
let mut dependent_node_names = HashSet::new();
collect_dependent_project_node_names(
project_graph,
@ -19,28 +19,27 @@ pub(super) fn find_all_project_node_dependencies<'a>(
fn collect_dependent_project_node_names<'a>(
project_graph: &'a ProjectGraph,
parent_node_name: &str,
dependent_node_names: &mut HashSet<&'a str>,
dependent_node_names: &mut HashSet<&'a String>,
exclude_external_dependencies: bool,
) {
let Some( dependencies ) = project_graph.dependencies.get(parent_node_name) else {
let Some(dependencies) = project_graph.dependencies.get(parent_node_name) else {
return;
};
for dependency in dependencies {
let dep = dependency.as_str();
// skip dependencies already added (avoid circular dependencies)
if dependent_node_names.contains(dep) {
if dependent_node_names.contains(dependency) {
continue;
}
if exclude_external_dependencies && project_graph.external_nodes.contains_key(dep) {
if exclude_external_dependencies && project_graph.external_nodes.contains_key(dependency) {
continue;
}
dependent_node_names.insert(dep);
dependent_node_names.insert(dependency);
collect_dependent_project_node_names(
project_graph,
dependency.as_str(),
dependency,
dependent_node_names,
exclude_external_dependencies,
);

1
packages/nx/src/native/transform-objects.ts
View File

@ -42,6 +42,7 @@ export function transformProjectGraphForRust(
graph.externalNodes ?? {}
)) {
externalNodes[projectName] = {
packageName: externalNode.data.packageName,
hash: externalNode.data.hash,
version: externalNode.data.version,
};