Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 9 additions & 2 deletions src/compiler/compiler.rs
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ use crate::dist::pkg;
use crate::lru_disk_cache;
use crate::mock_command::{CommandChild, CommandCreatorSync, RunCommand, exit_status};
use crate::server;
use crate::util::{fmt_duration_as_secs, run_input_output};
use crate::util::{filter_ccache_from_path, fmt_duration_as_secs, run_input_output};
use crate::{counted_array, dist};
use async_trait::async_trait;
use filetime::FileTime;
Expand Down Expand Up @@ -194,10 +194,13 @@ impl CompileCommandImpl for SingleCompileCommand {
env_vars,
cwd,
} = self;
// Filter out ccache directories from PATH to avoid double-caching
// when ccache is also installed on the system.
let env_vars = filter_ccache_from_path(env_vars.to_vec());
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Filter is only applied to SingleCompileCommand but NvccCompileCommand (in nvcc.rs) also implements CompileCommandImpl and may need the same treatment it is intentional ?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not intentional, I'll take a look.

let mut cmd = creator.clone().new_command_sync(executable);
cmd.args(arguments)
.env_clear()
.envs(env_vars.to_vec())
.envs(env_vars)
.current_dir(cwd);
run_input_output(cmd, None).await
}
Expand Down Expand Up @@ -1662,6 +1665,10 @@ compiler_version=__VERSION__
.to_vec();
let (tempdir, src) = write_temp_file(&pool, "testfile.c".as_ref(), test).await?;

// Filter out ccache directories from PATH to avoid double-caching
// when ccache is also installed on the system.
let env = filter_ccache_from_path(env);

let executable = executable.as_ref();
let mut cmd = creator.clone().new_command_sync(executable);
cmd.stdout(Stdio::piped())
Expand Down
24 changes: 18 additions & 6 deletions src/compiler/nvcc.rs
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ use crate::compiler::{
use crate::mock_command::{
CommandChild, CommandCreator, CommandCreatorSync, ExitStatusValue, RunCommand, exit_status,
};
use crate::util::{OsStrExt, run_input_output};
use crate::util::{OsStrExt, filter_ccache_from_path, run_input_output};
use crate::{counted_array, dist, protocol, server};
use async_trait::async_trait;
use fs::File;
Expand Down Expand Up @@ -181,6 +181,10 @@ impl CCompilerImpl for Nvcc {
_ => Err(anyhow!("PCH not supported by nvcc")),
}?;

// Filter out ccache directories from PATH to avoid double-caching
// when ccache is also installed on the system.
let env_vars = filter_ccache_from_path(env_vars.clone());

let initialize_cmd_and_args = || {
let mut command = creator.clone().new_command_sync(executable);
command
Expand Down Expand Up @@ -913,7 +917,7 @@ async fn select_nvcc_subcommands<T, F>(
creator: &T,
executable: &Path,
cwd: &Path,
env_vars: &mut Vec<(OsString, OsString)>,
env_vars: &mut [(OsString, OsString)],
remap_filenames: bool,
arguments: &[OsString],
select_subcommand: F,
Expand All @@ -938,13 +942,17 @@ where
);
}

// Filter out ccache directories from PATH to avoid double-caching
// when ccache is also installed on the system.
let mut env_vars = filter_ccache_from_path(env_vars.to_vec());

let mut nvcc_dryrun_cmd = creator.clone().new_command_sync(executable);

nvcc_dryrun_cmd
.args(&[arguments, &["--dryrun".into(), "--keep".into()][..]].concat())
.env_clear()
.current_dir(cwd)
.envs(env_vars.to_vec());
.envs(env_vars.clone());

let nvcc_dryrun_output = run_input_output(nvcc_dryrun_cmd, None).await?;

Expand Down Expand Up @@ -1257,14 +1265,18 @@ where
);
}

// Filter out ccache directories from PATH to avoid double-caching
// when ccache is also installed on the system.
let env_vars = filter_ccache_from_path(env_vars.to_vec());

let out = match cacheable {
Cacheable::No => {
let mut cmd = creator.clone().new_command_sync(exe);

cmd.args(args)
.current_dir(cwd)
.env_clear()
.envs(env_vars.to_vec());
.envs(env_vars.clone());

run_input_output(cmd, None)
.await
Expand All @@ -1275,11 +1287,11 @@ where
let args = dist::strings_to_osstrings(args);

match srvc
.compiler_info(exe.clone(), cwd.to_owned(), &args, env_vars)
.compiler_info(exe.clone(), cwd.to_owned(), &args, &env_vars)
.await
{
Err(err) => error_to_output(err),
Ok(compiler) => match compiler.parse_arguments(&args, cwd, env_vars) {
Ok(compiler) => match compiler.parse_arguments(&args, cwd, &env_vars) {
CompilerArguments::NotCompilation => Err(anyhow!("Not compilation")),
CompilerArguments::CannotCache(why, extra_info) => Err(extra_info
.map_or_else(
Expand Down
116 changes: 115 additions & 1 deletion src/util.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1015,11 +1015,125 @@ pub fn num_cpus() -> usize {
std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get)
}

/// Filter out ccache directories from PATH environment variable.
///
/// This prevents double-caching when ccache is installed and has
/// /usr/lib/ccache or /usr/lib64/ccache (or similar) in PATH, since those
/// directories contain wrapper scripts that would call ccache.
pub fn filter_ccache_from_path(env_vars: Vec<(OsString, OsString)>) -> Vec<(OsString, OsString)> {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Function allocates a new Vec even when no ccache dirs are present in PATH. Consider short-circuiting to return the input Vec unchanged when filtering is not needed

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sure

use std::env;

const CCACHE_DIR_COMPONENT: &str = "ccache";

let path_contains_ccache = |value: &OsString| -> bool {
std::env::split_paths(value).any(|p| {
p.components()
.any(|c| c.as_os_str() == CCACHE_DIR_COMPONENT)
})
};

// First pass: check if any PATH contains ccache
let has_ccache = env_vars
.iter()
.any(|(key, value)| key == "PATH" && path_contains_ccache(value));

if !has_ccache {
return env_vars;
}

// Second pass: filter out ccache directories from PATH
env_vars
.into_iter()
.map(|(key, value)| {
if key == "PATH" {
let filtered_path = env::split_paths(&value)
.filter(|p| {
!p.components()
.any(|c| c.as_os_str() == CCACHE_DIR_COMPONENT)
})
.collect::<Vec<_>>();
let new_value = env::join_paths(filtered_path).unwrap_or(value.clone());
(key, new_value)
} else {
(key, value)
}
})
.collect()
}

#[cfg(test)]
mod tests {
use super::{OsStrExt, TimeMacroFinder};
use super::{OsStrExt, TimeMacroFinder, filter_ccache_from_path};
use std::ffi::{OsStr, OsString};

#[test]
fn test_filter_ccache_from_path() {
use std::env;

// Create a PATH with ccache directories
let path_with_ccache = env::join_paths([
"/usr/bin",
"/usr/lib64/ccache",
"/usr/local/bin",
"/usr/lib/ccache",
"/home/user/bin",
])
.unwrap();

let env_vars = vec![
(OsString::from("HOME"), OsString::from("/home/user")),
(OsString::from("PATH"), path_with_ccache),
(OsString::from("LANG"), OsString::from("en_US.UTF-8")),
];

let filtered = filter_ccache_from_path(env_vars);

// Other env vars should be unchanged
assert_eq!(filtered.len(), 3);
assert_eq!(
filtered.iter().find(|(k, _)| k == "HOME").unwrap().1,
OsString::from("/home/user")
);
assert_eq!(
filtered.iter().find(|(k, _)| k == "LANG").unwrap().1,
OsString::from("en_US.UTF-8")
);

// PATH should have ccache directories removed
let new_path = &filtered.iter().find(|(k, _)| k == "PATH").unwrap().1;
let expected_path =
env::join_paths(["/usr/bin", "/usr/local/bin", "/home/user/bin"]).unwrap();
assert_eq!(new_path, &expected_path);
}

#[test]
fn test_filter_ccache_from_path_no_ccache() {
use std::env;

// Create a PATH without ccache directories
let path_without_ccache = env::join_paths(["/usr/bin", "/usr/local/bin"]).unwrap();

let env_vars = vec![(OsString::from("PATH"), path_without_ccache.clone())];

let filtered = filter_ccache_from_path(env_vars);

assert_eq!(filtered.len(), 1);
assert_eq!(filtered[0].1, path_without_ccache);
}

#[test]
fn test_filter_ccache_from_path_no_path() {
// No PATH variable at all
let env_vars = vec![
(OsString::from("HOME"), OsString::from("/home/user")),
(OsString::from("LANG"), OsString::from("en_US.UTF-8")),
];

let filtered = filter_ccache_from_path(env_vars.clone());

assert_eq!(filtered, env_vars);
}

#[test]
fn simple_starts_with() {
let a: &OsStr = "foo".as_ref();
Expand Down