Commit 93dcb5cf authored by Alexandre Pere's avatar Alexandre Pere

Adds on-local option

parent ac254b33
......@@ -37,7 +37,6 @@ tracing-attributes = "0.1.5"
tracing-futures = {version = "0.1.0", features=["futures-preview"]}
derivative = "1.0.3"
[lib]
name="liborchestra"
path="src/lib/mod.rs"
\ No newline at end of file
......@@ -263,12 +263,17 @@ pub async fn tar_remote_files (root: &PathBuf,
-> Result<Sha1Hash, String>{
// We create the files string
let files_string = files.iter()
let files_string;
if files.is_empty(){
files_string = format!("--files-from /dev/null");
} else {
files_string = files.iter()
.fold(String::new(), |mut s, path| {
s.push_str(path.to_str().unwrap());
s.push(' ');
s
});
}
// We create the archive
let command = RawCommand(format!("cd {} && tar -cvf {} {}",
root.to_str().unwrap(),
......
......@@ -104,6 +104,10 @@ fn main(){
.arg(clap::Arg::with_name("no-env-read")
.long("no-env-read")
.help("Do not read the local environment variables to apply it to the remote context."))
.arg(clap::Arg::with_name("on-local")
.short("L")
.long("on-local")
.help("This allows to reduce the transfers when the profile executes locally."))
.arg(clap::Arg::with_name("ARGUMENTS")
.help("Script arguments")
.multiple(true)
......@@ -180,6 +184,11 @@ fn main(){
.arg(clap::Arg::with_name("no-env-read")
.long("no-env-read")
.help("Do not read the local environment variables to apply it to the remote context."))
.arg(clap::Arg::with_name("on-local")
.short("L")
.long("on-local")
.help("This allows to reduce the transfers when the profile executes locally. \
Remote folders are overriden to follow the output folders."))
.arg(clap::Arg::with_name("post-command")
.short("p")
.long("post-command")
......@@ -253,6 +262,11 @@ fn main(){
.arg(clap::Arg::with_name("no-env-read")
.long("no-env-read")
.help("Do not read the local environment variables to apply it to the remote context."))
.arg(clap::Arg::with_name("on-local")
.short("L")
.long("on-local")
.help("This allows to reduce the transfers when the profile executes locally. \
Remote folders are overriden to follow the output folders."))
.arg(clap::Arg::with_name("post-command")
.short("p")
.long("post-command")
......
......@@ -66,7 +66,12 @@ pub fn batch(matches: clap::ArgMatches<'static>) -> Result<Exit, Exit>{
// We setup a few variables that will be used afterward.
info!("Reading arguments");
let leave = LeaveConfig::from(matches.value_of("leave").unwrap());
let leave;
if matches.is_present("on-local"){
leave = LeaveConfig::Everything;
} else{
leave = LeaveConfig::from(matches.value_of("leave").unwrap());
}
push_env(&mut store, "RUNAWAY_LEAVE", format!("{}", leave));
debug!("Leave option set to {}", leave);
let script = PathBuf::from(matches.value_of("SCRIPT").unwrap());
......@@ -77,7 +82,12 @@ pub fn batch(matches: clap::ArgMatches<'static>) -> Result<Exit, Exit>{
let repeats: usize = matches.value_of("repeats").unwrap().parse().unwrap();
debug!("Number of repeats set to {}", repeats);
let arguments_iter = repeat_iter(extract_args_iter(&matches)?, repeats);
let remotes_iter = repeat_iter(extract_remote_folders_iter(&matches)?, repeats);
let remotes_iter;
if matches.is_present("on-local"){
remotes_iter = repeat_iter(extract_output_folders_iter(&matches)?, repeats);
} else{
remotes_iter = repeat_iter(extract_remote_folders_iter(&matches)?, repeats);
}
let outputs_iter = repeat_iter(extract_output_folders_iter(&matches)?, repeats);
// We compute some paths
......@@ -91,7 +101,7 @@ pub fn batch(matches: clap::ArgMatches<'static>) -> Result<Exit, Exit>{
// We generate globs for file sending and fetching
info!("Reading ignore files");
let (mut send_ignore_globs, fetch_ignore_globs) = misc::get_send_fetch_ignores_globs(
let (mut send_ignore_globs, mut fetch_ignore_globs) = misc::get_send_fetch_ignores_globs(
&local_folder,
matches.value_of("send-ignore").unwrap(),
matches.value_of("fetch-ignore").unwrap()
......@@ -99,6 +109,9 @@ pub fn batch(matches: clap::ArgMatches<'static>) -> Result<Exit, Exit>{
send_ignore_globs.push(primitives::Glob(format!("**/{}", SEND_ARCH_RPATH)));
send_ignore_globs.push(primitives::Glob(matches.value_of("send-ignore").unwrap().into()));
send_ignore_globs.push(primitives::Glob(matches.value_of("fetch-ignore").unwrap().into()));
if matches.is_present("on-local"){
fetch_ignore_globs = vec!(primitives::Glob("*".into()));
}
debug!("Sendignore globs set to {}", send_ignore_globs.iter()
.fold(String::new(), |mut acc, s| {acc.push_str(&format!("\n{}", s.0)); acc}));
debug!("Fetchignore globs set to {}", fetch_ignore_globs.iter()
......@@ -183,7 +196,8 @@ pub fn batch(matches: clap::ArgMatches<'static>) -> Result<Exit, Exit>{
output_folder,
&fut_leave,
&fut_fetch_ignore_globs,
&fut_fetch_include_globs
&fut_fetch_include_globs,
fut_matches.is_present("on-local")
).await?;
unpacks_fetch_post_proc(&fut_matches, local_fetch_archive, store, remote_fetch_hash, execution_code)
};
......@@ -225,6 +239,14 @@ pub fn batch(matches: clap::ArgMatches<'static>) -> Result<Exit, Exit>{
//------------------------------------------------------------------------------------------ HELPERS
// Returns the absolute path if it is not absolute
fn absolutize(p: PathBuf) -> PathBuf{
if p.is_absolute(){
p
} else{
std::env::current_dir().unwrap().join(p)
}
}
// This type allows to return an iterator that owns a piece of data. I don't know how to write the
// next function without that, as the boxed iterator would reference to content read from the file
......@@ -398,6 +420,7 @@ async fn perform_on_node(store: EnvironmentStore,
leave: &LeaveConfig,
fetch_ignore_globs: &Vec<Glob<String>>,
fetch_include_globs: &Vec<Glob<String>>,
on_local: bool
) -> Result<(PathBuf, EnvironmentStore, Sha1Hash, i32), Exit>{
......@@ -421,7 +444,13 @@ async fn perform_on_node(store: EnvironmentStore,
);
// We generate the remote folder and unpack data into it
let remote_folder= PathBuf::from(substitute_environment(&store, remote_folder_pattern.as_str()));
let remote_folder;
if on_local{
let remote_path = PathBuf::from(substitute_environment(&store, output_folder_pattern.as_str()));
remote_folder = absolutize(remote_path);
} else {
remote_folder = PathBuf::from(substitute_environment(&store, remote_folder_pattern.as_str()));
}
push_env(&mut store, "RUNAWAY_PWD", remote_folder.to_str().unwrap());
let (remote_files_before, _) = unpacks_send_on_node(
&remote_folder,
......@@ -471,7 +500,7 @@ async fn perform_on_node(store: EnvironmentStore,
// We pack data to fetch
debug!("Compressing data to be fetched");
let remote_fetch_archive = remote_folder.join(FETCH_ARCH_RPATH);
let remote_fetch_archive = remote_folder.join(".to_fetch.tar");
let remote_fetch_hash = to_exit!(primitives::tar_remote_files(&remote_folder,
&files_to_fetch,
&remote_fetch_archive,
......@@ -481,8 +510,13 @@ async fn perform_on_node(store: EnvironmentStore,
// We generate output folder
let local_output_string = substitute_environment(&execution_context.envs, output_folder_pattern.as_str());
let local_output_folder = PathBuf::from(local_output_string);
let local_output_folder;
if on_local{
local_output_folder = remote_folder.clone();
} else {
let local_output_string = substitute_environment(&execution_context.envs, output_folder_pattern.as_str());
local_output_folder = to_exit!(PathBuf::from(local_output_string).canonicalize(), Exit::OutputFolder)?;
}
debug!("Local output folder set to: {}", local_output_folder.to_str().unwrap());
if !local_output_folder.exists(){
debug!("Creating output folder");
......
......@@ -53,7 +53,12 @@ pub fn exec(matches: clap::ArgMatches) -> Result<Exit, Exit>{
// We setup some parameters
info!("Reading arguments");
let leave = LeaveConfig::from(matches.value_of("leave").unwrap());
let leave;
if matches.is_present("on-local"){
leave = LeaveConfig::Everything;
} else{
leave = LeaveConfig::from(matches.value_of("leave").unwrap());
}
push_env(&mut store, "RUNAWAY_LEAVE", format!("{}", leave));
debug!("Leave option set to {}", leave);
let parameters = matches.value_of("ARGUMENTS").unwrap_or("").to_owned();
......@@ -72,12 +77,15 @@ pub fn exec(matches: clap::ArgMatches) -> Result<Exit, Exit>{
// We list files to send
info!("Reading ignore files");
let (mut send_ignore_globs, fetch_ignore_globs) = misc::get_send_fetch_ignores_globs(&local_folder,
let (mut send_ignore_globs, mut fetch_ignore_globs) = misc::get_send_fetch_ignores_globs(&local_folder,
matches.value_of("send-ignore").unwrap(),
matches.value_of("fetch-ignore").unwrap())?;
send_ignore_globs.push(primitives::Glob(format!("**/{}", SEND_ARCH_RPATH)));
send_ignore_globs.push(primitives::Glob(matches.value_of("send-ignore").unwrap().into()));
send_ignore_globs.push(primitives::Glob(matches.value_of("fetch-ignore").unwrap().into()));
if matches.is_present("on-local"){
fetch_ignore_globs = vec!(primitives::Glob("*".into()));
}
debug!("Sendignore globs set to {}", send_ignore_globs.iter()
.fold(String::new(), |mut acc, s| {acc.push_str(&format!("\n{}", s.0)); acc}));
debug!("Fetchignore globs set to {}", fetch_ignore_globs.iter()
......@@ -158,10 +166,15 @@ pub fn exec(matches: clap::ArgMatches) -> Result<Exit, Exit>{
// We substitute the remote folder and create it if needed
let remote_folder= PathBuf::from(
substitute_environment(&store,
matches.value_of("remote-folder").unwrap())
);
let remote_folder;
if matches.is_present("on-local"){
let remote_path = PathBuf::from(
substitute_environment(&store, matches.value_of("output-folder").unwrap()));
remote_folder = to_exit!(remote_path.canonicalize(), Exit::OutputFolder)?;
} else {
remote_folder = PathBuf::from(
substitute_environment(&store, matches.value_of("remote-folder").unwrap()));
}
debug!("Remote folder set to {}", remote_folder.to_str().unwrap());
if remote_send_archive.is_relative(){return Err(Exit::WrongRemoteFolderString)}
let remote_folder_exists = to_exit!(primitives::remote_folder_exists(&remote_folder, &node).await,
......@@ -240,7 +253,7 @@ pub fn exec(matches: clap::ArgMatches) -> Result<Exit, Exit>{
// We pack data to fetch
info!("Compressing data to be fetched");
let remote_fetch_archive = remote_folder.join(FETCH_ARCH_RPATH);
let remote_fetch_archive = remote_folder.join(".to_fetch.tar");
let remote_fetch_hash = to_exit!(primitives::tar_remote_files(&remote_folder,
&files_to_fetch,
&remote_fetch_archive,
......@@ -250,8 +263,13 @@ pub fn exec(matches: clap::ArgMatches) -> Result<Exit, Exit>{
// We fetch data back
let local_output_string = substitute_environment(&execution_context.envs, matches.value_of("output-folder").unwrap());
let local_output_folder = to_exit!(PathBuf::from(local_output_string).canonicalize(), Exit::OutputFolder)?;
let local_output_folder;
if matches.is_present("on-local"){
local_output_folder = remote_folder.clone();
} else{
let local_output_string = substitute_environment(&execution_context.envs, matches.value_of("output-folder").unwrap());
local_output_folder = to_exit!(PathBuf::from(local_output_string).canonicalize(), Exit::OutputFolder)?;
}
debug!("Local output folder set to: {}", local_output_folder.to_str().unwrap());
if !local_output_folder.exists(){
debug!("Creating output folder");
......
......@@ -63,7 +63,12 @@ pub fn sched(matches: clap::ArgMatches<'static>) -> Result<Exit, Exit>{
// We setup a few variables that will be used afterward.
info!("Reading arguments");
let leave = LeaveConfig::from(matches.value_of("leave").unwrap());
let leave;
if matches.is_present("on-local"){
leave = LeaveConfig::Everything;
} else{
leave = LeaveConfig::from(matches.value_of("leave").unwrap());
}
push_env(&mut store, "RUNAWAY_LEAVE", format!("{}", leave));
debug!("Leave option set to {}", leave);
let script = PathBuf::from(matches.value_of("SCRIPT").unwrap());
......@@ -85,7 +90,7 @@ pub fn sched(matches: clap::ArgMatches<'static>) -> Result<Exit, Exit>{
// We generate globs for file sending and fetching
info!("Reading ignore files");
let (mut send_ignore_globs, fetch_ignore_globs) = misc::get_send_fetch_ignores_globs(
let (mut send_ignore_globs, mut fetch_ignore_globs) = misc::get_send_fetch_ignores_globs(
&local_folder,
matches.value_of("send-ignore").unwrap(),
matches.value_of("fetch-ignore").unwrap()
......@@ -93,6 +98,9 @@ pub fn sched(matches: clap::ArgMatches<'static>) -> Result<Exit, Exit>{
send_ignore_globs.push(primitives::Glob(format!("**/{}", SEND_ARCH_RPATH)));
send_ignore_globs.push(primitives::Glob(matches.value_of("send-ignore").unwrap().into()));
send_ignore_globs.push(primitives::Glob(matches.value_of("fetch-ignore").unwrap().into()));
if matches.is_present("on-local"){
fetch_ignore_globs = vec!(primitives::Glob("*".into()));
}
debug!("Sendignore globs set to {}", send_ignore_globs.iter()
.fold(String::new(), |mut acc, s| {acc.push_str(&format!("\n{}", s.0)); acc}));
debug!("Fetchignore globs set to {}", fetch_ignore_globs.iter()
......@@ -214,7 +222,8 @@ pub fn sched(matches: clap::ArgMatches<'static>) -> Result<Exit, Exit>{
&outputs_template,
&leave,
&fetch_ignore_globs,
&fetch_include_globs
&fetch_include_globs,
matches.is_present("on-local"),
).await?;
let ret = unpacks_fetch_post_proc(&matches, local_fetch_archive, store.clone(), remote_fetch_hash, execution_code);
if let Some(EnvironmentValue(features)) = store.get(&EnvironmentKey("RUNAWAY_FEATURES".into())) {
......@@ -271,6 +280,15 @@ pub fn sched(matches: clap::ArgMatches<'static>) -> Result<Exit, Exit>{
//------------------------------------------------------------------------------------------ HELPERS
// Returns the absolute path if it is not absolute
fn absolutize(p: PathBuf) -> PathBuf{
if p.is_absolute(){
p
} else{
std::env::current_dir().unwrap().join(p)
}
}
// This type allows to return an iterator that owns a piece of data. I don't know how to write the
// next function without that, as the boxed iterator would reference to content read from the file
// that is owned by the function.
......@@ -355,6 +373,7 @@ async fn perform_on_node(store: EnvironmentStore,
leave: &LeaveConfig,
fetch_ignore_globs: &Vec<Glob<String>>,
fetch_include_globs: &Vec<Glob<String>>,
on_local: bool
) -> Result<(PathBuf, EnvironmentStore, Sha1Hash, i32), Exit>{
......@@ -369,7 +388,13 @@ async fn perform_on_node(store: EnvironmentStore,
// We generate the remote folder and unpack data into it
let remote_folder= PathBuf::from(substitute_environment(&store, remote_folder_pattern));
let remote_folder;
if on_local{
let remote_path = PathBuf::from(substitute_environment(&store, output_folder_pattern));
remote_folder = absolutize(remote_path);
} else {
remote_folder = PathBuf::from(substitute_environment(&store, remote_folder_pattern));
}
push_env(&mut store, "RUNAWAY_PWD", remote_folder.to_str().unwrap());
let (remote_files_before, _) = unpacks_send_on_node(
&remote_folder,
......@@ -419,7 +444,7 @@ async fn perform_on_node(store: EnvironmentStore,
// We pack data to fetch
debug!("Compressing data to be fetched");
let remote_fetch_archive = remote_folder.join(FETCH_ARCH_RPATH);
let remote_fetch_archive = remote_folder.join(".to_fetch.tar");
let remote_fetch_hash = to_exit!(primitives::tar_remote_files(&remote_folder,
&files_to_fetch,
&remote_fetch_archive,
......@@ -429,8 +454,14 @@ async fn perform_on_node(store: EnvironmentStore,
// We generate output folder
let local_output_string = substitute_environment(&execution_context.envs, output_folder_pattern);
let local_output_folder = PathBuf::from(local_output_string);
let local_output_folder;
if on_local{
local_output_folder = remote_folder.clone();
} else {
let local_output_string = substitute_environment(&execution_context.envs, output_folder_pattern);
local_output_folder = to_exit!(PathBuf::from(local_output_string).canonicalize(), Exit::OutputFolder)?;
}
debug!("Local output folder set to: {}", local_output_folder.to_str().unwrap());
if !local_output_folder.exists(){
debug!("Creating output folder");
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment