Mentions légales du service

Skip to content
Snippets Groups Projects
Commit e79bf4d9 authored by GUILLOTEAU Quentin's avatar GUILLOTEAU Quentin
Browse files

Revert "update snakefile for filesize" 

This reverts commit 375e78be.
parent 375e78be
Branches
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Snakefile
+ 23
− 112
View file @ e79bf4d9
PATH_HERE="/home/quentin/ghq/gitlab.inria.fr/qguillot/eval-folding-pfs" # PATH_HERE="/home/quentin/ghq/gitlab.inria.fr/qguillot/eval-folding-pfs"
# PATH_HERE="/home/qguilloteau/eval-folding-pfs" PATH_HERE="/home/qguilloteau/eval-folding-pfs"
RESULT_FOLDER=f"{PATH_HERE}/data" RESULT_FOLDER=f"{PATH_HERE}/data"
NIX_COMPANION_FILE=f"{PATH_HERE}/nix-user-chroot.sh" NIX_COMPANION_FILE=f"{PATH_HERE}/nix-user-chroot.sh"
...@@ -14,43 +14,10 @@ FLAVOURS = [ ...@@ -14,43 +14,10 @@ FLAVOURS = [
] ]
NB_NODES = [ NB_NODES = [
5, 9, 13, 17, 21, 25 4,
# 3, 6,
# 5, 8,
# 9, 16
# 13,
# 17,
# 21
# 4,
# 6,
# 8,
# 12,
# 16,
# 24
]
DARSHAN_YEARS = [
2018,
2019,
2020,
2021
]
SECTIONS_PAPER = [
"01_introduction",
"02_cigri_example",
"03_folding",
"04_methodo",
"05_eval"
]
FIGS = [
"darshan_analysis.pdf",
"boxplot_times.pdf",
"ratio_folding.pdf",
"ratio_folding_above_below.pdf",
"speedup.pdf",
"speedup_min.pdf"
] ]
...@@ -58,14 +25,14 @@ rule all: ...@@ -58,14 +25,14 @@ rule all:
input: input:
# expand(["nxc/build/composition::{file}"], file=FLAVOURS), # expand(["nxc/build/composition::{file}"], file=FLAVOURS),
# expand(["{result_folder}/csv_zip/results_csv_ior_{nb_nodes}_nodes_{flavour}.zip"], result_folder=RESULT_FOLDER, flavour=FLAVOURS, nb_nodes=NB_NODES), # expand(["{result_folder}/csv_zip/results_csv_ior_{nb_nodes}_nodes_{flavour}.zip"], result_folder=RESULT_FOLDER, flavour=FLAVOURS, nb_nodes=NB_NODES),
# expand(["{path_here}/data/json_zip/results_ior_{nb_nodes}_nodes_{flavour}.zip"], path_here=PATH_HERE, flavour=FLAVOURS, nb_nodes=NB_NODES), expand(["{path_here}/data/json_zip/results_ior_{nb_nodes}_nodes_{flavour}.zip"], path_here=PATH_HERE, flavour=FLAVOURS, nb_nodes=NB_NODES),
# expand(["{result_folder}/csv/{nb_nodes}_{flavour}.csv"], result_folder=RESULT_FOLDER, flavour=FLAVOURS, nb_nodes=NB_NODES), # expand(["{result_folder}/csv/{nb_nodes}_{flavour}.csv"], result_folder=RESULT_FOLDER, flavour=FLAVOURS, nb_nodes=NB_NODES)
# expand(["data/repeat/iter_{iter}/csv_zip/results_csv_ior_{nb_nodes}_nodes_{flavour}.zip"], iter=[1, 2, 3, 4, 5], nb_nodes=8, flavour=FLAVOURS), # expand(["data/repeat/iter_{iter}/csv_zip/results_csv_ior_{nb_nodes}_nodes_{flavour}.zip"], iter=[1, 2, 3, 4, 5], nb_nodes=8, flavour=FLAVOURS),
# expand(["data/repeat/iter_{iter}/csv/{nb_nodes}_{flavour}.csv"], iter=[1, 2, 3, 4, 5], nb_nodes=8, flavour=FLAVOURS), # expand(["data/repeat/iter_{iter}/csv/{nb_nodes}_{flavour}.csv"], iter=[1, 2, 3, 4, 5], nb_nodes=8, flavour=FLAVOURS),
# expand(["darshan_data/filtered_csv/darshan_{year}.csv"], year=DARSHAN_YEARS), # "Rplots.pdf",
"darshan_data/csv_full/darshan.csv", # "analysis/rmd/main.pdf",
"paper/main.pdf", # "analysis/repeat.pdf"
expand(["figs/{name}"], name=FIGS) # expand(["{result_folder}/results_ior_{nb_nodes}_{flavour}.csv"], result_folder=RESULT_FOLDER, flavour=FLAVOURS, nb_nodes=NB_NODES),
rule build_nxc_image: rule build_nxc_image:
input: input:
...@@ -90,18 +57,18 @@ rule run_ior: ...@@ -90,18 +57,18 @@ rule run_ior:
rule json_to_csv: rule json_to_csv:
input: input:
"analysis/ior_json_to_csv.py", "analysis/ior_json_to_csv.py",
"{PATH_HERE}/data/json_zip/results_ior_{nb_nodes}_nodes_{flavour}.zip" "{RESULT_FOLDER}/results_ior_{nb_nodes}_nodes_{flavour}.zip"
output: output:
"{PATH_HERE}/data/csv_zip/results_csv_ior_{nb_nodes}_nodes_{flavour}.zip" "{RESULT_FOLDER}/csv_zip/results_csv_ior_{nb_nodes}_nodes_{flavour}.zip"
shell: shell:
"nix develop .#python --command python3 {input} {output}" "nix develop .#python --command python3 {input} {output}"
rule generate_csv: rule generate_csv:
input: input:
"{PATH_HERE}/analysis/zip_to_csv.R", "analysis/zip_to_csv.R",
"{PATH_HERE}/data/csv_zip/results_csv_ior_{nb_nodes}_nodes_{flavour}.zip" "data/csv_zip/results_csv_ior_{nb_nodes}_nodes_{flavour}.zip"
output: output:
"{PATH_HERE}/data/csv/{nb_nodes}_{flavour}.csv" "data/csv/{nb_nodes}_{flavour}.csv"
shell: shell:
"nix develop .#rshell --command Rscript {input} {output}" "nix develop .#rshell --command Rscript {input} {output}"
...@@ -126,8 +93,8 @@ rule repeat_analysis: ...@@ -126,8 +93,8 @@ rule repeat_analysis:
rule general_analysis: rule general_analysis:
input: input:
"analysis/analysis2.R", "analysis/analysis.R",
expand(["{path_here}/data/csv/{nb_nodes}_{flavour}.csv"], path_here=PATH_HERE, flavour=FLAVOURS, nb_nodes=NB_NODES) expand(["{result_folder}/csv/{nb_nodes}_{flavour}.csv"], result_folder=RESULT_FOLDER, flavour=FLAVOURS, nb_nodes=NB_NODES)
output: output:
"Rplots.pdf" "Rplots.pdf"
shell: shell:
...@@ -135,69 +102,13 @@ rule general_analysis: ...@@ -135,69 +102,13 @@ rule general_analysis:
rule paper: rule paper:
input: input:
"paper/main.tex", "analysis/rmd/main.Rmd",
"paper/references.bib" expand(["{result_folder}/csv/{nb_nodes}_{flavour}.csv"], result_folder=RESULT_FOLDER, flavour=FLAVOURS, nb_nodes=NB_NODES)
output: output:
"paper/main.pdf" "analysis/rmd/main.pdf"
shell: shell:
"cd paper; nix develop .#paper --command rubber -d main.tex" "nix develop .#rmdshell --command Rscript -e 'rmarkdown::render(\"analysis/rmd/main.Rmd\", \"pdf_document\")'"
rule paper_tex:
input:
"paper/main.md",
expand(["paper/sections/{section_name}.md"], section_name = SECTIONS_PAPER)
output:
"paper/main.tex"
shell:
"nix develop .#paper --command pandoc --from=markdown --to=latex --template=paper/template.tex --output={output} {input}"
rule darshan_decompress:
input:
"darshan_data/ANL-ALCF-DARSHAN-THETA_{year}0101_{year}1231.csv.gz"
output:
"darshan_data/csv/darshan_{year}.csv"
shell:
"gunzip -k {input} -c > {output}"
rule darshan_filter:
input:
"analysis/darshan_filtering.R",
"darshan_data/csv/darshan_{year}.csv"
output:
"darshan_data/filtered_csv/darshan_{year}.csv"
shell:
"nix develop .#rshell --command Rscript {input} {wildcards.year} {output}"
rule darshan_merged:
input:
"analysis/darshan_merged.R",
expand(["darshan_data/filtered_csv/darshan_{year}.csv"], year=DARSHAN_YEARS)
output:
"darshan_data/csv_full/darshan.csv"
shell:
"nix develop .#rshell --command Rscript {input} {output}"
rule darsahn_analysis:
input:
"analysis/darshan_analysis.R",
"darshan_data/csv_full/darshan.csv"
output:
"figs/darshan_analysis.pdf"
shell:
"nix develop .#rshell --command Rscript {input} {output}"
rule generate_plots:
input:
"analysis/plot_{plot_name}.R",
expand(["{path_here}/data/csv/{nb_nodes}_{flavour}.csv"], path_here=PATH_HERE, flavour=FLAVOURS, nb_nodes=NB_NODES)
output:
"figs/{plot_name}.pdf"
shell:
"nix develop .#rshell --command Rscript {input} {output}"
# rule r_analysis: # rule r_analysis:
......
flake.nix
+ 0
− 3
View file @ e79bf4d9
...@@ -52,9 +52,6 @@ ...@@ -52,9 +52,6 @@
buildInputs = with pkgs; [ myRstudio pandoc texlive.combined.scheme-full ]; buildInputs = with pkgs; [ myRstudio pandoc texlive.combined.scheme-full ];
}; };
paper = pkgs.mkShell {
buildInputs = with pkgs; [ pandoc texlive.combined.scheme-full rubber ];
};
}; };
}; };
} }
......
nxc/composition.nix
+ 1
− 1
View file @ e79bf4d9
...@@ -21,7 +21,7 @@ ...@@ -21,7 +21,7 @@
environment.systemPackages = with pkgs; [ ior openmpi scripts.start_ior_nodes scripts.generate_ior_config ]; environment.systemPackages = with pkgs; [ ior openmpi scripts.start_ior_nodes scripts.generate_ior_config ];
fileSystems."/data" = { fileSystems."/data" = {
device = "server:/srv/shared"; device = "server:/";
fsType = "nfs"; fsType = "nfs";
}; };
}; };
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment