update script ior

Snakefile

@@ -14,16 +14,26 @@ FLAVOURS = [
 ]
 
 NB_NODES = [
-    4
+    2,
+    4,
+    6,
+    8,
+    12,
+    16
 ]
 
 
 rule all:
     input:
         # expand(["nxc/build/composition::{file}"], file=FLAVOURS),
-        expand(["{result_folder}/results_ior_{nb_nodes}_nodes_{flavour}.zip"], result_folder=RESULT_FOLDER, flavour=FLAVOURS, nb_nodes=NB_NODES),
+        expand(["{result_folder}/csv_zip/results_csv_ior_{nb_nodes}_nodes_{flavour}.zip"], result_folder=RESULT_FOLDER, flavour=FLAVOURS, nb_nodes=NB_NODES),
+        expand(["{result_folder}/csv/{nb_nodes}_{flavour}.csv"], result_folder=RESULT_FOLDER, flavour=FLAVOURS, nb_nodes=NB_NODES),
+        expand(["data/repeat/iter_{iter}/csv_zip/results_csv_ior_{nb_nodes}_nodes_{flavour}.zip"], iter=[1, 2, 3, 4, 5], nb_nodes=8, flavour=FLAVOURS),
+        expand(["data/repeat/iter_{iter}/csv/{nb_nodes}_{flavour}.csv"], iter=[1, 2, 3, 4, 5], nb_nodes=8, flavour=FLAVOURS),
+        "Rplots.pdf",
+        "analysis/rmd/main.pdf",
+        "analysis/repeat.pdf"
         # expand(["{result_folder}/results_ior_{nb_nodes}_{flavour}.csv"], result_folder=RESULT_FOLDER, flavour=FLAVOURS, nb_nodes=NB_NODES),
-        # "analysis/Rplots.pdf"
 
 rule build_nxc_image:
     input:
@@ -37,8 +47,6 @@ rule build_nxc_image:
     shell:
         "cd nxc; nix develop --command nxc build -f {wildcards.flavour}"
 
-
-
 rule run_ior:
     input:
         "nxc/build/composition::{flavour}"
@@ -50,17 +58,65 @@ rule run_ior:
 rule json_to_csv:
     input:
         "analysis/ior_json_to_csv.py",
-        "{RESULT_FOLDER}/results_ior_{nb_nodes}_{flavour}.json"
+        "{RESULT_FOLDER}/results_ior_{nb_nodes}_nodes_{flavour}.zip"
     output:
-        "{RESULT_FOLDER}/results_ior_{nb_nodes}_{flavour}.csv"
+        "{RESULT_FOLDER}/csv_zip/results_csv_ior_{nb_nodes}_nodes_{flavour}.zip"
     shell:
         "nix develop .#python --command python3 {input} {output}"
 
-rule r_analysis:
+rule generate_csv:
+    input:
+        "analysis/zip_to_csv.R",
+        "data/csv_zip/results_csv_ior_{nb_nodes}_nodes_{flavour}.zip"
+    output:
+        "data/csv/{nb_nodes}_{flavour}.csv"
+    shell:
+        "nix develop .#rshell --command Rscript {input} {output}"
+
+rule generate_repeat_csv:
+    input:
+        "analysis/zip_to_csv.R",
+        "data/repeat/iter_{iter}/csv_zip/results_csv_ior_{nb_nodes}_nodes_{flavour}.zip"
+    output:
+        "data/repeat/iter_{iter}/csv/{nb_nodes}_{flavour}.csv"
+    shell:
+        "nix develop .#rshell --command Rscript {input} {output}"
+
+rule repeat_analysis:
     input:
-        "analysis/script.R",
-        expand(["{result_folder}/results_ior_{nb_nodes}_{flavour}.csv"], result_folder=RESULT_FOLDER, flavour=FLAVOURS, nb_nodes=NB_NODES)
+        "analysis/repeat.R",
+        expand(["data/repeat/iter_{iter}/csv/{nb_nodes}_{flavour}.csv"], iter=[1, 2, 3, 4, 5], nb_nodes=8, flavour=FLAVOURS)
     output:
-        "analysis/Rplots.pdf"
+        "analysis/repeat.pdf"
     shell:
-        "cd analysis; nix develop .#rshell --command Rscript script.R"
+        "nix develop .#rshell --command Rscript {input} {output}"
+    
+
+rule general_analysis:
+    input:
+        "analysis/analysis.R",
+        expand(["{result_folder}/csv/{nb_nodes}_{flavour}.csv"], result_folder=RESULT_FOLDER, flavour=FLAVOURS, nb_nodes=NB_NODES)
+    output:
+        "Rplots.pdf"
+    shell:
+        "nix develop .#rshell --command Rscript {input}"
+
+rule paper:
+    input:
+        "analysis/rmd/main.Rmd",
+        expand(["{result_folder}/csv/{nb_nodes}_{flavour}.csv"], result_folder=RESULT_FOLDER, flavour=FLAVOURS, nb_nodes=NB_NODES)
+    output:
+        "analysis/rmd/main.pdf"
+    shell:
+        "nix develop .#rmdshell --command Rscript -e 'rmarkdown::render(\"analysis/rmd/main.Rmd\", \"pdf_document\")'"
+
+
+
+# rule r_analysis:
+#     input:
+#         "analysis/script.R",
+#         expand(["{result_folder}/results_ior_{nb_nodes}_{flavour}.csv"], result_folder=RESULT_FOLDER, flavour=FLAVOURS, nb_nodes=NB_NODES)
+#     output:
+#         "analysis/Rplots.pdf"
+#     shell:
+#         "cd analysis; nix develop .#rshell --command Rscript script.R"

analysis/zip_to_csv.R

+library(tidyverse)
+
+args = commandArgs(trailingOnly=TRUE)
+zip_filename = args[1]
+outname = args[2]
+
+files <- unzip(zip_filename, list=T)
+
+names <- files$Name
+
+total_nodes <- length(names)
+
+get_df <- function(filename) {
+    read_csv(unz(zip_filename, filename))
+}
+
+names %>%
+    map_df(get_df) %>%
+    mutate(total_nodes = total_nodes) %>%
+    write_csv(outname)
+

flake.nix

@@ -15,7 +15,18 @@
     ];
 
     myR = pkgs.rWrapper.override { packages = rPkgs; };
-    myRstudio = pkgs.rstudioWrapper.override { packages = rPkgs; };
+
+    rmdPkgs = with pkgs.rPackages; [
+        rmarkdown
+        markdown
+        knitr
+        tinytex
+        magick
+        codetools
+      ];
+
+      myRmd = pkgs.rWrapper.override { packages = rPkgs ++ rmdPkgs; };
+      myRstudio = pkgs.rstudioWrapper.override { packages = rPkgs ++ rmdPkgs; };
   in
   {
     devShells.${system} = {
@@ -32,6 +43,15 @@
       rshell = pkgs.mkShell {
         buildInputs = [ myR ];
       };
+
+      rmdshell = pkgs.mkShell {
+        buildInputs = with pkgs; [ myRmd pandoc texlive.combined.scheme-full ];
+      };
+
+      rmddev = pkgs.mkShell {
+        buildInputs = with pkgs; [ myRstudio pandoc texlive.combined.scheme-full ];
+      };
+
     };
   };
 }

nxc/script_ior.nix

@@ -10,10 +10,10 @@ IOR START
     hintsFileName=hintsFile
     multiFile=0
     interTestDelay=5
-    readFile=1
+    readFile=0
     writeFile=1
-    filePerProc=0
-    checkWrite=0
+    filePerProc=1
+    checkWrite=1
     checkRead=0
     keepFile=1
     quitOnError=0
@@ -26,7 +26,7 @@ IOR START
     preallocate=0
     useFileView=0
     keepFileWithError=0
-    setTimeStampSignature=0
+    setTimeStampSignature=1
     useSharedFilePointer=0
     useStridedDatatype=0
     uniqueDir=0
@@ -38,14 +38,19 @@ IOR START
     useO_DIRECT=0
     showHints=0
 
-    repetitions=10
+    repetitions=5
     numTasks=${builtins.toString numTasks}
     segmentCount=4
     blockSize=128M
     transferSize=4M
 
-    summaryFile=${nfsMountPoint}/results_ior.json
+    summaryFile=/data/results_ior.json
     summaryFormat=JSON
+RUN
+    writeFile=0
+    readFile=1
+    checkWrite=0
+    checkRead=1
 RUN
 IOR STOP
 ''