From b2197778d2a8b81a97b1e8fc1b7e77c445303592 Mon Sep 17 00:00:00 2001
From: Michael Eischer <eischer@cs.fau.de>
Date: Wed, 1 Jul 2020 19:28:12 +0200
Subject: [PATCH] Add option to distribute and enqueue the repository parent
folder
---
scripts/queue/upload.py | 16 +++++++++----
scripts/test/benchmark.py | 48 +++++++++++++++++++++++++++------------
2 files changed, 45 insertions(+), 19 deletions(-)
diff --git a/scripts/queue/upload.py b/scripts/queue/upload.py
index 78cac3e..4b9b0bb 100755
--- a/scripts/queue/upload.py
+++ b/scripts/queue/upload.py
@@ -15,13 +15,15 @@ def execute(cmd_args) -> None:
parser = argparse.ArgumentParser()
parser.add_argument("-p", "--prio", help="Use priority queue",
dest="queue", default="normal", action="store_const", const="prio")
+ parser.add_argument("--parent-folder", help="Enqueue the whole repository parent folder",
+ action="store_true")
parser.add_argument("experiment_name", help="name of the experiment")
args = parser.parse_args(cmd_args)
- return queue(args.queue, args.experiment_name)
+ return queue(args.queue, args.experiment_name, args.parent_folder)
-def queue(queue_name, experiment_name):
+def queue(queue_name: str, experiment_name: str, distribute_parent_folder: bool) -> None:
scripts_dir = os.path.join(MAIN_DIR, "scripts")
queue_file = os.path.join(scripts_dir, "config", "queue")
config = load_config(queue_file)
@@ -35,16 +37,22 @@ def queue(queue_name, experiment_name):
subprocess.run(["ssh", "{}@{}".format(config["queue.user"], config["queue.host"])]
+ quote("/bin/bash", "-c", " ".join(quote("mkdir", "-p", config["queue.upload"]))),
stdout=subprocess.DEVNULL, cwd=MAIN_DIR, check=True)
+
+ src = "./" if not distribute_parent_folder else "../"
+ basepath = ""
+ if distribute_parent_folder:
+ basepath = "/" + os.path.basename(MAIN_DIR)
+
subprocess.run(["rsync", "-ahe", "ssh", "--delete",
"--exclude=bin", "--exclude=out", "--exclude=tmp",
"--exclude=results", "--exclude=remote-runner",
"--exclude=.git", "--exclude=.idea", "--exclude=.mypy_cache", "--exclude=.terraform",
- "./", "{}@{}:{}/".format(config["queue.user"], config["queue.host"], config["queue.upload"])],
+ src, "{}@{}:{}/".format(config["queue.user"], config["queue.host"], config["queue.upload"])],
cwd=MAIN_DIR, check=True)
print("Queueing experiment")
tar_cmd = quote("tar", "-cf", queue_fn, "-C", config["queue.upload"], ".")
- cp_cmd = quote("cp", "{}/scripts/generated-config/commands".format(config["queue.upload"]), commands_fn)
+ cp_cmd = quote("cp", "{}/scripts/generated-config/commands".format(config["queue.upload"] + basepath), commands_fn)
touch_cmd = quote("touch", marker_fn)
cmd = tar_cmd + ["&&"] + cp_cmd + ["&&"] + touch_cmd
ssh_cmd = ["ssh", "-t", "{}@{}".format(config["queue.user"], config["queue.host"])] \
diff --git a/scripts/test/benchmark.py b/scripts/test/benchmark.py
index 56c7862..bcbf5ba 100755
--- a/scripts/test/benchmark.py
+++ b/scripts/test/benchmark.py
@@ -11,7 +11,7 @@ import time
import traceback
from configparser import ConfigParser
from datetime import datetime
-from typing import List, Sequence, Set
+from typing import List, Sequence, Set, Optional
MAIN_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
DEFAULT_RESULTS_DIR = os.path.realpath(os.path.join(MAIN_DIR, "results"))
@@ -291,6 +291,9 @@ class GenericTestRun(TestRun):
self.config_files += [self.server_config, self.refit_default_config, self.refit_overrides_config]
# additional files to store in results directory, paths should start with self.main_dir or self.scripts_dir
self.extra_files = []
+ # set to True to also distribute the parent folder. All test run commands still
+ # use this repository as their working directory
+ self.distribute_parent_folder = False
def get_servers(self, server_types: Sequence[str]) -> List[str]:
servers = [] # type: List[str]
@@ -321,9 +324,14 @@ class GenericTestRun(TestRun):
os.unlink(build_log)
return True
- def get_clean_paths(self):
+ def get_clean_paths(self, ignore_parent: bool = False):
local_path = remote_path = self.config["remote.path"] # type: str
+ if self.distribute_parent_folder and not ignore_parent:
+ base = os.path.basename(MAIN_DIR)
+ local_path = os.path.join(local_path, base)
+ remote_path = os.path.join(remote_path, base)
+
if local_path.startswith("~/"):
# expand to ensure local commands are executed consistently
local_path = os.path.expanduser(local_path)
@@ -337,7 +345,7 @@ class GenericTestRun(TestRun):
return local_path, remote_path
def stage_distr(self) -> bool:
- local_path, remote_path = self.get_clean_paths()
+ local_path, remote_path = self.get_clean_paths(ignore_parent=True)
# create folders in parallel
mkdirs = []
@@ -353,13 +361,14 @@ class GenericTestRun(TestRun):
# start rsyncs in parallel
rsyncs = []
+ src = "./" if not self.distribute_parent_folder else "../"
for server in self.get_unique_servers():
if server == "localhost":
- rsyncs.append(self.start_rsync("./", local_path,
+ rsyncs.append(self.start_rsync(src, local_path,
delete=True, remote=False))
else:
- rsyncs.append(self.start_rsync("./", "{}@{}:{}".format(self.config["remote.user"],
- server, shlex.quote(remote_path)),
+ rsyncs.append(self.start_rsync(src, "{}@{}:{}".format(self.config["remote.user"],
+ server, shlex.quote(remote_path)),
delete=True))
return self.wait_procs(rsyncs)
@@ -479,7 +488,9 @@ def execute(cls, cmd_args):
queue_parser = subparsers.add_parser('queue', help='queue benchmark')
queue_parser.add_argument("-p", "--prio", help="Use priority queue", action="store_true")
queue_parser.add_argument("-m", "--multi", help="Generate variants", action="store_true")
- queue_parser.add_argument("-n", "--dry-run", help="Prepare only benchmakr execution", action="store_true")
+ queue_parser.add_argument("-n", "--dry-run", help="Prepare only benchmark execution", action="store_true")
+ queue_parser.add_argument("--parent-folder", help="Enqueue the whole repository parent folder",
+ action="store_true")
queue_parser.add_argument("testfile", help="test script which controls the benchmark execution")
queue_parser.add_argument("scenario", help="name of the current test scenario")
queue_parser.add_argument("timer", help="duration of single test run", type=int)
@@ -557,7 +568,8 @@ def execute(cls, cmd_args):
return 1
elif args.cmd == "queue":
experiment_name = gen_experiment_name(args)
- success = prepare(args, experiment_name) and (args.dry_run or queue(args.prio, experiment_name))
+ success = prepare(args, experiment_name) and (
+ args.dry_run or queue(args.prio, experiment_name, args.parent_folder))
if success:
return 0
else:
@@ -610,18 +622,22 @@ def prepare(args, experiment_name):
cmds = []
results_path = "~/\"{}/{}\"".format(config["queue.results"], experiment_name)
+ basepath = ""
+ if args.parent_folder:
+ basepath = os.path.join(os.path.basename(MAIN_DIR), "")
+
if args.multi:
for variant in variants:
- cmds.append("cd scripts"
+ cmds.append("cd {}scripts"
" && ./test/helper/config-variants-generator.py -s {}"
" -c ./generated-config/refit-overrides -d test"
" -- {} ./{}.py run -r {} {} '{{}}' {}"
- .format(variant, args.scenario, args.testfile, results_path,
+ .format(basepath, variant, args.scenario, args.testfile, results_path,
args.testfile, args.timer, results_path))
else:
- cmds.append("cd scripts/test"
+ cmds.append("cd {}scripts/test"
" && ./{}.py run -r {} {} {} {}"
- .format(args.testfile, results_path, args.testfile, args.scenario,
+ .format(basepath, args.testfile, results_path, args.testfile, args.scenario,
args.timer, results_path))
with open(os.path.join(genconfig_dir, "commands"), "w") as f:
@@ -637,7 +653,7 @@ def config_helper(cmd, output, scripts_dir):
f.write(ret.stdout)
-def load_variants(variant_file: str):
+def load_variants(variant_file: str) -> Optional[Sequence[str]]:
variants = []
with open(variant_file, "r") as f:
for line in f:
@@ -659,10 +675,12 @@ def load_variants(variant_file: str):
return variants
-def queue(prio, experiment_name):
+def queue(prio: bool, experiment_name: str, parent_folder: bool) -> bool:
queue_flag = []
if prio:
- queue_flag = ["--prio"]
+ queue_flag += ["--prio"]
+ if parent_folder:
+ queue_flag += ["--parent-folder"]
subprocess.run(["scripts/queue/upload.py", experiment_name] + queue_flag, cwd=MAIN_DIR, check=True)
return True
--
GitLab