From e88f34d6878bbb33714a03b116c4c3921120f85f Mon Sep 17 00:00:00 2001 From: Florian Schmaus <flow@cs.fau.de> Date: Tue, 8 Jun 2021 20:44:40 +0200 Subject: [PATCH] Intermediate Commit (2021-06-08 20:44) --- benchmark-runner/Makefile | 8 ++ benchmark-runner/build.sc | 6 +- .../main/src/de/fau/cs/mazstab/Mazstab.scala | 23 +++- .../de/fau/cs/mazstab/MazstabPgfPlots.scala | 103 +++++++++++++++++- .../fau/cs/mazstab/MazstabPostProcess.scala | 38 +++++-- 5 files changed, 161 insertions(+), 17 deletions(-) diff --git a/benchmark-runner/Makefile b/benchmark-runner/Makefile index 08fe621..64568e0 100644 --- a/benchmark-runner/Makefile +++ b/benchmark-runner/Makefile @@ -16,3 +16,11 @@ format: .PHONY: scalafix scalafix: mill _.fix + +.PHONY: show-updates +show-updates: + mill mill.scalalib.Dependency/showUpdates + +.PHONY: show-deps +show-deps: + mill _.ivyDepsTree diff --git a/benchmark-runner/build.sc b/benchmark-runner/build.sc index 41da343..0387585 100644 --- a/benchmark-runner/build.sc +++ b/benchmark-runner/build.sc @@ -17,12 +17,12 @@ object main extends ScalaModule ivy"com.nrinaudo::kantan.csv:0.6.1", ivy"net.jcazevedo::moultingyaml:0.4.2", ivy"com.lihaoyi:::ammonite:2.3.8-124-2da846d2", - ivy"org.eclipse.jgit:org.eclipse.jgit:5.11.0.202103091610-r", - ivy"org.rogach::scallop:4.0.2", + ivy"org.eclipse.jgit:org.eclipse.jgit:5.12.0.202106070339-r", + ivy"org.rogach::scallop:4.0.3", ) def scalacOptions = Seq( - "-Ywarn-unused" + "-Ywarn-unused", ) } diff --git a/benchmark-runner/main/src/de/fau/cs/mazstab/Mazstab.scala b/benchmark-runner/main/src/de/fau/cs/mazstab/Mazstab.scala index 2f180e3..84cb904 100644 --- a/benchmark-runner/main/src/de/fau/cs/mazstab/Mazstab.scala +++ b/benchmark-runner/main/src/de/fau/cs/mazstab/Mazstab.scala @@ -13,7 +13,15 @@ class MazstabContext(val conf: MazstabConf) { .experimentDirectory() / "post-processed" / Mazstab.getTimestamp def getPostProcessedDirectory() = { - os.makeDir(postProcessedDirectory) + if (!os.isDir(postProcessedDirectory)) { + os.makeDir.all(postProcessedDirectory) + + val lastPostProcessedSymlink = postProcessedDirectory / up / "latest" + if (os.isLink(lastPostProcessedSymlink)) + os.remove(lastPostProcessedSymlink) + os.symlink(lastPostProcessedSymlink, postProcessedDirectory) + } + postProcessedDirectory } @@ -43,18 +51,27 @@ object Mazstab { val context = new MazstabContext(conf) if (conf.subcommand != Some(conf.postProcess)) { + val experimentDir = conf.experimentDirectory() val mazstabRepository = org.eclipse.jgit.storage.file.FileRepositoryBuilder .create(conf.rootDirectory().toIO) val startupMessage = s"""MazStab starting" -Experiment directory: ${conf.experimentDirectory()} +Experiment directory: ${experimentDir} MazStab root: ${conf.rootDirectory()} (${mazstabRepository.getBranch})""" println(startupMessage) // Note that we use the 'all' variant here, to ensure that // directories like out/ are also created. - os.makeDir.all(conf.experimentDirectory()) + os.makeDir.all(experimentDir) + + val rootOutDir = conf.rootDirectory() / "out" + if (experimentDir.startsWith(rootOutDir)) { + val rootOutLatestSynmlink = rootOutDir / "latest" + if (os.isLink(rootOutLatestSynmlink)) + os.remove(rootOutLatestSynmlink) + os.symlink(rootOutLatestSynmlink, experimentDir) + } val buildDir = buildMazstab(conf) diff --git a/benchmark-runner/main/src/de/fau/cs/mazstab/MazstabPgfPlots.scala b/benchmark-runner/main/src/de/fau/cs/mazstab/MazstabPgfPlots.scala index 681fb69..f970d77 100644 --- a/benchmark-runner/main/src/de/fau/cs/mazstab/MazstabPgfPlots.scala +++ b/benchmark-runner/main/src/de/fau/cs/mazstab/MazstabPgfPlots.scala @@ -2,22 +2,117 @@ // Copyright © 2021 Florian Schmaus package de.fau.cs.mazstab +import scala.collection._ +import scala.util.Using +import java.nio.charset.StandardCharsets + object MazstabPgfPlots { val pgfPlotHeader = """ \documentclass{standalone} + \input{common.tex} \begin{document} \begin{tikzpicture} -""" +""".drop(1) val pgfPlotFooter = """ \end{tikzpicture} \end{document} -""" +""".drop(1) + + val bs = '\\' + + val commonTexContent = s""" +${bs}usepackage{tikz} +${bs}usepackage{pgfplots} +${bs}usepackage{siunitx} + +${bs}usepgfplotslibrary{groupplots} + +\\pgfplotsset{compat=1.17} +""".drop(1) + + val makefileContent = """ +PLOTS_TEX := $(filter-out common.tex, $(wildcard *.tex)) +PLOTS_PDF := $(PLOTS_TEX:.tex=.pdf) +.PHONY: all +all: $(PLOTS_PDF) + +%.pdf: %.tex + rubber -m lualatex $< + +NPROC := $(shell nproc) +JOBS := $(shell echo $$(( $(NPROC) + 6))) +LOAD := $(shell echo "$(NPROC) * 1.35" | bc) +MAKE_PARALLEL_ARGS := -j$(JOBS) -l$(LOAD) + +.PHONY: parallel +parallel: + $(MAKE) $(MAKE_PARALLEL_ARGS) all + +""".drop(1) + + val test = """fooo + |bar + |baz""".stripMargin def apply( context: MazstabContext, - processedResults: Map[BenchmarkRunDescriptor, BenchmarkRunResult] - ) = {} + results: MazstabPostProcess.Result + ) = { + val pgfplotsDirectory = context.getPostProcessedDirectory() / "pgfsplots" + os.makeDir(pgfplotsDirectory) + + os.write(pgfplotsDirectory / "common.tex", commonTexContent) + os.write(pgfplotsDirectory / "Makefile", makefileContent) + + for (benchmark <- context.experimentDescription.benchmarks) { + val rtsResults = results._2.get(benchmark).get + val plotFile = pgfplotsDirectory / s"${benchmark}.tex" + Using(new java.io.FileWriter(plotFile.toIO, StandardCharsets.UTF_8)) { + writer => + generateSingleBenchmarkPlot(benchmark, rtsResults, writer) + } + } + + Mazstab.executeAndAwait( + "make", + "-C", + pgfplotsDirectory.toString, + "parallel" + ) + } + + def generateSingleBenchmarkPlot( + benchmark: String, + rtsResults: Map[String, SortedMap[Int, BenchmarkRunResult]], + writer: java.io.Writer + ): Unit = { + writer append pgfPlotHeader + + writer append s"% Benchmark ${benchmark}\n" + + writer append "\\begin{axis}\n" + + for ((rts, rtsResult) <- rtsResults) { + writer append """ + +\addplot+ table [x=cores, y=duration] { +cores duration +""".drop(1) + + for ((nthread, result) <- rtsResult) { + writer append s"$nthread ${result.postProcessedRun.averageDurationMicros}\n" + } + + writer append "};\n" + + writer append s"\\addlegendentry{${rts}}\n" + } + + writer append "\\end{axis}\n" + + writer append pgfPlotFooter + } } diff --git a/benchmark-runner/main/src/de/fau/cs/mazstab/MazstabPostProcess.scala b/benchmark-runner/main/src/de/fau/cs/mazstab/MazstabPostProcess.scala index 40605cf..64cd846 100644 --- a/benchmark-runner/main/src/de/fau/cs/mazstab/MazstabPostProcess.scala +++ b/benchmark-runner/main/src/de/fau/cs/mazstab/MazstabPostProcess.scala @@ -25,11 +25,16 @@ case class PostProcessedRun(averageDurationMicros: Double) object MazstabPostProcess { + type Result = ( + Map[BenchmarkRunDescriptor, BenchmarkRunResult], + Map[String, Map[String, SortedMap[Int, BenchmarkRunResult]]] + ) + def apply( context: MazstabContext - ): immutable.Map[BenchmarkRunDescriptor, BenchmarkRunResult] = { - val processedResults = - mutable.Map[BenchmarkRunDescriptor, BenchmarkRunResult]() + ): Result = { + val resultsMapBuilder = + Map.newBuilder[BenchmarkRunDescriptor, BenchmarkRunResult] val csvFiles = context.getRawDataDirectory.toIO.listFiles .filter(_.isFile) @@ -67,12 +72,31 @@ object MazstabPostProcess { val benchmarkRunResult = BenchmarkRunResult(postProcessedRun, rawBenchmarkRuns, rawWarmupRuns) - val previous = processedResults.put(descriptor, benchmarkRunResult) - if (!previous.isEmpty) throw new Exception() + resultsMapBuilder += (descriptor -> benchmarkRunResult) + } + + val processedResults = resultsMapBuilder.result + + val benchmarksMap = + Map.newBuilder[String, Map[String, SortedMap[Int, BenchmarkRunResult]]] + for (benchmark <- context.experimentDescription.benchmarks) { + val rtsMap = Map.newBuilder[String, SortedMap[Int, BenchmarkRunResult]] + + for (rts <- context.experimentDescription.runtimeSystems) { + val threadsMap = SortedMap.newBuilder[Int, BenchmarkRunResult] + + processedResults + .filter(_._1.benchmarkName == benchmark) + .filter(_._1.rts == rts) + .foreach(p => threadsMap += (p._1.nthreads -> p._2)) + + rtsMap += (rts -> threadsMap.result) + } + + benchmarksMap += (benchmark -> rtsMap.result) } - // Convert mutable map to immutable using 'toMap'. - processedResults.toMap + (processedResults, benchmarksMap.result) } def postProcess(csvLines: List[MazstabCsvLine]): PostProcessedRun = { -- GitLab