From 824b575dc55f555a5273fddc21fc0b375a6ad0d7 Mon Sep 17 00:00:00 2001 From: Jonathan Woodruff Date: Thu, 20 Mar 2025 16:59:25 +0000 Subject: [PATCH] Add benchmarks directory and add script to run benchmarks in it. --- .gitmodules | 3 + Tests/Run_benchmarks.py | 385 +++++++++++++++++++++++++++++ Tests/Run_regression.py | 13 +- Tests/benchmarks | 1 + builds/Resources/Include_Common.mk | 12 + 5 files changed, 407 insertions(+), 7 deletions(-) create mode 100755 Tests/Run_benchmarks.py create mode 160000 Tests/benchmarks diff --git a/.gitmodules b/.gitmodules index 76c0b05..6799cf1 100644 --- a/.gitmodules +++ b/.gitmodules @@ -16,3 +16,6 @@ [submodule "libs/WindCoreInterface"] path = libs/WindCoreInterface url = https://github.com/CTSRD-CHERI/WindCoreInterface.git +[submodule "Tests/benchmarks"] + path = Tests/benchmarks + url = https://github.com/CTSRD-CHERI/toooba-sim-benchmarks.git diff --git a/Tests/Run_benchmarks.py b/Tests/Run_benchmarks.py new file mode 100755 index 0000000..c324ca5 --- /dev/null +++ b/Tests/Run_benchmarks.py @@ -0,0 +1,385 @@ +#!/usr/bin/python3 + +# Copyright (c) 2018-2019 Bluespec, Inc. +# See LICENSE for license details + +usage_line = ( + " Usage:\n" + " $ \n" + "\n" + " Runs the RISC-V \n" + " on ISA tests: ELF files taken from /isa and its sub-directories.\n" + "\n" + " For each ELF file FOO, saves simulation output in /FOO.log. \n" + "\n" + " If is given, it must be one of the following:\n" + " v1: Print instruction trace during simulation\n" + " v2: Print pipeline stage state during simulation\n" + "\n" + " If is given, it must be an integer\n" + " Specifies the number of parallel processes used\n" + " (creates temporary separate working directories worker_0, worker_1, ...)\n" + " By default uses the number of CPUs listed in /proc/cpuinfo - 4.\n" + " In any case, limits it to 8.\n" + "\n" + " If is given, it must be an integer and must follow an explicit \n" + " Specifies the number of seconds to wait for each command run.\n" + " Defaults to 60s\n." + "\n" + " Example:\n" + " $ .exe_HW_sim ~somebody/GitHub/Toooba ./Logs v1 4\n" + " will run the verilator simulation executable on the following RISC-V ISA tests:\n" + " ~somebody/GitHub/Tests/benchmarks/*.bin\n" + " and will leave a transcript of each test's simulation output in files like\n" + " ./Logs/rv32ui-p-add.log\n" + " Each log will contain an instruction trace (because of the 'v1' arg).\n" + " It will use 4 processes in parallel to run the regressions.\n" + " (creating temporary working directories worker_0, ..., worker_4)\n" +) + +import sys +import os +import stat +import subprocess + +import multiprocessing + +# ================================================================ +# DEBUGGING ONLY: This exclude list allows skipping some specific test + +exclude_list = [] + +n_workers_max = 8 + +timeout = 6000 + +def print_hello (): + print ("hello") +# ================================================================ + +def main (argv = None): + print ("Use flag --help or --h for a help message") + if ((len (argv) <= 1) or + (argv [1] == '-h') or (argv [1] == '--help') or + (len (argv) < 4)): + + sys.stdout.write (usage_line) + sys.stdout.write ("\n") + return 0 + + # Simulation executable + if not (os.path.exists (argv [1])): + sys.stderr.write ("ERROR: The given simulation path does not seem to exist?\n") + sys.stderr.write (" Simulation path: " + sim_path + "\n") + sys.exit (1) + args_dict = {'sim_path': os.path.abspath (os.path.normpath (argv [1]))} + + # Repo in which to find ELFs and elf_to_hex executable + if (not os.path.exists (argv [2])): + sys.stderr.write ("ERROR: repo directory ({0}) does not exist?\n".format (argv [2])) + sys.stdout.write ("\n") + sys.stdout.write (usage_line) + sys.stdout.write ("\n") + return 1 + repo = os.path.abspath (os.path.normpath (argv [2])) + + elfs_path = os.path.join (repo, "Tests", "benchmarks") + if (not os.path.exists (elfs_path)): + sys.stderr.write ("ERROR: BINs directory ({0}) does not exist?\n".format (elfs_path)) + sys.stdout.write ("\n") + sys.stdout.write (usage_line) + sys.stdout.write ("\n") + return 1 + args_dict ['elfs_path'] = elfs_path + + # Logs directory + logs_path = os.path.abspath (os.path.normpath (argv [3])) + if not (os.path.exists (logs_path) and os.path.isdir (logs_path)): + print ("Creating dir: " + logs_path) + os.mkdir (logs_path) + args_dict ['logs_path'] = logs_path + + # Optional verbosity + verbosity = 0 + j = 4 + if len (argv) >= 5: + if argv [4] == "v1": + verbosity = 1 + j = 5 + elif argv [4] == "v2": + verbosity = 2 + j = 5 + args_dict ['verbosity'] = verbosity + + # Optional parallelism; limit it to 8 + if len (argv [j:]) != 0 and isdecimal (argv [j]): + n_workers = int (argv [j]) + j = 6 + else: + n_workers = multiprocessing.cpu_count () - 4 + n_workers = max(min (n_workers_max, n_workers), 1) + sys.stdout.write ("Using {0} worker processes\n".format (n_workers)) + + global timeout + if len(argv[j:]) != 0 and isdecimal (argv[j]): + timeout = int(argv[j]) + j = 7 + sys.stdout.write (f"Using {timeout}s timeout\n") + + # End of command-line arg processing + # ================================================================ + + # elf_to_hex executable + elf_to_hex_exe = os.path.join (repo, "Tests", "elf_to_hex", "elf_to_hex") + if (not os.path.exists (elf_to_hex_exe)): + sys.stderr.write ("ERROR: elf_to_hex executable does not exist?\n") + sys.stderr.write (" at {0}\n".format (elf_to_hex_exe)) + sys.stdout.write ("\n") + sys.stdout.write (usage_line) + sys.stdout.write ("\n") + return 1 + args_dict ['elf_to_hex_exe'] = elf_to_hex_exe + + sys.stdout.write ("Parameters:\n") + for key in iter (args_dict): + sys.stdout.write (" {0:<16}: {1}\n".format (key, args_dict [key])) + + def fn_filter_regular_file (level, filename): + (dirname, basename) = os.path.split (filename) + + # TEMPORARY FILTER WHILE DEBUGGING: + if basename in exclude_list: + sys.stdout.write ("WARNING: TEMPORARY FILTER IN EFFECT; REMOVE AFTER DEBUGGING\n") + sys.stdout.write (" This test is in exclude_list: {0}\n".format (basename)) + return False + + # Ignore filename if it is not a "*.bin" + if basename.find (".bin") != -1: + return True + + return False + + def fn_filter_dir (level, filename): + return True + + # Traverse the elfs_path and collect filenames of relevant isa tests + filenames = traverse (fn_filter_dir, fn_filter_regular_file, 0, elfs_path) + n_tests = len (filenames) + sys.stdout.write ("{0} relevant benchmarks found under {1}\n".format (n_tests, elfs_path)) + if n_tests == 0: + return 0 + args_dict ['filenames'] = filenames + args_dict ['n_tests'] = n_tests + + # Create a shared counter to index into the list of filenames + index = multiprocessing.Value ('L', 0) # Unsigned long (4 bytes) + args_dict ['index'] = index + + # Create a shared array for each worker's (n_executed, n_passed) results + results = multiprocessing.Array ('L', [ 0 for j in range (2 * n_workers) ]) + args_dict ['results'] = results + + # Create a TAP file to output individual test results in TAP format + tap_out = open("../isa_test_report.tap", "w") + tap_out.write("1.." + str(n_tests) + "\n") + tap_out.close() + + # Create n workers + sys.stdout.write ("Creating {0} workers (sub-processes)\n".format (n_workers)) + workers = [multiprocessing.Process (target = do_worker, + #args = (w, args_dict, tap_out)) + args = (w, args_dict)) + for w in range (n_workers)] + + # Start the workers + for worker in workers: worker.start () + + # Wait for all workers to finish + for worker in workers: worker.join () + + # Collect all results + num_executed = 0 + num_passed = 0 + with results.get_lock (): + for w in range (n_workers): + n_e = results [2 * w] + n_p = results [2 * w + 1] + sys.stdout.write ("Worker {0} executed {1} tests, of which {2} passed\n" + .format (w, n_e, n_p)) + num_executed = num_executed + n_e + num_passed = num_passed + n_p + + # Write final statistics + sys.stdout.write ("Total benchmarks: {0} tests\n".format (n_tests)) + sys.stdout.write ("Executed: {0} tests\n".format (num_executed)) + sys.stdout.write ("PASS: {0} tests\n".format (num_passed)) + sys.stdout.write ("FAIL: {0} tests\n".format (num_executed - num_passed)) + return 0 + +# ================================================================ +# Recursively traverse the dir tree below path and collect filenames +# that pass the given filter functions + +def traverse (fn_filter_dir, fn_filter_regular_file, level, path): + st = os.stat (path) + is_dir = stat.S_ISDIR (st.st_mode) + is_regular = stat.S_ISREG (st.st_mode) + + if is_dir and fn_filter_dir (level, path): + files = [] + for entry in os.listdir (path): + path1 = os.path.join (path, entry) + files.extend (traverse (fn_filter_dir, fn_filter_regular_file, level + 1, path1)) + return files + + elif is_regular and fn_filter_regular_file (level, path): + return [path] + + else: + return [] + +# ================================================================ +# For each ELF file, execute it in the RISC-V simulator + +def do_worker (worker_num, args_dict): + tmpdir = "./worker_" + "{0}".format (worker_num) + if not os.path.exists (tmpdir): + os.mkdir (tmpdir) + elif not os.path.isdir (tmpdir): + sys.stdout.write ("ERROR: Worker {0}: {1} exists but is not a dir".format (worker_num, tmpdir)) + return + + os.chdir (tmpdir) + sys.stdout.write ("Worker {0} using dir: {1}\n".format (worker_num, tmpdir)) + + n_tests = args_dict ['n_tests'] + filenames = args_dict ['filenames'] + index = args_dict ['index'] + results = args_dict ['results'] + + num_executed = 0 + num_passed = 0 + + while True: + # Get a unique index into the filenames, and get the filename + with index.get_lock(): + my_index = index.value + index.value = my_index + 1 + if my_index >= n_tests: + # All done + with results.get_lock(): + results [2 * worker_num] = num_executed + results [2 * worker_num + 1] = num_passed + return + filename = filenames [my_index] + + (message, passed) = do_benchmark (args_dict, filename) + num_executed = num_executed + 1 + + if passed: + num_passed = num_passed + 1 + pass_fail = "PASS" + else: + pass_fail = "FAIL" + + message = message + ("Worker {0}: Test: {1} {2} [So far: total {3}, executed {4}, PASS {5}, FAIL {6}]\n" + .format (worker_num, + os.path.basename (filename), + pass_fail, + n_tests, + num_executed, + num_passed, + num_executed - num_passed)) + sys.stdout.write (message) + # Create a TAP file to output individual test result in TAP format + tap_out = open("../benchmark_report.tap", "a+") + tap_out.write(("ok" if passed else "not ok") + " " + str(my_index + 1) + " - " + filenames[my_index] + "\n") + tap_out.close() + +# ================================================================ +# For each ELF file, execute it in the RISC-V simulator + +def do_benchmark (args_dict, full_filename): + message = "" + + (dirname, basename) = os.path.split (full_filename) + + # Construct the commands for sub-process execution + command1 = [args_dict ['elf_to_hex_exe'], full_filename, "Mem.hex"] + + command2 = [args_dict ['sim_path'], "+tohost"] + if (args_dict ['verbosity'] == 1): command2.append ("+v1") + elif (args_dict ['verbosity'] == 2): command2.append ("+v2") + + message = message + " Exec:" + for x in command1: + message = message + (" {0}".format (x)) + message = message + "\n" + + message = message + (" Exec:") + for x in command2: + message = message + (" {0}".format (x)) + message = message + ("\n") + + # Save stdouts in log file + log_filename = os.path.join (args_dict ['logs_path'], basename + ".log") + message = message + (" Writing log: {0}\n".format (log_filename)) + + with open (log_filename, 'w') as fd: + completed_process1 = run_command (command1, fd) + + if completed_process1 is not None and completed_process1.returncode == 0: + completed_process2 = run_command (command2, fd) + passed = completed_process2 is not None and \ + completed_process2.returncode == 0 and \ + not completed_process2.stdout.endswith ("FAIL 1") + else: + passed = False + + # If Tandem Verification trace file was created, save it as well + if os.path.exists ("./trace_out.dat"): + trace_filename = os.path.join (args_dict ['logs_path'], basename + ".trace_data") + os.rename ("./trace_out.dat", trace_filename) + message = message + (" Trace output saved in: {0}\n".format (trace_filename)) + + return (message, passed) + +# ================================================================ +# This is a wrapper around 'subprocess.run' because of an annoying +# incompatible change in moving from Python 3.5 to 3.6 + +def run_command (command, log_fd): + command_str = " ".join(command) + log_fd.write (f"Running: {command_str}\n") + try: + python_minor_version = sys.version_info [1] + if python_minor_version < 6: + # Python 3.5 and earlier + result = subprocess.run (args = command, + bufsize = 0, + stdout = subprocess.PIPE, + stderr = subprocess.STDOUT, + universal_newlines = True, + timeout=timeout) + else: + # Python 3.6 and later + result = subprocess.run (args = command, + bufsize = 0, + stdout = subprocess.PIPE, + stderr = subprocess.STDOUT, + encoding='utf-8', + timeout=timeout) + log_fd.write(f"Finished with exit code {result.returncode}\n") + log_fd.write("Stdout:\n") + log_fd.write (result.stdout) + return result + except subprocess.TimeoutExpired: + sys.stderr.write(f"TIMEOUT: {command_str} !\n") + log_fd.write("TIMEOUT!\n") + return None + +# ================================================================ +# For non-interactive invocations, call main() and use its return value +# as the exit code. +if __name__ == '__main__': + sys.exit (main (sys.argv)) diff --git a/Tests/Run_regression.py b/Tests/Run_regression.py index 70ae343..044ed3d 100755 --- a/Tests/Run_regression.py +++ b/Tests/Run_regression.py @@ -202,12 +202,12 @@ def main (argv = None): # Create a TAP file to output individual test results in TAP format tap_out = open("../isa_test_report.tap", "w") tap_out.write("1.." + str(n_tests) + "\n") - tap_out.flush() + tap_out.close() # Create n workers sys.stdout.write ("Creating {0} workers (sub-processes)\n".format (n_workers)) workers = [multiprocessing.Process (target = do_worker, - args = (w, args_dict, tap_out)) + args = (w, args_dict)) for w in range (n_workers)] # Start the workers @@ -228,9 +228,6 @@ def main (argv = None): num_executed = num_executed + n_e num_passed = num_passed + n_p - # Close tap_out file - tap_out.close() - # Write final statistics sys.stdout.write ("Total tests: {0} tests\n".format (n_tests)) sys.stdout.write ("Executed: {0} tests\n".format (num_executed)) @@ -335,7 +332,7 @@ def traverse (fn_filter_dir, fn_filter_regular_file, level, path): # ================================================================ # For each ELF file, execute it in the RISC-V simulator -def do_worker (worker_num, args_dict, tap_out): +def do_worker (worker_num, args_dict): tmpdir = "./worker_" + "{0}".format (worker_num) if not os.path.exists (tmpdir): os.mkdir (tmpdir) @@ -362,7 +359,6 @@ def do_worker (worker_num, args_dict, tap_out): if my_index >= n_tests: # All done with results.get_lock(): - tap_out.flush() results [2 * worker_num] = num_executed results [2 * worker_num + 1] = num_passed return @@ -386,7 +382,10 @@ def do_worker (worker_num, args_dict, tap_out): num_passed, num_executed - num_passed)) sys.stdout.write (message) + # Create a TAP file to output individual test result in TAP format + tap_out = open("../isa_test_report.tap", "a+") tap_out.write(("ok" if passed else "not ok") + " " + str(my_index + 1) + " - " + filenames[my_index] + "\n") + tap_out.close() # ================================================================ # For each ELF file, execute it in the RISC-V simulator diff --git a/Tests/benchmarks b/Tests/benchmarks new file mode 160000 index 0000000..1091c9b --- /dev/null +++ b/Tests/benchmarks @@ -0,0 +1 @@ +Subproject commit 1091c9b98e12c905f2fa2570575db5689acbf336 diff --git a/builds/Resources/Include_Common.mk b/builds/Resources/Include_Common.mk index 6ea0fa8..a2140d1 100644 --- a/builds/Resources/Include_Common.mk +++ b/builds/Resources/Include_Common.mk @@ -25,6 +25,7 @@ help: @echo '' @echo ' make test Runs simulation executable on rv32ui-p-add or rv64ui-p-add' @echo ' make isa_tests Runs simulation executable on all relevant standard RISC-V ISA tests' + @echo ' make benchmarks Runs simulation executable on a set of benchmarks' @echo '' @echo ' make clean Remove intermediate build-files unnecessary for execution' @echo ' make full_clean Restore to pristine state (pre-building anything)' @@ -54,6 +55,7 @@ BSC_COMPILATION_FLAGS += \ -D Near_Mem_Caches \ -D FABRIC64 \ -D BLUESIM \ + -D RVFI \ -D PERFORMANCE_MONITORING \ -keep-fires -aggressive-conditions -no-warn-action-shadowing -check-assert \ -suppress-warnings G0020 -steps-max-intervals 10000000 \ @@ -96,6 +98,16 @@ isa_tests: $(REPO)/Tests/Run_regression.py ./exe_HW_sim $(REPO) ./Logs $(ARCH) @echo "Finished running regressions; saved logs in Logs/" +# ================================================================ +# Benchmarks + +.PHONY: benchmarks +benchmarks: + @echo "Running benchmarks; saving logs in Logs/" + $(REPO)/Tests/Run_benchmarks.py ./exe_HW_sim $(REPO) ./Logs + @echo "Finished running benchmarks" + $(REPO)/Tests/benchmarks/report_log.sh Logs/*.bin.log + # ================================================================ # Generate Bluespec CHERI tag controller source file CAPSIZE = 128