added sections of the evaluation report
This commit is contained in:
482
docs/evaluation/evaluation.html
Normal file
482
docs/evaluation/evaluation.html
Normal file
@@ -0,0 +1,482 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
|
||||
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
|
||||
<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en">
|
||||
<head>
|
||||
<!-- 2025-01-12 Sun 17:26 -->
|
||||
<meta http-equiv="Content-Type" content="text/html;charset=utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>‎</title>
|
||||
<meta name="author" content="Akilan" />
|
||||
<meta name="generator" content="Org Mode" />
|
||||
<style>
|
||||
#content { max-width: 60em; margin: auto; }
|
||||
.title { text-align: center;
|
||||
margin-bottom: .2em; }
|
||||
.subtitle { text-align: center;
|
||||
font-size: medium;
|
||||
font-weight: bold;
|
||||
margin-top:0; }
|
||||
.todo { font-family: monospace; color: red; }
|
||||
.done { font-family: monospace; color: green; }
|
||||
.priority { font-family: monospace; color: orange; }
|
||||
.tag { background-color: #eee; font-family: monospace;
|
||||
padding: 2px; font-size: 80%; font-weight: normal; }
|
||||
.timestamp { color: #bebebe; }
|
||||
.timestamp-kwd { color: #5f9ea0; }
|
||||
.org-right { margin-left: auto; margin-right: 0px; text-align: right; }
|
||||
.org-left { margin-left: 0px; margin-right: auto; text-align: left; }
|
||||
.org-center { margin-left: auto; margin-right: auto; text-align: center; }
|
||||
.underline { text-decoration: underline; }
|
||||
#postamble p, #preamble p { font-size: 90%; margin: .2em; }
|
||||
p.verse { margin-left: 3%; }
|
||||
pre {
|
||||
border: 1px solid #e6e6e6;
|
||||
border-radius: 3px;
|
||||
background-color: #f2f2f2;
|
||||
padding: 8pt;
|
||||
font-family: monospace;
|
||||
overflow: auto;
|
||||
margin: 1.2em;
|
||||
}
|
||||
pre.src {
|
||||
position: relative;
|
||||
overflow: auto;
|
||||
}
|
||||
pre.src:before {
|
||||
display: none;
|
||||
position: absolute;
|
||||
top: -8px;
|
||||
right: 12px;
|
||||
padding: 3px;
|
||||
color: #555;
|
||||
background-color: #f2f2f299;
|
||||
}
|
||||
pre.src:hover:before { display: inline; margin-top: 14px;}
|
||||
/* Languages per Org manual */
|
||||
pre.src-asymptote:before { content: 'Asymptote'; }
|
||||
pre.src-awk:before { content: 'Awk'; }
|
||||
pre.src-authinfo::before { content: 'Authinfo'; }
|
||||
pre.src-C:before { content: 'C'; }
|
||||
/* pre.src-C++ doesn't work in CSS */
|
||||
pre.src-clojure:before { content: 'Clojure'; }
|
||||
pre.src-css:before { content: 'CSS'; }
|
||||
pre.src-D:before { content: 'D'; }
|
||||
pre.src-ditaa:before { content: 'ditaa'; }
|
||||
pre.src-dot:before { content: 'Graphviz'; }
|
||||
pre.src-calc:before { content: 'Emacs Calc'; }
|
||||
pre.src-emacs-lisp:before { content: 'Emacs Lisp'; }
|
||||
pre.src-fortran:before { content: 'Fortran'; }
|
||||
pre.src-gnuplot:before { content: 'gnuplot'; }
|
||||
pre.src-haskell:before { content: 'Haskell'; }
|
||||
pre.src-hledger:before { content: 'hledger'; }
|
||||
pre.src-java:before { content: 'Java'; }
|
||||
pre.src-js:before { content: 'Javascript'; }
|
||||
pre.src-latex:before { content: 'LaTeX'; }
|
||||
pre.src-ledger:before { content: 'Ledger'; }
|
||||
pre.src-lisp:before { content: 'Lisp'; }
|
||||
pre.src-lilypond:before { content: 'Lilypond'; }
|
||||
pre.src-lua:before { content: 'Lua'; }
|
||||
pre.src-matlab:before { content: 'MATLAB'; }
|
||||
pre.src-mscgen:before { content: 'Mscgen'; }
|
||||
pre.src-ocaml:before { content: 'Objective Caml'; }
|
||||
pre.src-octave:before { content: 'Octave'; }
|
||||
pre.src-org:before { content: 'Org mode'; }
|
||||
pre.src-oz:before { content: 'OZ'; }
|
||||
pre.src-plantuml:before { content: 'Plantuml'; }
|
||||
pre.src-processing:before { content: 'Processing.js'; }
|
||||
pre.src-python:before { content: 'Python'; }
|
||||
pre.src-R:before { content: 'R'; }
|
||||
pre.src-ruby:before { content: 'Ruby'; }
|
||||
pre.src-sass:before { content: 'Sass'; }
|
||||
pre.src-scheme:before { content: 'Scheme'; }
|
||||
pre.src-screen:before { content: 'Gnu Screen'; }
|
||||
pre.src-sed:before { content: 'Sed'; }
|
||||
pre.src-sh:before { content: 'shell'; }
|
||||
pre.src-sql:before { content: 'SQL'; }
|
||||
pre.src-sqlite:before { content: 'SQLite'; }
|
||||
/* additional languages in org.el's org-babel-load-languages alist */
|
||||
pre.src-forth:before { content: 'Forth'; }
|
||||
pre.src-io:before { content: 'IO'; }
|
||||
pre.src-J:before { content: 'J'; }
|
||||
pre.src-makefile:before { content: 'Makefile'; }
|
||||
pre.src-maxima:before { content: 'Maxima'; }
|
||||
pre.src-perl:before { content: 'Perl'; }
|
||||
pre.src-picolisp:before { content: 'Pico Lisp'; }
|
||||
pre.src-scala:before { content: 'Scala'; }
|
||||
pre.src-shell:before { content: 'Shell Script'; }
|
||||
pre.src-ebnf2ps:before { content: 'ebfn2ps'; }
|
||||
/* additional language identifiers per "defun org-babel-execute"
|
||||
in ob-*.el */
|
||||
pre.src-cpp:before { content: 'C++'; }
|
||||
pre.src-abc:before { content: 'ABC'; }
|
||||
pre.src-coq:before { content: 'Coq'; }
|
||||
pre.src-groovy:before { content: 'Groovy'; }
|
||||
/* additional language identifiers from org-babel-shell-names in
|
||||
ob-shell.el: ob-shell is the only babel language using a lambda to put
|
||||
the execution function name together. */
|
||||
pre.src-bash:before { content: 'bash'; }
|
||||
pre.src-csh:before { content: 'csh'; }
|
||||
pre.src-ash:before { content: 'ash'; }
|
||||
pre.src-dash:before { content: 'dash'; }
|
||||
pre.src-ksh:before { content: 'ksh'; }
|
||||
pre.src-mksh:before { content: 'mksh'; }
|
||||
pre.src-posh:before { content: 'posh'; }
|
||||
/* Additional Emacs modes also supported by the LaTeX listings package */
|
||||
pre.src-ada:before { content: 'Ada'; }
|
||||
pre.src-asm:before { content: 'Assembler'; }
|
||||
pre.src-caml:before { content: 'Caml'; }
|
||||
pre.src-delphi:before { content: 'Delphi'; }
|
||||
pre.src-html:before { content: 'HTML'; }
|
||||
pre.src-idl:before { content: 'IDL'; }
|
||||
pre.src-mercury:before { content: 'Mercury'; }
|
||||
pre.src-metapost:before { content: 'MetaPost'; }
|
||||
pre.src-modula-2:before { content: 'Modula-2'; }
|
||||
pre.src-pascal:before { content: 'Pascal'; }
|
||||
pre.src-ps:before { content: 'PostScript'; }
|
||||
pre.src-prolog:before { content: 'Prolog'; }
|
||||
pre.src-simula:before { content: 'Simula'; }
|
||||
pre.src-tcl:before { content: 'tcl'; }
|
||||
pre.src-tex:before { content: 'TeX'; }
|
||||
pre.src-plain-tex:before { content: 'Plain TeX'; }
|
||||
pre.src-verilog:before { content: 'Verilog'; }
|
||||
pre.src-vhdl:before { content: 'VHDL'; }
|
||||
pre.src-xml:before { content: 'XML'; }
|
||||
pre.src-nxml:before { content: 'XML'; }
|
||||
/* add a generic configuration mode; LaTeX export needs an additional
|
||||
(add-to-list 'org-latex-listings-langs '(conf " ")) in .emacs */
|
||||
pre.src-conf:before { content: 'Configuration File'; }
|
||||
|
||||
table { border-collapse:collapse; }
|
||||
caption.t-above { caption-side: top; }
|
||||
caption.t-bottom { caption-side: bottom; }
|
||||
td, th { vertical-align:top; }
|
||||
th.org-right { text-align: center; }
|
||||
th.org-left { text-align: center; }
|
||||
th.org-center { text-align: center; }
|
||||
td.org-right { text-align: right; }
|
||||
td.org-left { text-align: left; }
|
||||
td.org-center { text-align: center; }
|
||||
dt { font-weight: bold; }
|
||||
.footpara { display: inline; }
|
||||
.footdef { margin-bottom: 1em; }
|
||||
.figure { padding: 1em; }
|
||||
.figure p { text-align: center; }
|
||||
.equation-container {
|
||||
display: table;
|
||||
text-align: center;
|
||||
width: 100%;
|
||||
}
|
||||
.equation {
|
||||
vertical-align: middle;
|
||||
}
|
||||
.equation-label {
|
||||
display: table-cell;
|
||||
text-align: right;
|
||||
vertical-align: middle;
|
||||
}
|
||||
.inlinetask {
|
||||
padding: 10px;
|
||||
border: 2px solid gray;
|
||||
margin: 10px;
|
||||
background: #ffffcc;
|
||||
}
|
||||
#org-div-home-and-up
|
||||
{ text-align: right; font-size: 70%; white-space: nowrap; }
|
||||
textarea { overflow-x: auto; }
|
||||
.linenr { font-size: smaller }
|
||||
.code-highlighted { background-color: #ffff00; }
|
||||
.org-info-js_info-navigation { border-style: none; }
|
||||
#org-info-js_console-label
|
||||
{ font-size: 10px; font-weight: bold; white-space: nowrap; }
|
||||
.org-info-js_search-highlight
|
||||
{ background-color: #ffff00; color: #000000; font-weight: bold; }
|
||||
.org-svg { }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div id="content" class="content">
|
||||
<div id="table-of-contents" role="doc-toc">
|
||||
<h2>Table of Contents</h2>
|
||||
<div id="text-table-of-contents" role="doc-toc">
|
||||
<ul>
|
||||
<li><a href="#org490606f">1. Evaluation</a>
|
||||
<ul>
|
||||
<li><a href="#org2f83090">1.1. Expirement setup</a>
|
||||
<ul>
|
||||
<li><a href="#org0d440d4">1.1.1. Performance counters used</a></li>
|
||||
<li><a href="#org9cacc1c">1.1.2. Benchmarks</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li><a href="#orgcc0820e">1.2. Results</a></li>
|
||||
<li><a href="#org1d8ebaa">1.3. Usability</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
<div id="outline-container-org490606f" class="outline-2">
|
||||
<h2 id="org490606f"><span class="section-number-2">1.</span> Evaluation</h2>
|
||||
<div class="outline-text-2" id="text-1">
|
||||
<p>
|
||||
We conducted tests of the FAT Pointer-based range addresses against Jemalloc,
|
||||
the default memory allocator for CHERIBSD(, ), to assess the performance improvements
|
||||
enabled by a CHERI-based huge page-aware allocator. Specifically, we evaluated
|
||||
the reduction in TLB misses and its impact on overall
|
||||
performance metrics, such as wall clock runtime.
|
||||
</p>
|
||||
|
||||
<p>
|
||||
To comprehensively analyze the proposed allocator, we categorized benchmarks into
|
||||
two classes which are micro and macro benchmarks. Micro benchmarks comprise smaller
|
||||
C programs designed to target specific allocator patterns, enabling us to evaluate
|
||||
detailed aspects of the allocator's behavior. Macro benchmarks, on the other hand,
|
||||
encompass larger, real-world C programs, allowing us to assess the allocator's
|
||||
performance in more practical, real-world scenarios.
|
||||
</p>
|
||||
|
||||
<p>
|
||||
The experiment setup details the software stack used for evaluation. It includes
|
||||
the specific configurations, compiler options, and system environment tailored
|
||||
to benchmark the proposed allocator. This ensures consistency and repeatability
|
||||
in our results, providing a solid foundation for meaningful comparisons.
|
||||
</p>
|
||||
|
||||
<p>
|
||||
We further elaborated on the two classes of benchmarks executed. Micro benchmarks
|
||||
focused on particular allocation and deallocation patterns, such as sequential and
|
||||
random memory accesses, to stress-test the allocator under controlled conditions.
|
||||
Macro benchmarks involved real-world applications, offering insights into how
|
||||
the allocator performs with complex memory allocation demands, large datasets,
|
||||
and varying execution contexts.
|
||||
</p>
|
||||
|
||||
<p>
|
||||
The results section presents the outcomes of our benchmarks, highlighting key metrics
|
||||
such as TLB miss rates, memory usage, and runtime performance. We observed that the
|
||||
proposed allocator demonstrated significant improvements in reducing TLB misses,
|
||||
leading to noticeable enhancements in runtime efficiency for both micro and macro
|
||||
benchmarks. The behavior of specific allocation patterns and their impact on memory
|
||||
performance is detailed, providing a nuanced understanding of the allocator's effectiveness.
|
||||
</p>
|
||||
|
||||
<p>
|
||||
Based on the evaluated results, the usability of the proposed allocator shows promise
|
||||
for applications requiring optimized memory management and reduced overhead from TLB misses.
|
||||
However, limitations were also identified, such as scenarios where the allocator's performance
|
||||
gains were marginal or where it introduced additional complexity in memory management. These
|
||||
limitations provide a roadmap for future optimizations and refinements of the allocator design.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div id="outline-container-org2f83090" class="outline-3">
|
||||
<h3 id="org2f83090"><span class="section-number-3">1.1.</span> Expirement setup</h3>
|
||||
<div class="outline-text-3" id="text-1-1">
|
||||
<p>
|
||||
The CHERI Morello board was used to evaluate the proposed memory allocator.
|
||||
Morello implements the ARM A76 with enhanced server-class memory, featuring a
|
||||
quad-core ARM CPU with capability extensions. The L1 and L2 caches were modified
|
||||
to proliferate the capability bit, ensuring compatibility with CHERI's capability-based
|
||||
memory model. When compiling the C programs for benchmarking, the Benchmark ABI was
|
||||
used as recommended by the CHERI community. This compilation mode was enabled using
|
||||
the Clang compiler.
|
||||
</p>
|
||||
|
||||
<p>
|
||||
The Benchmark ABI was specifically designed because the Morello branch predictor
|
||||
was not expanded to predict bounds. Consequently, a capability-based jump introduces
|
||||
stalls in later PCC-dependent instructions until bounds are established. This issue
|
||||
is particularly significant during dynamically linked calls and returns between
|
||||
libraries, where bounds are changed to cover the called or returned-to library.
|
||||
Such stalls can negatively affect performance, making the Benchmark ABI an essential
|
||||
consideration for this evaluation.
|
||||
</p>
|
||||
|
||||
<p>
|
||||
Each C program was executed using two different memory allocators. The first was
|
||||
the modified C allocator, imported as a header file. This approach was necessary
|
||||
because the Benchmark ABI shared object file exhibited unexpected behavior,
|
||||
failing to overwrite the C program at runtime with the intended malloc functions.
|
||||
The second allocator was the standard OS memory allocator, which, in the case of
|
||||
CHERIBSD, is Jemalloc.
|
||||
</p>
|
||||
|
||||
<p>
|
||||
Performance measurements were carried out using ARM performance counters to
|
||||
ensure accurate evaluation. These counters provided detailed metrics, allowing
|
||||
us to compare the performance of the two allocators and assess the impact of
|
||||
the proposed changes.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div id="outline-container-org0d440d4" class="outline-4">
|
||||
<h4 id="org0d440d4"><span class="section-number-4">1.1.1.</span> Performance counters used</h4>
|
||||
<div class="outline-text-4" id="text-1-1-1">
|
||||
<!-- This HTML table template is generated by emacs 29.1 -->
|
||||
<table border="1">
|
||||
<tr>
|
||||
<td align="left" valign="top">
|
||||
Performance counter
|
||||
</td>
|
||||
<td align="left" valign="top">
|
||||
Description
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td align="left" valign="top">
|
||||
Wall clock <br />
|
||||
<br />
|
||||
<br />
|
||||
(p/l1d_tlb_rd) L1 data TLB reads <br />
|
||||
<br />
|
||||
(p/l2d_tlb_rd) L2 data TLB reads <br />
|
||||
<br />
|
||||
(p/l1d_tlb_refill) L1 data TLB refills <br />
|
||||
<br />
|
||||
<br />
|
||||
<br />
|
||||
<br />
|
||||
<br />
|
||||
<br />
|
||||
<br />
|
||||
<br />
|
||||
<br />
|
||||
(p/cpu_cycles) CPU cycles <br />
|
||||
<br />
|
||||
<br />
|
||||
<br />
|
||||
<br />
|
||||
<br />
|
||||
<br />
|
||||
(p/dtlb_walk) Data TLB walks <br />
|
||||
<br />
|
||||
<br />
|
||||
(p/ll_cache_miss_rd) Last level cache miss reads <br />
|
||||
<br />
|
||||
<br />
|
||||
|
||||
</td>
|
||||
<td align="left" valign="top">
|
||||
The actual time taken from the start of a <br />
|
||||
computer program to the end. <br />
|
||||
<br />
|
||||
Level 1 data TLB access, read <br />
|
||||
<br />
|
||||
Level 2 data TLB access, read <br />
|
||||
<br />
|
||||
Level 1 data TLB refill. <br />
|
||||
The Level 1 data TLB refill <br />
|
||||
counter tracks each access to <br />
|
||||
the L1D_TLB that results <br />
|
||||
in a refill of the Level 1 data <br />
|
||||
or unified TLB. This includes any <br />
|
||||
access that requires a memory lookup <br />
|
||||
due to a translation table walk <br />
|
||||
or accessing another level of TLB cache. <br />
|
||||
<br />
|
||||
The CPU CYCLES counter increases with <br />
|
||||
every clock cycle. However, it can be <br />
|
||||
affected by changes in clock frequency, <br />
|
||||
such as when WFI (Wait for Interrupt) <br />
|
||||
or WFE (Wait for Event) <br />
|
||||
instructions pause the clock. <br />
|
||||
<br />
|
||||
Data TLB access with at least <br />
|
||||
one translation table walk. <br />
|
||||
<br />
|
||||
Last level cache miss, read <br />
|
||||
(This refers to every miss in the <br />
|
||||
Last level cache that occurs <br />
|
||||
during a memory read operation.)
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div id="outline-container-org9cacc1c" class="outline-4">
|
||||
<h4 id="org9cacc1c"><span class="section-number-4">1.1.2.</span> Benchmarks</h4>
|
||||
<div class="outline-text-4" id="text-1-1-2">
|
||||
<p>
|
||||
The benchmarks are classified into 2 classes:
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<ol class="org-ol">
|
||||
<li><a id="orge18d7e4"></a>Micro benchmark<br />
|
||||
<div class="outline-text-5" id="text-1-1-2-1">
|
||||
<ul class="org-ul">
|
||||
<li>GLIBC: The Glibc benchmark evaluates the performance of
|
||||
malloc and free functions in single-threaded, multi-threaded,
|
||||
and emulated multi-threading scenarios using various block sizes and
|
||||
allocation patterns. It simulates real-world memory usage by partially
|
||||
deallocating blocks in FIFO order and fully deallocating them in LIFO order.
|
||||
Results are gathered across configurations to analyze performance variations.</li>
|
||||
<li>MemAccess: This benchmark by Alex Bordei evaluates the performance impact of
|
||||
memory access patterns by constructing and traversing a doubly
|
||||
linked list with varying working set sizes. It supports sequential or
|
||||
randomized structures, optional node operations, and multithreaded
|
||||
traversal using pthreads. The program dynamically allocates memory and systematically
|
||||
doubles the working set size to analyze memory hierarchy behavior.</li>
|
||||
</ul>
|
||||
</div>
|
||||
</li>
|
||||
|
||||
<li><a id="org697b307"></a>Macro runs<br />
|
||||
<div class="outline-text-5" id="text-1-1-2-2">
|
||||
<ul class="org-ul">
|
||||
<li>Kmeans: Kmeans implements a parallelized K-means clustering algorithm that
|
||||
assigns data points to clusters based on proximity to centroids,
|
||||
iteratively updating them until convergence. The computation is
|
||||
distributed across threads using the pthread library, dynamically
|
||||
assigning tasks to optimize performance. Parameters like data size
|
||||
and clusters are configurable, and the program ensures efficient
|
||||
memory management and synchronization.</li>
|
||||
<li>Richards: Richards is a task scheduling benchmark that simulates a
|
||||
multitasking environment with tasks of varying types and priorities,
|
||||
communicating through queued packets. The schedule function manages
|
||||
task execution based on state and priority, tracking processed packets
|
||||
and held tasks for performance evaluation. Configurable iterations and
|
||||
timing help measure system performance and ensure correctness.</li>
|
||||
</ul>
|
||||
</div>
|
||||
</li>
|
||||
</ol>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div id="outline-container-orgcc0820e" class="outline-3">
|
||||
<h3 id="orgcc0820e"><span class="section-number-3">1.2.</span> Results</h3>
|
||||
<div class="outline-text-3" id="text-1-2">
|
||||
|
||||
<div id="org0a8604a" class="figure">
|
||||
<p><img src="./diagrams/allbenchmarks.png" alt="allbenchmarks.png" align="right" />
|
||||
</p>
|
||||
</div>
|
||||
|
||||
|
||||
<div id="org7ada87f" class="figure">
|
||||
<p><img src="./diagrams/kmeans.png" alt="kmeans.png" align="right" />
|
||||
</p>
|
||||
</div>
|
||||
|
||||
|
||||
<div id="orgedd36f2" class="figure">
|
||||
<p><img src="./diagrams/glibc.png" alt="glibc.png" align="right" />
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div id="outline-container-org1d8ebaa" class="outline-3">
|
||||
<h3 id="org1d8ebaa"><span class="section-number-3">1.3.</span> Usability</h3>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div id="postamble" class="status">
|
||||
<p class="author">Author: Akilan</p>
|
||||
<p class="date">Created: 2025-01-12 Sun 17:26</p>
|
||||
<p class="validation"><a href="https://validator.w3.org/check?uri=referer">Validate</a></p>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
Reference in New Issue
Block a user