From 9b2318a9ba6399eca2f09ea3131e36c59f5675cf Mon Sep 17 00:00:00 2001 From: Jonathan Woodruff Date: Thu, 9 Dec 2021 12:27:57 +0000 Subject: [PATCH 01/24] Simpelest strategy for making wrongSpec not conflict with loads of rules. This triggers some assertions in the ROB what should be dealt with properly if this is going to work. --- .../RISCY_OOO/procs/lib/GlobalSpecUpdate.bsv | 44 ++++++++++++------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/src_Core/RISCY_OOO/procs/lib/GlobalSpecUpdate.bsv b/src_Core/RISCY_OOO/procs/lib/GlobalSpecUpdate.bsv index f17bb98..dbdf7c1 100644 --- a/src_Core/RISCY_OOO/procs/lib/GlobalSpecUpdate.bsv +++ b/src_Core/RISCY_OOO/procs/lib/GlobalSpecUpdate.bsv @@ -1,6 +1,6 @@ // Copyright (c) 2017 Massachusetts Institute of Technology -// +// // Permission is hereby granted, free of charge, to any person // obtaining a copy of this software and associated documentation // files (the "Software"), to deal in the Software without @@ -8,10 +8,10 @@ // modify, merge, publish, distribute, sublicense, and/or sell copies // of the Software, and to permit persons to whom the Software is // furnished to do so, subject to the following conditions: -// +// // The above copyright notice and this permission notice shall be // included in all copies or substantial portions of the Software. -// +// // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, // EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND @@ -26,6 +26,13 @@ import HasSpecBits::*; import GetPut::*; import Vector::*; import ReorderBuffer::*; +import FIFO::*; + +typedef struct { + Bool kill_all; + SpecTag spec_tag; + InstTag inst_tag; +} IncorrectSpec deriving(Bits, Eq, FShow); interface GlobalSpecUpdate#(numeric type correctSpecPortNum, numeric type conflictWrongSpecPortNum); interface Vector#(correctSpecPortNum, Put#(SpecTag)) correctSpec; @@ -50,6 +57,9 @@ module mkGlobalSpecUpdate#( Vector#(correctSpecPortNum, RWire#(void)) spec_conflict <- replicateM(mkRWire); // let the caller of conflictWrongSpec to be conflict with wrong spec Vector#(conflictWrongSpecPortNum, RWire#(void)) wrongSpec_conflict <- replicateM(mkRWire); + // must be a single-element fifo to ensure all pushing rules cannot fire while we are waiting + // to kill. + FIFO#(IncorrectSpec) incorrectSpec_ff <- mkFIFO1; (* fire_when_enabled, no_implicit_conditions *) rule canon_correct_spec; @@ -63,6 +73,20 @@ module mkGlobalSpecUpdate#( rob.correctSpeculation(mask); endrule + rule do_incorrect_spec; + IncorrectSpec x <- toGet(incorrectSpec_ff).get; + ifc.incorrectSpeculation(x.kill_all, x.spec_tag); + rob.incorrectSpeculation(x.kill_all, x.spec_tag, x.inst_tag); + // conflict with correct spec + for(Integer i = 0; i < valueof(correctSpecPortNum); i = i+1) begin + spec_conflict[i].wset(?); + end + // conflict with the caller of conflictWrongSpec + for(Integer i = 0; i < valueof(conflictWrongSpecPortNum); i = i+1) begin + wrongSpec_conflict[i].wset(?); + end + endrule + Vector#(correctSpecPortNum, Put#(SpecTag)) correctVec = ?; for(Integer i = 0; i < valueof(correctSpecPortNum); i = i+1) begin correctVec[i] = (interface Put; @@ -85,18 +109,8 @@ module mkGlobalSpecUpdate#( interface correctSpec = correctVec; - method Action incorrectSpec(Bool kill_all, SpecTag spec_tag, InstTag inst_tag); - ifc.incorrectSpeculation(kill_all, spec_tag); - rob.incorrectSpeculation(kill_all, spec_tag, inst_tag); - // conflict with correct spec - for(Integer i = 0; i < valueof(correctSpecPortNum); i = i+1) begin - spec_conflict[i].wset(?); - end - // conflict with the caller of conflictWrongSpec - for(Integer i = 0; i < valueof(conflictWrongSpecPortNum); i = i+1) begin - wrongSpec_conflict[i].wset(?); - end - endmethod + method Action incorrectSpec(Bool kill_all, SpecTag spec_tag, InstTag inst_tag) + = incorrectSpec_ff.enq(IncorrectSpec{kill_all: kill_all, spec_tag: spec_tag, inst_tag: inst_tag}); interface conflictWrongSpec = conflictWrongVec; endmodule From f619f4b0a1b88116c5394716bf7db6ee49a7a7bb Mon Sep 17 00:00:00 2001 From: Jonathan Woodruff Date: Mon, 15 Nov 2021 11:50:57 +0000 Subject: [PATCH 02/24] Make the associative BTB compressed. That is, only store the bottom 16 bits of the target if the upper bits of pc and nextPc match. Have a single "way" dedicated to full targets. --- src_Core/RISCY_OOO/procs/lib/Btb.bsv | 42 ++++++++++++++++++---------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/src_Core/RISCY_OOO/procs/lib/Btb.bsv b/src_Core/RISCY_OOO/procs/lib/Btb.bsv index 3c572d3..335d1ab 100644 --- a/src_Core/RISCY_OOO/procs/lib/Btb.bsv +++ b/src_Core/RISCY_OOO/procs/lib/Btb.bsv @@ -58,6 +58,7 @@ endinterface // Local BTB Typedefs typedef 1 PcLsbsIgnore; typedef 1024 BtbEntries; +typedef Bit#(16) CompressedTarget; typedef 2 BtbAssociativity; typedef Bit#(TLog#(SupSizeX2)) BtbBank; // Total entries/lanes of superscalar lookup/associativity @@ -94,9 +95,11 @@ module mkBtbCore(NextAddrPred#(hashSz)) Add#(1, a__, TDiv#(tagSz, hashSz)), Add#(b__, tagSz, TMul#(TDiv#(tagSz, hashSz), hashSz))); // Read and Write ordering doesn't matter since this is a predictor - Reg#(BtbBank) firstBank_reg <- mkRegU; - Vector#(SupSizeX2, MapSplit#(HashedTag#(hashSz), BtbIndex, VnD#(CapMem), BtbAssociativity)) - records <- replicateM(mkMapLossyBRAM); + Reg#(CapMem) addr_reg <- mkRegU; + Vector#(SupSizeX2, MapSplit#(HashedTag#(hashSz), BtbIndex, VnD#(CapMem), 1)) + fullRecords <- replicateM(mkMapLossyBRAM); + Vector#(SupSizeX2, MapSplit#(HashedTag#(hashSz), BtbIndex, VnD#(CompressedTarget), BtbAssociativity)) + compressedRecords <- replicateM(mkMapLossyBRAM); RWire#(BtbUpdate) updateEn <- mkRWire; function BtbAddr getBtbAddr(CapMem pc) = unpack(truncateLSB(getAddr(pc))); @@ -114,26 +117,34 @@ module mkBtbCore(NextAddrPred#(hashSz)) let taken = upd.taken; /*$display("MapUpdate in BTB - pc %x, bank: %x, taken: %x, next: %x, time: %t", pc, getBank(pc), taken, nextPc, $time);*/ - records[getBank(pc)].update(lookupKey(pc), VnD{v:taken, d:nextPc}); + CompressedTarget shortMask = -1; + CapMem mask = ~zeroExtend(shortMask); + if ((pc&mask) == (nextPc&mask)) + compressedRecords[getBank(pc)].update(lookupKey(pc), VnD{v:taken, d:truncate(nextPc)}); + else + fullRecords[getBank(pc)].update(lookupKey(pc), VnD{v:taken, d:nextPc}); endrule method Action put_pc(CapMem pc); - BtbAddr addr = getBtbAddr(pc); - firstBank_reg <= addr.bank; + addr_reg <= pc; // Start SupSizeX2 BTB lookups, but ensure to lookup in the appropriate // bank for the alignment of each potential branch. for (Integer i = 0; i < valueOf(SupSizeX2); i = i + 1) begin - BtbAddr a = unpack(pack(addr) + fromInteger(i)); - records[a.bank].lookupStart(MapKeyIndex{key: hash(a.tag), index: a.index}); + BtbAddr a = unpack(pack(getBtbAddr(pc)) + fromInteger(i)); + fullRecords[a.bank].lookupStart(MapKeyIndex{key: hash(a.tag), index: a.index}); + compressedRecords[a.bank].lookupStart(MapKeyIndex{key: hash(a.tag), index: a.index}); end endmethod method Vector#(SupSizeX2, Maybe#(CapMem)) pred; Vector#(SupSizeX2, Maybe#(CapMem)) ppcs = replicate(Invalid); - for (Integer i = 0; i < valueOf(SupSizeX2); i = i + 1) - if (records[i].lookupRead matches tagged Valid .record) - ppcs[i] = record.v ? Valid(record.d):Invalid; - ppcs = rotateBy(ppcs,unpack(-firstBank_reg)); // Rotate firstBank down to zeroeth element. + for (Integer i = 0; i < valueOf(SupSizeX2); i = i + 1) begin + if (fullRecords[i].lookupRead matches tagged Valid .r) + ppcs[i] = r.v ? Valid(r.d):Invalid; + if (compressedRecords[i].lookupRead matches tagged Valid .r) + ppcs[i] = r.v ? Valid({truncateLSB(addr_reg),r.d}):Invalid; + end + ppcs = rotateBy(ppcs,unpack(-getBtbAddr(addr_reg).bank)); // Rotate firstBank down to zeroeth element. return ppcs; endmethod @@ -143,9 +154,12 @@ module mkBtbCore(NextAddrPred#(hashSz)) `ifdef SECURITY method Action flush method Action flush; - for (Integer i = 0; i < valueOf(SupSizeX2); i = i + 1) records[i].clear; + for (Integer i = 0; i < valueOf(SupSizeX2); i = i + 1) begin + fullRecords[i].clear; + compressedRecords[i].clear; + end endmethod - method flush_done = records[0].clearDone; + method flush_done = fullRecords[0].clearDone; `else method flush = noAction; method flush_done = True; From b84562a027ad78f821cd1a62d1314501364ba394 Mon Sep 17 00:00:00 2001 From: Jonathan Woodruff Date: Fri, 7 Jan 2022 09:52:43 +0000 Subject: [PATCH 03/24] Improve scheduling for LatencyTimer to improve timing path discovered on DE10. --- src_Core/RISCY_OOO/procs/lib/LatencyTimer.bsv | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src_Core/RISCY_OOO/procs/lib/LatencyTimer.bsv b/src_Core/RISCY_OOO/procs/lib/LatencyTimer.bsv index 56b3853..2cdfb04 100644 --- a/src_Core/RISCY_OOO/procs/lib/LatencyTimer.bsv +++ b/src_Core/RISCY_OOO/procs/lib/LatencyTimer.bsv @@ -23,6 +23,7 @@ import Vector::*; import Types::*; +import ConfigReg::*; // a timer to track latency of multiple misses @@ -39,8 +40,8 @@ module mkLatencyTimer(LatencyTimer#(num, timeWidth)) provisos( Alias#(idxT, Bit#(TLog#(num))), Alias#(timeT, Bit#(timeWidth)) ); - Reg#(Vector#(num, timeT)) timer <- mkReg(replicate(0)); - Reg#(Vector#(num, Bool)) started <- mkReg(replicate(False)); // for checking purposes + Reg#(Vector#(num, timeT)) timer <- mkConfigReg(replicate(0)); + Reg#(Vector#(num, Bool)) started <- mkConfigReg(replicate(False)); // for checking purposes RWire#(idxT) startEn <- mkRWire; RWire#(idxT) doneEn <- mkRWire; From e600fd7d38c238a7d0ff468b2df86e790f8ebb29 Mon Sep 17 00:00:00 2001 From: Jonathan Woodruff Date: Sat, 8 Jan 2022 10:15:22 +0000 Subject: [PATCH 04/24] Delay BTB update by a cycle for timing. This should not be a primary degredation of performance since the redirection gets on seperately from this update which will only affect future predictions. --- src_Core/RISCY_OOO/procs/lib/Btb.bsv | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src_Core/RISCY_OOO/procs/lib/Btb.bsv b/src_Core/RISCY_OOO/procs/lib/Btb.bsv index 335d1ab..a1d9059 100644 --- a/src_Core/RISCY_OOO/procs/lib/Btb.bsv +++ b/src_Core/RISCY_OOO/procs/lib/Btb.bsv @@ -38,6 +38,7 @@ import Types::*; import ProcTypes::*; import ConfigReg::*; +import DReg::*; import Map::*; import Vector::*; import CHERICC_Fat::*; @@ -100,7 +101,7 @@ module mkBtbCore(NextAddrPred#(hashSz)) fullRecords <- replicateM(mkMapLossyBRAM); Vector#(SupSizeX2, MapSplit#(HashedTag#(hashSz), BtbIndex, VnD#(CompressedTarget), BtbAssociativity)) compressedRecords <- replicateM(mkMapLossyBRAM); - RWire#(BtbUpdate) updateEn <- mkRWire; + Reg#(Maybe#(BtbUpdate)) updateEn <- mkDReg(Invalid); function BtbAddr getBtbAddr(CapMem pc) = unpack(truncateLSB(getAddr(pc))); function BtbBank getBank(CapMem pc) = getBtbAddr(pc).bank; @@ -111,7 +112,7 @@ module mkBtbCore(NextAddrPred#(hashSz)) // no flush, accept update (* fire_when_enabled, no_implicit_conditions *) - rule canonUpdate(updateEn.wget matches tagged Valid .upd); + rule canonUpdate(updateEn matches tagged Valid .upd); let pc = upd.pc; let nextPc = upd.nextPc; let taken = upd.taken; @@ -149,7 +150,7 @@ module mkBtbCore(NextAddrPred#(hashSz)) endmethod method Action update(CapMem pc, CapMem nextPc, Bool taken); - updateEn.wset(BtbUpdate {pc: pc, nextPc: nextPc, taken: taken}); + updateEn <= Valid(BtbUpdate {pc: pc, nextPc: nextPc, taken: taken}); endmethod `ifdef SECURITY From 1d6633f42353bca472c00dcceb06a36e6e5e3620 Mon Sep 17 00:00:00 2001 From: Jonathan Woodruff Date: Mon, 10 Jan 2022 17:18:01 +0000 Subject: [PATCH 05/24] Buffer redirection from Commit to reduce conflicts in the pipeline. This is a general improvement, and should be upstreamed to master if there are no issues discovered. --- .../RISCY_OOO/procs/RV64G_OOO/CommitStage.bsv | 48 +++++++++++++------ 1 file changed, 34 insertions(+), 14 deletions(-) diff --git a/src_Core/RISCY_OOO/procs/RV64G_OOO/CommitStage.bsv b/src_Core/RISCY_OOO/procs/RV64G_OOO/CommitStage.bsv index 168b20b..faaa514 100644 --- a/src_Core/RISCY_OOO/procs/RV64G_OOO/CommitStage.bsv +++ b/src_Core/RISCY_OOO/procs/RV64G_OOO/CommitStage.bsv @@ -118,7 +118,8 @@ interface CommitInput; method Action setReconcileD; // recocile D$ // redirect method Action killAll; - method Action redirectPc(CapMem trap_pc + method Action redirectPc( + CapMem trap_pc `ifdef RVFI_DII , Dii_Parcel_Id dii_pid `endif @@ -157,6 +158,13 @@ typedef struct { SpecBits specBits; } RenameErrInfo deriving(Bits, Eq, FShow); +typedef struct { + CapMem trap_pc; +`ifdef RVFI_DII + Dii_Parcel_Id dii_pid; +`endif +} RedirectInfo deriving(Bits, Eq, FShow); + interface CommitStage; // performance method Data getPerf(ComStagePerfType t); @@ -519,6 +527,8 @@ module mkCommitStage#(CommitInput inIfc)(CommitStage); // cycle handles trap, redirect and handles system consistency Reg#(Maybe#(CommitTrap)) commitTrap <- mkReg(Invalid); // saves new pc here + FIFO#(RedirectInfo) redirectQ <- mkFIFO; + // maintain system consistency when system state (CSR) changes or for security function Action makeSystemConsistent(Bool flushTlb, Bool flushSecurity, @@ -780,11 +790,11 @@ module mkCommitStage#(CommitInput inIfc)(CommitStage); // trap handling & redirect let trap_updates <- csrf.trap(trap.trap, cast(trap.pc), trap.addr, trap.orig_inst); CapPipe new_pc = cast(trap_updates.new_pcc); - inIfc.redirectPc(cast(new_pc) + redirectQ.enq(RedirectInfo{trap_pc: cast(new_pc) `ifdef RVFI_DII - , trap.x.dii_pid + (is_16b_inst(trap.orig_inst) ? 1 : 2) + , dii_pid: trap.x.dii_pid + (is_16b_inst(trap.orig_inst) ? 1 : 2) `endif - ); + }); `ifdef RVFI Rvfi_Traces rvfis = replicate(tagged Invalid); rvfis[0] = genRVFI(trap.x, traceCnt, getTSB(), getAddr(new_pc)); @@ -825,11 +835,11 @@ module mkCommitStage#(CommitInput inIfc)(CommitStage); // kill everything, redirect, and increment epoch inIfc.killAll; - inIfc.redirectPc(x.pc -`ifdef RVFI_DII - , x.dii_pid -`endif - ); + redirectQ.enq(RedirectInfo{trap_pc: x.pc + `ifdef RVFI_DII + , dii_pid: x.dii_pid + `endif + }); inIfc.incrementEpoch; // the killed Ld should have claimed phy reg, we should not commit it; @@ -945,11 +955,12 @@ module mkCommitStage#(CommitInput inIfc)(CommitStage); m_ret_updates = tagged Valid ret_updates; `endif end - inIfc.redirectPc(next_pc -`ifdef RVFI_DII - , x.dii_pid + (is_16b_inst(x.orig_inst) ? 1 : 2) -`endif - ); + redirectQ.enq(RedirectInfo{trap_pc: next_pc + `ifdef RVFI_DII + , x.dii_pid + (is_16b_inst(x.orig_inst) ? 1 : 2) + `endif + }); + `ifdef RVFI Rvfi_Traces rvfis = replicate(tagged Invalid); @@ -1343,6 +1354,15 @@ module mkCommitStage#(CommitInput inIfc)(CommitStage); `endif endrule + rule pass_redirect; + RedirectInfo ri <- toGet(redirectQ).get; + inIfc.redirectPc(ri.trap_pc +`ifdef RVFI_DII + , ri.rii_pid +`endif + ); + endrule + // ================================================================ // INTERFACE From 1e9ee9d12174758bb03e1e2e4a2d0a00935ed4a3 Mon Sep 17 00:00:00 2001 From: Jonathan Woodruff Date: Thu, 9 Dec 2021 12:27:57 +0000 Subject: [PATCH 06/24] Simpelest strategy for making wrongSpec not conflict with loads of rules. This triggers some assertions in the ROB what should be dealt with properly if this is going to work. --- .../RISCY_OOO/procs/lib/GlobalSpecUpdate.bsv | 44 ++++++++++++------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/src_Core/RISCY_OOO/procs/lib/GlobalSpecUpdate.bsv b/src_Core/RISCY_OOO/procs/lib/GlobalSpecUpdate.bsv index f17bb98..dbdf7c1 100644 --- a/src_Core/RISCY_OOO/procs/lib/GlobalSpecUpdate.bsv +++ b/src_Core/RISCY_OOO/procs/lib/GlobalSpecUpdate.bsv @@ -1,6 +1,6 @@ // Copyright (c) 2017 Massachusetts Institute of Technology -// +// // Permission is hereby granted, free of charge, to any person // obtaining a copy of this software and associated documentation // files (the "Software"), to deal in the Software without @@ -8,10 +8,10 @@ // modify, merge, publish, distribute, sublicense, and/or sell copies // of the Software, and to permit persons to whom the Software is // furnished to do so, subject to the following conditions: -// +// // The above copyright notice and this permission notice shall be // included in all copies or substantial portions of the Software. -// +// // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, // EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND @@ -26,6 +26,13 @@ import HasSpecBits::*; import GetPut::*; import Vector::*; import ReorderBuffer::*; +import FIFO::*; + +typedef struct { + Bool kill_all; + SpecTag spec_tag; + InstTag inst_tag; +} IncorrectSpec deriving(Bits, Eq, FShow); interface GlobalSpecUpdate#(numeric type correctSpecPortNum, numeric type conflictWrongSpecPortNum); interface Vector#(correctSpecPortNum, Put#(SpecTag)) correctSpec; @@ -50,6 +57,9 @@ module mkGlobalSpecUpdate#( Vector#(correctSpecPortNum, RWire#(void)) spec_conflict <- replicateM(mkRWire); // let the caller of conflictWrongSpec to be conflict with wrong spec Vector#(conflictWrongSpecPortNum, RWire#(void)) wrongSpec_conflict <- replicateM(mkRWire); + // must be a single-element fifo to ensure all pushing rules cannot fire while we are waiting + // to kill. + FIFO#(IncorrectSpec) incorrectSpec_ff <- mkFIFO1; (* fire_when_enabled, no_implicit_conditions *) rule canon_correct_spec; @@ -63,6 +73,20 @@ module mkGlobalSpecUpdate#( rob.correctSpeculation(mask); endrule + rule do_incorrect_spec; + IncorrectSpec x <- toGet(incorrectSpec_ff).get; + ifc.incorrectSpeculation(x.kill_all, x.spec_tag); + rob.incorrectSpeculation(x.kill_all, x.spec_tag, x.inst_tag); + // conflict with correct spec + for(Integer i = 0; i < valueof(correctSpecPortNum); i = i+1) begin + spec_conflict[i].wset(?); + end + // conflict with the caller of conflictWrongSpec + for(Integer i = 0; i < valueof(conflictWrongSpecPortNum); i = i+1) begin + wrongSpec_conflict[i].wset(?); + end + endrule + Vector#(correctSpecPortNum, Put#(SpecTag)) correctVec = ?; for(Integer i = 0; i < valueof(correctSpecPortNum); i = i+1) begin correctVec[i] = (interface Put; @@ -85,18 +109,8 @@ module mkGlobalSpecUpdate#( interface correctSpec = correctVec; - method Action incorrectSpec(Bool kill_all, SpecTag spec_tag, InstTag inst_tag); - ifc.incorrectSpeculation(kill_all, spec_tag); - rob.incorrectSpeculation(kill_all, spec_tag, inst_tag); - // conflict with correct spec - for(Integer i = 0; i < valueof(correctSpecPortNum); i = i+1) begin - spec_conflict[i].wset(?); - end - // conflict with the caller of conflictWrongSpec - for(Integer i = 0; i < valueof(conflictWrongSpecPortNum); i = i+1) begin - wrongSpec_conflict[i].wset(?); - end - endmethod + method Action incorrectSpec(Bool kill_all, SpecTag spec_tag, InstTag inst_tag) + = incorrectSpec_ff.enq(IncorrectSpec{kill_all: kill_all, spec_tag: spec_tag, inst_tag: inst_tag}); interface conflictWrongSpec = conflictWrongVec; endmodule From 7ba5ddec8b4e9b46736a1dc778c07cc5e98b42a5 Mon Sep 17 00:00:00 2001 From: Jonathan Woodruff Date: Tue, 11 Jan 2022 00:06:49 +0000 Subject: [PATCH 07/24] A version that seems to actually run. The buffer in GlobalSpecUpdate has to be a SpecFifo. --- src_Core/CPU/Core.bsv | 9 ++- .../procs/RV64G_OOO/AluExePipeline.bsv | 6 +- .../RISCY_OOO/procs/RV64G_OOO/CommitStage.bsv | 13 +-- .../RISCY_OOO/procs/lib/GlobalSpecUpdate.bsv | 37 +++++---- .../RISCY_OOO/procs/lib/ReorderBuffer.bsv | 6 +- src_Core/RISCY_OOO/procs/lib/SpecFifo.bsv | 5 ++ src_Core/RISCY_OOO/procs/lib/SplitLSQ.bsv | 80 ++++--------------- 7 files changed, 65 insertions(+), 91 deletions(-) diff --git a/src_Core/CPU/Core.bsv b/src_Core/CPU/Core.bsv index 7172cf9..5c29e37 100644 --- a/src_Core/CPU/Core.bsv +++ b/src_Core/CPU/Core.bsv @@ -223,6 +223,7 @@ interface CoreFixPoint; interface MemExePipeline memExeIfc; method Action killAll; // kill everything: used by commit stage interface Reg#(Bool) doStatsIfc; + method Bool pendingIncorrectSpec; endinterface `ifdef CONTRACTS_VERIFY @@ -412,7 +413,7 @@ module mkCore#(CoreId coreId)(Core); method setRegReadyAggr = writeAggr(aluWrAggrPort(i)); interface sendBypass = sendBypassIfc; method writeRegFile = writeCons(aluWrConsPort(i)); - method Action redirect(CapMem new_pc, SpecTag spec_tag, InstTag inst_tag); + method Action redirect(CapMem new_pc, SpecTag spec_tag, InstTag inst_tag, SpecBits spec_bits); if (verbose) begin $display("[ALU redirect - %d] ", i, fshow(new_pc), "; ", fshow(spec_tag), "; ", fshow(inst_tag)); @@ -423,7 +424,7 @@ module mkCore#(CoreId coreId)(Core); , inst_tag.dii_next_pid `endif ); - globalSpecUpdate.incorrectSpec(False, spec_tag, inst_tag); + globalSpecUpdate.incorrectSpec(False, spec_tag, inst_tag, spec_bits); endmethod method correctSpec = globalSpecUpdate.correctSpec[finishAluCorrectSpecPort(i)].put; method doStats = doStatsReg._read; @@ -510,9 +511,10 @@ module mkCore#(CoreId coreId)(Core); interface fpuMulDivExeIfc = fpuMulDivExe; interface memExeIfc = memExe; method Action killAll; - globalSpecUpdate.incorrectSpec(True, ?, ?); + globalSpecUpdate.incorrectSpec(True, ?, ?, 0); endmethod interface doStatsIfc = doStatsReg; + method pendingIncorrectSpec = globalSpecUpdate.pendingIncorrectSpec; endmodule CoreFixPoint coreFix <- moduleFix(mkCoreFixPoint); @@ -647,6 +649,7 @@ module mkCore#(CoreId coreId)(Core); method stbEmpty = stb.isEmpty; method stqEmpty = lsq.stqEmpty; method lsqSetAtCommit = lsq.setAtCommit; + method pauseCommit = coreFix.pendingIncorrectSpec; method tlbNoPendingReq = iTlb.noPendingReq && dTlb.noPendingReq; method setFlushTlbs; diff --git a/src_Core/RISCY_OOO/procs/RV64G_OOO/AluExePipeline.bsv b/src_Core/RISCY_OOO/procs/RV64G_OOO/AluExePipeline.bsv index b39fc2c..acb59be 100644 --- a/src_Core/RISCY_OOO/procs/RV64G_OOO/AluExePipeline.bsv +++ b/src_Core/RISCY_OOO/procs/RV64G_OOO/AluExePipeline.bsv @@ -194,7 +194,7 @@ interface AluExeInput; // write reg file & set conservative sb method Action writeRegFile(PhyRIndx dst, CapPipe data); // redirect - method Action redirect(CapMem new_pc, SpecTag spec_tag, InstTag inst_tag); + method Action redirect(CapMem new_pc, SpecTag spec_tag, InstTag inst_tag, SpecBits spec_bits); // spec update method Action correctSpec(SpecTag t); @@ -469,7 +469,7 @@ module mkAluExePipeline#(AluExeInput inIfc)(AluExePipeline); if (x.controlFlow.mispredict) (* nosplit *) begin // wrong branch predictin, we must have spec tag doAssert(isValid(x.spec_tag), "mispredicted branch must have spec tag"); - inIfc.redirect(cast(x.controlFlow.nextPc), validValue(x.spec_tag), x.tag); + inIfc.redirect(cast(x.controlFlow.nextPc), validValue(x.spec_tag), x.tag, exeToFin.spec_bits); // must be a branch, train branch predictor doAssert(x.iType == Jr || x.iType == CJALR || x.iType == CCall || x.iType == Br, "only jr, br, cjalr, and ccall can mispredict"); inIfc.fetch_train_predictors(FetchTrainBP { @@ -481,6 +481,8 @@ module mkAluExePipeline#(AluExeInput inIfc)(AluExePipeline); mispred: True, isCompressed: x.isCompressed }); + $display("alu mispredict pc¤: %x, nextPc: %x, %d", + x.controlFlow.pc, x.controlFlow.nextPc, cur_cycle); `ifdef PERF_COUNT // performance counter if(inIfc.doStats) begin diff --git a/src_Core/RISCY_OOO/procs/RV64G_OOO/CommitStage.bsv b/src_Core/RISCY_OOO/procs/RV64G_OOO/CommitStage.bsv index 168b20b..397543a 100644 --- a/src_Core/RISCY_OOO/procs/RV64G_OOO/CommitStage.bsv +++ b/src_Core/RISCY_OOO/procs/RV64G_OOO/CommitStage.bsv @@ -108,6 +108,8 @@ interface CommitInput; interface Vector#(SupSize, Put#(LdStQTag)) lsqSetAtCommit; // TLB has stopped processing now method Bool tlbNoPendingReq; + // Pause committing, probably for buffered wrongSpec + method Bool pauseCommit; // set flags method Action setFlushTlbs; method Action setUpdateVMInfo; @@ -518,6 +520,7 @@ module mkCommitStage#(CommitInput inIfc)(CommitStage); // we commit trap in two cycles: first cycle deq ROB and flush; second // cycle handles trap, redirect and handles system consistency Reg#(Maybe#(CommitTrap)) commitTrap <- mkReg(Invalid); // saves new pc here + Bool pauseCommit = isValid(commitTrap) || inIfc.pauseCommit; // maintain system consistency when system state (CSR) changes or for security function Action makeSystemConsistent(Bool flushTlb, @@ -650,7 +653,7 @@ module mkCommitStage#(CommitInput inIfc)(CommitStage); `ifdef INCLUDE_GDB_CONTROL (rg_run_state == RUN_STATE_RUNNING) &&& `endif - !isValid(commitTrap) &&& + !pauseCommit &&& rob.deqPort[0].deq_data.trap matches tagged Valid .trap ); rob.deqPort[0].deq; @@ -815,7 +818,7 @@ module mkCommitStage#(CommitInput inIfc)(CommitStage); `ifdef INCLUDE_GDB_CONTROL (rg_run_state == RUN_STATE_RUNNING) &&& `endif - !isValid(commitTrap) &&& + !pauseCommit &&& !isValid(rob.deqPort[0].deq_data.trap) &&& rob.deqPort[0].deq_data.ldKilled matches tagged Valid .killBy ); @@ -856,7 +859,7 @@ module mkCommitStage#(CommitInput inIfc)(CommitStage); `ifdef INCLUDE_GDB_CONTROL (rg_run_state == RUN_STATE_RUNNING) && `endif - !isValid(commitTrap) && + !pauseCommit && !isValid(rob.deqPort[0].deq_data.trap) && !isValid(rob.deqPort[0].deq_data.ldKilled) && rob.deqPort[0].deq_data.rob_inst_state == Executed && @@ -1031,7 +1034,7 @@ module mkCommitStage#(CommitInput inIfc)(CommitStage); // Lr/Sc/Amo/MMIO cannot proceed to executed until we notify LSQ that it // has reached the commit stage rule notifyLSQCommit( - !isValid(commitTrap) && + !pauseCommit && !isValid(rob.deqPort[0].deq_data.trap) && !isValid(rob.deqPort[0].deq_data.ldKilled) && rob.deqPort[0].deq_data.rob_inst_state != Executed && @@ -1052,7 +1055,7 @@ module mkCommitStage#(CommitInput inIfc)(CommitStage); `ifdef INCLUDE_GDB_CONTROL (rg_run_state == RUN_STATE_RUNNING) && `endif - !isValid(commitTrap) && + !pauseCommit && !isValid(rob.deqPort[0].deq_data.trap) && !isValid(rob.deqPort[0].deq_data.ldKilled) && rob.deqPort[0].deq_data.rob_inst_state == Executed && diff --git a/src_Core/RISCY_OOO/procs/lib/GlobalSpecUpdate.bsv b/src_Core/RISCY_OOO/procs/lib/GlobalSpecUpdate.bsv index dbdf7c1..8234e3b 100644 --- a/src_Core/RISCY_OOO/procs/lib/GlobalSpecUpdate.bsv +++ b/src_Core/RISCY_OOO/procs/lib/GlobalSpecUpdate.bsv @@ -26,7 +26,7 @@ import HasSpecBits::*; import GetPut::*; import Vector::*; import ReorderBuffer::*; -import FIFO::*; +import SpecFifo::*; typedef struct { Bool kill_all; @@ -36,13 +36,14 @@ typedef struct { interface GlobalSpecUpdate#(numeric type correctSpecPortNum, numeric type conflictWrongSpecPortNum); interface Vector#(correctSpecPortNum, Put#(SpecTag)) correctSpec; - method Action incorrectSpec(Bool kill_all, SpecTag spec_tag, InstTag inst_tag); + method Action incorrectSpec(Bool kill_all, SpecTag spec_tag, InstTag inst_tag, SpecBits spec_bits); // Some rules (e.g. doFinishFpuMulDiv) in Core.bsv may not conflict with wrong spec // and is ordered before rules that calls incorrectSpec // this creates cycles in scheduling // To break the cycle, such rules can call the following interface // to manually create a conflict with rules that do incorrectSpec interface Vector#(conflictWrongSpecPortNum, Put#(void)) conflictWrongSpec; + method Bool pendingIncorrectSpec; endinterface module mkGlobalSpecUpdate#( @@ -54,12 +55,12 @@ module mkGlobalSpecUpdate#( // record correct spec tags Vector#(correctSpecPortNum, RWire#(SpecTag)) correctSpecTag <- replicateM(mkRWire); // make wrong spec conflict with correct spec - Vector#(correctSpecPortNum, RWire#(void)) spec_conflict <- replicateM(mkRWire); + Vector#(correctSpecPortNum, PulseWire) spec_conflict <- replicateM(mkPulseWire); // let the caller of conflictWrongSpec to be conflict with wrong spec - Vector#(conflictWrongSpecPortNum, RWire#(void)) wrongSpec_conflict <- replicateM(mkRWire); + Vector#(conflictWrongSpecPortNum, PulseWire) wrongSpec_conflict <- replicateM(mkPulseWire); // must be a single-element fifo to ensure all pushing rules cannot fire while we are waiting // to kill. - FIFO#(IncorrectSpec) incorrectSpec_ff <- mkFIFO1; + SpecFifo#(2,IncorrectSpec,1,1) incorrectSpec_ff <- mkSpecFifoCF(True); (* fire_when_enabled, no_implicit_conditions *) rule canon_correct_spec; @@ -69,31 +70,32 @@ module mkGlobalSpecUpdate#( mask[tag] = 0; end end + incorrectSpec_ff.specUpdate.correctSpeculation(mask); ifc.correctSpeculation(mask); rob.correctSpeculation(mask); endrule rule do_incorrect_spec; - IncorrectSpec x <- toGet(incorrectSpec_ff).get; + IncorrectSpec x = incorrectSpec_ff.first.data; + incorrectSpec_ff.deq; + incorrectSpec_ff.specUpdate.incorrectSpeculation(x.kill_all, x.spec_tag); ifc.incorrectSpeculation(x.kill_all, x.spec_tag); rob.incorrectSpeculation(x.kill_all, x.spec_tag, x.inst_tag); // conflict with correct spec for(Integer i = 0; i < valueof(correctSpecPortNum); i = i+1) begin - spec_conflict[i].wset(?); + spec_conflict[i].send; end // conflict with the caller of conflictWrongSpec for(Integer i = 0; i < valueof(conflictWrongSpecPortNum); i = i+1) begin - wrongSpec_conflict[i].wset(?); + wrongSpec_conflict[i].send; end endrule Vector#(correctSpecPortNum, Put#(SpecTag)) correctVec = ?; for(Integer i = 0; i < valueof(correctSpecPortNum); i = i+1) begin correctVec[i] = (interface Put; - method Action put(SpecTag t); + method Action put(SpecTag t) if (!spec_conflict[i]); correctSpecTag[i].wset(t); - // conflict with wrong spec - spec_conflict[i].wset(?); endmethod endinterface); end @@ -101,16 +103,21 @@ module mkGlobalSpecUpdate#( Vector#(conflictWrongSpecPortNum, Put#(void)) conflictWrongVec = ?; for(Integer i = 0; i < valueof(conflictWrongSpecPortNum); i = i+1) begin conflictWrongVec[i] = (interface Put; - method Action put(void x); - wrongSpec_conflict[i].wset(?); + method Action put(void x) if (!wrongSpec_conflict[i]); + noAction; endmethod endinterface); end interface correctSpec = correctVec; - method Action incorrectSpec(Bool kill_all, SpecTag spec_tag, InstTag inst_tag) - = incorrectSpec_ff.enq(IncorrectSpec{kill_all: kill_all, spec_tag: spec_tag, inst_tag: inst_tag}); + method Action incorrectSpec(Bool kill_all, SpecTag spec_tag, InstTag inst_tag, SpecBits spec_bits) + = incorrectSpec_ff.enq(ToSpecFifo{ + data: IncorrectSpec{kill_all: kill_all, spec_tag: spec_tag, inst_tag: inst_tag}, + spec_bits: spec_bits + }); interface conflictWrongSpec = conflictWrongVec; + + method Bool pendingIncorrectSpec = incorrectSpec_ff.notEmpty; endmodule diff --git a/src_Core/RISCY_OOO/procs/lib/ReorderBuffer.bsv b/src_Core/RISCY_OOO/procs/lib/ReorderBuffer.bsv index 1ca4aa7..194f570 100644 --- a/src_Core/RISCY_OOO/procs/lib/ReorderBuffer.bsv +++ b/src_Core/RISCY_OOO/procs/lib/ReorderBuffer.bsv @@ -674,7 +674,7 @@ module mkSupReorderBuffer#( Add#(1, a__, aluExeNum), Add#(1, b__, fpuMulDivExeNum) ); - Bool verbose = False; + Bool verbose = True; // doCommit rule: deq < wrongSpec (overwrite deq in doCommit) < doRenaming rule: enq Integer valid_deq_port = 0; @@ -756,6 +756,7 @@ module mkSupReorderBuffer#( // move deqP & reset valid deqP[i] <= getNextPtr(deqP[i]); valid[i][deqP[i]][valid_deq_port] <= False; + $display("deq[%d][%d]", i, deqP[i]); end end // update firstDeqWay: find the first deq port that is not enabled @@ -935,6 +936,9 @@ module mkSupReorderBuffer#( end endfunction for(Integer i = 0; i < valueof(SingleScalarSize); i = i+1) begin + if (in_kill_range(fromInteger(i)) != (row[w][i].dependsOn_wrongSpec(specTag) && valid[w][i][valid_wrongSpec_port])) + $display("enqP: %d, enqPNext: %d, w: %d, i: %d, wrongSpec: %d, valid: %d, cur_cycle: %d", + enqP[w], enqPNext[w], w, i, row[w][i].dependsOn_wrongSpec(specTag), valid[w][i][valid_wrongSpec_port], cur_cycle); doAssert( in_kill_range(fromInteger(i)) == (row[w][i].dependsOn_wrongSpec(specTag) && valid[w][i][valid_wrongSpec_port]), diff --git a/src_Core/RISCY_OOO/procs/lib/SpecFifo.bsv b/src_Core/RISCY_OOO/procs/lib/SpecFifo.bsv index 818a340..8eb8c0e 100644 --- a/src_Core/RISCY_OOO/procs/lib/SpecFifo.bsv +++ b/src_Core/RISCY_OOO/procs/lib/SpecFifo.bsv @@ -43,6 +43,7 @@ interface SpecFifo#( method Action enq(ToSpecFifo#(t) x); method Action deq; method ToSpecFifo#(t) first; + method Bool notEmpty; interface SpeculationUpdate specUpdate; endinterface @@ -166,6 +167,8 @@ module mkSpecFifo#( }; endmethod + method Bool notEmpty = valid[deqP][sched.validDeqPort]; + interface SpeculationUpdate specUpdate; method Action correctSpeculation(SpecBits mask); // clear spec bits for all entries @@ -310,6 +313,8 @@ module mkSpecFifoCF#( }; endmethod + method Bool notEmpty = valid[1][deqP]; + interface SpeculationUpdate specUpdate; method correctSpeculation = correctSpecF.enq; method incorrectSpeculation(kill_all, specTag) = diff --git a/src_Core/RISCY_OOO/procs/lib/SplitLSQ.bsv b/src_Core/RISCY_OOO/procs/lib/SplitLSQ.bsv index 24fa28c..6513a8c 100644 --- a/src_Core/RISCY_OOO/procs/lib/SplitLSQ.bsv +++ b/src_Core/RISCY_OOO/procs/lib/SplitLSQ.bsv @@ -949,17 +949,7 @@ module mkSplitLSQ(SplitLSQ); // make wrongSpec conflict with all others (but not correctSpec method and // findIssue) - RWire#(void) wrongSpec_hit_conflict <- mkRWire; - RWire#(void) wrongSpec_enqIss_conflict <- mkRWire; - RWire#(void) wrongSpec_enq_conflict <- mkRWire; - RWire#(void) wrongSpec_cacheEvict_conflict <- mkRWire; - RWire#(void) wrongSpec_update_conflict <- mkRWire; - RWire#(void) wrongSpec_issue_conflict <- mkRWire; - RWire#(void) wrongSpec_respLd_conflict <- mkRWire; - RWire#(void) wrongSpec_deqLd_conflict <- mkRWire; - RWire#(void) wrongSpec_deqSt_conflict <- mkRWire; - RWire#(void) wrongSpec_verify_conflict <- mkRWire; - RWire#(void) wrongSpec_wakeBySB_conflict <- mkRWire; + PulseWire wrongSpec_conflict <- mkPulseWire; // make wrongSpec more urgent than firstSt (resolve bsc error) Wire#(Bool) wrongSpec_urgent_firstSt <- mkDWire(True); Map#(Bit#(10),Bit#(6),Int#(3),2) ldKillMap <- mkMapLossy(minBound); @@ -1145,7 +1135,7 @@ module mkSplitLSQ(SplitLSQ); end endrule - rule enqIssueQ(issueLdInfo.wget matches tagged Valid .info); + rule enqIssueQ(issueLdInfo.wget matches tagged Valid .info &&& !wrongSpec_conflict); if(verbose) begin $display("[LSQ - enqIss] ", fshow(info)); end @@ -1187,8 +1177,6 @@ module mkSplitLSQ(SplitLSQ); spec_bits: ld_specBits_enqIss[info.tag] }); ld_inIssueQ_enqIss[info.tag] <= True; - // make conflict with incorrect spec - wrongSpec_enqIss_conflict.wset(?); endrule // Verify SQ entry one by one @@ -1211,7 +1199,8 @@ module mkSplitLSQ(SplitLSQ); // NOTE that when SQ is full and all verified, verifyP will point to a // valid and verified entry rule verifySt(st_valid_verify[st_verifyP_verify] && - !st_verified_verify[st_verifyP_verify]); + !st_verified_verify[st_verifyP_verify] && + !wrongSpec_conflict); StQTag verP = st_verifyP_verify; // check if the entry can be verified. We should not fire this rule if @@ -1261,9 +1250,6 @@ module mkSplitLSQ(SplitLSQ); joinActions(map(setVerified, idxVec)); if(verbose) $display("[LSQ - verifySt] st_verifyP %d", verP); - - // make conflict with incorrect spec - wrongSpec_verify_conflict.wset(?); endrule `ifdef BSIM @@ -1437,10 +1423,7 @@ module mkSplitLSQ(SplitLSQ); endcase); endmethod - method ActionValue#(LSQHitInfo) getHit(LdStQTag t); - // Conflict with wrong spec. This makes cache pipelineResp rule - // conflict with wrong spec, and can help avoid scheduling cycle. - wrongSpec_hit_conflict.wset(?); + method ActionValue#(LSQHitInfo) getHit(LdStQTag t) if (!wrongSpec_conflict); return (case(t) matches tagged Ld .tag: (LSQHitInfo { waitWPResp: ld_waitWPResp_hit[tag], @@ -1465,7 +1448,7 @@ module mkSplitLSQ(SplitLSQ); MemInst mem_inst, Maybe#(PhyDst) dst, SpecBits spec_bits, - Bit#(16) pc_hash) if(ld_can_enq_wire); + Bit#(16) pc_hash) if(ld_can_enq_wire && !wrongSpec_conflict); if(verbose) begin $display("[LSQ - enqLd] enqP %d; ", ld_enqP, "; ", fshow(inst_tag), @@ -1518,14 +1501,12 @@ module mkSplitLSQ(SplitLSQ); ld_olderSt_enq[ld_enqP] <= Invalid; ld_olderStVerified_enq[ld_enqP] <= False; end - // make conflict with incorrect spec - wrongSpec_enq_conflict.wset(?); endmethod method Action enqSt(InstTag inst_tag, MemInst mem_inst, Maybe#(PhyDst) dst, - SpecBits spec_bits) if(st_can_enq_wire); + SpecBits spec_bits) if(st_can_enq_wire && !wrongSpec_conflict); if(verbose) begin $display("[LSQ - enqSt] enqP %d; ", st_enqP, "; ", fshow(inst_tag), @@ -1554,8 +1535,6 @@ module mkSplitLSQ(SplitLSQ); st_verified_enq[st_enqP] <= False; st_specBits_enq[st_enqP] <= spec_bits; st_atCommit_enq[st_enqP] <= False; - // make conflict with incorrect spec - wrongSpec_enq_conflict.wset(?); endmethod method Action updateData(StQTag t, MemTaggedData d); @@ -1570,7 +1549,7 @@ module mkSplitLSQ(SplitLSQ); method ActionValue#(LSQUpdateAddrResult) updateAddr( LdStQTag lsqTag, Maybe#(Trap) fault, Bool allowCap, Addr pa, Bool mmio, ByteOrTagEn shift_be - ); + ) if (!wrongSpec_conflict); // index vec for vector functions Vector#(LdQSize, LdQTag) idxVec = genWith(fromInteger); @@ -1736,9 +1715,6 @@ module mkSplitLSQ(SplitLSQ); end end - // make conflict with incorrect spec - wrongSpec_update_conflict.wset(?); - // return waiting for wp resp bit: for deciding whether the updating Ld // can be issued return LSQUpdateAddrResult { @@ -1753,7 +1729,7 @@ module mkSplitLSQ(SplitLSQ); method ActionValue#(LSQIssueLdResult) issueLd(LdQTag tag, Addr pa, ByteOrTagEn shift_be, - SBSearchRes sbRes); + SBSearchRes sbRes) if (!wrongSpec_conflict); if(verbose) begin $display("[LSQ - issueLd] ", fshow(tag), "; ", fshow(pa), "; ", fshow(shift_be), "; ", fshow(sbRes)); @@ -2022,9 +1998,6 @@ module mkSplitLSQ(SplitLSQ); end `endif - // make conflict with incorrect spec - wrongSpec_issue_conflict.wset(?); - return issRes; endmethod @@ -2041,7 +2014,7 @@ module mkSplitLSQ(SplitLSQ); return issueLdQ.first.data; endmethod - method ActionValue#(LSQRespLdResult) respLd(LdQTag t, MemTaggedData alignedData); + method ActionValue#(LSQRespLdResult) respLd(LdQTag t, MemTaggedData alignedData) if (!wrongSpec_conflict); let res = LSQRespLdResult { wrongPath: False, dst: Invalid, @@ -2090,8 +2063,6 @@ module mkSplitLSQ(SplitLSQ); $display("[LSQ - respLd] ", fshow(t), "; ", fshow(alignedData), "; ", fshow(res)); end - // make conflict with incorrect spec - wrongSpec_respLd_conflict.wset(?); // return return res; endmethod @@ -2116,7 +2087,7 @@ module mkSplitLSQ(SplitLSQ); }; endmethod - method Action deqLd if(deqLdGuard); + method Action deqLd if(deqLdGuard && !wrongSpec_conflict); LdQTag deqP = ld_deqP_deqLd; if(verbose) $display("[LSQ - deqLd] deqP %d", deqP); @@ -2163,9 +2134,6 @@ module mkSplitLSQ(SplitLSQ); endfunction Vector#(LdQSize, LdQTag) idxVec = genWith(fromInteger); joinActions(map(setReady, idxVec)); - - // make conflict with incorrect spec - wrongSpec_deqLd_conflict.wset(?); endmethod method StQDeqEntry firstSt if(deqStGuard && wrongSpec_urgent_firstSt); @@ -2186,7 +2154,7 @@ module mkSplitLSQ(SplitLSQ); }; endmethod - method Action deqSt if(deqStGuard); + method Action deqSt if(deqStGuard && !wrongSpec_conflict); StQTag deqP = st_deqP; if(verbose) $display("[LSQ - deqSt] deqP %d", deqP); @@ -2234,13 +2202,10 @@ module mkSplitLSQ(SplitLSQ); endfunction Vector#(LdQSize, LdQTag) idxVec = genWith(fromInteger); joinActions(map(resetSt, idxVec)); - - // make conflict with incorrect spec - wrongSpec_deqSt_conflict.wset(?); endmethod `ifdef TSO_MM - method Action cacheEvict(LineAddr lineAddr); + method Action cacheEvict(LineAddr lineAddr) if (!wrongSpec_conflict); if(verbose) $display("[LSQ - cacheEvict] ", fshow(lineAddr)); // kill a load if it satisfies the following conditions: // (1) valid @@ -2270,14 +2235,11 @@ module mkSplitLSQ(SplitLSQ); doAssert(!ld_isMMIO_evict[killTag], "cannot kill MMIO"); doAssert(ld_memFunc[killTag] == Ld, "can only kill Ld"); end - - // make conflict with incorrect spec - wrongSpec_cacheEvict_conflict.wset(?); endmethod `else - method Action wakeupLdStalledBySB(SBIndex sbIdx); + method Action wakeupLdStalledBySB(SBIndex sbIdx) if (!wrongSpec_conflict); if(verbose) begin $display("[LSQ - wakeupBySB] ", fshow(sbIdx)); end @@ -2292,8 +2254,6 @@ module mkSplitLSQ(SplitLSQ); endfunction Vector#(LdQSize, LdQTag) idxVec = genWith(fromInteger); joinActions(map(setReady, idxVec)); - // make conflict with incorrect spec - wrongSpec_wakeBySB_conflict.wset(?); endmethod `endif @@ -2438,17 +2398,7 @@ module mkSplitLSQ(SplitLSQ); end // make conflict with others - wrongSpec_hit_conflict.wset(?); - wrongSpec_enqIss_conflict.wset(?); - wrongSpec_enq_conflict.wset(?); - wrongSpec_update_conflict.wset(?); - wrongSpec_issue_conflict.wset(?); - wrongSpec_respLd_conflict.wset(?); - wrongSpec_deqLd_conflict.wset(?); - wrongSpec_deqSt_conflict.wset(?); - wrongSpec_verify_conflict.wset(?); - wrongSpec_cacheEvict_conflict.wset(?); - wrongSpec_wakeBySB_conflict.wset(?); + wrongSpec_conflict.send(); // more urgent than firstSt wrongSpec_urgent_firstSt <= True; endmethod From d1de1ab95296f725b933b241010ae1ebc7321da9 Mon Sep 17 00:00:00 2001 From: Jonathan Woodruff Date: Wed, 12 Jan 2022 13:13:39 +0000 Subject: [PATCH 08/24] Also pause branch execution when there is a pending wrong speculation. --- src_Core/CPU/Core.bsv | 1 + .../procs/RV64G_OOO/AluExePipeline.bsv | 6 ++++-- src_Core/RISCY_OOO/procs/lib/SpecFifo.bsv | 18 +++++++----------- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/src_Core/CPU/Core.bsv b/src_Core/CPU/Core.bsv index 5c29e37..0783e1e 100644 --- a/src_Core/CPU/Core.bsv +++ b/src_Core/CPU/Core.bsv @@ -426,6 +426,7 @@ module mkCore#(CoreId coreId)(Core); ); globalSpecUpdate.incorrectSpec(False, spec_tag, inst_tag, spec_bits); endmethod + method Bool pauseExecute = globalSpecUpdate.pendingIncorrectSpec; method correctSpec = globalSpecUpdate.correctSpec[finishAluCorrectSpecPort(i)].put; method doStats = doStatsReg._read; `ifdef PERFORMANCE_MONITORING diff --git a/src_Core/RISCY_OOO/procs/RV64G_OOO/AluExePipeline.bsv b/src_Core/RISCY_OOO/procs/RV64G_OOO/AluExePipeline.bsv index acb59be..efb03a5 100644 --- a/src_Core/RISCY_OOO/procs/RV64G_OOO/AluExePipeline.bsv +++ b/src_Core/RISCY_OOO/procs/RV64G_OOO/AluExePipeline.bsv @@ -195,6 +195,8 @@ interface AluExeInput; method Action writeRegFile(PhyRIndx dst, CapPipe data); // redirect method Action redirect(CapMem new_pc, SpecTag spec_tag, InstTag inst_tag, SpecBits spec_bits); + // pending invalidation could pause execute/redirections. + method Bool pauseExecute; // spec update method Action correctSpec(SpecTag t); @@ -354,7 +356,7 @@ module mkAluExePipeline#(AluExeInput inIfc)(AluExePipeline); }); endrule - rule doExeAlu; + rule doExeAlu(!inIfc.pauseExecute); regToExeQ.deq; let regToExe = regToExeQ.first; let x = regToExe.data; @@ -417,7 +419,7 @@ module mkAluExePipeline#(AluExeInput inIfc)(AluExePipeline); }); endrule - rule doFinishAlu; + rule doFinishAlu(!inIfc.pauseExecute); exeToFinQ.deq; let exeToFin = exeToFinQ.first; let x = exeToFin.data; diff --git a/src_Core/RISCY_OOO/procs/lib/SpecFifo.bsv b/src_Core/RISCY_OOO/procs/lib/SpecFifo.bsv index 8eb8c0e..b043b71 100644 --- a/src_Core/RISCY_OOO/procs/lib/SpecFifo.bsv +++ b/src_Core/RISCY_OOO/procs/lib/SpecFifo.bsv @@ -25,7 +25,7 @@ import ProcTypes::*; import HasSpecBits::*; import Vector::*; import Ehr::*; -import FIFOF::*; +import DReg::*; import Assert::*; import Types::*; import ConfigReg::*; @@ -223,8 +223,8 @@ module mkSpecFifoCF#( Vector#(size, Reg#(t)) row <- replicateM(mkConfigRegU); Ehr#(3, Vector#(size, SpecBits)) specBits <- mkEhr(?); - FIFOF#(SpecBits) correctSpecF <- mkUGFIFOF; - FIFOF#(IncorrectSpeculation) incorrectSpecF <- mkUGFIFOF; + Reg#(Maybe#(SpecBits)) correctSpec <- mkDReg(Invalid); + Reg#(Maybe#(IncorrectSpeculation)) incorrectSpec <- mkDReg(Invalid); Reg#(idxT) enqP <- mkConfigReg(0); Ehr#(2, idxT) deqP_ehr <- mkEhr(0); @@ -245,17 +245,13 @@ module mkSpecFifoCF#( Vector#(size, SpecBits) newSpecBits = specBits[0]; Vector#(size, Bool) newValid = valid[0]; // Fold in CorrectSpec update: - if (correctSpecF.notEmpty) begin - SpecBits mask = correctSpecF.first(); - correctSpecF.deq(); + if (correctSpec matches tagged Valid .mask) begin // clear spec bits for all entries for (Integer i=0; i Date: Wed, 12 Jan 2022 16:08:33 +0000 Subject: [PATCH 09/24] Timing optimisation for Tournament Predictor. Start global-history-indexed lookups in the previous cycle. This is a general timing optimisation that should be pushed upstream at some point. --- src_Core/RISCY_OOO/procs/lib/Btb.bsv | 5 ++++- src_Core/RISCY_OOO/procs/lib/TourPred.bsv | 17 +++++++++++++---- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src_Core/RISCY_OOO/procs/lib/Btb.bsv b/src_Core/RISCY_OOO/procs/lib/Btb.bsv index a1d9059..2b92d72 100644 --- a/src_Core/RISCY_OOO/procs/lib/Btb.bsv +++ b/src_Core/RISCY_OOO/procs/lib/Btb.bsv @@ -131,7 +131,10 @@ module mkBtbCore(NextAddrPred#(hashSz)) // Start SupSizeX2 BTB lookups, but ensure to lookup in the appropriate // bank for the alignment of each potential branch. for (Integer i = 0; i < valueOf(SupSizeX2); i = i + 1) begin - BtbAddr a = unpack(pack(getBtbAddr(pc)) + fromInteger(i)); + // Only add lower bits for timing. + BtbAddr a = getBtbAddr(pc); + a = unpack({a.tag, {a.index,a.bank} + fromInteger(i)}); + //BtbAddr a = unpack(pack(getBtbAddr(pc)) + fromInteger(i)); fullRecords[a.bank].lookupStart(MapKeyIndex{key: hash(a.tag), index: a.index}); compressedRecords[a.bank].lookupStart(MapKeyIndex{key: hash(a.tag), index: a.index}); end diff --git a/src_Core/RISCY_OOO/procs/lib/TourPred.bsv b/src_Core/RISCY_OOO/procs/lib/TourPred.bsv index d751f0d..7cd680b 100644 --- a/src_Core/RISCY_OOO/procs/lib/TourPred.bsv +++ b/src_Core/RISCY_OOO/procs/lib/TourPred.bsv @@ -114,6 +114,8 @@ module mkTourPred(DirPredictor#(TourTrainInfo)); endfunction TourGlobalHist curGHist = gHistReg.history; // global history: MSB is the latest branch + Reg#(Vector#(SupSize, Bool)) globalTakenVec <- mkRegU; + Reg#(Vector#(SupSize, Bool)) useLocalVec <- mkRegU; Vector#(SupSize, DirPred#(TourTrainInfo)) predIfc; for(Integer i = 0; i < valueof(SupSize); i = i+1) begin @@ -127,12 +129,11 @@ module mkTourPred(DirPredictor#(TourTrainInfo)); // all previous branch in this cycle must be not taken // otherwise this branch should be on wrong path // because all inst in same cycle are fetched consecutively - TourGlobalHist globalHist = curGHist >> predCnt[i]; // get global prediction - Bool globalTaken = isTaken(globalBht.sub(globalHist)); + Bool globalTaken = globalTakenVec[predCnt[i]]; // make choice - Bool useLocal = isTaken(choiceBht.sub(globalHist)); + Bool useLocal = useLocalVec[predCnt[i]]; Bool taken = useLocal ? localTaken : globalTaken; // record prediction @@ -145,7 +146,7 @@ module mkTourPred(DirPredictor#(TourTrainInfo)); return DirPredResult { taken: taken, train: TourTrainInfo { - globalHist: globalHist, + globalHist: curGHist >> predCnt[i], localHist: localHist, globalTaken: globalTaken, localTaken: localTaken @@ -158,6 +159,14 @@ module mkTourPred(DirPredictor#(TourTrainInfo)); (* fire_when_enabled, no_implicit_conditions *) rule canonGlobalHist; gHistReg.addHistory(predRes[valueof(SupSize)], predCnt[valueof(SupSize)]); + // Buffer useLocalVec + // Reproduce next history; this would ideally be done in GlobalBrHistReg to avoid duplicating logic. + TourGlobalHist nHist = truncate({predRes[valueof(SupSize)], curGHist} >> predCnt[valueof(SupSize)]); + function Bool globalTakenLookup (Integer i) = isTaken(globalBht.sub(nHist >> i)); + function Bool useLocalLookup (Integer i) = isTaken(choiceBht.sub(nHist >> i)); + globalTakenVec <= genWith(globalTakenLookup); + useLocalVec <= genWith(useLocalLookup); + // Reset counters and prediction. predRes[valueof(SupSize)] <= 0; predCnt[valueof(SupSize)] <= 0; endrule From 1f8181d5958b07c1f17f326bc14daaac09bc2fe3 Mon Sep 17 00:00:00 2001 From: Jonathan Woodruff Date: Thu, 13 Jan 2022 11:14:20 +0000 Subject: [PATCH 10/24] Move to (recently merged) non-pipelined Divide. --- src_Core/RISCY_OOO/procs/lib/Fpu.bsv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src_Core/RISCY_OOO/procs/lib/Fpu.bsv b/src_Core/RISCY_OOO/procs/lib/Fpu.bsv index b3894c4..d7ff598 100644 --- a/src_Core/RISCY_OOO/procs/lib/Fpu.bsv +++ b/src_Core/RISCY_OOO/procs/lib/Fpu.bsv @@ -95,7 +95,7 @@ module mkDoubleDiv(Server#(Tuple3#(Double, Double, FpuRoundMode), Tuple2#(Double `ifdef USE_XILINX_FPU let fpu <- mkXilinxFpDiv; `else - let int_div <- mkDivider(1); // [sizhuo] size in RVFpu: 2 + let int_div <- mkNonPipelinedDivider(3); // [sizhuo] size in RVFpu: 2 let fpu <- mkFloatingPointDivider(int_div); `endif return fpu; From dac5c9910082c0c3b82edb446bdbcc4b5036e119 Mon Sep 17 00:00:00 2001 From: Jonathan Woodruff Date: Fri, 14 Jan 2022 11:53:40 +0000 Subject: [PATCH 11/24] Perform a number of decode and prediction operations unconditionally on the vector of instructions outside of the decode loop to improve timing. Specifically, the branch direction prediction is lifted out and depends only on whether there is agreement between direction prediction and a branch being predicted by fetch, not on the actual predicted next PC. This appears to keep the global history clean enough to perform on-par with the old design in CoreMark. This is a general timing improvement and should be upstreamed. --- .../RISCY_OOO/procs/RV64G_OOO/FetchStage.bsv | 51 ++++++++++++------- 1 file changed, 33 insertions(+), 18 deletions(-) diff --git a/src_Core/RISCY_OOO/procs/RV64G_OOO/FetchStage.bsv b/src_Core/RISCY_OOO/procs/RV64G_OOO/FetchStage.bsv index 9851594..536a330 100644 --- a/src_Core/RISCY_OOO/procs/RV64G_OOO/FetchStage.bsv +++ b/src_Core/RISCY_OOO/procs/RV64G_OOO/FetchStage.bsv @@ -658,16 +658,44 @@ module mkFetchStage(FetchStage); // Note that only 1 redirection may happen in a cycle Maybe#(IType) redirectInst = Invalid; `endif + // Vector functions to generate all PCs, predicted next PCs, decode all instructions, + // and perform direction prediction for all. Taking these out of the following loop + // makes them unconditional, not depending on previous iterations, improving timing. + function t valid (Maybe#(t) v) = v.Valid; + function CapMem getPc(Integer i) = decompressPc(valid(decodeIn[i]).pc); + Vector#(SupSize, CapMem) pcs = genWith(getPc); + function CapMem getPpc(Integer i) = decompressPc(valid(decodeIn[i]).ppc); + Vector#(SupSize, CapMem) ppcs = genWith(getPpc); + function DecodeResult getDecodeResult(Integer i) = decode(valid(decodeIn[i]).inst, getFlags(pcs[i])==1); + Vector#(SupSize, DecodeResult) decode_results = genWith(getDecodeResult); + function DecodedInst getDInst(Integer i) = decode_results[i].dInst; + Vector#(SupSize, DecodedInst) dInsts = genWith(getDInst); + Vector#(SupSize, DirPredResult#(DirPredTrainInfo)) pred_ress = + replicate(DirPredResult{taken: False, train: ?}); + Bool fetch_branch_misprediction = False; + for (Integer i = 0; i < valueof(SupSize); i=i+1) begin + if(dInsts[i].iType == Br && !fetch_branch_misprediction) begin + pred_ress[i] <- dirPred.pred[i].pred(getAddr(pcs[i])); + fetch_branch_misprediction = (pred_ress[i].taken != valid(decodeIn[i]).pred_jump); + end + end + function Maybe#(CapMem) getBranchNextPc(Integer i) = + decodeBrPred(pcs[i], dInsts[i], pred_ress[i].taken, (valid(decodeIn[i]).inst_kind == Inst_32b)); + Vector#(SupSize, Maybe#(CapMem)) nextPcsBranch = genWith(getBranchNextPc); for (Integer i = 0; i < valueof(SupSize); i=i+1) begin if (decodeIn[i] matches tagged Valid .in) begin let cause = in.cause; - CapMem pc = decompressPc(in.pc); - CapMem ppc = decompressPc(in.ppc); + CapMem pc = pcs[i];//decompressPc(in.pc); + CapMem ppc = ppcs[i];//decompressPc(in.ppc); pcBlocks.rPort[i].remove(in.pc.idx); if (verbose) $display("Decode: %0d in = ", i, fshow (in)); + let decode_result = decode_results[i];//decode(in.inst, getFlags(pc)==1); // Decode 32b inst, or 32b expansion of 16b inst + let dInst = dInsts[i];//decode_result.dInst; + let regs = decode_result.regs; + // do decode and branch prediction // Drop here if does not match the decode_epoch. if (in.decode_epoch == decode_epoch_local && in.mispred_first_half) begin @@ -684,28 +712,15 @@ module mkFetchStage(FetchStage); end else if (in.decode_epoch == decode_epoch_local) begin doAssert(in.main_epoch == f_main_epoch, "main epoch must match"); - let decode_result = decode(in.inst, getFlags(pc)==1); // Decode 32b inst, or 32b expansion of 16b inst - // update cause if decode exception and no earlier (TLB) exception if (!isValid(cause)) begin cause = decode_result.illegalInst ? tagged Valid excIllegalInst : tagged Invalid; end - let dInst = decode_result.dInst; - let regs = decode_result.regs; - DirPredTrainInfo dp_train = ?; // dir pred training bookkeeping - // update predicted next pc if (!isValid(cause)) begin // direction predict - Bool pred_taken = False; - if(dInst.iType == Br) begin - let pred_res <- dirPred.pred[i].pred(getAddr(pc)); - pred_taken = pred_res.taken; - dp_train = pred_res.train; - end - Maybe#(CapMem) nextPc = decodeBrPred(pc, dInst, pred_taken, (in.inst_kind == Inst_32b)); - + Maybe#(CapMem) nextPc = nextPcsBranch[i]; // return address stack link reg is x1 or x5 function Bool linkedR(Maybe#(ArchRIndx) register); Bool res = False; @@ -750,7 +765,7 @@ module mkFetchStage(FetchStage); end if(verbose) begin $display("Branch prediction: ", fshow(dInst.iType), " ; ", fshow(pc), " ; ", - fshow(ppc), " ; ", fshow(pred_taken), " ; ", fshow(nextPc)); + fshow(ppc), " ; ", fshow(nextPc)); end // If we don't have a good guess about where we are going, don't proceed. @@ -783,7 +798,7 @@ module mkFetchStage(FetchStage); `endif ppc: ppc, main_epoch: in.main_epoch, - dpTrain: dp_train, + dpTrain: pred_ress[i].train, inst: in.inst, dInst: dInst, orig_inst: in.orig_inst, From e497091e184bee596b5bb70e471912fc186eb870 Mon Sep 17 00:00:00 2001 From: Jonathan Woodruff Date: Tue, 18 Jan 2022 11:05:09 +0000 Subject: [PATCH 12/24] A timing optimisation for the direction predictor. Add a "nextPc" interface so that lookup can begin in the previous cycle. Remove the "pc" operands in the vector of lookup interfaces, but rather derive the PC from lookup from the first PC being looked up in that cycle. That is, by adding 4*(interface number) to the beginning lookup PC. As this might not actually be the PC of the instruction, pass the index you used with the training info so that you are certain to train with the same index you looked up. This has less than 1% overhead in CoreMark. This is a general improvement for reasonable timing which should be upstreamed. --- .../RISCY_OOO/procs/RV64G_OOO/FetchStage.bsv | 28 +++++++++++++------ src_Core/RISCY_OOO/procs/lib/Bht.bsv | 19 ++++++++----- src_Core/RISCY_OOO/procs/lib/BrPred.bsv | 8 ++++-- src_Core/RISCY_OOO/procs/lib/GSelectPred.bsv | 21 ++++++++++---- src_Core/RISCY_OOO/procs/lib/GSharePred.bsv | 23 ++++++++++----- src_Core/RISCY_OOO/procs/lib/TourPred.bsv | 20 +++++++++---- .../RISCY_OOO/procs/lib/TourPredSecure.bsv | 26 ++++++++++------- 7 files changed, 100 insertions(+), 45 deletions(-) diff --git a/src_Core/RISCY_OOO/procs/RV64G_OOO/FetchStage.bsv b/src_Core/RISCY_OOO/procs/RV64G_OOO/FetchStage.bsv index 536a330..4cb507a 100644 --- a/src_Core/RISCY_OOO/procs/RV64G_OOO/FetchStage.bsv +++ b/src_Core/RISCY_OOO/procs/RV64G_OOO/FetchStage.bsv @@ -358,6 +358,10 @@ module mkFetchStage(FetchStage); Integer pc_fetch3_port = 2; Integer pc_redirect_port = 3; Integer pc_final_port = 4; + // To track the next expected PC in Decode for early lookups for prediction. + Ehr#(TAdd#(SupSize, 2), Addr) decode_pc_reg <- mkEhr(?); + Integer decode_pc_redirect_port = valueOf(SupSize); + Integer decode_pc_final_port = valueOf(SupSize) + 1; // PC compression structure holding an indexed set of PC blocks so that only indexes need be tracked. IndexedMultiset#(PcIdx, PcMSB, SupSizeX2) pcBlocks <- mkIndexedMultisetQueue; @@ -659,7 +663,7 @@ module mkFetchStage(FetchStage); Maybe#(IType) redirectInst = Invalid; `endif // Vector functions to generate all PCs, predicted next PCs, decode all instructions, - // and perform direction prediction for all. Taking these out of the following loop + // and perform direction predictions. Taking these out of the following loop // makes them unconditional, not depending on previous iterations, improving timing. function t valid (Maybe#(t) v) = v.Valid; function CapMem getPc(Integer i) = decompressPc(valid(decodeIn[i]).pc); @@ -675,7 +679,7 @@ module mkFetchStage(FetchStage); Bool fetch_branch_misprediction = False; for (Integer i = 0; i < valueof(SupSize); i=i+1) begin if(dInsts[i].iType == Br && !fetch_branch_misprediction) begin - pred_ress[i] <- dirPred.pred[i].pred(getAddr(pcs[i])); + pred_ress[i] <- dirPred.pred[i].pred; fetch_branch_misprediction = (pred_ress[i].taken != valid(decodeIn[i]).pred_jump); end end @@ -686,14 +690,14 @@ module mkFetchStage(FetchStage); for (Integer i = 0; i < valueof(SupSize); i=i+1) begin if (decodeIn[i] matches tagged Valid .in) begin let cause = in.cause; - CapMem pc = pcs[i];//decompressPc(in.pc); - CapMem ppc = ppcs[i];//decompressPc(in.ppc); + CapMem pc = pcs[i]; + CapMem ppc = ppcs[i]; pcBlocks.rPort[i].remove(in.pc.idx); if (verbose) $display("Decode: %0d in = ", i, fshow (in)); - let decode_result = decode_results[i];//decode(in.inst, getFlags(pc)==1); // Decode 32b inst, or 32b expansion of 16b inst - let dInst = dInsts[i];//decode_result.dInst; + let decode_result = decode_results[i]; // Decode 32b inst, or 32b expansion of 16b inst + let dInst = dInsts[i]; let regs = decode_result.regs; // do decode and branch prediction @@ -792,6 +796,7 @@ module mkFetchStage(FetchStage); `endif end end // if (!isValid(cause)) + decode_pc_reg[i] <= getAddr(ppc); let out = FromFetchStage{pc: pc, `ifdef RVFI_DII dii_pid: in.dii_pid, @@ -823,8 +828,8 @@ module mkFetchStage(FetchStage); end // for (Integer i = 0; i < valueof(SupSize); i=i+1) // update PC and epoch - if(redirectPc matches tagged Valid .nextPc) begin - pc_reg[pc_decode_port] <= nextPc; + if(redirectPc matches tagged Valid .rp) begin + pc_reg[pc_decode_port] <= rp; end `ifdef RVFI_DII doAssert(isValid(redirectPc) == isValid(redirectDiiPid), "PC and DII redirections always happen together"); @@ -850,6 +855,10 @@ module mkFetchStage(FetchStage); `endif endrule + rule reportDecodePc; + dirPred.nextPc(decode_pc_reg[decode_pc_final_port]); + endrule + // train next addr pred: we use a wire to catch outputs of napTrainByDecQ. // This prevents napTrainByDecQ from clogging doDecode rule when // superscalar size is large @@ -917,6 +926,7 @@ module mkFetchStage(FetchStage); dii_pid_reg[pc_redirect_port] <= dii_pid; if (verbose) $display("%t Redirect: dii_pid_reg %d", $time(), dii_pid); `endif + decode_pc_reg[decode_pc_redirect_port] <= getAddr(new_pc); f_main_epoch <= (f_main_epoch == fromInteger(valueOf(NumEpochs)-1)) ? 0 : f_main_epoch + 1; // redirect comes, stop stalling for redirect waitForRedirect[1] <= False; @@ -958,7 +968,7 @@ module mkFetchStage(FetchStage); //end if (iType == Br) begin // Train the direction predictor for all branches - dirPred.update(getAddr(pc), taken, dpTrain, mispred); + dirPred.update(taken, dpTrain, mispred); end // train next addr pred when mispred if(mispred) begin diff --git a/src_Core/RISCY_OOO/procs/lib/Bht.bsv b/src_Core/RISCY_OOO/procs/lib/Bht.bsv index e87cb08..7560fb4 100644 --- a/src_Core/RISCY_OOO/procs/lib/Bht.bsv +++ b/src_Core/RISCY_OOO/procs/lib/Bht.bsv @@ -42,18 +42,21 @@ import BrPred::*; export BhtTrainInfo; export mkBht; - -typedef Bit#(0) BhtTrainInfo; // no training info needs to be remembered +export BhtEntries; +export BhtIndex; // Local BHT Typedefs typedef 128 BhtEntries; typedef Bit#(TLog#(BhtEntries)) BhtIndex; +typedef BhtIndex BhtTrainInfo; + (* synthesize *) module mkBht(DirPredictor#(BhtTrainInfo)); // Read and Write ordering doesn't matter since this is a predictor // mkRegFileWCF is the RegFile version of mkConfigReg RegFile#(BhtIndex, Bit#(2)) hist <- mkRegFileWCF(0,fromInteger(valueOf(BhtEntries)-1)); + Reg#(Addr) pc_reg <- mkRegU; function BhtIndex getIndex(Addr pc); return truncate(pc >> 2); @@ -62,22 +65,24 @@ module mkBht(DirPredictor#(BhtTrainInfo)); Vector#(SupSize, DirPred#(BhtTrainInfo)) predIfc; for(Integer i = 0; i < valueof(SupSize); i = i+1) begin predIfc[i] = (interface DirPred; - method ActionValue#(DirPredResult#(BhtTrainInfo)) pred(Addr pc); - let index = getIndex(pc); + method ActionValue#(DirPredResult#(BhtTrainInfo)) pred; + let index = getIndex(offsetPc(pc_reg, i)); Bit#(2) cnt = hist.sub(index); Bool taken = cnt[1] == 1; return DirPredResult { taken: taken, - train: 0 + train: index }; endmethod endinterface); end + method nextPc = pc_reg._write; + interface pred = predIfc; - method Action update(Addr pc, Bool taken, BhtTrainInfo train, Bool mispred); - let index = getIndex(pc); + method Action update(Bool taken, BhtTrainInfo train, Bool mispred); + let index = train; let current_hist = hist.sub(index); Bit#(2) next_hist; if(taken) begin diff --git a/src_Core/RISCY_OOO/procs/lib/BrPred.bsv b/src_Core/RISCY_OOO/procs/lib/BrPred.bsv index 8d074a4..76f1b8e 100644 --- a/src_Core/RISCY_OOO/procs/lib/BrPred.bsv +++ b/src_Core/RISCY_OOO/procs/lib/BrPred.bsv @@ -67,18 +67,22 @@ endfunction // general types for direction predictor +// Function to offset PC by the probable size of an instruction without a full add delay. +function Addr offsetPc(Addr pc, Integer i) = {truncateLSB(pc), pc[7:0] + (fromInteger(i)*4)}; + typedef struct { Bool taken; trainInfoT train; // info that a branch must keep for future training } DirPredResult#(type trainInfoT) deriving(Bits, Eq, FShow); interface DirPred#(type trainInfoT); - method ActionValue#(DirPredResult#(trainInfoT)) pred(Addr pc); + method ActionValue#(DirPredResult#(trainInfoT)) pred; endinterface interface DirPredictor#(type trainInfoT); + method Action nextPc(Addr nextPc); interface Vector#(SupSize, DirPred#(trainInfoT)) pred; - method Action update(Addr pc, Bool taken, trainInfoT train, Bool mispred); + method Action update(Bool taken, trainInfoT train, Bool mispred); method Action flush; method Bool flush_done; endinterface diff --git a/src_Core/RISCY_OOO/procs/lib/GSelectPred.bsv b/src_Core/RISCY_OOO/procs/lib/GSelectPred.bsv index 41b9f12..adf70ca 100644 --- a/src_Core/RISCY_OOO/procs/lib/GSelectPred.bsv +++ b/src_Core/RISCY_OOO/procs/lib/GSelectPred.bsv @@ -46,6 +46,9 @@ export GSelectGHistSz; export GSelectGHist; export GSelectTrainInfo; export mkGSelectPred; +export PCIndexSz; +export BhtIndexSz; +export BhtIndex; // 1KB gselect predictor @@ -60,6 +63,7 @@ typedef Bit#(BhtIndexSz) BhtIndex; // bookkeeping info a branch should keep for future training typedef struct { GSelectGHist gHist; + BhtIndex index; } GSelectTrainInfo deriving(Bits, Eq, FShow); // global history @@ -83,6 +87,9 @@ module mkGSelectPred(DirPredictor#(GSelectTrainInfo)); Ehr#(TAdd#(1, SupSize), Bit#(TLog#(TAdd#(SupSize, 1)))) predCnt <- mkEhr(0); Ehr#(TAdd#(1, SupSize), Bit#(SupSize)) predRes <- mkEhr(0); + // Lookup PC + Reg#(Addr) pc_reg <- mkRegU; + function BhtIndex getIndex(Addr pc, GSelectGHist gHist); Bit#(PCIndexSz) pcIdx = truncate(pc >> 2); return {gHist, pcIdx}; @@ -106,13 +113,14 @@ module mkGSelectPred(DirPredictor#(GSelectTrainInfo)); Vector#(SupSize, DirPred#(GSelectTrainInfo)) predIfc; for(Integer i = 0; i < valueof(SupSize); i = i+1) begin predIfc[i] = (interface DirPred; - method ActionValue#(DirPredResult#(GSelectTrainInfo)) pred(Addr pc); + method ActionValue#(DirPredResult#(GSelectTrainInfo)) pred; // get the global history // all previous branch in this cycle must be not taken // otherwise this branch should be on wrong path // because all inst in same cycle are fetched consecutively GSelectGHist gHist = curGHist >> predCnt[i]; - Bool taken = isTaken(tab.sub(getIndex(pc, gHist))); + BhtIndex index = getIndex(offsetPc(pc_reg, i), gHist); + Bool taken = isTaken(tab.sub(index)); // record pred result predCnt[i] <= predCnt[i] + 1; @@ -124,7 +132,8 @@ module mkGSelectPred(DirPredictor#(GSelectTrainInfo)); return DirPredResult { taken: taken, train: GSelectTrainInfo { - gHist: gHist + gHist: gHist, + index: index } }; endmethod @@ -138,16 +147,18 @@ module mkGSelectPred(DirPredictor#(GSelectTrainInfo)); predCnt[valueof(SupSize)] <= 0; endrule + method nextPc = pc_reg._write; + interface pred = predIfc; - method Action update(Addr pc, Bool taken, GSelectTrainInfo train, Bool mispred); + method Action update(Bool taken, GSelectTrainInfo train, Bool mispred); // update history if mispred if(mispred) begin GSelectGHist newHist = truncate({pack(taken), train.gHist} >> 1); globalHist.redirect(newHist); end // update sat cnt - let index = getIndex(pc, train.gHist); + let index = train.index; Bit#(2) cnt = tab.sub(index); tab.upd(index, updateCnt(cnt, taken)); endmethod diff --git a/src_Core/RISCY_OOO/procs/lib/GSharePred.bsv b/src_Core/RISCY_OOO/procs/lib/GSharePred.bsv index 395ba46..b95541f 100644 --- a/src_Core/RISCY_OOO/procs/lib/GSharePred.bsv +++ b/src_Core/RISCY_OOO/procs/lib/GSharePred.bsv @@ -46,6 +46,8 @@ export GShareGHistSz; export GShareGHist; export GShareTrainInfo; export mkGSharePred; +export BhtIndexSz; +export BhtIndex; // 16KB gshare predictor (to match BOOM evaluation paper) @@ -60,6 +62,7 @@ typedef Bit#(BhtIndexSz) BhtIndex; // bookkeeping info a branch should keep for future training typedef struct { GShareGHist gHist; + BhtIndex index; } GShareTrainInfo deriving(Bits, Eq, FShow); // global history @@ -83,6 +86,9 @@ module mkGSharePred(DirPredictor#(GShareTrainInfo)); Ehr#(TAdd#(1, SupSize), Bit#(TLog#(TAdd#(SupSize, 1)))) predCnt <- mkEhr(0); Ehr#(TAdd#(1, SupSize), Bit#(SupSize)) predRes <- mkEhr(0); + // Lookup PC + Reg#(Addr) pc_reg <- mkRegU; + function BhtIndex getIndex(Addr pc, GShareGHist gHist); Bit#(PCIndexSz) pcIdx = truncate(pc >> 2); return gHist ^ pcIdx; @@ -106,13 +112,14 @@ module mkGSharePred(DirPredictor#(GShareTrainInfo)); Vector#(SupSize, DirPred#(GShareTrainInfo)) predIfc; for(Integer i = 0; i < valueof(SupSize); i = i+1) begin predIfc[i] = (interface DirPred; - method ActionValue#(DirPredResult#(GShareTrainInfo)) pred(Addr pc); + method ActionValue#(DirPredResult#(GShareTrainInfo)) pred; // get the global history // all previous branch in this cycle must be not taken // otherwise this branch should be on wrong path // because all inst in same cycle are fetched consecutively GShareGHist gHist = curGHist >> predCnt[i]; - Bool taken = isTaken(tab.sub(getIndex(pc, gHist))); + BhtIndex index = getIndex(offsetPc(pc_reg, i), gHist); + Bool taken = isTaken(tab.sub(index)); // record pred result predCnt[i] <= predCnt[i] + 1; @@ -124,7 +131,8 @@ module mkGSharePred(DirPredictor#(GShareTrainInfo)); return DirPredResult { taken: taken, train: GShareTrainInfo { - gHist: gHist + gHist: gHist, + index: index } }; endmethod @@ -138,18 +146,19 @@ module mkGSharePred(DirPredictor#(GShareTrainInfo)); predCnt[valueof(SupSize)] <= 0; endrule + method nextPc = pc_reg._write; + interface pred = predIfc; - method Action update(Addr pc, Bool taken, GShareTrainInfo train, Bool mispred); + method Action update(Bool taken, GShareTrainInfo train, Bool mispred); // update history if mispred if(mispred) begin GShareGHist newHist = truncate({pack(taken), train.gHist} >> 1); globalHist.redirect(newHist); end // update sat cnt - let index = getIndex(pc, train.gHist); - Bit#(2) cnt = tab.sub(index); - tab.upd(index, updateCnt(cnt, taken)); + Bit#(2) cnt = tab.sub(train.index); + tab.upd(train.index, updateCnt(cnt, taken)); endmethod method flush = noAction; diff --git a/src_Core/RISCY_OOO/procs/lib/TourPred.bsv b/src_Core/RISCY_OOO/procs/lib/TourPred.bsv index 7cd680b..ede7bb9 100644 --- a/src_Core/RISCY_OOO/procs/lib/TourPred.bsv +++ b/src_Core/RISCY_OOO/procs/lib/TourPred.bsv @@ -50,6 +50,8 @@ export TourTrainInfo(..); export TourGHistReg(..); export mkTourGHistReg; export mkTourPred; +export PCIndexSz; +export PCIndex; // 4KB tournament predictor @@ -66,6 +68,7 @@ typedef struct { TourLocalHist localHist; Bool globalTaken; Bool localTaken; + PCIndex pcIndex; } TourTrainInfo deriving(Bits, Eq, FShow); // global history reg @@ -90,6 +93,9 @@ module mkTourPred(DirPredictor#(TourTrainInfo)); // choice sat counters: large (taken) -- use local, small (not taken) -- use global RegFile#(TourGlobalHist, Bit#(2)) choiceBht <- mkRegFileWCF(0, maxBound); + // Lookup PC + Reg#(Addr) pc_reg <- mkRegU; + // EHR to record predict results in this cycle Ehr#(TAdd#(1, SupSize), SupCnt) predCnt <- mkEhr(0); Ehr#(TAdd#(1, SupSize), Bit#(SupSize)) predRes <- mkEhr(0); @@ -120,9 +126,10 @@ module mkTourPred(DirPredictor#(TourTrainInfo)); Vector#(SupSize, DirPred#(TourTrainInfo)) predIfc; for(Integer i = 0; i < valueof(SupSize); i = i+1) begin predIfc[i] = (interface DirPred; - method ActionValue#(DirPredResult#(TourTrainInfo)) pred(Addr pc); + method ActionValue#(DirPredResult#(TourTrainInfo)) pred; + PCIndex pcIndex = getPCIndex(offsetPc(pc_reg, i)); // get local history & prediction - TourLocalHist localHist = localHistTab.sub(getPCIndex(pc)); + TourLocalHist localHist = localHistTab.sub(pcIndex); Bool localTaken = isTaken(localBht.sub(localHist)); // get the global history @@ -149,7 +156,8 @@ module mkTourPred(DirPredictor#(TourTrainInfo)); globalHist: curGHist >> predCnt[i], localHist: localHist, globalTaken: globalTaken, - localTaken: localTaken + localTaken: localTaken, + pcIndex: pcIndex } }; endmethod @@ -171,16 +179,18 @@ module mkTourPred(DirPredictor#(TourTrainInfo)); predCnt[valueof(SupSize)] <= 0; endrule + method nextPc = pc_reg._write; + interface pred = predIfc; - method Action update(Addr pc, Bool taken, TourTrainInfo train, Bool mispred); + method Action update(Bool taken, TourTrainInfo train, Bool mispred); // update history if mispred if(mispred) begin TourGlobalHist newHist = truncateLSB({pack(taken), train.globalHist}); gHistReg.redirect(newHist); end // update local history (assume only 1 branch for an PC in flight) - localHistTab.upd(getPCIndex(pc), truncateLSB({pack(taken), train.localHist})); + localHistTab.upd(train.pcIndex, truncateLSB({pack(taken), train.localHist})); // update local sat cnt let localCnt = localBht.sub(train.localHist); localBht.upd(train.localHist, updateCnt(localCnt, taken)); diff --git a/src_Core/RISCY_OOO/procs/lib/TourPredSecure.bsv b/src_Core/RISCY_OOO/procs/lib/TourPredSecure.bsv index 747b531..d97b46f 100644 --- a/src_Core/RISCY_OOO/procs/lib/TourPredSecure.bsv +++ b/src_Core/RISCY_OOO/procs/lib/TourPredSecure.bsv @@ -73,7 +73,6 @@ typedef TExp#(LgGlobalVecSz) GlobalVecSz; typedef Bit#(GlobalVecSz) GlobalVecSelect; typedef struct { - Addr pc; Bool taken; TourTrainInfo train; Bool mispred; @@ -93,6 +92,9 @@ module mkTourPredSecure(DirPredictor#(TourTrainInfo)); // choice sat counters: large (taken) -- use local, small (not taken) -- use global RegFile#(TabIndex, Vector#(GlobalVecSz, Bit#(2))) choiceBht <- mkRegFileWCF(0, maxBound); + // Lookup PC + Reg#(Addr) pc_reg <- mkRegU; + // EHR to record predict results in this cycle Ehr#(TAdd#(1, SupSize), SupCnt) predCnt <- mkEhr(0); Ehr#(TAdd#(1, SupSize), Bit#(SupSize)) predRes <- mkEhr(0); @@ -103,8 +105,9 @@ module mkTourPredSecure(DirPredictor#(TourTrainInfo)); Reg#(Bool) flushDone <- mkReg(True); Reg#(TabIndex) flushIndex <- mkReg(0); - function Tuple2#(TabIndex, LocalHistVecSelect) getPCIndex(Addr pc); - PCIndex pcIdx = truncate(pc >> 2); + function PCIndex getPCIndex(Addr pc) = truncate(pc >> 2); + + function Tuple2#(TabIndex, LocalHistVecSelect) getPCIndices(PCIndex pcIdx); TabIndex tabIdx = truncateLSB(pcIdx); LocalHistVecSelect sel = truncate(pcIdx); return tuple2(tabIdx, sel); @@ -142,9 +145,10 @@ module mkTourPredSecure(DirPredictor#(TourTrainInfo)); Vector#(SupSize, DirPred#(TourTrainInfo)) predIfc; for(Integer i = 0; i < valueof(SupSize); i = i+1) begin predIfc[i] = (interface DirPred; - method ActionValue#(DirPredResult#(TourTrainInfo)) pred(Addr pc); + method ActionValue#(DirPredResult#(TourTrainInfo)) pred; // get local history - let {localHistTabIdx, localHistVecSel} = getPCIndex(pc); + PCIndex pcIndex = getPCIndex(offsetPc(pc_reg, i)); + let {localHistTabIdx, localHistVecSel} = getPCIndices(pcIndex); Vector#(LocalHistVecSz, TourLocalHist) localHistVec = localHistTab.sub(localHistTabIdx); TourLocalHist localHist = localHistVec[localHistVecSel]; // get local prediction @@ -180,7 +184,8 @@ module mkTourPredSecure(DirPredictor#(TourTrainInfo)); globalHist: globalHist, localHist: localHist, globalTaken: globalTaken, - localTaken: localTaken + localTaken: localTaken, + pcIndex: pcIndex } }; endmethod @@ -197,7 +202,6 @@ module mkTourPredSecure(DirPredictor#(TourTrainInfo)); // no flush, accept update (* fire_when_enabled, no_implicit_conditions *) rule canonUpdate(flushDone &&& updateEn.wget matches tagged Valid .upd); - let pc = upd.pc; let taken = upd.taken; let train = upd.train; let mispred = upd.mispred; @@ -209,7 +213,7 @@ module mkTourPredSecure(DirPredictor#(TourTrainInfo)); end // update local history (assume only 1 branch for an PC in flight) - let {localHistTabIdx, localHistVecSel} = getPCIndex(pc); + let {localHistTabIdx, localHistVecSel} = getPCIndices(train.pcIndex); Vector#(LocalHistVecSz, TourLocalHist) localHistVec = localHistTab.sub(localHistTabIdx); localHistVec[localHistVecSel] = truncateLSB({pack(taken), train.localHist}); localHistTab.upd(localHistTabIdx, localHistVec); @@ -251,10 +255,12 @@ module mkTourPredSecure(DirPredictor#(TourTrainInfo)); end endrule + method nextPc = pc_reg._write; + interface pred = predIfc; - method Action update(Addr pc, Bool taken, TourTrainInfo train, Bool mispred); - updateEn.wset(TourUpdate {pc: pc, taken: taken, train: train, mispred: mispred}); + method Action update(Bool taken, TourTrainInfo train, Bool mispred); + updateEn.wset(TourUpdate {taken: taken, train: train, mispred: mispred}); endmethod method Action flush if(flushDone); From ef6945cc07d16fa78c3458c6f7303821ee382bd0 Mon Sep 17 00:00:00 2001 From: Jonathan Woodruff Date: Wed, 19 Jan 2022 15:28:45 +0000 Subject: [PATCH 13/24] More readable version of previous optimisation, with input from Alexandre. --- .../RISCY_OOO/procs/RV64G_OOO/FetchStage.bsv | 56 ++++++++++--------- 1 file changed, 30 insertions(+), 26 deletions(-) diff --git a/src_Core/RISCY_OOO/procs/RV64G_OOO/FetchStage.bsv b/src_Core/RISCY_OOO/procs/RV64G_OOO/FetchStage.bsv index 4cb507a..c2b9a2b 100644 --- a/src_Core/RISCY_OOO/procs/RV64G_OOO/FetchStage.bsv +++ b/src_Core/RISCY_OOO/procs/RV64G_OOO/FetchStage.bsv @@ -235,6 +235,14 @@ typedef struct { Bool mispred_first_half; } InstrFromFetch3 deriving(Bits, Eq, FShow); +typedef struct { + CapMem pc; + CapMem ppc; + DecodeResult result; + DirPredResult#(DirPredTrainInfo) dir_pred; + Maybe#(CapMem) dir_ppc; +} PreDecode deriving(Bits, Eq, FShow); + function InstrFromFetch3 fetch3_2_instC(Fetch3ToDecode in, Instruction inst, Bit#(32) orig_inst) = InstrFromFetch3 { pc: in.pc, @@ -662,42 +670,38 @@ module mkFetchStage(FetchStage); // Note that only 1 redirection may happen in a cycle Maybe#(IType) redirectInst = Invalid; `endif - // Vector functions to generate all PCs, predicted next PCs, decode all instructions, - // and perform direction predictions. Taking these out of the following loop - // makes them unconditional, not depending on previous iterations, improving timing. + // A loop to prepare values for the main decode loop. + // These allow us to perform direction prediction with minimal dependencies + // between instructions to avoid a critical path. function t valid (Maybe#(t) v) = v.Valid; - function CapMem getPc(Integer i) = decompressPc(valid(decodeIn[i]).pc); - Vector#(SupSize, CapMem) pcs = genWith(getPc); - function CapMem getPpc(Integer i) = decompressPc(valid(decodeIn[i]).ppc); - Vector#(SupSize, CapMem) ppcs = genWith(getPpc); - function DecodeResult getDecodeResult(Integer i) = decode(valid(decodeIn[i]).inst, getFlags(pcs[i])==1); - Vector#(SupSize, DecodeResult) decode_results = genWith(getDecodeResult); - function DecodedInst getDInst(Integer i) = decode_results[i].dInst; - Vector#(SupSize, DecodedInst) dInsts = genWith(getDInst); - Vector#(SupSize, DirPredResult#(DirPredTrainInfo)) pred_ress = - replicate(DirPredResult{taken: False, train: ?}); + Vector#(SupSize, PreDecode) pd = ?; Bool fetch_branch_misprediction = False; for (Integer i = 0; i < valueof(SupSize); i=i+1) begin - if(dInsts[i].iType == Br && !fetch_branch_misprediction) begin - pred_ress[i] <- dirPred.pred[i].pred; - fetch_branch_misprediction = (pred_ress[i].taken != valid(decodeIn[i]).pred_jump); - end + pd[i].pc = decompressPc(valid(decodeIn[i]).pc); + pd[i].ppc = decompressPc(valid(decodeIn[i]).ppc); + pd[i].result = decode(valid(decodeIn[i]).inst, getFlags(pd[i].pc)==1); + // Estimate when we won't use later instructions in the bundle to avoid + // poluting the global history. Depending on the full "local_epoch" logic + // in the main loop is too slow timing-wise. + if(pd[i].result.dInst.iType == Br && !fetch_branch_misprediction) begin + pd[i].dir_pred <- dirPred.pred[i].pred; + fetch_branch_misprediction = (pd[i].dir_pred.taken != valid(decodeIn[i]).pred_jump); + end + pd[i].dir_ppc = decodeBrPred(pd[i].pc, pd[i].result.dInst, pd[i].dir_pred.taken, (valid(decodeIn[i]).inst_kind == Inst_32b)); end - function Maybe#(CapMem) getBranchNextPc(Integer i) = - decodeBrPred(pcs[i], dInsts[i], pred_ress[i].taken, (valid(decodeIn[i]).inst_kind == Inst_32b)); - Vector#(SupSize, Maybe#(CapMem)) nextPcsBranch = genWith(getBranchNextPc); for (Integer i = 0; i < valueof(SupSize); i=i+1) begin if (decodeIn[i] matches tagged Valid .in) begin let cause = in.cause; - CapMem pc = pcs[i]; - CapMem ppc = ppcs[i]; + PreDecode p = pd[i]; + CapMem pc = p.pc; + CapMem ppc = p.ppc; pcBlocks.rPort[i].remove(in.pc.idx); if (verbose) $display("Decode: %0d in = ", i, fshow (in)); - let decode_result = decode_results[i]; // Decode 32b inst, or 32b expansion of 16b inst - let dInst = dInsts[i]; + let decode_result = p.result; // Decode 32b inst, or 32b expansion of 16b inst + let dInst = decode_result.dInst; let regs = decode_result.regs; // do decode and branch prediction @@ -724,7 +728,7 @@ module mkFetchStage(FetchStage); // update predicted next pc if (!isValid(cause)) begin // direction predict - Maybe#(CapMem) nextPc = nextPcsBranch[i]; + Maybe#(CapMem) nextPc = p.dir_ppc; // return address stack link reg is x1 or x5 function Bool linkedR(Maybe#(ArchRIndx) register); Bool res = False; @@ -803,7 +807,7 @@ module mkFetchStage(FetchStage); `endif ppc: ppc, main_epoch: in.main_epoch, - dpTrain: pred_ress[i].train, + dpTrain: p.dir_pred.train, inst: in.inst, dInst: dInst, orig_inst: in.orig_inst, From 678c5bcf4992d4e90bd4e04580543175dddf2e29 Mon Sep 17 00:00:00 2001 From: Jonathan Woodruff Date: Thu, 20 Jan 2022 17:35:16 +0000 Subject: [PATCH 14/24] Fix RVFI_DII which had inadvertently been broken. --- src_Core/RISCY_OOO/procs/RV64G_OOO/CommitStage.bsv | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src_Core/RISCY_OOO/procs/RV64G_OOO/CommitStage.bsv b/src_Core/RISCY_OOO/procs/RV64G_OOO/CommitStage.bsv index faaa514..4e743fc 100644 --- a/src_Core/RISCY_OOO/procs/RV64G_OOO/CommitStage.bsv +++ b/src_Core/RISCY_OOO/procs/RV64G_OOO/CommitStage.bsv @@ -957,7 +957,7 @@ module mkCommitStage#(CommitInput inIfc)(CommitStage); end redirectQ.enq(RedirectInfo{trap_pc: next_pc `ifdef RVFI_DII - , x.dii_pid + (is_16b_inst(x.orig_inst) ? 1 : 2) + , dii_pid: x.dii_pid + (is_16b_inst(x.orig_inst) ? 1 : 2) `endif }); @@ -1358,7 +1358,7 @@ module mkCommitStage#(CommitInput inIfc)(CommitStage); RedirectInfo ri <- toGet(redirectQ).get; inIfc.redirectPc(ri.trap_pc `ifdef RVFI_DII - , ri.rii_pid + , ri.dii_pid `endif ); endrule From 6ea5d34d062adcf59255495207daed8c027dba8d Mon Sep 17 00:00:00 2001 From: Jonathan Woodruff Date: Tue, 25 Jan 2022 17:05:34 +0000 Subject: [PATCH 15/24] Bump Bluestuff. --- libs/BlueStuff | 2 +- src_Core/Core/CoreW.bsv | 7 ++++--- src_Testbench/SoC/SoC_Top.bsv | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/libs/BlueStuff b/libs/BlueStuff index df2529d..a6e2273 160000 --- a/libs/BlueStuff +++ b/libs/BlueStuff @@ -1 +1 @@ -Subproject commit df2529dd4298fe77cfba638001f4a960f82fbe02 +Subproject commit a6e2273920cadf21c54c16865d00dbd345b59b13 diff --git a/src_Core/Core/CoreW.bsv b/src_Core/Core/CoreW.bsv index ba38045..9d058d6 100644 --- a/src_Core/Core/CoreW.bsv +++ b/src_Core/Core/CoreW.bsv @@ -59,6 +59,7 @@ import Cur_Cycle :: *; import GetPut_Aux :: *; import Routable :: *; import AXI4 :: *; +import AXI4_Utils :: *; import TagControllerAXI :: *; import CacheCore :: *; @@ -171,7 +172,7 @@ module mkCoreW #(Reset dm_power_on_reset) Proc_IFC proc <- mkProc (reset_by all_harts_reset); // handle uncached interface - let proc_uncached = extendIDFields (zeroMasterUserFields (proc.master1), 0); + let proc_uncached = prepend_AXI4_Master_id (0, zero_AXI4_Master_user (proc.master1)); // Bridge for uncached expernal bus transactions. let uncached_mem_shim <- mkAXI4ShimFF(reset_by all_harts_reset); @@ -388,7 +389,7 @@ module mkCoreW #(Reset dm_power_on_reset) //let slave_vector = newVector; slave_vector[default_slave_num] = uncached_mem_shim.slave; slave_vector[llc_slave_num] = proc.debug_module_mem_server; - slave_vector[plic_slave_num] = zeroSlaveUserFields (plic.axi4_slave); + slave_vector[plic_slave_num] = zero_AXI4_Slave_user (plic.axi4_slave); function Vector#(Num_Slaves_2x3, Bool) route_2x3 (Bit#(Wd_Addr) addr); Vector#(Num_Slaves_2x3, Bool) res = replicate(False); @@ -459,7 +460,7 @@ module mkCoreW #(Reset dm_power_on_reset) interface cpu_imem_master = tagController.master; // Uncached master to Fabric master interface - interface cpu_dmem_master = extendIDFields(zeroMasterUserFields(uncached_mem_shim.master), 0); + interface cpu_dmem_master = prepend_AXI4_Master_id(0, zero_AXI4_Master_user(uncached_mem_shim.master)); // ---------------------------------------------------------------- // External interrupt sources diff --git a/src_Testbench/SoC/SoC_Top.bsv b/src_Testbench/SoC/SoC_Top.bsv index 3fbd43f..4bb7e24 100644 --- a/src_Testbench/SoC/SoC_Top.bsv +++ b/src_Testbench/SoC/SoC_Top.bsv @@ -204,7 +204,7 @@ module mkSoC_Top #(Reset dm_power_on_reset) route_vector[mem0_controller_slave_num] = soc_map.m_mem0_controller_addr_range; // Fabric to UART0 - slave_vector[uart0_slave_num] = zeroSlaveUserFields(uart0.slave); + slave_vector[uart0_slave_num] = zero_AXI4_Slave_user(uart0.slave); route_vector[uart0_slave_num] = soc_map.m_uart0_addr_range; `ifdef INCLUDE_ACCEL0 From 4cfd652b0d5eb3fa3a607e1307faaa5fca314aac Mon Sep 17 00:00:00 2001 From: Jonathan Woodruff Date: Tue, 25 Jan 2022 20:42:16 +0000 Subject: [PATCH 16/24] Use mkRegOR in register file. --- src_Core/RISCY_OOO/procs/lib/PhysRFile.bsv | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src_Core/RISCY_OOO/procs/lib/PhysRFile.bsv b/src_Core/RISCY_OOO/procs/lib/PhysRFile.bsv index 90c5d2d..d1e1f7a 100644 --- a/src_Core/RISCY_OOO/procs/lib/PhysRFile.bsv +++ b/src_Core/RISCY_OOO/procs/lib/PhysRFile.bsv @@ -47,6 +47,7 @@ import ProcTypes::*; import Vector::*; import Ehr::*; import ConfigReg::*; +import SpecialRegs::*; interface RFileWr#(type d); method Action wr( PhyRIndx rindx, d data ); @@ -73,7 +74,9 @@ module mkRFile#(d defaultRegisterValue, Bool lazy)( RFile#(wrNum, rdNum, d) ) pr // phy reg init val must be 0: because x0 is renamed to phy reg 0, // which must be 0 at all time - Vector#(NumPhyReg, Ehr#(ehrPortNum, d)) rfile <- replicateM(mkEhr(defaultRegisterValue)); + // Using a mkRegOR here assumes there will be a single write per register per cycle. + // As each register is allocated to a single instruction which will execute once, this should always be true. + Vector#(NumPhyReg, Vector#(ehrPortNum, Reg#(d))) rfile <- replicateM(mkRegOR(defaultRegisterValue)); Vector#(NumPhyReg, d) rdData = ?; if(lazy) begin From 1ee961bb83241a5ac67714364d5058366805a7c4 Mon Sep 17 00:00:00 2001 From: Jonathan Woodruff Date: Thu, 27 Jan 2022 17:13:11 +0000 Subject: [PATCH 17/24] Simplify timing optimisation by including in the root of the original loop. --- .../RISCY_OOO/procs/RV64G_OOO/FetchStage.bsv | 49 +++++-------------- 1 file changed, 13 insertions(+), 36 deletions(-) diff --git a/src_Core/RISCY_OOO/procs/RV64G_OOO/FetchStage.bsv b/src_Core/RISCY_OOO/procs/RV64G_OOO/FetchStage.bsv index c2b9a2b..17e5e4e 100644 --- a/src_Core/RISCY_OOO/procs/RV64G_OOO/FetchStage.bsv +++ b/src_Core/RISCY_OOO/procs/RV64G_OOO/FetchStage.bsv @@ -235,14 +235,6 @@ typedef struct { Bool mispred_first_half; } InstrFromFetch3 deriving(Bits, Eq, FShow); -typedef struct { - CapMem pc; - CapMem ppc; - DecodeResult result; - DirPredResult#(DirPredTrainInfo) dir_pred; - Maybe#(CapMem) dir_ppc; -} PreDecode deriving(Bits, Eq, FShow); - function InstrFromFetch3 fetch3_2_instC(Fetch3ToDecode in, Instruction inst, Bit#(32) orig_inst) = InstrFromFetch3 { pc: in.pc, @@ -670,40 +662,25 @@ module mkFetchStage(FetchStage); // Note that only 1 redirection may happen in a cycle Maybe#(IType) redirectInst = Invalid; `endif - // A loop to prepare values for the main decode loop. - // These allow us to perform direction prediction with minimal dependencies - // between instructions to avoid a critical path. - function t valid (Maybe#(t) v) = v.Valid; - Vector#(SupSize, PreDecode) pd = ?; - Bool fetch_branch_misprediction = False; + Bool likely_epoch_change = False; for (Integer i = 0; i < valueof(SupSize); i=i+1) begin - pd[i].pc = decompressPc(valid(decodeIn[i]).pc); - pd[i].ppc = decompressPc(valid(decodeIn[i]).ppc); - pd[i].result = decode(valid(decodeIn[i]).inst, getFlags(pd[i].pc)==1); - // Estimate when we won't use later instructions in the bundle to avoid - // poluting the global history. Depending on the full "local_epoch" logic - // in the main loop is too slow timing-wise. - if(pd[i].result.dInst.iType == Br && !fetch_branch_misprediction) begin - pd[i].dir_pred <- dirPred.pred[i].pred; - fetch_branch_misprediction = (pd[i].dir_pred.taken != valid(decodeIn[i]).pred_jump); + CapMem pc = decompressPc(validValue(decodeIn[i]).pc); + CapMem ppc = decompressPc(validValue(decodeIn[i]).ppc); + let decode_result = decode(validValue(decodeIn[i]).inst, getFlags(pc)==1); // Decode 32b inst, or 32b expansion of 16b inst + let dInst = decode_result.dInst; + let regs = decode_result.regs; + DirPredResult#(DirPredTrainInfo) dir_pred = DirPredResult{taken: False, train: ?}; + if(decode_result.dInst.iType == Br && !likely_epoch_change) begin + dir_pred <- dirPred.pred[i].pred; + likely_epoch_change = (dir_pred.taken != validValue(decodeIn[i]).pred_jump); end - pd[i].dir_ppc = decodeBrPred(pd[i].pc, pd[i].result.dInst, pd[i].dir_pred.taken, (valid(decodeIn[i]).inst_kind == Inst_32b)); - end - - for (Integer i = 0; i < valueof(SupSize); i=i+1) begin + Maybe#(CapMem) dir_ppc = decodeBrPred(pc, decode_result.dInst, dir_pred.taken, (validValue(decodeIn[i]).inst_kind == Inst_32b)); if (decodeIn[i] matches tagged Valid .in) begin let cause = in.cause; - PreDecode p = pd[i]; - CapMem pc = p.pc; - CapMem ppc = p.ppc; pcBlocks.rPort[i].remove(in.pc.idx); if (verbose) $display("Decode: %0d in = ", i, fshow (in)); - let decode_result = p.result; // Decode 32b inst, or 32b expansion of 16b inst - let dInst = decode_result.dInst; - let regs = decode_result.regs; - // do decode and branch prediction // Drop here if does not match the decode_epoch. if (in.decode_epoch == decode_epoch_local && in.mispred_first_half) begin @@ -728,7 +705,7 @@ module mkFetchStage(FetchStage); // update predicted next pc if (!isValid(cause)) begin // direction predict - Maybe#(CapMem) nextPc = p.dir_ppc; + Maybe#(CapMem) nextPc = dir_ppc; // return address stack link reg is x1 or x5 function Bool linkedR(Maybe#(ArchRIndx) register); Bool res = False; @@ -807,7 +784,7 @@ module mkFetchStage(FetchStage); `endif ppc: ppc, main_epoch: in.main_epoch, - dpTrain: p.dir_pred.train, + dpTrain: dir_pred.train, inst: in.inst, dInst: dInst, orig_inst: in.orig_inst, From 3543056f82954b6b5c204badf7994ddaed97b03e Mon Sep 17 00:00:00 2001 From: gameboo Date: Fri, 28 Jan 2022 12:13:26 +0000 Subject: [PATCH 18/24] Added a CORE_MINI configuration (with @jdw57) --- .../RISCY_OOO/procs/RV64G_OOO/ProcConfig.bsv | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src_Core/RISCY_OOO/procs/RV64G_OOO/ProcConfig.bsv b/src_Core/RISCY_OOO/procs/RV64G_OOO/ProcConfig.bsv index 871a586..d84e1ae 100644 --- a/src_Core/RISCY_OOO/procs/RV64G_OOO/ProcConfig.bsv +++ b/src_Core/RISCY_OOO/procs/RV64G_OOO/ProcConfig.bsv @@ -167,6 +167,30 @@ // ==== CORE SIZE ==== // +`ifdef CORE_MINI + + // superscalar + `define sizeSup 2 + + // ROB + `define ROB_SIZE 32 + + // speculation + `define NUM_EPOCHS 4 + `define NUM_SPEC_TAGS 4 + + // LSQ + `define LDQ_SIZE 8 + `define STQ_SIZE 4 + `define SB_SIZE 2 + + // reservation station sizes + `define RS_ALU_SIZE 8 + `define RS_MEM_SIZE 4 + `define RS_FPUMULDIV_SIZE 4 + +`endif + `ifdef CORE_TINY // superscalar From c1c4a46b1ec467a1d88800811121e296a6f9f693 Mon Sep 17 00:00:00 2001 From: Jonathan Woodruff Date: Thu, 9 Dec 2021 12:27:57 +0000 Subject: [PATCH 19/24] Simpelest strategy for making wrongSpec not conflict with loads of rules. This triggers some assertions in the ROB what should be dealt with properly if this is going to work. --- .../RISCY_OOO/procs/lib/GlobalSpecUpdate.bsv | 44 ++++++++++++------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/src_Core/RISCY_OOO/procs/lib/GlobalSpecUpdate.bsv b/src_Core/RISCY_OOO/procs/lib/GlobalSpecUpdate.bsv index f17bb98..dbdf7c1 100644 --- a/src_Core/RISCY_OOO/procs/lib/GlobalSpecUpdate.bsv +++ b/src_Core/RISCY_OOO/procs/lib/GlobalSpecUpdate.bsv @@ -1,6 +1,6 @@ // Copyright (c) 2017 Massachusetts Institute of Technology -// +// // Permission is hereby granted, free of charge, to any person // obtaining a copy of this software and associated documentation // files (the "Software"), to deal in the Software without @@ -8,10 +8,10 @@ // modify, merge, publish, distribute, sublicense, and/or sell copies // of the Software, and to permit persons to whom the Software is // furnished to do so, subject to the following conditions: -// +// // The above copyright notice and this permission notice shall be // included in all copies or substantial portions of the Software. -// +// // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, // EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND @@ -26,6 +26,13 @@ import HasSpecBits::*; import GetPut::*; import Vector::*; import ReorderBuffer::*; +import FIFO::*; + +typedef struct { + Bool kill_all; + SpecTag spec_tag; + InstTag inst_tag; +} IncorrectSpec deriving(Bits, Eq, FShow); interface GlobalSpecUpdate#(numeric type correctSpecPortNum, numeric type conflictWrongSpecPortNum); interface Vector#(correctSpecPortNum, Put#(SpecTag)) correctSpec; @@ -50,6 +57,9 @@ module mkGlobalSpecUpdate#( Vector#(correctSpecPortNum, RWire#(void)) spec_conflict <- replicateM(mkRWire); // let the caller of conflictWrongSpec to be conflict with wrong spec Vector#(conflictWrongSpecPortNum, RWire#(void)) wrongSpec_conflict <- replicateM(mkRWire); + // must be a single-element fifo to ensure all pushing rules cannot fire while we are waiting + // to kill. + FIFO#(IncorrectSpec) incorrectSpec_ff <- mkFIFO1; (* fire_when_enabled, no_implicit_conditions *) rule canon_correct_spec; @@ -63,6 +73,20 @@ module mkGlobalSpecUpdate#( rob.correctSpeculation(mask); endrule + rule do_incorrect_spec; + IncorrectSpec x <- toGet(incorrectSpec_ff).get; + ifc.incorrectSpeculation(x.kill_all, x.spec_tag); + rob.incorrectSpeculation(x.kill_all, x.spec_tag, x.inst_tag); + // conflict with correct spec + for(Integer i = 0; i < valueof(correctSpecPortNum); i = i+1) begin + spec_conflict[i].wset(?); + end + // conflict with the caller of conflictWrongSpec + for(Integer i = 0; i < valueof(conflictWrongSpecPortNum); i = i+1) begin + wrongSpec_conflict[i].wset(?); + end + endrule + Vector#(correctSpecPortNum, Put#(SpecTag)) correctVec = ?; for(Integer i = 0; i < valueof(correctSpecPortNum); i = i+1) begin correctVec[i] = (interface Put; @@ -85,18 +109,8 @@ module mkGlobalSpecUpdate#( interface correctSpec = correctVec; - method Action incorrectSpec(Bool kill_all, SpecTag spec_tag, InstTag inst_tag); - ifc.incorrectSpeculation(kill_all, spec_tag); - rob.incorrectSpeculation(kill_all, spec_tag, inst_tag); - // conflict with correct spec - for(Integer i = 0; i < valueof(correctSpecPortNum); i = i+1) begin - spec_conflict[i].wset(?); - end - // conflict with the caller of conflictWrongSpec - for(Integer i = 0; i < valueof(conflictWrongSpecPortNum); i = i+1) begin - wrongSpec_conflict[i].wset(?); - end - endmethod + method Action incorrectSpec(Bool kill_all, SpecTag spec_tag, InstTag inst_tag) + = incorrectSpec_ff.enq(IncorrectSpec{kill_all: kill_all, spec_tag: spec_tag, inst_tag: inst_tag}); interface conflictWrongSpec = conflictWrongVec; endmodule From 4fbb7b56480a715ea591d9c308d4f2032171c790 Mon Sep 17 00:00:00 2001 From: Jonathan Woodruff Date: Tue, 11 Jan 2022 00:06:49 +0000 Subject: [PATCH 20/24] A version that seems to actually run. The buffer in GlobalSpecUpdate has to be a SpecFifo. --- src_Core/CPU/Core.bsv | 9 ++- .../procs/RV64G_OOO/AluExePipeline.bsv | 6 +- .../RISCY_OOO/procs/RV64G_OOO/CommitStage.bsv | 13 +-- .../RISCY_OOO/procs/lib/GlobalSpecUpdate.bsv | 37 +++++---- .../RISCY_OOO/procs/lib/ReorderBuffer.bsv | 6 +- src_Core/RISCY_OOO/procs/lib/SpecFifo.bsv | 5 ++ src_Core/RISCY_OOO/procs/lib/SplitLSQ.bsv | 80 ++++--------------- 7 files changed, 65 insertions(+), 91 deletions(-) diff --git a/src_Core/CPU/Core.bsv b/src_Core/CPU/Core.bsv index 7172cf9..5c29e37 100644 --- a/src_Core/CPU/Core.bsv +++ b/src_Core/CPU/Core.bsv @@ -223,6 +223,7 @@ interface CoreFixPoint; interface MemExePipeline memExeIfc; method Action killAll; // kill everything: used by commit stage interface Reg#(Bool) doStatsIfc; + method Bool pendingIncorrectSpec; endinterface `ifdef CONTRACTS_VERIFY @@ -412,7 +413,7 @@ module mkCore#(CoreId coreId)(Core); method setRegReadyAggr = writeAggr(aluWrAggrPort(i)); interface sendBypass = sendBypassIfc; method writeRegFile = writeCons(aluWrConsPort(i)); - method Action redirect(CapMem new_pc, SpecTag spec_tag, InstTag inst_tag); + method Action redirect(CapMem new_pc, SpecTag spec_tag, InstTag inst_tag, SpecBits spec_bits); if (verbose) begin $display("[ALU redirect - %d] ", i, fshow(new_pc), "; ", fshow(spec_tag), "; ", fshow(inst_tag)); @@ -423,7 +424,7 @@ module mkCore#(CoreId coreId)(Core); , inst_tag.dii_next_pid `endif ); - globalSpecUpdate.incorrectSpec(False, spec_tag, inst_tag); + globalSpecUpdate.incorrectSpec(False, spec_tag, inst_tag, spec_bits); endmethod method correctSpec = globalSpecUpdate.correctSpec[finishAluCorrectSpecPort(i)].put; method doStats = doStatsReg._read; @@ -510,9 +511,10 @@ module mkCore#(CoreId coreId)(Core); interface fpuMulDivExeIfc = fpuMulDivExe; interface memExeIfc = memExe; method Action killAll; - globalSpecUpdate.incorrectSpec(True, ?, ?); + globalSpecUpdate.incorrectSpec(True, ?, ?, 0); endmethod interface doStatsIfc = doStatsReg; + method pendingIncorrectSpec = globalSpecUpdate.pendingIncorrectSpec; endmodule CoreFixPoint coreFix <- moduleFix(mkCoreFixPoint); @@ -647,6 +649,7 @@ module mkCore#(CoreId coreId)(Core); method stbEmpty = stb.isEmpty; method stqEmpty = lsq.stqEmpty; method lsqSetAtCommit = lsq.setAtCommit; + method pauseCommit = coreFix.pendingIncorrectSpec; method tlbNoPendingReq = iTlb.noPendingReq && dTlb.noPendingReq; method setFlushTlbs; diff --git a/src_Core/RISCY_OOO/procs/RV64G_OOO/AluExePipeline.bsv b/src_Core/RISCY_OOO/procs/RV64G_OOO/AluExePipeline.bsv index b39fc2c..acb59be 100644 --- a/src_Core/RISCY_OOO/procs/RV64G_OOO/AluExePipeline.bsv +++ b/src_Core/RISCY_OOO/procs/RV64G_OOO/AluExePipeline.bsv @@ -194,7 +194,7 @@ interface AluExeInput; // write reg file & set conservative sb method Action writeRegFile(PhyRIndx dst, CapPipe data); // redirect - method Action redirect(CapMem new_pc, SpecTag spec_tag, InstTag inst_tag); + method Action redirect(CapMem new_pc, SpecTag spec_tag, InstTag inst_tag, SpecBits spec_bits); // spec update method Action correctSpec(SpecTag t); @@ -469,7 +469,7 @@ module mkAluExePipeline#(AluExeInput inIfc)(AluExePipeline); if (x.controlFlow.mispredict) (* nosplit *) begin // wrong branch predictin, we must have spec tag doAssert(isValid(x.spec_tag), "mispredicted branch must have spec tag"); - inIfc.redirect(cast(x.controlFlow.nextPc), validValue(x.spec_tag), x.tag); + inIfc.redirect(cast(x.controlFlow.nextPc), validValue(x.spec_tag), x.tag, exeToFin.spec_bits); // must be a branch, train branch predictor doAssert(x.iType == Jr || x.iType == CJALR || x.iType == CCall || x.iType == Br, "only jr, br, cjalr, and ccall can mispredict"); inIfc.fetch_train_predictors(FetchTrainBP { @@ -481,6 +481,8 @@ module mkAluExePipeline#(AluExeInput inIfc)(AluExePipeline); mispred: True, isCompressed: x.isCompressed }); + $display("alu mispredict pc¤: %x, nextPc: %x, %d", + x.controlFlow.pc, x.controlFlow.nextPc, cur_cycle); `ifdef PERF_COUNT // performance counter if(inIfc.doStats) begin diff --git a/src_Core/RISCY_OOO/procs/RV64G_OOO/CommitStage.bsv b/src_Core/RISCY_OOO/procs/RV64G_OOO/CommitStage.bsv index 4e743fc..cdc1fd1 100644 --- a/src_Core/RISCY_OOO/procs/RV64G_OOO/CommitStage.bsv +++ b/src_Core/RISCY_OOO/procs/RV64G_OOO/CommitStage.bsv @@ -108,6 +108,8 @@ interface CommitInput; interface Vector#(SupSize, Put#(LdStQTag)) lsqSetAtCommit; // TLB has stopped processing now method Bool tlbNoPendingReq; + // Pause committing, probably for buffered wrongSpec + method Bool pauseCommit; // set flags method Action setFlushTlbs; method Action setUpdateVMInfo; @@ -526,6 +528,7 @@ module mkCommitStage#(CommitInput inIfc)(CommitStage); // we commit trap in two cycles: first cycle deq ROB and flush; second // cycle handles trap, redirect and handles system consistency Reg#(Maybe#(CommitTrap)) commitTrap <- mkReg(Invalid); // saves new pc here + Bool pauseCommit = isValid(commitTrap) || inIfc.pauseCommit; FIFO#(RedirectInfo) redirectQ <- mkFIFO; @@ -660,7 +663,7 @@ module mkCommitStage#(CommitInput inIfc)(CommitStage); `ifdef INCLUDE_GDB_CONTROL (rg_run_state == RUN_STATE_RUNNING) &&& `endif - !isValid(commitTrap) &&& + !pauseCommit &&& rob.deqPort[0].deq_data.trap matches tagged Valid .trap ); rob.deqPort[0].deq; @@ -825,7 +828,7 @@ module mkCommitStage#(CommitInput inIfc)(CommitStage); `ifdef INCLUDE_GDB_CONTROL (rg_run_state == RUN_STATE_RUNNING) &&& `endif - !isValid(commitTrap) &&& + !pauseCommit &&& !isValid(rob.deqPort[0].deq_data.trap) &&& rob.deqPort[0].deq_data.ldKilled matches tagged Valid .killBy ); @@ -866,7 +869,7 @@ module mkCommitStage#(CommitInput inIfc)(CommitStage); `ifdef INCLUDE_GDB_CONTROL (rg_run_state == RUN_STATE_RUNNING) && `endif - !isValid(commitTrap) && + !pauseCommit && !isValid(rob.deqPort[0].deq_data.trap) && !isValid(rob.deqPort[0].deq_data.ldKilled) && rob.deqPort[0].deq_data.rob_inst_state == Executed && @@ -1042,7 +1045,7 @@ module mkCommitStage#(CommitInput inIfc)(CommitStage); // Lr/Sc/Amo/MMIO cannot proceed to executed until we notify LSQ that it // has reached the commit stage rule notifyLSQCommit( - !isValid(commitTrap) && + !pauseCommit && !isValid(rob.deqPort[0].deq_data.trap) && !isValid(rob.deqPort[0].deq_data.ldKilled) && rob.deqPort[0].deq_data.rob_inst_state != Executed && @@ -1063,7 +1066,7 @@ module mkCommitStage#(CommitInput inIfc)(CommitStage); `ifdef INCLUDE_GDB_CONTROL (rg_run_state == RUN_STATE_RUNNING) && `endif - !isValid(commitTrap) && + !pauseCommit && !isValid(rob.deqPort[0].deq_data.trap) && !isValid(rob.deqPort[0].deq_data.ldKilled) && rob.deqPort[0].deq_data.rob_inst_state == Executed && diff --git a/src_Core/RISCY_OOO/procs/lib/GlobalSpecUpdate.bsv b/src_Core/RISCY_OOO/procs/lib/GlobalSpecUpdate.bsv index dbdf7c1..8234e3b 100644 --- a/src_Core/RISCY_OOO/procs/lib/GlobalSpecUpdate.bsv +++ b/src_Core/RISCY_OOO/procs/lib/GlobalSpecUpdate.bsv @@ -26,7 +26,7 @@ import HasSpecBits::*; import GetPut::*; import Vector::*; import ReorderBuffer::*; -import FIFO::*; +import SpecFifo::*; typedef struct { Bool kill_all; @@ -36,13 +36,14 @@ typedef struct { interface GlobalSpecUpdate#(numeric type correctSpecPortNum, numeric type conflictWrongSpecPortNum); interface Vector#(correctSpecPortNum, Put#(SpecTag)) correctSpec; - method Action incorrectSpec(Bool kill_all, SpecTag spec_tag, InstTag inst_tag); + method Action incorrectSpec(Bool kill_all, SpecTag spec_tag, InstTag inst_tag, SpecBits spec_bits); // Some rules (e.g. doFinishFpuMulDiv) in Core.bsv may not conflict with wrong spec // and is ordered before rules that calls incorrectSpec // this creates cycles in scheduling // To break the cycle, such rules can call the following interface // to manually create a conflict with rules that do incorrectSpec interface Vector#(conflictWrongSpecPortNum, Put#(void)) conflictWrongSpec; + method Bool pendingIncorrectSpec; endinterface module mkGlobalSpecUpdate#( @@ -54,12 +55,12 @@ module mkGlobalSpecUpdate#( // record correct spec tags Vector#(correctSpecPortNum, RWire#(SpecTag)) correctSpecTag <- replicateM(mkRWire); // make wrong spec conflict with correct spec - Vector#(correctSpecPortNum, RWire#(void)) spec_conflict <- replicateM(mkRWire); + Vector#(correctSpecPortNum, PulseWire) spec_conflict <- replicateM(mkPulseWire); // let the caller of conflictWrongSpec to be conflict with wrong spec - Vector#(conflictWrongSpecPortNum, RWire#(void)) wrongSpec_conflict <- replicateM(mkRWire); + Vector#(conflictWrongSpecPortNum, PulseWire) wrongSpec_conflict <- replicateM(mkPulseWire); // must be a single-element fifo to ensure all pushing rules cannot fire while we are waiting // to kill. - FIFO#(IncorrectSpec) incorrectSpec_ff <- mkFIFO1; + SpecFifo#(2,IncorrectSpec,1,1) incorrectSpec_ff <- mkSpecFifoCF(True); (* fire_when_enabled, no_implicit_conditions *) rule canon_correct_spec; @@ -69,31 +70,32 @@ module mkGlobalSpecUpdate#( mask[tag] = 0; end end + incorrectSpec_ff.specUpdate.correctSpeculation(mask); ifc.correctSpeculation(mask); rob.correctSpeculation(mask); endrule rule do_incorrect_spec; - IncorrectSpec x <- toGet(incorrectSpec_ff).get; + IncorrectSpec x = incorrectSpec_ff.first.data; + incorrectSpec_ff.deq; + incorrectSpec_ff.specUpdate.incorrectSpeculation(x.kill_all, x.spec_tag); ifc.incorrectSpeculation(x.kill_all, x.spec_tag); rob.incorrectSpeculation(x.kill_all, x.spec_tag, x.inst_tag); // conflict with correct spec for(Integer i = 0; i < valueof(correctSpecPortNum); i = i+1) begin - spec_conflict[i].wset(?); + spec_conflict[i].send; end // conflict with the caller of conflictWrongSpec for(Integer i = 0; i < valueof(conflictWrongSpecPortNum); i = i+1) begin - wrongSpec_conflict[i].wset(?); + wrongSpec_conflict[i].send; end endrule Vector#(correctSpecPortNum, Put#(SpecTag)) correctVec = ?; for(Integer i = 0; i < valueof(correctSpecPortNum); i = i+1) begin correctVec[i] = (interface Put; - method Action put(SpecTag t); + method Action put(SpecTag t) if (!spec_conflict[i]); correctSpecTag[i].wset(t); - // conflict with wrong spec - spec_conflict[i].wset(?); endmethod endinterface); end @@ -101,16 +103,21 @@ module mkGlobalSpecUpdate#( Vector#(conflictWrongSpecPortNum, Put#(void)) conflictWrongVec = ?; for(Integer i = 0; i < valueof(conflictWrongSpecPortNum); i = i+1) begin conflictWrongVec[i] = (interface Put; - method Action put(void x); - wrongSpec_conflict[i].wset(?); + method Action put(void x) if (!wrongSpec_conflict[i]); + noAction; endmethod endinterface); end interface correctSpec = correctVec; - method Action incorrectSpec(Bool kill_all, SpecTag spec_tag, InstTag inst_tag) - = incorrectSpec_ff.enq(IncorrectSpec{kill_all: kill_all, spec_tag: spec_tag, inst_tag: inst_tag}); + method Action incorrectSpec(Bool kill_all, SpecTag spec_tag, InstTag inst_tag, SpecBits spec_bits) + = incorrectSpec_ff.enq(ToSpecFifo{ + data: IncorrectSpec{kill_all: kill_all, spec_tag: spec_tag, inst_tag: inst_tag}, + spec_bits: spec_bits + }); interface conflictWrongSpec = conflictWrongVec; + + method Bool pendingIncorrectSpec = incorrectSpec_ff.notEmpty; endmodule diff --git a/src_Core/RISCY_OOO/procs/lib/ReorderBuffer.bsv b/src_Core/RISCY_OOO/procs/lib/ReorderBuffer.bsv index 1ca4aa7..194f570 100644 --- a/src_Core/RISCY_OOO/procs/lib/ReorderBuffer.bsv +++ b/src_Core/RISCY_OOO/procs/lib/ReorderBuffer.bsv @@ -674,7 +674,7 @@ module mkSupReorderBuffer#( Add#(1, a__, aluExeNum), Add#(1, b__, fpuMulDivExeNum) ); - Bool verbose = False; + Bool verbose = True; // doCommit rule: deq < wrongSpec (overwrite deq in doCommit) < doRenaming rule: enq Integer valid_deq_port = 0; @@ -756,6 +756,7 @@ module mkSupReorderBuffer#( // move deqP & reset valid deqP[i] <= getNextPtr(deqP[i]); valid[i][deqP[i]][valid_deq_port] <= False; + $display("deq[%d][%d]", i, deqP[i]); end end // update firstDeqWay: find the first deq port that is not enabled @@ -935,6 +936,9 @@ module mkSupReorderBuffer#( end endfunction for(Integer i = 0; i < valueof(SingleScalarSize); i = i+1) begin + if (in_kill_range(fromInteger(i)) != (row[w][i].dependsOn_wrongSpec(specTag) && valid[w][i][valid_wrongSpec_port])) + $display("enqP: %d, enqPNext: %d, w: %d, i: %d, wrongSpec: %d, valid: %d, cur_cycle: %d", + enqP[w], enqPNext[w], w, i, row[w][i].dependsOn_wrongSpec(specTag), valid[w][i][valid_wrongSpec_port], cur_cycle); doAssert( in_kill_range(fromInteger(i)) == (row[w][i].dependsOn_wrongSpec(specTag) && valid[w][i][valid_wrongSpec_port]), diff --git a/src_Core/RISCY_OOO/procs/lib/SpecFifo.bsv b/src_Core/RISCY_OOO/procs/lib/SpecFifo.bsv index 818a340..8eb8c0e 100644 --- a/src_Core/RISCY_OOO/procs/lib/SpecFifo.bsv +++ b/src_Core/RISCY_OOO/procs/lib/SpecFifo.bsv @@ -43,6 +43,7 @@ interface SpecFifo#( method Action enq(ToSpecFifo#(t) x); method Action deq; method ToSpecFifo#(t) first; + method Bool notEmpty; interface SpeculationUpdate specUpdate; endinterface @@ -166,6 +167,8 @@ module mkSpecFifo#( }; endmethod + method Bool notEmpty = valid[deqP][sched.validDeqPort]; + interface SpeculationUpdate specUpdate; method Action correctSpeculation(SpecBits mask); // clear spec bits for all entries @@ -310,6 +313,8 @@ module mkSpecFifoCF#( }; endmethod + method Bool notEmpty = valid[1][deqP]; + interface SpeculationUpdate specUpdate; method correctSpeculation = correctSpecF.enq; method incorrectSpeculation(kill_all, specTag) = diff --git a/src_Core/RISCY_OOO/procs/lib/SplitLSQ.bsv b/src_Core/RISCY_OOO/procs/lib/SplitLSQ.bsv index 24fa28c..6513a8c 100644 --- a/src_Core/RISCY_OOO/procs/lib/SplitLSQ.bsv +++ b/src_Core/RISCY_OOO/procs/lib/SplitLSQ.bsv @@ -949,17 +949,7 @@ module mkSplitLSQ(SplitLSQ); // make wrongSpec conflict with all others (but not correctSpec method and // findIssue) - RWire#(void) wrongSpec_hit_conflict <- mkRWire; - RWire#(void) wrongSpec_enqIss_conflict <- mkRWire; - RWire#(void) wrongSpec_enq_conflict <- mkRWire; - RWire#(void) wrongSpec_cacheEvict_conflict <- mkRWire; - RWire#(void) wrongSpec_update_conflict <- mkRWire; - RWire#(void) wrongSpec_issue_conflict <- mkRWire; - RWire#(void) wrongSpec_respLd_conflict <- mkRWire; - RWire#(void) wrongSpec_deqLd_conflict <- mkRWire; - RWire#(void) wrongSpec_deqSt_conflict <- mkRWire; - RWire#(void) wrongSpec_verify_conflict <- mkRWire; - RWire#(void) wrongSpec_wakeBySB_conflict <- mkRWire; + PulseWire wrongSpec_conflict <- mkPulseWire; // make wrongSpec more urgent than firstSt (resolve bsc error) Wire#(Bool) wrongSpec_urgent_firstSt <- mkDWire(True); Map#(Bit#(10),Bit#(6),Int#(3),2) ldKillMap <- mkMapLossy(minBound); @@ -1145,7 +1135,7 @@ module mkSplitLSQ(SplitLSQ); end endrule - rule enqIssueQ(issueLdInfo.wget matches tagged Valid .info); + rule enqIssueQ(issueLdInfo.wget matches tagged Valid .info &&& !wrongSpec_conflict); if(verbose) begin $display("[LSQ - enqIss] ", fshow(info)); end @@ -1187,8 +1177,6 @@ module mkSplitLSQ(SplitLSQ); spec_bits: ld_specBits_enqIss[info.tag] }); ld_inIssueQ_enqIss[info.tag] <= True; - // make conflict with incorrect spec - wrongSpec_enqIss_conflict.wset(?); endrule // Verify SQ entry one by one @@ -1211,7 +1199,8 @@ module mkSplitLSQ(SplitLSQ); // NOTE that when SQ is full and all verified, verifyP will point to a // valid and verified entry rule verifySt(st_valid_verify[st_verifyP_verify] && - !st_verified_verify[st_verifyP_verify]); + !st_verified_verify[st_verifyP_verify] && + !wrongSpec_conflict); StQTag verP = st_verifyP_verify; // check if the entry can be verified. We should not fire this rule if @@ -1261,9 +1250,6 @@ module mkSplitLSQ(SplitLSQ); joinActions(map(setVerified, idxVec)); if(verbose) $display("[LSQ - verifySt] st_verifyP %d", verP); - - // make conflict with incorrect spec - wrongSpec_verify_conflict.wset(?); endrule `ifdef BSIM @@ -1437,10 +1423,7 @@ module mkSplitLSQ(SplitLSQ); endcase); endmethod - method ActionValue#(LSQHitInfo) getHit(LdStQTag t); - // Conflict with wrong spec. This makes cache pipelineResp rule - // conflict with wrong spec, and can help avoid scheduling cycle. - wrongSpec_hit_conflict.wset(?); + method ActionValue#(LSQHitInfo) getHit(LdStQTag t) if (!wrongSpec_conflict); return (case(t) matches tagged Ld .tag: (LSQHitInfo { waitWPResp: ld_waitWPResp_hit[tag], @@ -1465,7 +1448,7 @@ module mkSplitLSQ(SplitLSQ); MemInst mem_inst, Maybe#(PhyDst) dst, SpecBits spec_bits, - Bit#(16) pc_hash) if(ld_can_enq_wire); + Bit#(16) pc_hash) if(ld_can_enq_wire && !wrongSpec_conflict); if(verbose) begin $display("[LSQ - enqLd] enqP %d; ", ld_enqP, "; ", fshow(inst_tag), @@ -1518,14 +1501,12 @@ module mkSplitLSQ(SplitLSQ); ld_olderSt_enq[ld_enqP] <= Invalid; ld_olderStVerified_enq[ld_enqP] <= False; end - // make conflict with incorrect spec - wrongSpec_enq_conflict.wset(?); endmethod method Action enqSt(InstTag inst_tag, MemInst mem_inst, Maybe#(PhyDst) dst, - SpecBits spec_bits) if(st_can_enq_wire); + SpecBits spec_bits) if(st_can_enq_wire && !wrongSpec_conflict); if(verbose) begin $display("[LSQ - enqSt] enqP %d; ", st_enqP, "; ", fshow(inst_tag), @@ -1554,8 +1535,6 @@ module mkSplitLSQ(SplitLSQ); st_verified_enq[st_enqP] <= False; st_specBits_enq[st_enqP] <= spec_bits; st_atCommit_enq[st_enqP] <= False; - // make conflict with incorrect spec - wrongSpec_enq_conflict.wset(?); endmethod method Action updateData(StQTag t, MemTaggedData d); @@ -1570,7 +1549,7 @@ module mkSplitLSQ(SplitLSQ); method ActionValue#(LSQUpdateAddrResult) updateAddr( LdStQTag lsqTag, Maybe#(Trap) fault, Bool allowCap, Addr pa, Bool mmio, ByteOrTagEn shift_be - ); + ) if (!wrongSpec_conflict); // index vec for vector functions Vector#(LdQSize, LdQTag) idxVec = genWith(fromInteger); @@ -1736,9 +1715,6 @@ module mkSplitLSQ(SplitLSQ); end end - // make conflict with incorrect spec - wrongSpec_update_conflict.wset(?); - // return waiting for wp resp bit: for deciding whether the updating Ld // can be issued return LSQUpdateAddrResult { @@ -1753,7 +1729,7 @@ module mkSplitLSQ(SplitLSQ); method ActionValue#(LSQIssueLdResult) issueLd(LdQTag tag, Addr pa, ByteOrTagEn shift_be, - SBSearchRes sbRes); + SBSearchRes sbRes) if (!wrongSpec_conflict); if(verbose) begin $display("[LSQ - issueLd] ", fshow(tag), "; ", fshow(pa), "; ", fshow(shift_be), "; ", fshow(sbRes)); @@ -2022,9 +1998,6 @@ module mkSplitLSQ(SplitLSQ); end `endif - // make conflict with incorrect spec - wrongSpec_issue_conflict.wset(?); - return issRes; endmethod @@ -2041,7 +2014,7 @@ module mkSplitLSQ(SplitLSQ); return issueLdQ.first.data; endmethod - method ActionValue#(LSQRespLdResult) respLd(LdQTag t, MemTaggedData alignedData); + method ActionValue#(LSQRespLdResult) respLd(LdQTag t, MemTaggedData alignedData) if (!wrongSpec_conflict); let res = LSQRespLdResult { wrongPath: False, dst: Invalid, @@ -2090,8 +2063,6 @@ module mkSplitLSQ(SplitLSQ); $display("[LSQ - respLd] ", fshow(t), "; ", fshow(alignedData), "; ", fshow(res)); end - // make conflict with incorrect spec - wrongSpec_respLd_conflict.wset(?); // return return res; endmethod @@ -2116,7 +2087,7 @@ module mkSplitLSQ(SplitLSQ); }; endmethod - method Action deqLd if(deqLdGuard); + method Action deqLd if(deqLdGuard && !wrongSpec_conflict); LdQTag deqP = ld_deqP_deqLd; if(verbose) $display("[LSQ - deqLd] deqP %d", deqP); @@ -2163,9 +2134,6 @@ module mkSplitLSQ(SplitLSQ); endfunction Vector#(LdQSize, LdQTag) idxVec = genWith(fromInteger); joinActions(map(setReady, idxVec)); - - // make conflict with incorrect spec - wrongSpec_deqLd_conflict.wset(?); endmethod method StQDeqEntry firstSt if(deqStGuard && wrongSpec_urgent_firstSt); @@ -2186,7 +2154,7 @@ module mkSplitLSQ(SplitLSQ); }; endmethod - method Action deqSt if(deqStGuard); + method Action deqSt if(deqStGuard && !wrongSpec_conflict); StQTag deqP = st_deqP; if(verbose) $display("[LSQ - deqSt] deqP %d", deqP); @@ -2234,13 +2202,10 @@ module mkSplitLSQ(SplitLSQ); endfunction Vector#(LdQSize, LdQTag) idxVec = genWith(fromInteger); joinActions(map(resetSt, idxVec)); - - // make conflict with incorrect spec - wrongSpec_deqSt_conflict.wset(?); endmethod `ifdef TSO_MM - method Action cacheEvict(LineAddr lineAddr); + method Action cacheEvict(LineAddr lineAddr) if (!wrongSpec_conflict); if(verbose) $display("[LSQ - cacheEvict] ", fshow(lineAddr)); // kill a load if it satisfies the following conditions: // (1) valid @@ -2270,14 +2235,11 @@ module mkSplitLSQ(SplitLSQ); doAssert(!ld_isMMIO_evict[killTag], "cannot kill MMIO"); doAssert(ld_memFunc[killTag] == Ld, "can only kill Ld"); end - - // make conflict with incorrect spec - wrongSpec_cacheEvict_conflict.wset(?); endmethod `else - method Action wakeupLdStalledBySB(SBIndex sbIdx); + method Action wakeupLdStalledBySB(SBIndex sbIdx) if (!wrongSpec_conflict); if(verbose) begin $display("[LSQ - wakeupBySB] ", fshow(sbIdx)); end @@ -2292,8 +2254,6 @@ module mkSplitLSQ(SplitLSQ); endfunction Vector#(LdQSize, LdQTag) idxVec = genWith(fromInteger); joinActions(map(setReady, idxVec)); - // make conflict with incorrect spec - wrongSpec_wakeBySB_conflict.wset(?); endmethod `endif @@ -2438,17 +2398,7 @@ module mkSplitLSQ(SplitLSQ); end // make conflict with others - wrongSpec_hit_conflict.wset(?); - wrongSpec_enqIss_conflict.wset(?); - wrongSpec_enq_conflict.wset(?); - wrongSpec_update_conflict.wset(?); - wrongSpec_issue_conflict.wset(?); - wrongSpec_respLd_conflict.wset(?); - wrongSpec_deqLd_conflict.wset(?); - wrongSpec_deqSt_conflict.wset(?); - wrongSpec_verify_conflict.wset(?); - wrongSpec_cacheEvict_conflict.wset(?); - wrongSpec_wakeBySB_conflict.wset(?); + wrongSpec_conflict.send(); // more urgent than firstSt wrongSpec_urgent_firstSt <= True; endmethod From a56deab46c81236e779db97d2b02553a393f25ed Mon Sep 17 00:00:00 2001 From: Jonathan Woodruff Date: Thu, 9 Dec 2021 12:27:57 +0000 Subject: [PATCH 21/24] Simpelest strategy for making wrongSpec not conflict with loads of rules. This triggers some assertions in the ROB what should be dealt with properly if this is going to work. --- .../RISCY_OOO/procs/lib/GlobalSpecUpdate.bsv | 20 ++++++------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/src_Core/RISCY_OOO/procs/lib/GlobalSpecUpdate.bsv b/src_Core/RISCY_OOO/procs/lib/GlobalSpecUpdate.bsv index 8234e3b..28ae322 100644 --- a/src_Core/RISCY_OOO/procs/lib/GlobalSpecUpdate.bsv +++ b/src_Core/RISCY_OOO/procs/lib/GlobalSpecUpdate.bsv @@ -54,10 +54,8 @@ module mkGlobalSpecUpdate#( ); // record correct spec tags Vector#(correctSpecPortNum, RWire#(SpecTag)) correctSpecTag <- replicateM(mkRWire); - // make wrong spec conflict with correct spec - Vector#(correctSpecPortNum, PulseWire) spec_conflict <- replicateM(mkPulseWire); - // let the caller of conflictWrongSpec to be conflict with wrong spec - Vector#(conflictWrongSpecPortNum, PulseWire) wrongSpec_conflict <- replicateM(mkPulseWire); + // make wrong spec conflict with correct spec and conflictWrongSpec + PulseWire spec_conflict <- mkPulseWire; // must be a single-element fifo to ensure all pushing rules cannot fire while we are waiting // to kill. SpecFifo#(2,IncorrectSpec,1,1) incorrectSpec_ff <- mkSpecFifoCF(True); @@ -81,20 +79,14 @@ module mkGlobalSpecUpdate#( incorrectSpec_ff.specUpdate.incorrectSpeculation(x.kill_all, x.spec_tag); ifc.incorrectSpeculation(x.kill_all, x.spec_tag); rob.incorrectSpeculation(x.kill_all, x.spec_tag, x.inst_tag); - // conflict with correct spec - for(Integer i = 0; i < valueof(correctSpecPortNum); i = i+1) begin - spec_conflict[i].send; - end - // conflict with the caller of conflictWrongSpec - for(Integer i = 0; i < valueof(conflictWrongSpecPortNum); i = i+1) begin - wrongSpec_conflict[i].send; - end + // conflict with correct spec and conflictWrongSpec + spec_conflict.send; endrule Vector#(correctSpecPortNum, Put#(SpecTag)) correctVec = ?; for(Integer i = 0; i < valueof(correctSpecPortNum); i = i+1) begin correctVec[i] = (interface Put; - method Action put(SpecTag t) if (!spec_conflict[i]); + method Action put(SpecTag t) if (!spec_conflict); correctSpecTag[i].wset(t); endmethod endinterface); @@ -103,7 +95,7 @@ module mkGlobalSpecUpdate#( Vector#(conflictWrongSpecPortNum, Put#(void)) conflictWrongVec = ?; for(Integer i = 0; i < valueof(conflictWrongSpecPortNum); i = i+1) begin conflictWrongVec[i] = (interface Put; - method Action put(void x) if (!wrongSpec_conflict[i]); + method Action put(void x) if (!spec_conflict); noAction; endmethod endinterface); From ad5d5aca0d381724b097d1a3a64c355470424e6c Mon Sep 17 00:00:00 2001 From: Jonathan Woodruff Date: Wed, 12 Jan 2022 13:13:39 +0000 Subject: [PATCH 22/24] Also pause branch execution when there is a pending wrong speculation. --- src_Core/CPU/Core.bsv | 1 + .../procs/RV64G_OOO/AluExePipeline.bsv | 6 ++++-- src_Core/RISCY_OOO/procs/lib/SpecFifo.bsv | 18 +++++++----------- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/src_Core/CPU/Core.bsv b/src_Core/CPU/Core.bsv index 5c29e37..0783e1e 100644 --- a/src_Core/CPU/Core.bsv +++ b/src_Core/CPU/Core.bsv @@ -426,6 +426,7 @@ module mkCore#(CoreId coreId)(Core); ); globalSpecUpdate.incorrectSpec(False, spec_tag, inst_tag, spec_bits); endmethod + method Bool pauseExecute = globalSpecUpdate.pendingIncorrectSpec; method correctSpec = globalSpecUpdate.correctSpec[finishAluCorrectSpecPort(i)].put; method doStats = doStatsReg._read; `ifdef PERFORMANCE_MONITORING diff --git a/src_Core/RISCY_OOO/procs/RV64G_OOO/AluExePipeline.bsv b/src_Core/RISCY_OOO/procs/RV64G_OOO/AluExePipeline.bsv index acb59be..efb03a5 100644 --- a/src_Core/RISCY_OOO/procs/RV64G_OOO/AluExePipeline.bsv +++ b/src_Core/RISCY_OOO/procs/RV64G_OOO/AluExePipeline.bsv @@ -195,6 +195,8 @@ interface AluExeInput; method Action writeRegFile(PhyRIndx dst, CapPipe data); // redirect method Action redirect(CapMem new_pc, SpecTag spec_tag, InstTag inst_tag, SpecBits spec_bits); + // pending invalidation could pause execute/redirections. + method Bool pauseExecute; // spec update method Action correctSpec(SpecTag t); @@ -354,7 +356,7 @@ module mkAluExePipeline#(AluExeInput inIfc)(AluExePipeline); }); endrule - rule doExeAlu; + rule doExeAlu(!inIfc.pauseExecute); regToExeQ.deq; let regToExe = regToExeQ.first; let x = regToExe.data; @@ -417,7 +419,7 @@ module mkAluExePipeline#(AluExeInput inIfc)(AluExePipeline); }); endrule - rule doFinishAlu; + rule doFinishAlu(!inIfc.pauseExecute); exeToFinQ.deq; let exeToFin = exeToFinQ.first; let x = exeToFin.data; diff --git a/src_Core/RISCY_OOO/procs/lib/SpecFifo.bsv b/src_Core/RISCY_OOO/procs/lib/SpecFifo.bsv index 8eb8c0e..b043b71 100644 --- a/src_Core/RISCY_OOO/procs/lib/SpecFifo.bsv +++ b/src_Core/RISCY_OOO/procs/lib/SpecFifo.bsv @@ -25,7 +25,7 @@ import ProcTypes::*; import HasSpecBits::*; import Vector::*; import Ehr::*; -import FIFOF::*; +import DReg::*; import Assert::*; import Types::*; import ConfigReg::*; @@ -223,8 +223,8 @@ module mkSpecFifoCF#( Vector#(size, Reg#(t)) row <- replicateM(mkConfigRegU); Ehr#(3, Vector#(size, SpecBits)) specBits <- mkEhr(?); - FIFOF#(SpecBits) correctSpecF <- mkUGFIFOF; - FIFOF#(IncorrectSpeculation) incorrectSpecF <- mkUGFIFOF; + Reg#(Maybe#(SpecBits)) correctSpec <- mkDReg(Invalid); + Reg#(Maybe#(IncorrectSpeculation)) incorrectSpec <- mkDReg(Invalid); Reg#(idxT) enqP <- mkConfigReg(0); Ehr#(2, idxT) deqP_ehr <- mkEhr(0); @@ -245,17 +245,13 @@ module mkSpecFifoCF#( Vector#(size, SpecBits) newSpecBits = specBits[0]; Vector#(size, Bool) newValid = valid[0]; // Fold in CorrectSpec update: - if (correctSpecF.notEmpty) begin - SpecBits mask = correctSpecF.first(); - correctSpecF.deq(); + if (correctSpec matches tagged Valid .mask) begin // clear spec bits for all entries for (Integer i=0; i Date: Tue, 1 Feb 2022 09:10:48 +0000 Subject: [PATCH 23/24] Enable tracing of physical addresses for RVFI (if PADDR_RVFI is defined). This is done by adding a vector of padder lookup interfaces to the load store queue. This should help with targeted memory translation debugging, and also with compatability with the current Sail RVFI implemenation which is probably incorrect with respect to the RVFI spec which calls for virtual addresses in the maddr field. --- src_Core/CPU/Core.bsv | 1 + .../RISCY_OOO/procs/RV64G_OOO/CommitStage.bsv | 14 ++- src_Core/RISCY_OOO/procs/lib/SplitLSQ.bsv | 95 ++++++++++++++++--- 3 files changed, 91 insertions(+), 19 deletions(-) diff --git a/src_Core/CPU/Core.bsv b/src_Core/CPU/Core.bsv index 0783e1e..7ce2467 100644 --- a/src_Core/CPU/Core.bsv +++ b/src_Core/CPU/Core.bsv @@ -650,6 +650,7 @@ module mkCore#(CoreId coreId)(Core); method stbEmpty = stb.isEmpty; method stqEmpty = lsq.stqEmpty; method lsqSetAtCommit = lsq.setAtCommit; + method lookupPAddr = lsq.lookupPAddr; method pauseCommit = coreFix.pendingIncorrectSpec; method tlbNoPendingReq = iTlb.noPendingReq && dTlb.noPendingReq; diff --git a/src_Core/RISCY_OOO/procs/RV64G_OOO/CommitStage.bsv b/src_Core/RISCY_OOO/procs/RV64G_OOO/CommitStage.bsv index cdc1fd1..8e14907 100644 --- a/src_Core/RISCY_OOO/procs/RV64G_OOO/CommitStage.bsv +++ b/src_Core/RISCY_OOO/procs/RV64G_OOO/CommitStage.bsv @@ -60,6 +60,7 @@ import RenameDebugIF::*; import CHERICap::*; import CHERICC_Fat::*; import ISA_Decls_CHERI::*; +import RegFile::*; // Just for the interface `ifdef PERFORMANCE_MONITORING import StatCounters::*; `endif @@ -106,6 +107,8 @@ interface CommitInput; method Bool stqEmpty; // notify LSQ that inst has reached commit interface Vector#(SupSize, Put#(LdStQTag)) lsqSetAtCommit; + // method for getting translated addresses for tracing. + interface Vector#(SupSize, RegFile#(LdStQTag, Addr)) lookupPAddr; // TLB has stopped processing now method Bool tlbNoPendingReq; // Pause committing, probably for buffered wrongSpec @@ -213,7 +216,7 @@ typedef struct { Data mtvec; } TraceStateBundle deriving(Bits, FShow); -function Maybe#(RVFI_DII_Execution#(DataSz,DataSz)) genRVFI(ToReorderBuffer rot, Dii_Id traceCnt, TraceStateBundle tsb, Data next_pc); +function Maybe#(RVFI_DII_Execution#(DataSz,DataSz)) genRVFI(ToReorderBuffer rot, Dii_Id traceCnt, TraceStateBundle tsb, Data next_pc, Addr paddr); Addr addr = 0; Data data = 0; Data wdata = 0; @@ -230,6 +233,9 @@ function Maybe#(RVFI_DII_Execution#(DataSz,DataSz)) genRVFI(ToReorderBuffer rot, case (rot.ppc_vaddr_csrData) matches tagged VAddr .vaddr: begin addr = vaddr; +`ifdef PADDR_RVFI + addr = paddr; +`endif case (rot.lsqTag) matches tagged Ld .l: rmask = rot.traceBundle.memByteEn; tagged St .s: begin @@ -800,7 +806,7 @@ module mkCommitStage#(CommitInput inIfc)(CommitStage); }); `ifdef RVFI Rvfi_Traces rvfis = replicate(tagged Invalid); - rvfis[0] = genRVFI(trap.x, traceCnt, getTSB(), getAddr(new_pc)); + rvfis[0] = genRVFI(trap.x, traceCnt, getTSB(), getAddr(new_pc), inIfc.lookupPAddr[0].sub(trap.x.lsqTag)); rvfiQ.enq(rvfis); traceCnt <= traceCnt + 1; `endif @@ -969,7 +975,7 @@ module mkCommitStage#(CommitInput inIfc)(CommitStage); Rvfi_Traces rvfis = replicate(tagged Invalid); x.ppc_vaddr_csrData = tagged PPC next_pc; CapPipe cp = cast(next_pc); - rvfis[0] = genRVFI(x, traceCnt, getTSB(), getOffset(cp)); + rvfis[0] = genRVFI(x, traceCnt, getTSB(), getOffset(cp), inIfc.lookupPAddr[0].sub(x.lsqTag)); rvfiQ.enq(rvfis); traceCnt <= traceCnt + 1; `endif @@ -1147,7 +1153,7 @@ module mkCommitStage#(CommitInput inIfc)(CommitStage); if (verbose) $display("%t : [doCommitNormalInst - %d] ", $time(), i, fshow(inst_tag), " ; ", fshow(x)); `ifdef RVFI CapPipe pipePc = cast(x.pc); - rvfis[i] = genRVFI(x, traceCnt + zeroExtend(whichTrace), getTSB(), getOffset(pipePc) + (is_16b_inst(x.orig_inst) ? 2:4)); + rvfis[i] = genRVFI(x, traceCnt + zeroExtend(whichTrace), getTSB(), getOffset(pipePc) + (is_16b_inst(x.orig_inst) ? 2:4), inIfc.lookupPAddr[i].sub(x.lsqTag)); whichTrace = whichTrace + 1; `endif diff --git a/src_Core/RISCY_OOO/procs/lib/SplitLSQ.bsv b/src_Core/RISCY_OOO/procs/lib/SplitLSQ.bsv index 6513a8c..e7d8cba 100644 --- a/src_Core/RISCY_OOO/procs/lib/SplitLSQ.bsv +++ b/src_Core/RISCY_OOO/procs/lib/SplitLSQ.bsv @@ -51,6 +51,7 @@ import StoreBuffer::*; import Exec::*; import FP_Utils::*; import CacheUtils::*; // For CLoadTags alignment +import RegFile::*; // Just for the interface // I don't want to export auxiliary functions, so manually export all types export LdQMemFunc(..); @@ -439,6 +440,8 @@ interface SplitLSQ; // Sc/Amo/Fence, and flush L1 cache. method StQDeqEntry firstSt; method Action deqSt; + // method for reporting the physical address of an entry used for tracing. + interface Vector#(SupSize, RegFile#(LdStQTag, Addr)) lookupPAddr; `ifdef TSO_MM // Kill loads when a cache line is evicted (TSO only) method Action cacheEvict(LineAddr a); @@ -949,7 +952,17 @@ module mkSplitLSQ(SplitLSQ); // make wrongSpec conflict with all others (but not correctSpec method and // findIssue) - PulseWire wrongSpec_conflict <- mkPulseWire; + RWire#(void) wrongSpec_hit_conflict <- mkRWire; + RWire#(void) wrongSpec_enqIss_conflict <- mkRWire; + RWire#(void) wrongSpec_enq_conflict <- mkRWire; + RWire#(void) wrongSpec_cacheEvict_conflict <- mkRWire; + RWire#(void) wrongSpec_update_conflict <- mkRWire; + RWire#(void) wrongSpec_issue_conflict <- mkRWire; + RWire#(void) wrongSpec_respLd_conflict <- mkRWire; + RWire#(void) wrongSpec_deqLd_conflict <- mkRWire; + RWire#(void) wrongSpec_deqSt_conflict <- mkRWire; + RWire#(void) wrongSpec_verify_conflict <- mkRWire; + RWire#(void) wrongSpec_wakeBySB_conflict <- mkRWire; // make wrongSpec more urgent than firstSt (resolve bsc error) Wire#(Bool) wrongSpec_urgent_firstSt <- mkDWire(True); Map#(Bit#(10),Bit#(6),Int#(3),2) ldKillMap <- mkMapLossy(minBound); @@ -1135,7 +1148,7 @@ module mkSplitLSQ(SplitLSQ); end endrule - rule enqIssueQ(issueLdInfo.wget matches tagged Valid .info &&& !wrongSpec_conflict); + rule enqIssueQ(issueLdInfo.wget matches tagged Valid .info); if(verbose) begin $display("[LSQ - enqIss] ", fshow(info)); end @@ -1177,6 +1190,8 @@ module mkSplitLSQ(SplitLSQ); spec_bits: ld_specBits_enqIss[info.tag] }); ld_inIssueQ_enqIss[info.tag] <= True; + // make conflict with incorrect spec + wrongSpec_enqIss_conflict.wset(?); endrule // Verify SQ entry one by one @@ -1199,8 +1214,7 @@ module mkSplitLSQ(SplitLSQ); // NOTE that when SQ is full and all verified, verifyP will point to a // valid and verified entry rule verifySt(st_valid_verify[st_verifyP_verify] && - !st_verified_verify[st_verifyP_verify] && - !wrongSpec_conflict); + !st_verified_verify[st_verifyP_verify]); StQTag verP = st_verifyP_verify; // check if the entry can be verified. We should not fire this rule if @@ -1250,6 +1264,9 @@ module mkSplitLSQ(SplitLSQ); joinActions(map(setVerified, idxVec)); if(verbose) $display("[LSQ - verifySt] st_verifyP %d", verP); + + // make conflict with incorrect spec + wrongSpec_verify_conflict.wset(?); endrule `ifdef BSIM @@ -1415,6 +1432,16 @@ module mkSplitLSQ(SplitLSQ); endinterface); end + RegFile#(LdStQTag, Addr) lookupAPAddr = (interface RegFile; + method Addr sub(LdStQTag t); + case (t) matches + tagged Ld .l: return ld_paddr_deqLd[l]; + tagged St .s: return st_paddr_deqSt[s]; + endcase + endmethod + method upd = ?; + endinterface); + method ByteOrTagEn getOrigBE(LdStQTag t); return (case(t) matches tagged Ld .tag: (ld_byteOrTagEn[tag]); @@ -1423,7 +1450,10 @@ module mkSplitLSQ(SplitLSQ); endcase); endmethod - method ActionValue#(LSQHitInfo) getHit(LdStQTag t) if (!wrongSpec_conflict); + method ActionValue#(LSQHitInfo) getHit(LdStQTag t); + // Conflict with wrong spec. This makes cache pipelineResp rule + // conflict with wrong spec, and can help avoid scheduling cycle. + wrongSpec_hit_conflict.wset(?); return (case(t) matches tagged Ld .tag: (LSQHitInfo { waitWPResp: ld_waitWPResp_hit[tag], @@ -1448,7 +1478,7 @@ module mkSplitLSQ(SplitLSQ); MemInst mem_inst, Maybe#(PhyDst) dst, SpecBits spec_bits, - Bit#(16) pc_hash) if(ld_can_enq_wire && !wrongSpec_conflict); + Bit#(16) pc_hash) if(ld_can_enq_wire); if(verbose) begin $display("[LSQ - enqLd] enqP %d; ", ld_enqP, "; ", fshow(inst_tag), @@ -1501,12 +1531,14 @@ module mkSplitLSQ(SplitLSQ); ld_olderSt_enq[ld_enqP] <= Invalid; ld_olderStVerified_enq[ld_enqP] <= False; end + // make conflict with incorrect spec + wrongSpec_enq_conflict.wset(?); endmethod method Action enqSt(InstTag inst_tag, MemInst mem_inst, Maybe#(PhyDst) dst, - SpecBits spec_bits) if(st_can_enq_wire && !wrongSpec_conflict); + SpecBits spec_bits) if(st_can_enq_wire); if(verbose) begin $display("[LSQ - enqSt] enqP %d; ", st_enqP, "; ", fshow(inst_tag), @@ -1535,6 +1567,8 @@ module mkSplitLSQ(SplitLSQ); st_verified_enq[st_enqP] <= False; st_specBits_enq[st_enqP] <= spec_bits; st_atCommit_enq[st_enqP] <= False; + // make conflict with incorrect spec + wrongSpec_enq_conflict.wset(?); endmethod method Action updateData(StQTag t, MemTaggedData d); @@ -1549,7 +1583,7 @@ module mkSplitLSQ(SplitLSQ); method ActionValue#(LSQUpdateAddrResult) updateAddr( LdStQTag lsqTag, Maybe#(Trap) fault, Bool allowCap, Addr pa, Bool mmio, ByteOrTagEn shift_be - ) if (!wrongSpec_conflict); + ); // index vec for vector functions Vector#(LdQSize, LdQTag) idxVec = genWith(fromInteger); @@ -1715,6 +1749,9 @@ module mkSplitLSQ(SplitLSQ); end end + // make conflict with incorrect spec + wrongSpec_update_conflict.wset(?); + // return waiting for wp resp bit: for deciding whether the updating Ld // can be issued return LSQUpdateAddrResult { @@ -1729,7 +1766,7 @@ module mkSplitLSQ(SplitLSQ); method ActionValue#(LSQIssueLdResult) issueLd(LdQTag tag, Addr pa, ByteOrTagEn shift_be, - SBSearchRes sbRes) if (!wrongSpec_conflict); + SBSearchRes sbRes); if(verbose) begin $display("[LSQ - issueLd] ", fshow(tag), "; ", fshow(pa), "; ", fshow(shift_be), "; ", fshow(sbRes)); @@ -1998,6 +2035,9 @@ module mkSplitLSQ(SplitLSQ); end `endif + // make conflict with incorrect spec + wrongSpec_issue_conflict.wset(?); + return issRes; endmethod @@ -2014,7 +2054,7 @@ module mkSplitLSQ(SplitLSQ); return issueLdQ.first.data; endmethod - method ActionValue#(LSQRespLdResult) respLd(LdQTag t, MemTaggedData alignedData) if (!wrongSpec_conflict); + method ActionValue#(LSQRespLdResult) respLd(LdQTag t, MemTaggedData alignedData); let res = LSQRespLdResult { wrongPath: False, dst: Invalid, @@ -2063,6 +2103,8 @@ module mkSplitLSQ(SplitLSQ); $display("[LSQ - respLd] ", fshow(t), "; ", fshow(alignedData), "; ", fshow(res)); end + // make conflict with incorrect spec + wrongSpec_respLd_conflict.wset(?); // return return res; endmethod @@ -2087,7 +2129,7 @@ module mkSplitLSQ(SplitLSQ); }; endmethod - method Action deqLd if(deqLdGuard && !wrongSpec_conflict); + method Action deqLd if(deqLdGuard); LdQTag deqP = ld_deqP_deqLd; if(verbose) $display("[LSQ - deqLd] deqP %d", deqP); @@ -2134,6 +2176,9 @@ module mkSplitLSQ(SplitLSQ); endfunction Vector#(LdQSize, LdQTag) idxVec = genWith(fromInteger); joinActions(map(setReady, idxVec)); + + // make conflict with incorrect spec + wrongSpec_deqLd_conflict.wset(?); endmethod method StQDeqEntry firstSt if(deqStGuard && wrongSpec_urgent_firstSt); @@ -2154,7 +2199,7 @@ module mkSplitLSQ(SplitLSQ); }; endmethod - method Action deqSt if(deqStGuard && !wrongSpec_conflict); + method Action deqSt if(deqStGuard); StQTag deqP = st_deqP; if(verbose) $display("[LSQ - deqSt] deqP %d", deqP); @@ -2202,10 +2247,15 @@ module mkSplitLSQ(SplitLSQ); endfunction Vector#(LdQSize, LdQTag) idxVec = genWith(fromInteger); joinActions(map(resetSt, idxVec)); + + // make conflict with incorrect spec + wrongSpec_deqSt_conflict.wset(?); endmethod + interface lookupPAddr = replicate(lookupAPAddr); + `ifdef TSO_MM - method Action cacheEvict(LineAddr lineAddr) if (!wrongSpec_conflict); + method Action cacheEvict(LineAddr lineAddr); if(verbose) $display("[LSQ - cacheEvict] ", fshow(lineAddr)); // kill a load if it satisfies the following conditions: // (1) valid @@ -2235,11 +2285,14 @@ module mkSplitLSQ(SplitLSQ); doAssert(!ld_isMMIO_evict[killTag], "cannot kill MMIO"); doAssert(ld_memFunc[killTag] == Ld, "can only kill Ld"); end + + // make conflict with incorrect spec + wrongSpec_cacheEvict_conflict.wset(?); endmethod `else - method Action wakeupLdStalledBySB(SBIndex sbIdx) if (!wrongSpec_conflict); + method Action wakeupLdStalledBySB(SBIndex sbIdx); if(verbose) begin $display("[LSQ - wakeupBySB] ", fshow(sbIdx)); end @@ -2254,6 +2307,8 @@ module mkSplitLSQ(SplitLSQ); endfunction Vector#(LdQSize, LdQTag) idxVec = genWith(fromInteger); joinActions(map(setReady, idxVec)); + // make conflict with incorrect spec + wrongSpec_wakeBySB_conflict.wset(?); endmethod `endif @@ -2398,7 +2453,17 @@ module mkSplitLSQ(SplitLSQ); end // make conflict with others - wrongSpec_conflict.send(); + wrongSpec_hit_conflict.wset(?); + wrongSpec_enqIss_conflict.wset(?); + wrongSpec_enq_conflict.wset(?); + wrongSpec_update_conflict.wset(?); + wrongSpec_issue_conflict.wset(?); + wrongSpec_respLd_conflict.wset(?); + wrongSpec_deqLd_conflict.wset(?); + wrongSpec_deqSt_conflict.wset(?); + wrongSpec_verify_conflict.wset(?); + wrongSpec_cacheEvict_conflict.wset(?); + wrongSpec_wakeBySB_conflict.wset(?); // more urgent than firstSt wrongSpec_urgent_firstSt <= True; endmethod From 05df76073b3e1e86e27c33fddeb8339b53caf3e0 Mon Sep 17 00:00:00 2001 From: Jonathan Woodruff Date: Tue, 1 Feb 2022 11:08:37 +0000 Subject: [PATCH 24/24] Revert some unintended changes from the last commit. --- src_Core/RISCY_OOO/procs/lib/SplitLSQ.bsv | 80 +++++------------------ 1 file changed, 15 insertions(+), 65 deletions(-) diff --git a/src_Core/RISCY_OOO/procs/lib/SplitLSQ.bsv b/src_Core/RISCY_OOO/procs/lib/SplitLSQ.bsv index e7d8cba..25838b1 100644 --- a/src_Core/RISCY_OOO/procs/lib/SplitLSQ.bsv +++ b/src_Core/RISCY_OOO/procs/lib/SplitLSQ.bsv @@ -952,17 +952,7 @@ module mkSplitLSQ(SplitLSQ); // make wrongSpec conflict with all others (but not correctSpec method and // findIssue) - RWire#(void) wrongSpec_hit_conflict <- mkRWire; - RWire#(void) wrongSpec_enqIss_conflict <- mkRWire; - RWire#(void) wrongSpec_enq_conflict <- mkRWire; - RWire#(void) wrongSpec_cacheEvict_conflict <- mkRWire; - RWire#(void) wrongSpec_update_conflict <- mkRWire; - RWire#(void) wrongSpec_issue_conflict <- mkRWire; - RWire#(void) wrongSpec_respLd_conflict <- mkRWire; - RWire#(void) wrongSpec_deqLd_conflict <- mkRWire; - RWire#(void) wrongSpec_deqSt_conflict <- mkRWire; - RWire#(void) wrongSpec_verify_conflict <- mkRWire; - RWire#(void) wrongSpec_wakeBySB_conflict <- mkRWire; + PulseWire wrongSpec_conflict <- mkPulseWire; // make wrongSpec more urgent than firstSt (resolve bsc error) Wire#(Bool) wrongSpec_urgent_firstSt <- mkDWire(True); Map#(Bit#(10),Bit#(6),Int#(3),2) ldKillMap <- mkMapLossy(minBound); @@ -1148,7 +1138,7 @@ module mkSplitLSQ(SplitLSQ); end endrule - rule enqIssueQ(issueLdInfo.wget matches tagged Valid .info); + rule enqIssueQ(issueLdInfo.wget matches tagged Valid .info &&& !wrongSpec_conflict); if(verbose) begin $display("[LSQ - enqIss] ", fshow(info)); end @@ -1190,8 +1180,6 @@ module mkSplitLSQ(SplitLSQ); spec_bits: ld_specBits_enqIss[info.tag] }); ld_inIssueQ_enqIss[info.tag] <= True; - // make conflict with incorrect spec - wrongSpec_enqIss_conflict.wset(?); endrule // Verify SQ entry one by one @@ -1214,7 +1202,8 @@ module mkSplitLSQ(SplitLSQ); // NOTE that when SQ is full and all verified, verifyP will point to a // valid and verified entry rule verifySt(st_valid_verify[st_verifyP_verify] && - !st_verified_verify[st_verifyP_verify]); + !st_verified_verify[st_verifyP_verify] && + !wrongSpec_conflict); StQTag verP = st_verifyP_verify; // check if the entry can be verified. We should not fire this rule if @@ -1264,9 +1253,6 @@ module mkSplitLSQ(SplitLSQ); joinActions(map(setVerified, idxVec)); if(verbose) $display("[LSQ - verifySt] st_verifyP %d", verP); - - // make conflict with incorrect spec - wrongSpec_verify_conflict.wset(?); endrule `ifdef BSIM @@ -1450,10 +1436,7 @@ module mkSplitLSQ(SplitLSQ); endcase); endmethod - method ActionValue#(LSQHitInfo) getHit(LdStQTag t); - // Conflict with wrong spec. This makes cache pipelineResp rule - // conflict with wrong spec, and can help avoid scheduling cycle. - wrongSpec_hit_conflict.wset(?); + method ActionValue#(LSQHitInfo) getHit(LdStQTag t) if (!wrongSpec_conflict); return (case(t) matches tagged Ld .tag: (LSQHitInfo { waitWPResp: ld_waitWPResp_hit[tag], @@ -1478,7 +1461,7 @@ module mkSplitLSQ(SplitLSQ); MemInst mem_inst, Maybe#(PhyDst) dst, SpecBits spec_bits, - Bit#(16) pc_hash) if(ld_can_enq_wire); + Bit#(16) pc_hash) if(ld_can_enq_wire && !wrongSpec_conflict); if(verbose) begin $display("[LSQ - enqLd] enqP %d; ", ld_enqP, "; ", fshow(inst_tag), @@ -1531,14 +1514,12 @@ module mkSplitLSQ(SplitLSQ); ld_olderSt_enq[ld_enqP] <= Invalid; ld_olderStVerified_enq[ld_enqP] <= False; end - // make conflict with incorrect spec - wrongSpec_enq_conflict.wset(?); endmethod method Action enqSt(InstTag inst_tag, MemInst mem_inst, Maybe#(PhyDst) dst, - SpecBits spec_bits) if(st_can_enq_wire); + SpecBits spec_bits) if(st_can_enq_wire && !wrongSpec_conflict); if(verbose) begin $display("[LSQ - enqSt] enqP %d; ", st_enqP, "; ", fshow(inst_tag), @@ -1567,8 +1548,6 @@ module mkSplitLSQ(SplitLSQ); st_verified_enq[st_enqP] <= False; st_specBits_enq[st_enqP] <= spec_bits; st_atCommit_enq[st_enqP] <= False; - // make conflict with incorrect spec - wrongSpec_enq_conflict.wset(?); endmethod method Action updateData(StQTag t, MemTaggedData d); @@ -1583,7 +1562,7 @@ module mkSplitLSQ(SplitLSQ); method ActionValue#(LSQUpdateAddrResult) updateAddr( LdStQTag lsqTag, Maybe#(Trap) fault, Bool allowCap, Addr pa, Bool mmio, ByteOrTagEn shift_be - ); + ) if (!wrongSpec_conflict); // index vec for vector functions Vector#(LdQSize, LdQTag) idxVec = genWith(fromInteger); @@ -1749,9 +1728,6 @@ module mkSplitLSQ(SplitLSQ); end end - // make conflict with incorrect spec - wrongSpec_update_conflict.wset(?); - // return waiting for wp resp bit: for deciding whether the updating Ld // can be issued return LSQUpdateAddrResult { @@ -1766,7 +1742,7 @@ module mkSplitLSQ(SplitLSQ); method ActionValue#(LSQIssueLdResult) issueLd(LdQTag tag, Addr pa, ByteOrTagEn shift_be, - SBSearchRes sbRes); + SBSearchRes sbRes) if (!wrongSpec_conflict); if(verbose) begin $display("[LSQ - issueLd] ", fshow(tag), "; ", fshow(pa), "; ", fshow(shift_be), "; ", fshow(sbRes)); @@ -2035,9 +2011,6 @@ module mkSplitLSQ(SplitLSQ); end `endif - // make conflict with incorrect spec - wrongSpec_issue_conflict.wset(?); - return issRes; endmethod @@ -2054,7 +2027,7 @@ module mkSplitLSQ(SplitLSQ); return issueLdQ.first.data; endmethod - method ActionValue#(LSQRespLdResult) respLd(LdQTag t, MemTaggedData alignedData); + method ActionValue#(LSQRespLdResult) respLd(LdQTag t, MemTaggedData alignedData) if (!wrongSpec_conflict); let res = LSQRespLdResult { wrongPath: False, dst: Invalid, @@ -2103,8 +2076,6 @@ module mkSplitLSQ(SplitLSQ); $display("[LSQ - respLd] ", fshow(t), "; ", fshow(alignedData), "; ", fshow(res)); end - // make conflict with incorrect spec - wrongSpec_respLd_conflict.wset(?); // return return res; endmethod @@ -2129,7 +2100,7 @@ module mkSplitLSQ(SplitLSQ); }; endmethod - method Action deqLd if(deqLdGuard); + method Action deqLd if(deqLdGuard && !wrongSpec_conflict); LdQTag deqP = ld_deqP_deqLd; if(verbose) $display("[LSQ - deqLd] deqP %d", deqP); @@ -2176,9 +2147,6 @@ module mkSplitLSQ(SplitLSQ); endfunction Vector#(LdQSize, LdQTag) idxVec = genWith(fromInteger); joinActions(map(setReady, idxVec)); - - // make conflict with incorrect spec - wrongSpec_deqLd_conflict.wset(?); endmethod method StQDeqEntry firstSt if(deqStGuard && wrongSpec_urgent_firstSt); @@ -2199,7 +2167,7 @@ module mkSplitLSQ(SplitLSQ); }; endmethod - method Action deqSt if(deqStGuard); + method Action deqSt if(deqStGuard && !wrongSpec_conflict); StQTag deqP = st_deqP; if(verbose) $display("[LSQ - deqSt] deqP %d", deqP); @@ -2247,15 +2215,12 @@ module mkSplitLSQ(SplitLSQ); endfunction Vector#(LdQSize, LdQTag) idxVec = genWith(fromInteger); joinActions(map(resetSt, idxVec)); - - // make conflict with incorrect spec - wrongSpec_deqSt_conflict.wset(?); endmethod interface lookupPAddr = replicate(lookupAPAddr); `ifdef TSO_MM - method Action cacheEvict(LineAddr lineAddr); + method Action cacheEvict(LineAddr lineAddr) if (!wrongSpec_conflict); if(verbose) $display("[LSQ - cacheEvict] ", fshow(lineAddr)); // kill a load if it satisfies the following conditions: // (1) valid @@ -2285,14 +2250,11 @@ module mkSplitLSQ(SplitLSQ); doAssert(!ld_isMMIO_evict[killTag], "cannot kill MMIO"); doAssert(ld_memFunc[killTag] == Ld, "can only kill Ld"); end - - // make conflict with incorrect spec - wrongSpec_cacheEvict_conflict.wset(?); endmethod `else - method Action wakeupLdStalledBySB(SBIndex sbIdx); + method Action wakeupLdStalledBySB(SBIndex sbIdx) if (!wrongSpec_conflict); if(verbose) begin $display("[LSQ - wakeupBySB] ", fshow(sbIdx)); end @@ -2307,8 +2269,6 @@ module mkSplitLSQ(SplitLSQ); endfunction Vector#(LdQSize, LdQTag) idxVec = genWith(fromInteger); joinActions(map(setReady, idxVec)); - // make conflict with incorrect spec - wrongSpec_wakeBySB_conflict.wset(?); endmethod `endif @@ -2453,17 +2413,7 @@ module mkSplitLSQ(SplitLSQ); end // make conflict with others - wrongSpec_hit_conflict.wset(?); - wrongSpec_enqIss_conflict.wset(?); - wrongSpec_enq_conflict.wset(?); - wrongSpec_update_conflict.wset(?); - wrongSpec_issue_conflict.wset(?); - wrongSpec_respLd_conflict.wset(?); - wrongSpec_deqLd_conflict.wset(?); - wrongSpec_deqSt_conflict.wset(?); - wrongSpec_verify_conflict.wset(?); - wrongSpec_cacheEvict_conflict.wset(?); - wrongSpec_wakeBySB_conflict.wset(?); + wrongSpec_conflict.send(); // more urgent than firstSt wrongSpec_urgent_firstSt <= True; endmethod