diff --git a/src_Core/RISCY_OOO/procs/RV64G_OOO/FetchStage.bsv b/src_Core/RISCY_OOO/procs/RV64G_OOO/FetchStage.bsv index b75e34c..c84cf6f 100644 --- a/src_Core/RISCY_OOO/procs/RV64G_OOO/FetchStage.bsv +++ b/src_Core/RISCY_OOO/procs/RV64G_OOO/FetchStage.bsv @@ -73,7 +73,7 @@ import CHERICC_Fat::*; import RVFI_DII_Types::*; import Types::*; `endif - +import IndexedMultiset::*; import Cur_Cycle :: *; // ================================================================ @@ -160,6 +160,20 @@ interface FetchStage; interface Perf#(DecStagePerfType) perf; endinterface +// PC "compression" types to facilitate storing common upper PC bits in a +// shared structure +typedef 12 PcLsbSz; // Defines PC block size for PCs that will share an index for upper bits. +typedef TLog#(TMul#(SupSize,4)) PcIdxSz; // Number of distinct PC blocks allowed in-flight in the Fetch pipeline. +typedef Bit#(PcLsbSz) PcLSB; +typedef Bit#(TSub#(SizeOf#(CapMem),PcLsbSz)) PcMSB; +typedef Bit#(PcIdxSz) PcIdx; +typedef struct { + PcLSB lsb; + PcIdx idx; +} PcCompressed deriving(Bits,Eq,FShow); +function PcCompressed compressPc(PcIdx i, CapMem a) = + PcCompressed{idx: i, lsb: truncate(a)}; + typedef struct { Addr pc; Epoch mainEp; @@ -168,21 +182,21 @@ typedef struct { } FetchDebugState deriving(Bits, Eq, FShow); typedef struct { - CapMem pc; + PcCompressed pc; `ifdef RVFI_DII Dii_Parcel_Id dii_pid; `endif - Maybe#(CapMem) pred_next_pc; + Maybe#(PcCompressed) pred_next_pc; Bool decode_epoch; Epoch main_epoch; } Fetch1ToFetch2 deriving(Bits, Eq, FShow); typedef struct { - CapMem pc; + PcCompressed pc; `ifdef RVFI_DII Dii_Parcel_Id dii_pid; `endif - Maybe#(CapMem) pred_next_pc; + Maybe#(PcCompressed) pred_next_pc; Maybe#(Exception) cause; Bool access_mmio; // inst fetch from MMIO Bool decode_epoch; @@ -190,8 +204,8 @@ typedef struct { } Fetch2ToFetch3 deriving(Bits, Eq, FShow); typedef struct { - CapMem pc; - Maybe#(CapMem) ppc; + PcCompressed pc; + Maybe#(PcCompressed) ppc; Maybe#(Exception) cause; Bit#(16) inst_frag; Bool decode_epoch; @@ -203,11 +217,11 @@ typedef struct { // Used purely internally in doDecode. typedef struct { - CapMem pc; + PcCompressed pc; `ifdef RVFI_DII Dii_Parcel_Id dii_pid; `endif - CapMem ppc; + PcCompressed ppc; Bool decode_epoch; Epoch main_epoch; Instruction inst; @@ -224,7 +238,7 @@ function InstrFromFetch3 fetch3_2_instC(Fetch3ToDecode in, Instruction inst, Bit `ifdef RVFI_DII dii_pid: in.dii_pid, `endif - ppc: fromMaybe(in.pc + 2, in.ppc), // This assumes we will call this function on the last fragment of any instruction. + ppc: fromMaybe(PcCompressed{lsb: in.pc.lsb + 2, idx: in.pc.idx}, in.ppc), // This assumes we will call this function on the last fragment of any instruction. decode_epoch: in.decode_epoch, main_epoch: in.main_epoch, inst: inst, @@ -340,6 +354,9 @@ module mkFetchStage(FetchStage); Integer pc_fetch3_port = 2; Integer pc_redirect_port = 3; + // PC compression structure holding an indexed set of PC blocks so that only indexes need be tracked. + IndexedMultiset#(PcIdx, PcMSB, SupSizeX2) pcBlocks <- mkIndexedMultisetQueue; + function CapMem decompressPc(PcCompressed p) = {pcBlocks.lookup(p.idx),p.lsb}; // Epochs Ehr#(2, Bool) decode_epoch <- mkEhr(False); Reg#(Epoch) f_main_epoch <- mkReg(0); // fetch estimate of main epoch @@ -445,12 +462,16 @@ module mkFetchStage(FetchStage); lastItlbReq <= getAddr(pc) & (~ align32b_mask); `endif + let pc_idxs <- pcBlocks.insertAndReserve(truncateLSB(pc), truncateLSB(next_fetch_pc)); + PcIdx pc_idx = pc_idxs.inserted; + PcIdx ppc_idx = pc_idxs.reserved; let out = Fetch1ToFetch2 { - pc: pc, + pc: compressPc(pc_idx, pc), `ifdef RVFI_DII dii_pid: dii_pid, `endif - pred_next_pc: pred_next_pc, + pred_next_pc: isValid(pred_next_pc) ? + Valid(compressPc(ppc_idx, validValue(pred_next_pc))) : Invalid, decode_epoch: decode_epoch[0], main_epoch: f_main_epoch}; @@ -562,37 +583,29 @@ module mkFetchStage(FetchStage); `else if(fetch3In.access_mmio) begin inst_d <- mmio.bootRomResp; - if(verbose) $display("get answer from MMIO 0x%0x", getAddr(fetch3In.pc), " ", fshow(inst_d)); + if(verbose) $display("get answer from MMIO 0x%0x", getAddr(decompressPc(fetch3In.pc)), " ", fshow(inst_d)); end else begin - if(verbose) $display("get answer from memory 0x%0x", getAddr(fetch3In.pc)); + if(verbose) $display("get answer from memory 0x%0x", getAddr(decompressPc(fetch3In.pc))); inst_d <- mem_server.response.get; end `endif end - if (drop_f22f3) begin - if (verbosity >= 2) begin - $display ("----------------"); - $display ("Fetch3: Drop: main_epoch: %d decode epoch: %d fetch3 epoch %d", f_main_epoch, decode_epoch[1]); - $display ("Fetch3: f22f3.first: ", fshow (f22f3.first)); - $display ("Fetch3: inst_d: ", fshow (inst_d)); - end - end - else begin - for (Integer i = 0; i < valueOf(SupSizeX2) && fromInteger(i) <= nbSupX2In; i = i + 1) begin - if (inst_d[i] matches tagged Valid .inst_frag) - f32d.enqS[i].enq (Fetch3ToDecode { - pc: addPc(fetch3In.pc, (2 * fromInteger(i))), + + for (Integer i = 0; i < valueOf(SupSizeX2) && fromInteger(i) <= nbSupX2In; i = i + 1) begin + PcCompressed pc = fetch3In.pc; + pc.lsb = pc.lsb + (2 * fromInteger(i)); + f32d.enqS[i].enq (Fetch3ToDecode { + pc: pc, `ifdef RVFI_DII - dii_pid: fetch3In.dii_pid + fromInteger(i), + dii_pid: fetch3In.dii_pid + fromInteger(i), `endif - ppc: (fromInteger(i)==nbSupX2In) ? fetch3In.pred_next_pc : Invalid, - inst_frag: inst_frag, - cause: fetch3In.cause, - decode_epoch: fetch3In.decode_epoch, - main_epoch: fetch3In.main_epoch - }); - end + ppc: (fromInteger(i)==nbSupX2In) ? fetch3In.pred_next_pc : Invalid, + inst_frag: validValue(inst_d[i]), + cause: fetch3In.cause, + decode_epoch: fetch3In.decode_epoch, + main_epoch: fetch3In.main_epoch + }); end endrule: doFetch3 @@ -600,7 +613,10 @@ module mkFetchStage(FetchStage); rule doDecodeFlush(f32d.deqS[0].canDeq && !isCurrent(f32d.deqS[0].first)); for (Integer i = 0; i < valueOf(SupSizeX2); i = i + 1) - if (f32d.deqS[i].canDeq &&& !isCurrent(f32d.deqS[i].first)) f32d.deqS[i].deq; + if (f32d.deqS[i].canDeq &&& !isCurrent(f32d.deqS[i].first)) begin + pcBlocks.rPort[i].remove(f32d.deqS[i].first.pc.idx); + f32d.deqS[i].deq; + end endrule: doDecodeFlush rule doDecode(f32d.deqS[0].canDeq && isCurrent(f32d.deqS[0].first)); @@ -621,7 +637,7 @@ module mkFetchStage(FetchStage); if (prev_frag_available &&& !is_16b_inst(prev_frag.inst_frag)) begin // 2nd half of 32-bit instruction new_pick = tagged Valid fetch3s_2_inst(frag, prev_frag); if (!validValue(new_pick).mispred_first_half) begin - doAssert(getAddr(prev_frag.pc)+2 == getAddr(frag.pc), "Attached fragments with non-contigious PCs"); + doAssert(getAddr(decompressPc(prev_frag.pc))+2 == getAddr(decompressPc(frag.pc)), "Attached fragments with non-contigious PCs"); `ifdef RVFI_DII doAssert(prev_frag.dii_pid+1 == frag.dii_pid, "Attached fragments with non-contigious DII IDs"); `endif @@ -662,7 +678,9 @@ module mkFetchStage(FetchStage); for (Integer i = 0; i < valueof(SupSize); i=i+1) begin if (decodeIn[i] matches tagged Valid .in) begin let cause = in.cause; - let ppc = in.ppc; + let pc = decompressPc(in.pc); + let ppc = decompressPc(in.ppc); + pcBlocks.rPort[i].remove(in.pc.idx); if (verbose) $display("Decode: %0d in = ", i, fshow (in)); @@ -673,17 +691,17 @@ module mkFetchStage(FetchStage); // uncompressed instruction, so we redirect to this PC and // train it to fetch the other half in future. if (in.decode_epoch == decode_epoch_local && in.mispred_first_half) begin - if (verbose) $display("mispredicted first half in decode: pc : %h", in.pc); + if (verbose) $display("mispredicted first half in decode: pc : %h", pc); decode_epoch_local = !decode_epoch_local; - redirectPc = Valid (in.pc); // record redirect to the first PC in this bundle. - trainNAP = Valid (TrainNAP {pc: in.pc, nextPc: addPc(in.pc, 2)}); + redirectPc = Valid (pc); // record redirect to the first PC in this bundle. + trainNAP = Valid (TrainNAP {pc: pc, nextPc: addPc(pc, 2)}); `ifdef RVFI_DII redirectDiiPid = Valid (in.dii_pid); `endif end else if (in.decode_epoch == decode_epoch_local) begin doAssert(in.main_epoch == f_main_epoch, "main epoch must match"); - let decode_result = decode(in.inst, getFlags(in.pc)==1); // Decode 32b inst, or 32b expansion of 16b inst + let decode_result = decode(in.inst, getFlags(pc)==1); // Decode 32b inst, or 32b expansion of 16b inst // update cause if decode exception and no earlier (TLB) exception if (!isValid(cause)) begin @@ -699,11 +717,11 @@ module mkFetchStage(FetchStage); // direction predict Bool pred_taken = False; if(dInst.iType == Br) begin - let pred_res <- dirPred.pred[i].pred(in.pc); + let pred_res <- dirPred.pred[i].pred(pc); pred_taken = pred_res.taken; dp_train = pred_res.train; end - Maybe#(CapMem) nextPc = decodeBrPred(in.pc, dInst, pred_taken, (in.inst_kind == Inst_32b)); + Maybe#(CapMem) nextPc = decodeBrPred(pc, dInst, pred_taken, (in.inst_kind == Inst_32b)); // return address stack link reg is x1 or x5 function Bool linkedR(Maybe#(ArchRIndx) register); @@ -715,7 +733,7 @@ module mkFetchStage(FetchStage); endfunction Bool dst_link = linkedR(regs.dst); Bool src1_link = linkedR(regs.src1); - CapMem push_addr = addPc(in.pc, ((in.inst_kind == Inst_32b) ? 4 : 2)); + CapMem push_addr = addPc(pc, ((in.inst_kind == Inst_32b) ? 4 : 2)); CapMem pop_addr = ras.ras[i].first; if (dInst.iType == J && dst_link) begin @@ -749,13 +767,13 @@ module mkFetchStage(FetchStage); end if(verbose) begin - $display("Branch prediction: ", fshow(dInst.iType), " ; ", fshow(in.pc), " ; ", + $display("Branch prediction: ", fshow(dInst.iType), " ; ", fshow(pc), " ; ", fshow(ppc), " ; ", fshow(pred_taken), " ; ", fshow(nextPc)); end // check previous mispred - if (nextPc matches tagged Valid .decode_pred_next_pc &&& (decode_pred_next_pc != in.ppc)) begin - if (verbose) $display("ppc and decodeppc : %h %h", in.ppc, decode_pred_next_pc); + if (nextPc matches tagged Valid .decode_pred_next_pc &&& (decode_pred_next_pc != ppc)) begin + if (verbose) $display("ppc and decodeppc : %h %h", ppc, decode_pred_next_pc); decode_epoch_local = !decode_epoch_local; redirectPc = Valid (decode_pred_next_pc); // record redirect next pc `ifdef RVFI_DII @@ -763,7 +781,7 @@ module mkFetchStage(FetchStage); `endif ppc = decode_pred_next_pc; // train next addr pred when mispredict - let last_x16_pc = addPc(in.pc, ((in.inst_kind == Inst_32b) ? 2 : 0)); + let last_x16_pc = addPc(pc, ((in.inst_kind == Inst_32b) ? 2 : 0)); trainNAP = Valid (TrainNAP {pc: last_x16_pc, nextPc: decode_pred_next_pc}); `ifdef PERF_COUNT // performance stats: record decode redirect @@ -772,7 +790,7 @@ module mkFetchStage(FetchStage); `endif end end // if (!isValid(cause)) - let out = FromFetchStage{pc: in.pc, + let out = FromFetchStage{pc: pc, `ifdef RVFI_DII dii_pid: in.dii_pid, `endif @@ -784,7 +802,7 @@ module mkFetchStage(FetchStage); orig_inst: in.orig_inst, regs: decode_result.regs, cause: cause, - tval: getAddr(in.pc) + ((in.cause_second_half) ? 2:0) + tval: getAddr(pc) + ((in.cause_second_half) ? 2:0) }; out_fifo.enqS[i].enq(out); if (verbosity >= 1) begin diff --git a/src_Core/RISCY_OOO/procs/lib/IndexedMultiset.bsv b/src_Core/RISCY_OOO/procs/lib/IndexedMultiset.bsv new file mode 100644 index 0000000..f567634 --- /dev/null +++ b/src_Core/RISCY_OOO/procs/lib/IndexedMultiset.bsv @@ -0,0 +1,230 @@ +/*- + * Copyright (c) 2020 Jonathan Woodruff + * All rights reserved. + * + * This software was developed by SRI International and the University of + * Cambridge Computer Laboratory (Department of Computer Science and + * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the + * DARPA SSITH research programme. + * + * @BERI_LICENSE_HEADER_START@ + * + * Licensed to BERI Open Systems C.I.C. (BERI) under one or more contributor + * license agreements. See the NOTICE file distributed with this work for + * additional information regarding copyright ownership. BERI licenses this + * file to you under the BERI Hardware-Software License, Version 1.0 (the + * "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at: + * + * http://www.beri-open-systems.org/legal/license-1-0.txt + * + * Unless required by applicable law or agreed to in writing, Work distributed + * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * @BERI_LICENSE_HEADER_END@ + */ + +import Vector::*; +import Assert::*; + +interface IndexedMultisetRemoveIfc#(type idxT); + method Action remove(idxT i); +endinterface +interface IndexedMultiset#(type idxT, type datT, numeric type remWidth); + method ActionValue#(idxT) insert(datT d); + // Reserved data values must be the same as the next inserted value or they will be overwritten. + method ActionValue#(IndexedMultisetIndices#(idxT)) insertAndReserve(datT ins, datT res); + method datT lookup(idxT i); + interface Vector#(remWidth, IndexedMultisetRemoveIfc#(idxT)) rPort; + method Action clearAll; +endinterface + +typedef struct { + datT data; + ctr count; +} IndexedMultisetRecord#(type datT, type ctr) deriving(Bits); + +typedef struct { + idxT inserted; + idxT reserved; +} IndexedMultisetIndices#(type idxT) deriving(Bits); + +typedef struct { + idxT idx; + datT dat; +} IndexedMultisetInsert#(type idxT, type datT) deriving(Bits); + +module mkIndexedMultiset(IndexedMultiset#(idxT, datT, remWidth)) + provisos ( + Bits#(datT, _datsz), + Eq#(datT), + Bits#(idxT, _idxsz), + PrimIndex#(idxT, a__), + Alias#(wide_idxT, Bit#(TAdd#(_idxsz,2))) // Allow for generous number of duplicates per data value; 4x index capacity. + ); + Vector#(TExp#(_idxsz), Reg#(IndexedMultisetRecord#(datT,wide_idxT))) + records <- replicateM(mkReg(unpack(0))); + let recsRead = readVReg(records); + RWire#(IndexedMultisetInsert#(idxT,datT)) insertW <- mkRWire; + RWire#(IndexedMultisetInsert#(idxT,datT)) reserveW <- mkRWire; + Vector#(remWidth, RWire#(idxT)) removeW <- replicateM(mkRWire); + PulseWire clearW <- mkPulseWire; + + (* no_implicit_conditions *) + rule canon; + Vector#(TExp#(_idxsz),IndexedMultisetRecord#(datT,wide_idxT)) recs = readVReg(records); + if (insertW.wget matches tagged Valid .r) begin + recs[r.idx].data = r.dat; + recs[r.idx].count = recs[r.idx].count + 1; + end + if (reserveW.wget matches tagged Valid .r) recs[r.idx].data = r.dat; + for (Integer i = 0; i < valueOf(remWidth); i = i + 1) + if (removeW[i].wget matches tagged Valid .r) recs[r].count = recs[r].count - 1; + if (clearW) recs = replicate(IndexedMultisetRecord{data: ?, count: 0}); + writeVReg(records, recs); + endrule + + function isEmpty(r) = (r.count == 0); + function isFull(r) = (r.count == ~0); + function dataMatch(d, r) = (r.data == d); + function ActionValue#(idxT) findIdx(datT d, Vector#(TExp#(_idxsz), IndexedMultisetRecord#(datT,wide_idxT)) recs) = + actionvalue + let mi = findIndex(dataMatch(d),recs); + if (!isValid(mi)) mi = findIndex(isEmpty,recs); + dynamicAssert(isValid(mi), "failed to insert in indexed multiset"); + return unpack(pack(validValue(mi))); + endactionvalue; + + Vector#(remWidth, IndexedMultisetRemoveIfc#(idxT)) removes; + for (Integer i = 0; i < valueOf(remWidth); i = i + 1) begin + removes[i] = interface IndexedMultisetRemoveIfc#(idxT); + method Action remove(idxT index); + dynamicAssert(recsRead[index].count != 0, "removed empty item in indexed multiset"); + removeW[i].wset(index); + endmethod + endinterface; + end + + Bool anyEmpty = any(isEmpty,recsRead); + Bool anyFull = any(isFull,recsRead); + method ActionValue#(idxT) insert(datT d) if (anyEmpty && !anyFull); + let i <- findIdx(d,recsRead); + insertW.wset(IndexedMultisetInsert{idx:i, dat:d}); + return i; + endmethod + + method ActionValue#(IndexedMultisetIndices#(idxT)) insertAndReserve(datT ins, datT res); + let insIdx <- findIdx(ins,recsRead); + insertW.wset(IndexedMultisetInsert{idx:insIdx, dat:ins}); + let newRecs = recsRead; + newRecs[insIdx].data = ins; + let resIdx <- findIdx(res,newRecs); + reserveW.wset(IndexedMultisetInsert{idx:resIdx, dat:res}); + return IndexedMultisetIndices{inserted: insIdx, reserved: resIdx}; + endmethod + + method datT lookup(idxT i) = records[i].data; + + interface rPort = removes; + + method Action clearAll = clearW.send; +endmodule + +// An implementation of Indexed Multiset that depends on inserting and removing in the same order. +module mkIndexedMultisetQueue(IndexedMultiset#(Bit#(idxTSz), datT, remWidth)) + provisos ( + Alias#(idxT, Bit#(idxTSz)), + Bits#(datT, _datsz), + Eq#(datT) + ); + Vector#(TExp#(idxTSz), Reg#(datT)) records <- replicateM(mkRegU); + Vector#(TExp#(idxTSz), datT) recsRead = readVReg(records); + + Reg#(Bit#(TAdd#(idxTSz,1))) lhead <- mkReg(0); + Reg#(Bit#(TAdd#(idxTSz,1))) ltail <- mkReg(0); + idxT head = truncate(lhead); + Bit#(TAdd#(idxTSz,1)) level = lhead - ltail; + //Bool empty = (level==0); + Bool full = (level==fromInteger(valueOf(TExp#(idxTSz)))); + Bool almostFull = (level>=fromInteger(valueOf(TExp#(idxTSz)))-1); + + RWire#(datT) insertW <- mkRWire; + RWire#(datT) reserveW <- mkRWire; + Vector#(remWidth, RWire#(idxT)) removeW <- replicateM(mkRWire); + PulseWire clearW <- mkPulseWire; + + function Bit#(TAdd#(idxTSz,1)) updateWidePointer(Bit#(TAdd#(idxTSz,1)) lIdx, idxT idx); + idxT diff = idx - truncate(lIdx); + return lIdx + zeroExtend(diff); + endfunction + (* no_implicit_conditions *) + rule canon; + Vector#(TExp#(idxTSz),datT) recs = recsRead; + Bit#(TAdd#(idxTSz,1)) nlhead = lhead; + Bit#(TAdd#(idxTSz,1)) nltail = ltail; + if (insertW.wget matches tagged Valid .d) begin + $display("insert dat:%x nlhead:%d", {pack(d),12'b0}, nlhead); + idxT i = truncate(nlhead); + recs[i] = d; + nlhead = nlhead + 1; + end + if (reserveW.wget matches tagged Valid .d) begin + $display("reserve dat:%x nlhead:%d", {pack(d),12'b0}, nlhead); + idxT i = truncate(nlhead); + recs[i] = d; + //nlhead = nlhead + 1; + end + for (Integer i = 0; i < valueOf(remWidth); i = i + 1) + if (removeW[i].wget matches tagged Valid .r) begin + $display("remove[%d] idx:%d", i, r); + nltail = updateWidePointer(nltail, r); + end + if (clearW) begin + nlhead = 0; + nltail = 0; + end + lhead <= nlhead; + ltail <= nltail; + writeVReg(records, recs); + $display("full:%x ltail:%d lhead:%d level:%d", full, ltail, lhead, level); + endrule + + Vector#(remWidth, IndexedMultisetRemoveIfc#(idxT)) removes; + for (Integer i = 0; i < valueOf(remWidth); i = i + 1) begin + removes[i] = interface IndexedMultisetRemoveIfc#(idxT); + method Action remove(idxT idx) = removeW[i].wset(idx); + endinterface; + end + + method ActionValue#(idxT) insert(datT d) if (!full); + Bool next = (recsRead[head-1]!=d); + if (next) insertW.wset(d); + return (next) ? head:head-1; + endmethod + // Reserved data values must be the same as the next inserted value or they will be overwritten. + method ActionValue#(IndexedMultisetIndices#(idxT)) insertAndReserve(datT ins, datT res) if (!almostFull); + idxT insIdx = head - 1; // Default, assuming a match. + idxT resIdx = head - 1; // Default, assuming a match. + if (recsRead[head - 1]!=ins) begin + insIdx = head; + insertW.wset(ins); // Increment head. + resIdx = head; // new default for reserved. + if (res!=ins) begin + resIdx = head + 1; + reserveW.wset(res); // Increment head again! + end + end else if (recsRead[head - 1]!=res) begin + resIdx = head; + reserveW.wset(res); + end + return IndexedMultisetIndices{inserted: insIdx, reserved: resIdx}; + endmethod + + method datT lookup(idxT i) = recsRead[i]; + + interface rPort = removes; + + method Action clearAll = clearW.send; +endmodule