From 60c1e31b0164aa7d01b2389917d7ba1571e573b6 Mon Sep 17 00:00:00 2001 From: James Clarke Date: Wed, 1 Jan 2020 22:22:17 +0000 Subject: [PATCH] Reduce needless fetch stalls in presence of compressed instructions Currently, our next address prediction cannot distinguish between a taken compressed branch to PC+4 and an uncompressed instruction that falls through. We can instead make the NAP machinery much more robust by keying it on the 16b parcels, with uncompressed branches having their taken prediction on the second 16b parcel. This also removes the need for the address prediction requests to be chained. Moreover, if we decode more than 2 instructions in one cycle due to decompression, we throw away any subsequent instructions and treat it like a branch miss, redirecting and thus restarting the pipeline from the first discarded PC. We should therefore instead save them for issuing on the next cycle and avoid the redirects. To ensure we don't needlessly reduce our IPC, if we have a partial issue's width of instructions saved, we should also support issuing instructions from the next ICache response if valid, which should be the case in hot correctly-predicted code paths, especially tight loops. As part of this change, we also keep the pending straddle state in Fetch3 rather than sending it to Decode only to have it be forwarded back. Combined, these two approaches ensure the fetch unit can maintain an IPC of 2 after it has had time to be correctly trained, regardless of the distribution of compressed instructions. --- src_Core/CPU/Core.bsv | 2 +- .../procs/RV64G_OOO/AluExePipeline.bsv | 9 +- .../RISCY_OOO/procs/RV64G_OOO/FetchStage.bsv | 477 ++++++++++++------ src_Core/RISCY_OOO/procs/lib/Btb.bsv | 10 +- 4 files changed, 326 insertions(+), 172 deletions(-) diff --git a/src_Core/CPU/Core.bsv b/src_Core/CPU/Core.bsv index 0796cfa..7f05d80 100644 --- a/src_Core/CPU/Core.bsv +++ b/src_Core/CPU/Core.bsv @@ -309,7 +309,7 @@ module mkCore#(CoreId coreId)(Core); let train <- toGet(trainBPQ[i]).get; fetchStage.train_predictors( train.pc, train.nextPc, train.iType, train.taken, - train.dpTrain, train.mispred + train.dpTrain, train.mispred, train.isCompressed ); endrule end diff --git a/src_Core/RISCY_OOO/procs/RV64G_OOO/AluExePipeline.bsv b/src_Core/RISCY_OOO/procs/RV64G_OOO/AluExePipeline.bsv index 9d57edf..8b3c99b 100644 --- a/src_Core/RISCY_OOO/procs/RV64G_OOO/AluExePipeline.bsv +++ b/src_Core/RISCY_OOO/procs/RV64G_OOO/AluExePipeline.bsv @@ -78,6 +78,7 @@ typedef struct { Maybe#(PhyDst) dst; InstTag tag; DirPredTrainInfo dpTrain; + Bool isCompressed; // result Data data; // alu compute result Maybe#(Data) csrData; // data to write CSR file @@ -123,6 +124,7 @@ typedef struct { Bool taken; DirPredTrainInfo dpTrain; Bool mispred; + Bool isCompressed; } FetchTrainBP deriving(Bits, Eq, FShow); interface AluExeInput; @@ -291,6 +293,7 @@ module mkAluExePipeline#(AluExeInput inIfc)(AluExePipeline); dst: x.dst, tag: x.tag, dpTrain: x.dpTrain, + isCompressed: x.orig_inst[1:0] != 2'b11, data: exec_result.data, csrData: isValid(x.dInst.csr) ? Valid (exec_result.csrData) : Invalid, controlFlow: exec_result.controlFlow, @@ -333,7 +336,8 @@ module mkAluExePipeline#(AluExeInput inIfc)(AluExePipeline); iType: x.iType, taken: x.controlFlow.taken, dpTrain: x.dpTrain, - mispred: True + mispred: True, + isCompressed: x.isCompressed }); `ifdef PERF_COUNT // performance counter @@ -361,7 +365,8 @@ module mkAluExePipeline#(AluExeInput inIfc)(AluExePipeline); iType: x.iType, taken: x.controlFlow.taken, dpTrain: x.dpTrain, - mispred: False + mispred: False, + isCompressed: x.isCompressed }); end end diff --git a/src_Core/RISCY_OOO/procs/RV64G_OOO/FetchStage.bsv b/src_Core/RISCY_OOO/procs/RV64G_OOO/FetchStage.bsv index 56c20c1..635d904 100644 --- a/src_Core/RISCY_OOO/procs/RV64G_OOO/FetchStage.bsv +++ b/src_Core/RISCY_OOO/procs/RV64G_OOO/FetchStage.bsv @@ -36,6 +36,7 @@ import MemoryTypes::*; import Types::*; import ProcTypes::*; import CCTypes::*; +import DefaultValue::*; import Ras::*; import EpochManager::*; import Performance::*; @@ -76,7 +77,7 @@ interface FetchStage; method Action done_flushing(); method Action train_predictors( Addr pc, Addr next_pc, IType iType, Bool taken, - DirPredTrainInfo dpTrain, Bool mispred + DirPredTrainInfo dpTrain, Bool mispred, Bool isCompressed ); // security @@ -100,26 +101,34 @@ typedef struct { typedef struct { Addr pc; - Addr pred_next_pc; + Maybe#(Addr) pred_next_pc; + Bool fetch3_epoch; Bool decode_epoch; Epoch main_epoch; } Fetch1ToFetch2 deriving(Bits, Eq, FShow); typedef struct { Addr pc; - Addr phys_pc; - Addr pred_next_pc; + Maybe#(Addr) pred_next_pc; Maybe#(Exception) cause; Addr tval; // in case of exception Bool access_mmio; // inst fetch from MMIO + Bool fetch3_epoch; Bool decode_epoch; Epoch main_epoch; } Fetch2ToFetch3 deriving(Bits, Eq, FShow); -// TODO: this name 'Fetch3ToDecode' is a misnomer. -// The struct passed from doFetch3 to doDecode is Fetch2ToFetch3 (same type as doFetch2 to doFetch3), -// and Fetch3ToDecode is used purely internally in doDecode. +typedef struct { + Addr pc; + Addr pred_next_pc; + Bool mispred_first_half; + Maybe#(Exception) cause; + Addr tval; // in case of exception + Bool decode_epoch; + Epoch main_epoch; +} Fetch3ToDecode deriving(Bits, Eq, FShow); +// Used purely internally in doDecode. typedef struct { Addr pc; Addr ppc; @@ -127,7 +136,7 @@ typedef struct { Epoch main_epoch; Instruction inst; Maybe#(Exception) cause; -} Fetch3ToDecode deriving(Bits, Eq, FShow); +} InstrFromFetch3 deriving(Bits, Eq, FShow); typedef struct { Addr pc; @@ -178,13 +187,25 @@ endfunction typedef TMul #(SupSize, 2) SupSizeX2; typedef Bit #(TLog #(TAdd #(SupSizeX2, 1))) SupCntX2; +// Merging up to SupSize-1 pending instructions with up to SupSizeX2 decoded +// instructions produces up to 3*SupSize-1 instructions; SupSize can be issued +// if present, leaving up to 2*SupSize-1 pending. +typedef TSub #(TMul #(SupSize, 2), 1) SupSizeX2S1; +typedef Bit #(TLog #(TAdd #(SupSizeX2S1, 1))) SupCntX2S1; +typedef TSub #(TMul #(SupSize, 3), 1) SupSizeX3S1; +typedef Bit #(TLog #(TAdd #(SupSizeX3S1, 1))) SupCntX3S1; + +// Appending the pending and decoded vectors produces an intermediate +// 4*SupSize-1 vector (with only up to 3*SupSize-1 elements non-empty). +typedef TSub #(TMul #(SupSize, 4), 1) SupSizeX4S1; +typedef Bit #(TLog #(TAdd #(SupSizeX4S1, 1))) SupCntX4S1; + // Parsing a sequence of 16-bit parcels returns a sequence of the // following kinds or items typedef enum {Inst_None, // When we run off the end of the sequence Inst_16b, // A 16b instruction - Inst_32b, // A 32b instruction - Inst_32b_Lsbs // Lower 16b of a 32b instr + Inst_32b // A 32b instruction } Inst_Kind deriving (Bits, Eq, FShow); @@ -199,6 +220,12 @@ typedef struct { } Inst_Item deriving (Bits, Eq, FShow); +instance DefaultValue #(Inst_Item); + function Inst_Item defaultValue = Inst_Item { + pc: 0, inst_kind: Inst_None, orig_inst: 0, inst: 0 + }; +endinstance + // Input 'inst_d' was fetched from memory: up to superscalar-size sequence of 32b parcels. // Convert this into 16b parcels, prior to re-parsing for possible mix of 32b and 16b instructions. // This is a pure function; ActionValue is used only to allow $displays for debugging. @@ -223,31 +250,53 @@ endfunction // Parse 16b parcels (v_x16) into a sequence of 16b or 32b instructions. // This is a pure function; ActionValue is used only to allow $displays for debugging. -function ActionValue #(Vector #(SupSize, Inst_Item)) +function ActionValue #(Tuple4 #(SupCntX2, + Vector #(SupSizeX2, Inst_Item), + Addr, + Maybe #(Tuple3 #(Addr, Bit #(16), Bool)))) fav_parse_insts (Bool verbose, Addr pc_start, - SupCntX2 n_x16s, - Vector #(SupSizeX2, Bit #(16)) v_x16); + Maybe #(Addr) pred_next_pc, + Maybe #(Tuple3 #(Addr, Bit #(16), Bool)) pending_straddle, + SupCntX2 n_x16s, + Vector #(SupSizeX2, Bit #(16)) v_x16); actionvalue - // Parse up to SupSize instructions (v_items) from fetched v_x16 parcels (v_x16). - Vector #(SupSize, Inst_Item) v_items = replicate (Inst_Item {pc: pc_start, - inst_kind: Inst_None, - orig_inst: 0, - inst: 0}); - SupCntX2 j = ((pc_start [1:0] == 2'b00) ? 0 : 1); // Start parse at parcel 0/1 depending on pc lsbs + // Parse up to SupSizeX2 instructions (v_items) from fetched v_x16 parcels (v_x16). + Vector #(SupSizeX2, Inst_Item) v_items = replicate (Inst_Item {pc: pc_start, + inst_kind: Inst_None, + orig_inst: 0, + inst: 0}); + Maybe #(Tuple3 #(Addr, Bit #(16), Bool)) next_straddle = tagged Invalid; + // Start parse at parcel 0/1 depending on pc lsbs and pending straddle + SupCntX2 j = ((pc_start [1:0] == 2'b00 || isValid(pending_straddle)) ? 0 : 1); Addr pc = pc_start; - for (Integer i = 0; i < valueOf (SupSize); i = i + 1) begin + Integer n_items = 0; + for (Integer i = 0; i < valueOf (SupSizeX2); i = i + 1) begin Inst_Kind inst_kind = Inst_None; Bit #(32) orig_inst = 0; Bit #(32) inst = 0; Addr next_pc = pc; if (j < n_x16s) begin - if (is_16b_inst (v_x16 [j])) begin + if (i == 0 &&& pending_straddle matches tagged Valid {.s_pc, .s_lsbs, .s_mispred}) begin + if (pc != s_pc + 2) begin + $display ("FetchStage.fav_parse_insts: straddle: pc mismatch: pc = 0x%0h but s_pc = 0x%0h", pc, s_pc); + dynamicAssert (False, "FetchStage.fav_parse_insts: straddle: pc mismatch"); + end + pc = s_pc; + inst_kind = Inst_32b; + orig_inst = { v_x16[0], s_lsbs }; + inst = orig_inst; + j = 1; + next_pc = s_pc + 4; + n_items = 1; + end + else if (is_16b_inst (v_x16 [j])) begin inst_kind = Inst_16b; orig_inst = zeroExtend (v_x16 [j]); inst = fv_decode_C (misa, misa_mxl_64, v_x16 [j]); // Expand 16b inst to 32b inst j = j + 1; next_pc = pc + 2; + n_items = i + 1; if (verbose) $display ("FetchStage.fav_parse_insts: C inst 0x%0h -> inst 0x%0h", orig_inst, inst); end @@ -258,12 +307,15 @@ function ActionValue #(Vector #(SupSize, Inst_Item)) inst = orig_inst; j = j + 2; next_pc = pc + 4; + n_items = i + 1; end else begin - inst_kind = Inst_32b_Lsbs; - orig_inst = zeroExtend (v_x16 [j]); + next_straddle = tagged Valid tuple3(pc, v_x16[j], isValid(pred_next_pc)); j = j + 1; - next_pc = pc + 2; + // Leave next_pc unchanged and clear pred_next_pc so we + // return the right predicted pc for the vector, which + // excludes the pending straddle. + pred_next_pc = tagged Invalid; end end else begin @@ -281,10 +333,12 @@ function ActionValue #(Vector #(SupSize, Inst_Item)) if (verbose) begin $display ("FetchStage.fav_parse_insts:"); $display (" v_x16: ", fshow (v_x16)); + $display (" n_items: %0d", n_items); $display (" v_items: ", fshow (v_items)); + $display (" next_straddle: ", fshow (next_straddle)); end - return v_items; + return tuple4(fromInteger(n_items), v_items, fromMaybe(pc, pred_next_pc), next_straddle); endactionvalue endfunction @@ -315,24 +369,28 @@ module mkFetchStage(FetchStage); // We stall until the flush is done Reg#(Bool) waitForFlush <- mkReg(False); - Ehr#(3, Addr) pc_reg <- mkEhr(0); + Ehr#(4, Addr) pc_reg <- mkEhr(0); Integer pc_fetch1_port = 0; Integer pc_decode_port = 1; - Integer pc_redirect_port = 2; + Integer pc_fetch3_port = 2; + Integer pc_redirect_port = 3; // Epochs + Reg#(Bool) fetch3_epoch <- mkReg(False); Ehr#(2, Bool) decode_epoch <- mkEhr(False); Reg#(Epoch) f_main_epoch <- mkReg(0); // fetch estimate of main epoch - // Regs/wires to hold the first half of an instruction that straddles a cache line boundary - Ehr #(3, Bool) ehr_pending_straddle <- mkEhr (False); - Ehr #(2, Addr) ehr_half_inst_pc <- mkEhr (?); // The PC of the straddling instruction - Ehr #(2, Bit #(16)) ehr_half_inst_lsbs <- mkEhr (?); // The 16 lsbs of the straddling instruction + // Reg to hold the first half of an instruction that straddles a cache line boundary + Ehr #(2, Maybe #(Tuple3 #(Addr, Bit #(16), Bool))) ehr_pending_straddle <- mkEhr(tagged Invalid); + // Reg to hold extra instructions from Fetch3 to send to decode the next cycle + Reg #(Vector #(SupSizeX2S1, Inst_Item)) rg_pending_decode <- mkReg(replicate(defaultValue)); + Reg #(SupCntX2S1) rg_pending_n_items <- mkRegU; + Reg #(Fetch3ToDecode) rg_pending_f32d <- mkRegU; // Pipeline Stage FIFOs - Fifo#(2, Tuple2#(Bit#(TLog#(SupSize)),Fetch1ToFetch2)) f12f2 <- mkCFFifo; - Fifo#(4, Tuple2#(Bit#(TLog#(SupSize)),Fetch2ToFetch3)) f22f3 <- mkCFFifo; // FIFO should match I$ latency - Fifo#(2, Tuple2#(Bit#(TLog#(SupSize)),Fetch2ToFetch3)) f32d <- mkCFFifo; + Fifo#(2, Tuple2#(Bit#(TLog#(SupSizeX2)),Fetch1ToFetch2)) f12f2 <- mkCFFifo; + Fifo#(4, Tuple2#(Bit#(TLog#(SupSizeX2)),Fetch2ToFetch3)) f22f3 <- mkCFFifo; // FIFO should match I$ latency + Fifo#(2, Tuple2#(Bit#(TLog#(SupSize)),Fetch3ToDecode)) f32d <- mkCFFifo; // Fifo#(2, Vector#(SupSize,Maybe#(Instruction))) instdata <- mkPipelineFifo(); // OLD // FIFO from rule doFetch3 to rule doDecode @@ -387,31 +445,27 @@ module mkFetchStage(FetchStage); // Predict the next fetch-PC based only on current PC (without // knowing the instructions). - // Note: this chains calls to nextAddrPred. If this is a critical-path problem, - // alternatively one could apply nextAddrPred in parallel at pc+2, pc+4, pc+6, ... - // and memo-ize them in a vector (TODO). - function ActionValue #(Tuple2 #(Integer, Addr)) fav_pred_next_pc (Addr pc); + function ActionValue #(Tuple2 #(Integer, Maybe #(Addr))) fav_pred_next_pc (Addr pc); actionvalue Addr prev_PC = pc; - Addr pred_next_pc = nextAddrPred.predPc (prev_PC); - Integer posLastSup = 0; + Maybe #(Addr) pred_next_pc = nextAddrPred.predPc (prev_PC); + Integer posLastSupX2 = 0; Bool done = False; - for (Integer i = 0; i < valueof (SupSize); i = i + 1) begin + for (Integer i = 0; i < valueOf (SupSizeX2); i = i + 1) begin if (! done) begin - Bool lastInstInCacheLine = (getLineInstOffset (prev_PC) == maxBound); - Bool isSeq16 = ((prev_PC + 2) == pred_next_pc); - Bool isSeq32 = ((prev_PC + 4) == pred_next_pc); - Bool isJump = ((! isSeq16) && (! isSeq32)); - done = ((i == (valueOf (SupSize) - 1)) || lastInstInCacheLine || isJump); - posLastSup = i; + Bool isLastX2 = (i == (valueOf (SupSizeX2) - 1)) || ((pc[1:0] != 2'b00) && (i == (valueOf (SupSizeX2) - 2))); + Bool lastInstInCacheLine = (getLineInstOffset (prev_PC) == maxBound) && (prev_PC[1:0] != 2'b00); + Bool isJump = isValid(pred_next_pc); + done = isLastX2 || lastInstInCacheLine || isJump; + posLastSupX2 = i; if (! done) begin - prev_PC = pred_next_pc; - pred_next_pc = nextAddrPred.predPc (pred_next_pc); + prev_PC = prev_PC + 2; + pred_next_pc = nextAddrPred.predPc (prev_PC); end end end - return tuple2 (posLastSup, pred_next_pc); + return tuple2 (posLastSupX2, pred_next_pc); endactionvalue endfunction @@ -445,8 +499,9 @@ module mkFetchStage(FetchStage); pc_reg[pc_fetch1_port] <= pred_next_pc; */ - match { .posLastSup, .pred_next_pc } <- fav_pred_next_pc (pc); - pc_reg[pc_fetch1_port] <= pred_next_pc; + match { .posLastSupX2, .pred_next_pc } <- fav_pred_next_pc (pc); + let next_fetch_pc = fromMaybe(pc + 2 * (fromInteger(posLastSupX2) + 1), pred_next_pc); + pc_reg[pc_fetch1_port] <= next_fetch_pc; // Send TLB request. // Mask to 32-bit alignment, even if 'C' is supported (where we may discard first 2 bytes) @@ -456,14 +511,16 @@ module mkFetchStage(FetchStage); let out = Fetch1ToFetch2 { pc: pc, pred_next_pc: pred_next_pc, + fetch3_epoch: fetch3_epoch, decode_epoch: decode_epoch[0], main_epoch: f_main_epoch}; - f12f2.enq(tuple2(fromInteger(posLastSup),out)); + let nbSupX2 = fromInteger(posLastSupX2) + (pc[1:0] == 2'b00 ? 0 : 1); + f12f2.enq(tuple2(nbSupX2,out)); if (verbose) $display("Fetch1: ", fshow(out)); endrule rule doFetch2; - let {nbSup,in} = f12f2.first; + let {nbSupX2,in} = f12f2.first; f12f2.deq; // Get TLB response @@ -483,6 +540,7 @@ module mkFetchStage(FetchStage); // cache line size, so all nbSup+1 insts can be fetched // from boot rom. It won't happen that insts fetched from // boot rom is less than requested. + Bit #(TLog #(SupSize)) nbSup = truncate(nbSupX2 >> 1); mmio.bootRomReq(phys_pc, nbSup); access_mmio = True; end @@ -500,112 +558,219 @@ module mkFetchStage(FetchStage); let out = Fetch2ToFetch3 { pc: in.pc, - phys_pc: phys_pc, pred_next_pc: in.pred_next_pc, cause: cause, tval: tval, access_mmio: access_mmio, + fetch3_epoch: in.fetch3_epoch, decode_epoch: in.decode_epoch, main_epoch: in.main_epoch }; - f22f3.enq(tuple2(nbSup,out)); + f22f3.enq(tuple2(nbSupX2,out)); if (verbosity > 0) begin $display ("----------------"); $display ("Fetch2: TLB response pyhs_pc 0x%0h cause ", phys_pc, fshow (cause)); - $display ("Fetch2: f2_tof3.enq: nbSup %0d out ", nbSup, fshow (out)); + $display ("Fetch2: f2_tof3.enq: nbSupX2 %0d out ", nbSupX2, fshow (out)); end endrule // Break out of i$ rule doFetch3; - let {nbSup, fetch3In} = f22f3.first; - f22f3.deq(); - if (verbosity > 0) - $display("Fetch3: fetch3In: ", fshow (fetch3In)); + let {nbSupX2In, fetch3In} = f22f3.first; + if (verbosity > 0) begin + if (f22f3.notEmpty) + $display("Fetch3: nbSupX2In: %0d fetch3In: ", nbSupX2In, fshow (fetch3In)); + else + $display("Fetch3: Nothing else from Fetch2"); + end + + SupCntX2S1 pending_n_items = rg_pending_n_items; + let out = rg_pending_f32d; + Maybe #(Tuple3 #(Addr, Bit #(16), Bool)) pending_straddle = ehr_pending_straddle[0]; + + if (pending_n_items > 0) begin + if (rg_pending_f32d.main_epoch != f_main_epoch || rg_pending_f32d.decode_epoch != decode_epoch[1]) begin + // Just drop it. Also drop any pending straddle, as that is + // associated with the same epoch. + pending_n_items = 0; + pending_straddle = tagged Invalid; + if (verbosity > 0) begin + $display ("----------------"); + $display ("Fetch3: Drop pending: main_epoch: %d decode epoch: %d", f_main_epoch, decode_epoch[1]); + $display ("Fetch3: rg_pending_n_items: ", fshow (rg_pending_n_items)); + $display ("Fetch3: rg_pending_f32d: ", fshow (rg_pending_f32d)); + $display ("Fetch3: rg_pending_decode: ", fshow (rg_pending_decode)); + end + end + end + + SupCntX2 parsed_n_items = 0; + Inst_Item inst_item_none = Inst_Item {pc: fetch3In.pc, inst_kind: Inst_None, orig_inst: 0, inst: 0}; + Vector #(SupSizeX2, Inst_Item) parsed_v_items = replicate (inst_item_none); + + let mispred_first_half = pending_straddle matches tagged Valid {.s_pc, .s_lsbs, .s_mispred} &&& s_mispred ? True : False; + let can_merge = pending_n_items > 0 + && pending_n_items < fromInteger(valueOf(SupSize)) + && f22f3.notEmpty + && !isValid(fetch3In.cause) + && fetch3In.main_epoch == rg_pending_f32d.main_epoch + && fetch3In.decode_epoch == rg_pending_f32d.decode_epoch + && !mispred_first_half; + + let drop_f22f3 = f22f3.notEmpty + && ( fetch3In.main_epoch != f_main_epoch + || fetch3In.decode_epoch != decode_epoch[1] + || fetch3In.fetch3_epoch != fetch3_epoch); + + let parse_f22f3 = !drop_f22f3 && (pending_n_items == 0 || can_merge); // Get ICache/MMIO response if no exception // In case of exception, we still need to process at least inst_data[0] // (it will be turned to an exception later), so inst_data[0] must be // valid. Vector#(SupSize,Maybe#(Instruction)) inst_d = replicate(tagged Valid (0)); - if(!isValid(fetch3In.cause)) begin - if(fetch3In.access_mmio) begin - if(verbose) $display("get answer from MMIO %d", fetch3In.pc); - inst_d <- mmio.bootRomResp; - end - else begin - if(verbose) $display("get answer from memory %d", fetch3In.pc); - inst_d <- mem_server.response.get; + if (drop_f22f3 || parse_f22f3) begin + f22f3.deq(); + if (!isValid(fetch3In.cause)) begin + if(fetch3In.access_mmio) begin + if(verbose) $display("get answer from MMIO %d", fetch3In.pc); + inst_d <- mmio.bootRomResp; + end + else begin + if(verbose) $display("get answer from memory %d", fetch3In.pc); + inst_d <- mem_server.response.get; + end end end - if (fetch3In.decode_epoch != decode_epoch[1]) begin - // Just drop it. - if (verbosity > 0) begin - $display ("----------------"); - $display ("Fetch3: Drop: decode epoch: %d", decode_epoch[1]); - $display ("Fetch3: f22f3.first: ", fshow (f22f3.first)); - $display ("Fetch3: inst_d: ", fshow (inst_d)); - end - end - else begin - // Re-interpret fetched 32b parcels (inst_d) as 16b parcels - match { .n_x16s, .v_x16 } <- fav_inst_d_to_x16s (inst_d); - Addr start_PC = fetch3In.pc; + if (drop_f22f3) begin + // Drop any pending straddle if this is for a different main or + // decode epoch since that invalidates our Fetch3 redirect, but + // otherwise keep it to flush the pipeline until we get the next + // half of the straddle. + if (fetch3In.main_epoch != f_main_epoch || fetch3In.decode_epoch != decode_epoch[1]) begin + pending_straddle = tagged Invalid; + end + if (verbosity > 0) begin + $display ("----------------"); + $display ("Fetch3: Drop: main_epoch: %d decode epoch: %d fetch3 epoch %d", f_main_epoch, decode_epoch[1], fetch3_epoch); + $display ("Fetch3: f22f3.first: ", fshow (f22f3.first)); + $display ("Fetch3: inst_d: ", fshow (inst_d)); + end + end + else if (parse_f22f3) begin + // Re-interpret fetched 32b parcels (inst_d) as 16b parcels + let { n_x16s, v_x16 } <- fav_inst_d_to_x16s (inst_d); + // Cap n_x16s, as otherwise we misattribute the bundle's PC + // prediction to a later instruction and erroneously think we + // took a branch miss. This condition is hit because the cache + // interface uses aligned 32b parcels and thus we can end up with + // an extra 16b parcel after the window we want. Note that + // nbSupX2In will still include the first 16b parcel even if our PC + // is misaligned, but this will be discarded by fav_parse_insts. + if (n_x16s > extend(nbSupX2In) + 1) + n_x16s = extend(nbSupX2In) + 1; - // Handle cache-line boundary straddling instruction, if one is pending - if (ehr_pending_straddle[1]) begin - if (fetch3In.pc != ehr_half_inst_pc[1] + 4) begin - $display ("----------------"); - $display ("Fetch3: straddle: pc mismatch"); - $display ("Fetch3: f22f3.first: ", fshow (f22f3.first)); - $display ("Fetch3: inst_d: ", fshow (inst_d)); - dynamicAssert (False, "Fetch3: straddle: pc mismatch"); - end - else begin - // Prepend onto the sequence: { first-half of the instruction , 0 } - v_x16 = shiftInAt0 (shiftInAt0 (v_x16, ehr_half_inst_lsbs[1]), 0); - let bound = valueOf (SupSizeX2) - 1; - if (n_x16s < (fromInteger (bound) - 1)) - n_x16s = n_x16s + 2; - else if (n_x16s < fromInteger (bound)) - n_x16s = n_x16s + 1; - start_PC = ehr_half_inst_pc[1]; - ehr_pending_straddle[1] <= False; - if (verbosity > 0) begin - $display ("----------------"); - $display ("Fetch3: straddle: prepend x16 %0h", ehr_half_inst_lsbs[1]); - $display ("Fetch3: f22f3.first: ", fshow (f22f3.first)); - $display ("Fetch3: inst_d: ", fshow (inst_d)); - $display ("Fetch3: v_x16: ", fshow (v_x16)); - end - end - end + // Parse v_x16 into 32-bit and 16-bit instructions + Addr pred_next_pc; + {parsed_n_items, parsed_v_items, pred_next_pc, pending_straddle} <- + fav_parse_insts (verbose, fetch3In.pc, fetch3In.pred_next_pc, pending_straddle, n_x16s, v_x16); - // Parse v_x16 into 32-bit and 16-bit instructions - Vector #(SupSize, Inst_Item) v_items <- fav_parse_insts (verbose, start_PC, n_x16s, v_x16); + if (pending_n_items == 0) begin + out = Fetch3ToDecode { + pc: fetch3In.pc, + pred_next_pc: pred_next_pc, + mispred_first_half: mispred_first_half, + cause: fetch3In.cause, + tval: fetch3In.tval, + decode_epoch: fetch3In.decode_epoch, + main_epoch: fetch3In.main_epoch + }; + end + else begin + out.pred_next_pc = pred_next_pc; + end - instdata.enq (v_items); - f32d.enq(f22f3.first); + // Redirect doFetch1 if we predicted a taken compressed branch + // but this is an uncompressed instruction. We will tell decode + // to retrain when we issue the full instruction next time. + if (pending_straddle matches tagged Valid {.s_pc, .s_lsbs, .s_mispred} + &&& s_mispred) begin + pc_reg[pc_fetch3_port] <= s_pc + 2; + fetch3_epoch <= ! fetch3_epoch; + end + end - if (verbosity > 0) begin - $display ("----------------"); - $display ("Fetch3: epoch inst: %d, epoch main : %d", fetch3In.main_epoch, f_main_epoch); - $display ("Fetch3: inst_d: ", fshow (inst_d)); - $display ("Fetch3: v_items: ", fshow (v_items)); - $display ("Fetch3: f32d.enq: ", fshow (f22f3.first)); - end - end + SupCntX2S1 next_pending_n_items = 0; + + if (pending_n_items > 0 || parse_f22f3) begin + SupCntX3S1 n_items = extend(pending_n_items) + extend(parsed_n_items); + Bit #(TLog #(SupSize)) nbSupOut = truncate(n_items - 1); + + let pending_spaces = fromInteger(valueOf(SupSizeX2S1)) - pending_n_items; + Vector #(SupSizeX2S1, Inst_Item) pending_items_ralign = + shiftOutFromN(inst_item_none, rg_pending_decode, pending_spaces); + // Appease bluespec compiler with seemingly-unnecessary extension; + // otherwise elaboration fails with: + // Error: "Vector.bs", line 791, column 33: (T0051) + // Literal 7 is not a valid Bit#(2). + // During elaboration of the body of rule `doFetch3' at + // ... + SupCntX4S1 pending_spaces_ext = extend(pending_spaces); + Vector #(SupSizeX3S1, Inst_Item) v_items = + take(shiftOutFrom0(inst_item_none, append(pending_items_ralign, parsed_v_items), pending_spaces_ext)); + + // Handle decoding more instructions than we can issue this cycle + if (n_items > fromInteger(valueOf(SupSize))) begin + nbSupOut = fromInteger(valueOf(SupSize) - 1); + + if (!isValid(out.cause)) begin + next_pending_n_items = truncate(n_items - fromInteger(valueOf(SupSize))); + rg_pending_decode <= drop(v_items); + rg_pending_f32d <= Fetch3ToDecode { + pc: v_items[valueOf(SupSize)].pc, + pred_next_pc: out.pred_next_pc, + mispred_first_half: False, + cause: tagged Invalid, + tval: 0, + decode_epoch: out.decode_epoch, + main_epoch: out.main_epoch + }; + end + + out.pred_next_pc = v_items[valueOf(SupSize)].pc; + end + + if (n_items > 0) begin + instdata.enq(take(v_items)); + f32d.enq(tuple2(nbSupOut, out)); + if (verbosity > 0) begin + $display ("----------------"); + $display ("Fetch3: epoch inst: %d, epoch main : %d", out.main_epoch, f_main_epoch); + $display ("Fetch3: inst_d: ", fshow (inst_d)); + $display ("Fetch3: v_items: ", fshow (v_items)); + $display ("Fetch3: f32d.enq: nbSup %0d out ", nbSupOut, fshow (out)); + end + end + else begin + // This means we started fetching from a line straddling + // instruction; need another cycle to have something to + // issue. + dynamicAssert(isValid(pending_straddle), "Decoded no instructions and no straddle!"); + end + end + rg_pending_n_items <= next_pending_n_items; + ehr_pending_straddle[0] <= pending_straddle; endrule: doFetch3 rule doDecode; - let {nbSup, fetch3In} = f32d.first; + let {nbSup, decodeIn} = f32d.first; f32d.deq(); let inst_data = instdata.first(); instdata.deq(); // The main_epoch check is required to make sure this stage doesn't // redirect the PC if a later stage already redirected the PC. - if (fetch3In.main_epoch == f_main_epoch) begin + if (decodeIn.main_epoch == f_main_epoch) begin Bool decode_epoch_local = decode_epoch[0]; // next value for decode epoch Maybe#(Addr) redirectPc = Invalid; // next pc redirect by branch predictor Maybe#(TrainNAP) trainNAP = Invalid; // training data sent to next addr pred @@ -616,49 +781,23 @@ module mkFetchStage(FetchStage); `endif for (Integer i = 0; i < valueof(SupSize); i=i+1) begin - if ((inst_data[i].inst_kind == Inst_32b_Lsbs) && (fromInteger(i) <= nbSup)) begin - if (fetch3In.decode_epoch == decode_epoch_local) begin - // Save the half-instruction and redirect doFetch1 to get the next cache line - ehr_pending_straddle[0] <= True; - ehr_half_inst_pc[0] <= inst_data[i].pc; - ehr_half_inst_lsbs[0] <= inst_data[i].orig_inst [15:0]; - decode_epoch_local = ! decode_epoch_local; - let next_PC = inst_data[i].pc + 4; - redirectPc = tagged Valid (next_PC); - // We don't train NAP because that's about the dynamic successor to this instruction, - // not about the second half of this instruction. - - if (verbosity > 0) begin - $display ("----------------"); - $display ("FetchStage.doDecode [%0d]: straddle. pc %0h x16 %0h redirecting to %0h new decode_epoch %d", - i, inst_data[i].pc, x16, next_PC, decode_epoch_local); - end - end - else begin - // just drop wrong path instructions - if (verbose) begin - $display ("FetchStage.doDecode [%0d]: Inst_32b_Lsbs: drop due to decode epoch", i); - $display (" inst_data = ", fshow (inst_data)); - end - end - end - else if (inst_data[i].inst_kind != Inst_None && (fromInteger(i) <= nbSup)) begin + if (inst_data[i].inst_kind != Inst_None && (fromInteger(i) <= nbSup)) begin // Inst_16b or Inst_32b // get the input to decode let inst_data_shifted = shiftInAtN (inst_data, ?); // for predicted PCs - let in = Fetch3ToDecode { + let in = InstrFromFetch3 { pc: inst_data[i].pc, // last inst, next pc may not be pc+2/pc+4 ppc: ((fromInteger(i) == nbSup) - ? fetch3In.pred_next_pc + ? decodeIn.pred_next_pc : inst_data_shifted[i].pc), - decode_epoch: fetch3In.decode_epoch, - main_epoch: fetch3In.main_epoch, + decode_epoch: decodeIn.decode_epoch, + main_epoch: decodeIn.main_epoch, inst: inst_data [i].inst, // original 32b inst, or expanded version of 16b inst - cause: fetch3In.cause + cause: decodeIn.cause }; let cause = in.cause; - Addr tval = fetch3In.tval; + Addr tval = decodeIn.tval; if (verbose) $display("Decode: %0d in = ", i, fshow (in)); @@ -672,7 +811,7 @@ module mkFetchStage(FetchStage); // update cause and tval if decode exception and no earlier (TLB) exception if (!isValid(cause)) begin cause = decode_result.illegalInst ? tagged Valid IllegalInst : tagged Invalid; - tval = fetch3In.tval; + tval = decodeIn.tval; end let dInst = decode_result.dInst; @@ -736,6 +875,13 @@ module mkFetchStage(FetchStage); fshow(in.ppc), " ; ", fshow(pred_taken), " ; ", fshow(nextPc)); end + if (i == 0 && decodeIn.mispred_first_half) begin + // We predicted a taken branch for PC, but this is an + // uncompressed instruction, so we train it to fetch + // the other half in future. + trainNAP = Valid (TrainNAP {pc: in.pc, nextPc: in.pc + 2}); + end + // check previous mispred if (nextPc matches tagged Valid .decode_pred_next_pc &&& decode_pred_next_pc != in.ppc) begin if (verbose) $display("ppc and decodeppc : %h %h", in.ppc, decode_pred_next_pc); @@ -743,7 +889,9 @@ module mkFetchStage(FetchStage); redirectPc = Valid (decode_pred_next_pc); // record redirect next pc in.ppc = decode_pred_next_pc; // train next addr pred when mispredict - trainNAP = Valid (TrainNAP {pc: in.pc, nextPc: decode_pred_next_pc}); + let last_x16_pc = in.pc + ((inst_data[i].inst_kind == Inst_32b) ? 2 : 0); + if (!decodeIn.mispred_first_half) + trainNAP = Valid (TrainNAP {pc: last_x16_pc, nextPc: decode_pred_next_pc}); `ifdef PERF_COUNT // performance stats: record decode redirect doAssert(redirectInst == Invalid, "at most 1 decode redirect per cycle"); @@ -798,7 +946,7 @@ module mkFetchStage(FetchStage); endcase end `endif - end // if (fetch3In.main_epoch == f_main_epoch) + end // if (decodeIn.main_epoch == f_main_epoch) else begin if (verbose) $display("drop in fetch3decode"); end @@ -819,7 +967,7 @@ module mkFetchStage(FetchStage); // only when misprediction happens, i.e., train by dec is already at // wrong path. TrainNAP train = fromMaybe(validValue(napTrainByDec.wget), napTrainByExe.wget); - nextAddrPred.update(train.pc, train.nextPc, train.nextPc != train.pc + 4); + nextAddrPred.update(train.pc, train.nextPc, train.nextPc != train.pc + 2); endrule // Security: we can flush when front end is empty, i.e. @@ -853,7 +1001,7 @@ module mkFetchStage(FetchStage); if (verbose) $display("Redirect: newpc %h, old f_main_epoch %d, new f_main_epoch %d",new_pc,f_main_epoch,f_main_epoch+1); pc_reg[pc_redirect_port] <= new_pc; f_main_epoch <= (f_main_epoch == fromInteger(valueOf(NumEpochs)-1)) ? 0 : f_main_epoch + 1; - ehr_pending_straddle[2] <= False; + ehr_pending_straddle[1] <= tagged Invalid; // redirect comes, stop stalling for redirect waitForRedirect <= False; setWaitRedirect_redirect_conflict.wset(?); // conflict with setWaitForRedirect @@ -873,7 +1021,7 @@ module mkFetchStage(FetchStage); method Action train_predictors( Addr pc, Addr next_pc, IType iType, Bool taken, - DirPredTrainInfo dpTrain, Bool mispred + DirPredTrainInfo dpTrain, Bool mispred, Bool isCompressed ); //if (iType == J || (iType == Br && next_pc < pc)) begin // // Only train the next address predictor for jumps and backward branches @@ -886,7 +1034,8 @@ module mkFetchStage(FetchStage); end // train next addr pred when mispred if(mispred) begin - napTrainByExe.wset(TrainNAP {pc: pc, nextPc: next_pc}); + let last_x16_pc = pc + (isCompressed ? 0 : 2); + napTrainByExe.wset(TrainNAP {pc: last_x16_pc, nextPc: next_pc}); end endmethod diff --git a/src_Core/RISCY_OOO/procs/lib/Btb.bsv b/src_Core/RISCY_OOO/procs/lib/Btb.bsv index 713013f..7ec95d4 100644 --- a/src_Core/RISCY_OOO/procs/lib/Btb.bsv +++ b/src_Core/RISCY_OOO/procs/lib/Btb.bsv @@ -31,8 +31,8 @@ export NextAddrPred(..); export mkBtb; interface NextAddrPred; - method Addr predPc(Addr pc); - method Action update(Addr pc, Addr nextPc, Bool taken); + method Maybe#(Addr) predPc(Addr pc); + method Action update(Addr pc, Addr brTarget, Bool taken); // security method Action flush; method Bool flush_done; @@ -96,13 +96,13 @@ module mkBtb(NextAddrPred); endrule `endif - method Addr predPc(Addr pc); + method Maybe#(Addr) predPc(Addr pc); BtbIndex index = getIndex(pc); BtbTag tag = getTag(pc); if(valid[index] && tag == tags.sub(index)) - return next_addrs.sub(index); + return tagged Valid next_addrs.sub(index); else - return (pc + 4); + return tagged Invalid; endmethod method Action update(Addr pc, Addr nextPc, Bool taken);