diff --git a/libs/TagController b/libs/TagController index 0af8406..c01eded 160000 --- a/libs/TagController +++ b/libs/TagController @@ -1 +1 @@ -Subproject commit 0af8406ac66734e4bac4f55904a47dd9dbd4163d +Subproject commit c01eded44999d0a1cabb0470628cbd208fc1984e diff --git a/src_Core/CPU/LLC_AXI4_Adapter.bsv b/src_Core/CPU/LLC_AXI4_Adapter.bsv index d4496e6..adf771f 100644 --- a/src_Core/CPU/LLC_AXI4_Adapter.bsv +++ b/src_Core/CPU/LLC_AXI4_Adapter.bsv @@ -85,19 +85,19 @@ module mkLLC_AXi4_Adapter #(MemFifoClient #(idT, childT) llc) // Functions to interact with the fabric // Send a read-request into the fabric - function Action fa_fabric_send_read_req (Fabric_Addr addr); + function Action fa_fabric_send_read_req (Fabric_Addr addr, Bool tag_req); action let mem_req_rd_addr = AXI4_ARFlit {arid: fabric_default_mid, araddr: addr, arlen: 0, // burst len = arlen+1 - arsize: 64, + arsize: tag_req ? 1 : 64, arburst: INCR, arlock: fabric_default_lock, arcache: fabric_default_arcache, arprot: fabric_default_prot, arqos: fabric_default_qos, arregion: fabric_default_region, - aruser: fabric_default_aruser}; + aruser: pack(tag_req)}; masterPortShim.slave.ar.put(mem_req_rd_addr); @@ -127,7 +127,7 @@ module mkLLC_AXi4_Adapter #(MemFifoClient #(idT, childT) llc) end Addr line_addr = {ld.addr [63:6], 6'h0 }; // Addr of containing cache line - fa_fabric_send_read_req (line_addr); + fa_fabric_send_read_req (line_addr, ld.tag_req); f_pending_reads.enq (ld); llc.toM.deq; endrule @@ -150,6 +150,9 @@ module mkLLC_AXi4_Adapter #(MemFifoClient #(idT, childT) llc) MemRsMsg #(idT, childT) resp = MemRsMsg {data: new_cline, child: ldreq.child, id: ldreq.id}; + if (ldreq.tag_req) begin + resp.data = CLine { tag: unpack(truncate(mem_rsp.rdata)), data: ?}; + end llc.rsFromM.enq (resp); if (cfg_verbosity > 1) $display (" Response to LLC: ", fshow (resp)); diff --git a/src_Core/Core/Fabric_Defs.bsv b/src_Core/Core/Fabric_Defs.bsv index 51c52dc..26de07f 100644 --- a/src_Core/Core/Fabric_Defs.bsv +++ b/src_Core/Core/Fabric_Defs.bsv @@ -73,7 +73,7 @@ typedef 512 Wd_Data; // Width of fabric 'user' datapaths. Carry capability tags on data lines. typedef 0 Wd_AW_User; typedef 0 Wd_B_User; -typedef 0 Wd_AR_User; +typedef 1 Wd_AR_User; typedef TMax#(TDiv#(Wd_Data, CLEN),1) Wd_W_User; typedef TMax#(TDiv#(Wd_Data, CLEN),1) Wd_R_User; diff --git a/src_Core/RISCY_OOO/coherence/src/CCTypes.bsv b/src_Core/RISCY_OOO/coherence/src/CCTypes.bsv index 76f3f1a..90a3038 100644 --- a/src_Core/RISCY_OOO/coherence/src/CCTypes.bsv +++ b/src_Core/RISCY_OOO/coherence/src/CCTypes.bsv @@ -47,9 +47,10 @@ import ClientServer::*; typedef enum { I = 2'd0, - S = 2'd1, - E = 2'd2, - M = 2'd3 + T = 2'd1, + S = 2'd2, + E = 2'd3, + M = 2'd4 } MESI deriving(Bits, Eq, FShow); typedef MESI Msi; @@ -317,6 +318,7 @@ typedef struct { Addr addr; childT child; // from which LLC/Dir idT id; // ld req id and other info need encoding + Bool tag_req; // request for cap tags, not data } LdMemRq#(type idT, type childT) deriving(Bits, Eq, FShow); typedef struct { // LdMemRq id with more info encoded to handle DMA req in LLC diff --git a/src_Core/RISCY_OOO/coherence/src/L1Bank.bsv b/src_Core/RISCY_OOO/coherence/src/L1Bank.bsv index b7f3d1b..8462fd2 100644 --- a/src_Core/RISCY_OOO/coherence/src/L1Bank.bsv +++ b/src_Core/RISCY_OOO/coherence/src/L1Bank.bsv @@ -779,10 +779,9 @@ endfunction // check tag match Bool tag_match = ram.info.tag == getTag(procRq.addr); // check enough cache state for hit - Bool enough_cs = enoughCacheState(ram.info.cs, procRq.toState); + Bool enough_cs_to_hit = enoughCacheState(ram.info.cs, procRq.toState); // check if cs is not I Bool cs_valid = ram.info.cs > I; - if(ram.info.owner matches tagged Valid .cOwner) begin if(cOwner != n) begin // owner is another cRq, so must just go through tag match @@ -806,7 +805,7 @@ endfunction "cRq swapped in by previous cRq, tag must match & cs > I" ); // Hit or Miss (but no replacement) - if(enough_cs) begin + if(enough_cs_to_hit) begin if (verbose) $display("%t L1 %m pipelineResp: cRq: own by itself, hit", $time); cRqHit(n, procRq); @@ -822,9 +821,9 @@ endfunction cRqScEarlyFail(True); end else begin - if (verbose) - $display("%t L1 %m pipelineResp: cRq: own by itself, miss no replace", $time); - cRqMissNoReplacement; + if (verbose) + $display("%t L1 %m pipelineResp: cRq: own by itself, miss no replace", $time); + cRqMissNoReplacement; end end end @@ -847,7 +846,7 @@ endfunction end else begin // Check hit or miss, replacment may be needed - if(tag_match && enough_cs) begin + if(tag_match && enough_cs_to_hit) begin // Hit doAssert(cs_valid, "hit, so cs must > I"); if (verbose) diff --git a/src_Core/RISCY_OOO/coherence/src/L1Pipe.bsv b/src_Core/RISCY_OOO/coherence/src/L1Pipe.bsv index 481edbf..c1d17fa 100644 --- a/src_Core/RISCY_OOO/coherence/src/L1Pipe.bsv +++ b/src_Core/RISCY_OOO/coherence/src/L1Pipe.bsv @@ -318,7 +318,7 @@ module mkL1Pipe( ); actionvalue doAssert(toState > oldCs, "should truly upgrade cs"); - doAssert((oldCs == I) == dataV, "valid resp data for upgrade from I"); + doAssert((oldCs < S) == dataV, "valid resp data when data already up to date"); return UpdateByUpCs {cs: toState}; endactionvalue endfunction diff --git a/src_Core/RISCY_OOO/coherence/src/LLBank.bsv b/src_Core/RISCY_OOO/coherence/src/LLBank.bsv index bf792a4..0fbe4ba 100644 --- a/src_Core/RISCY_OOO/coherence/src/LLBank.bsv +++ b/src_Core/RISCY_OOO/coherence/src/LLBank.bsv @@ -523,7 +523,7 @@ endfunction // send to pipeline pipeline.send(MRs (LLPipeMRsIn { addr: cRq.addr, - toState: cRq.toState == M ? M : E, // set upgrade state + toState: cRq.toState == M ? M : cRq.toState == T ? T : E, // set upgrade state data: respData, way: cSlot.way })); @@ -578,7 +578,8 @@ endfunction // child rq needs refill cache line, dma rq does not refill: isRqFromC(cRq.id), mshrIdx: n - } + }, + tag_req: cRq.toState == T }); toMQ.enq(msg); toMInfoQ.deq; // deq info @@ -623,7 +624,8 @@ endfunction id: LdMemRqId { refill: True, mshrIdx: n - } + }, + tag_req: cRq.toState == T }); toMQ.enq(msg); // whole thing is done, reset bit and deq info @@ -870,6 +872,7 @@ endfunction ); // decide upgrade state Msi toState = cRq.toState; + // XXX Add auto update to S from T here if(cRq.toState == S && cRq.canUpToE && ram.info.dir == replicate(I) && respLoadWithE(isMRs)) begin toState = E; end @@ -880,7 +883,7 @@ endfunction toState: toState }); cRqMshr.pipelineResp.setStateSlot(n, Done, ?); // we no longer need slot info - cRqMshr.pipelineResp.setData(n, ram.info.dir[cRq.child] == I ? Valid (ram.line) : Invalid); + cRqMshr.pipelineResp.setData(n, ram.info.dir[cRq.child] <= T ? Valid (ram.line) : Invalid); // update child dir dirT newDir = ram.info.dir; newDir[cRq.child] = toState; @@ -1066,25 +1069,26 @@ endfunction endfunction // function to process cRq from child miss without replacement (MSHR slot may have garbage) - function Action cRqFromCMissNoReplacement(Vector#(childNum, DirPend) dirPend); + function Action cRqFromCMissNoReplacement(Vector#(childNum, DirPend) dirPend, Bool dataReq); action doAssert(isRqFromC(cRq.id), "should be cRq from child"); // it is impossible in LLC to have slot.waitP == True in this function // because there is no pRq in LLC to interrupt a cRq cRqSlotT cSlot = pipeOutCSlot; doAssert(!cSlot.waitP, "waitP must be false"); - // in LLC, we req memory only when cur cs is I - if(ram.info.cs == I) begin + // in LLC, we req memory only when we don't have enough data + Bool reqMem = ram.info.cs == I || (dataReq && ram.info.cs == T); + if(reqMem) begin toMInfoQ.enq(ToMemInfo{ mshrIdx: n, t: Ld }); - doAssert(ram.info.dir == replicate(I), "dir should be all I"); + //doAssert(ram.info.dir == replicate(I), "dir should be all I"); end // update mshr (data field is irrelevant, should be already invalid) cRqMshr.pipelineResp.setStateSlot(n, WaitSt, LLCRqSlot { way: pipeOut.way, // use way from pipeline - waitP: ram.info.cs == I, + waitP: reqMem, repTag: ?, // no replacement dirPend: dirPend }); @@ -1227,7 +1231,7 @@ endfunction if(cRq.id matches tagged Child ._i) begin // req from child, get dir pend Vector#(childNum, DirPend) dirPend = getDirPendNonCompatForChild; - if(dirPend == replicate(Invalid)) begin + if(dirPend == replicate(Invalid) && (cRq.toState == T || ram.info.cs >= S)) begin if (verbose) $display("%t LL %m pipelineResp: cRq from child: own by itself, hit", $time); cRqFromCHit(n, cRq, False); @@ -1237,13 +1241,13 @@ endfunction $display("%t LL %m pipelineResp: cRq from child: own by itself, miss no replace: ", $time, fshow(dirPend) ); - cRqFromCMissNoReplacement(dirPend); + cRqFromCMissNoReplacement(dirPend, cRq.toState >= S); end end else begin // req from DMA, get dir pend Vector#(childNum, DirPend) dirPend = getDirPendNonCompatForDma; - if(dirPend == replicate(Invalid)) begin + if(dirPend == replicate(Invalid) && (cRq.toState == T || ram.info.cs >= S)) begin if (verbose) $display("%t LL %m pipelineResp: cRq from dma: own by itself, hit", $time); cRqFromDmaHit(n, cRq); @@ -1280,7 +1284,7 @@ endfunction if(ram.info.cs == I || ram.info.tag == getTag(cRq.addr)) begin // No Replacement necessary, check dir Vector#(childNum, DirPend) dirPend = getDirPendNonCompatForChild; - if(ram.info.cs > I && dirPend == replicate(Invalid)) begin + if(ram.info.cs > I && dirPend == replicate(Invalid) && (cRq.toState == T || ram.info.cs >= S)) begin if (verbose) $display("%t LL %m pipelineResp: cRq: no owner, hit", $time); cRqFromCHit(n, cRq, False); @@ -1290,7 +1294,7 @@ endfunction $display("%t LL %m pipelineResp: cRq: no owner, miss no replace: ", $time, fshow(dirPend) ); - cRqFromCMissNoReplacement(dirPend); + cRqFromCMissNoReplacement(dirPend, cRq.toState >= S); end end else begin @@ -1308,11 +1312,11 @@ endfunction // cRq from DMA if(ram.info.cs > I && ram.info.tag == getTag(cRq.addr)) begin // hit in LLC, check dir - if(dirPend == replicate(Invalid)) begin + if(dirPend == replicate(Invalid) && (cRq.toState == T || ram.info.cs >= S)) begin cRqFromDmaHit(n, cRq); end else begin - cRqFromDmaMissByChildren(dirPend); + cRqFromDmaMissByChildren(dirPend); // XXX this might need fixing up in the T->S case? end end else begin @@ -1358,7 +1362,7 @@ endfunction doAssert(ram.info.cs >= cRq.toState && ram.info.tag == getTag(cRq.addr), "mRs must be tag match & have enough cs" ); - doAssert(ram.info.dir == replicate(I), "all children must be I"); + //doAssert(ram.info.dir == replicate(I), "all children must be I"); doAssert(!cOwner.replacing, "mRs cannot hit on replacing line"); doAssert(cSlot.way == pipeOut.way, "mRs should hit on way in MSHR slot"); doAssert(cSlot.waitP, "mRs should match cRq which is waiting for it"); diff --git a/src_Core/RISCY_OOO/coherence/src/LLPipe.bsv b/src_Core/RISCY_OOO/coherence/src/LLPipe.bsv index 9880deb..fd3f316 100644 --- a/src_Core/RISCY_OOO/coherence/src/LLPipe.bsv +++ b/src_Core/RISCY_OOO/coherence/src/LLPipe.bsv @@ -276,7 +276,7 @@ module mkLLPipe( ); actionvalue doAssert(toState > oldCs, "should truly upgrade cs"); - doAssert((oldCs == I) && dataV, "LLC mRs always has data"); + doAssert((oldCs == I || (oldCs == T && toState >= S)) && dataV, "LLC mRs always has data"); return UpdateByUpCs {cs: toState}; endactionvalue endfunction diff --git a/src_Core/RISCY_OOO/procs/RV64G_OOO/MemExePipeline.bsv b/src_Core/RISCY_OOO/procs/RV64G_OOO/MemExePipeline.bsv index ac4cb10..4a8c473 100644 --- a/src_Core/RISCY_OOO/procs/RV64G_OOO/MemExePipeline.bsv +++ b/src_Core/RISCY_OOO/procs/RV64G_OOO/MemExePipeline.bsv @@ -1518,7 +1518,7 @@ module mkMemExePipeline#(MemExeInput inIfc)(MemExePipeline); dMem.procReq.req(ProcRq { id: zeroExtend(lsqTag), addr: addr, - toState: multicore ? S : E, // in case of single core, just fetch to E + toState: loadTags ? T : (multicore ? S : E), // in case of single core, just fetch to E op: Ld, byteEn: ?, data: ?,