feat(CoupledL2): release bypass and L2 escalated way persistence in L1

src/main/scala/coupledL2/Common.scala

@@ -326,18 +326,21 @@ class BlockInfo(implicit p: Parameters) extends L2Bundle {
 
 // used for nested C Release
 class NestedWriteback(implicit p: Parameters) extends L2Bundle {
-  val set = UInt(setBits.W)
-  val tag = UInt(tagBits.W)
+  val set = Input(UInt(setBits.W))
+  val tag = Input(UInt(tagBits.W))
   // Nested ReleaseData sets block dirty
-  val c_set_dirty = Bool()
+  val c_set_dirty = Input(Bool())
   // Nested Release sets block TIP
-  val c_set_tip = Bool()
+  val c_set_tip = Input(Bool())
   // Nested Snoop invalidates block
-  val b_inv_dirty = Bool()
+  val b_inv_dirty = Input(Bool())
+
+  val b_toB = chiOpt.map(_ => Input(Bool()))
+  val b_toN = chiOpt.map(_ => Input(Bool()))
+  val b_toClean = chiOpt.map(_ => Input(Bool()))
 
-  val b_toB = chiOpt.map(_ => Bool())
-  val b_toN = chiOpt.map(_ => Bool())
-  val b_toClean = chiOpt.map(_ => Bool())
+  // Nested Release hit replace
+  val replaceMatch = Output(Bool())
 }
 
 class PrefetchCtrlFromCore extends Bundle {

src/main/scala/coupledL2/Directory.scala

@@ -104,6 +104,13 @@ class MetaWrite(implicit p: Parameters) extends L2Bundle {
   val set = UInt(setBits.W)
   val wayOH = UInt(cacheParams.ways.W)
   val wmeta = new MetaEntry
+  val release = Valid(new Bundle() { // Release-only mode, write mask bundle
+    val dirty = Bool()
+    val clients = Bool()
+    val state = Bool()
+    val tagErr = Bool()
+    val dataErr = Bool()
+  }) 
 }
 
 class TagWrite(implicit p: Parameters) extends L2Bundle {
@@ -171,7 +178,7 @@ class Directory(implicit p: Parameters) extends L2Module {
     ))
   }
 
-  val metaArray = Module(new SRAMTemplate(new MetaEntry, sets, ways, singlePort = true, hasMbist = mbist, hasSramCtl = hasSramCtl))
+  val metaArray = Module(new SRAMTemplate(new MetaEntry, sets, ways, singlePort = true, useBitmask = true, hasMbist = mbist, hasSramCtl = hasSramCtl))
 
   val tagRead_s3 = Wire(Vec(ways, UInt(tagBits.W)))
   val metaRead = Wire(Vec(ways, new MetaEntry()))
@@ -234,12 +241,23 @@ class Directory(implicit p: Parameters) extends L2Module {
   errorRead := bankTagError
 
   // Meta R/W
+  val metaMaskNone = Fill(io.metaWReq.bits.wmeta.getWidth, 0.U)
+  val metaMaskAll = Fill(io.metaWReq.bits.wmeta.getWidth, 1.U)
+
+  val metaMaskRelease = WireInit(metaMaskNone.asTypeOf(new MetaEntry))
+  metaMaskRelease.dirty := Fill(metaMaskRelease.dirty.getWidth, io.metaWReq.bits.release.bits.dirty) // masking 'dirty'
+  metaMaskRelease.clients := Fill(metaMaskRelease.clients.getWidth, io.metaWReq.bits.release.bits.clients) // masking 'clients'
+  metaMaskRelease.state := Fill(metaMaskRelease.state.getWidth, io.metaWReq.bits.release.bits.state) // masking 'state'
+  metaMaskRelease.tagErr := Fill(metaMaskRelease.tagErr.getWidth, io.metaWReq.bits.release.bits.tagErr) // masking 'tagErr'
+  metaMaskRelease.dataErr := Fill(metaMaskRelease.dataErr.getWidth, io.metaWReq.bits.release.bits.dataErr) // masking 'dataErr'
+
   metaRead := metaArray.io.r(io.read.fire, io.read.bits.set).resp.data
   metaArray.io.w(
     metaWen,
     io.metaWReq.bits.wmeta,
     io.metaWReq.bits.set,
-    io.metaWReq.bits.wayOH
+    io.metaWReq.bits.wayOH,
+    Mux(io.metaWReq.bits.release.valid, metaMaskRelease.asUInt, metaMaskAll)
   )
 
   val metaAll_s3 = RegEnable(metaRead, 0.U.asTypeOf(metaRead), reqValid_s2)

src/main/scala/coupledL2/GrantBuffer.scala

@@ -94,6 +94,7 @@ class GrantBuffer(implicit p: Parameters) extends L2Module {
     d.data := data
     d.corrupt := task.corrupt || task.denied
     d.echo.lift(IsKeywordKey).foreach(_ := false.B)
+    d.user.lift(WayKey).foreach(_ := task.way)
     d
   }
 

src/main/scala/coupledL2/L2Param.scala

@@ -46,6 +46,10 @@ case class L1Param
   val needResolveAlias = aliasBitsOpt.nonEmpty
 }
 
+// Pass way in [ L2 -> D -> L1 ] and [ L1 -> C -> L2 ]
+case object WayKey extends ControlKey[UInt]("way")
+case class WayField() extends BundleField[UInt](WayKey, Output(UInt(4.W)), _ := 0.U(4.W))
+
 // Pass PMA and uncached memory attribute from PBMT to MMIOBridge
 case object MemBackTypeMM extends ControlKey[Bool]("memBackType_MM")
 case class MemBackTypeMMField() extends BundleField[Bool](MemBackTypeMM, Output(Bool()), _ := false.B)
@@ -82,10 +86,10 @@ case class L2Param(
   // Client
   echoField: Seq[BundleFieldBase] = Nil,
   reqField: Seq[BundleFieldBase] = Nil,
-  respKey: Seq[BundleKeyBase] = Seq(IsHitKey),
+  respKey: Seq[BundleKeyBase] = Seq(IsHitKey, WayKey),
   // Manager
-  reqKey: Seq[BundleKeyBase] = Seq(AliasKey, VaddrKey, PrefetchKey, ReqSourceKey),
-  respField: Seq[BundleFieldBase] = Nil,
+  reqKey: Seq[BundleKeyBase] = Seq(AliasKey, VaddrKey, PrefetchKey, ReqSourceKey, WayKey),
+  respField: Seq[BundleFieldBase] = Seq(WayField()),
 
   innerBuf: TLBufferParams = TLBufferParams(),
   outerBuf: TLBufferParams = TLBufferParams(
@@ -134,6 +138,8 @@ case class L2Param(
   // Enable new clint
   EnablePrivateClint: Boolean = false
 ) {
+  require(ways <= 16, "L1 way record for L2: only support up to 16 ways")
+
   def toCacheParams: CacheParameters = CacheParameters(
     name = name,
     sets = sets,

src/main/scala/coupledL2/RequestArb.scala

@@ -150,30 +150,46 @@ class RequestArb(implicit p: Parameters) extends L2Module
 
   // TODO: A Hint is allowed to enter if !s2_ready for mcp2_stall
 
-  val sink_ready_basic = io.dirRead_s1.ready && resetFinish && !mshr_task_s1.valid && s2_ready
+  val sink_ready_nodir = resetFinish && !mshr_task_s1.valid && s2_ready
+  val sink_ready_basic = io.dirRead_s1.ready && sink_ready_nodir
 
   io.sinkA.ready := sink_ready_basic && !block_A && !sinkValids(1) && !sinkValids(0) // SinkC prior to SinkA & SinkB
   io.sinkB.ready := sink_ready_basic && !block_B && !sinkValids(0) // SinkB prior to SinkA
-  io.sinkC.ready := sink_ready_basic && !block_C
+
+  if (enableCHI)
+    io.sinkC.ready := sink_ready_nodir && !block_C
+  else
+    io.sinkC.ready := sink_ready_basic && !block_C
 
   val chnl_task_s1 = Wire(Valid(new TaskBundle()))
-  chnl_task_s1.valid := io.dirRead_s1.ready && sinkValids.orR && resetFinish
   chnl_task_s1.bits := ParallelPriorityMux(sinkValids, Seq(C_task, B_task, A_task))
 
+  if (enableCHI)
+    chnl_task_s1.valid := (io.sinkC.fire || io.dirRead_s1.ready) && sinkValids.orR && resetFinish
+  else
+    chnl_task_s1.valid := io.dirRead_s1.ready && sinkValids.orR && resetFinish
+
   // mshr_task_s1 is s1_[reg]
   // task_s1 is [wire] to s2_reg
   val task_s1 = Mux(mshr_task_s1.valid, mshr_task_s1, chnl_task_s1)
-  val s1_to_s2_valid = task_s1.valid && !mshr_replRead_stall
+  val s1_to_s2_valid = Wire(Bool()) 
+
+  if (enableCHI)
+    s1_to_s2_valid := task_s1.valid && (!mshr_replRead_stall || io.sinkC.fire)
+  else
+    s1_to_s2_valid := task_s1.valid && !mshr_replRead_stall
 
-  s1_cango  := task_s1.valid && !mshr_replRead_stall
+  s1_cango  := s1_to_s2_valid
   s1_fire   := s1_cango && s2_ready
 
   io.taskInfo_s1.valid := s1_fire
   io.taskInfo_s1.bits := task_s1.bits
 
   /* Meta read request */
-  // ^ only sinkA/B/C tasks need to read directory
-  io.dirRead_s1.valid := s2_ready && (chnl_task_s1.valid && !mshr_task_s1.valid || s1_needs_replRead && !io.fromMainPipe.blockG_s1)
+  // ^ only sinkA/B tasks need to read directory, and sinkC under full-TileLink
+  io.dirRead_s1.valid := s2_ready && 
+    (if (enableCHI) !io.sinkC.fire else true.B) && 
+    (chnl_task_s1.valid && !mshr_task_s1.valid || s1_needs_replRead && !io.fromMainPipe.blockG_s1)
   io.dirRead_s1.bits.set := task_s1.bits.set
   io.dirRead_s1.bits.tag := task_s1.bits.tag
   // invalid way which causes mshr_retry
@@ -198,10 +214,17 @@ class RequestArb(implicit p: Parameters) extends L2Module
 
   /* ========  Stage 2 ======== */
   val s1_AHint_fire = io.sinkA.fire && io.sinkA.bits.opcode === Hint
-  // any req except AHint might access DS, and continuous DS accesses are prohibited
-  val ds_mcp2_stall = RegNext(s1_fire && !s1_AHint_fire)
+  val s1_CRelease_fire = io.sinkC.fire && io.sinkC.bits.opcode === Release
+  // any req except:
+  //  1. (A) Hint
+  //  2. (C) Release
+  // might access DS, and continuous DS accesses are prohibited
+  val ds_mcp2_stall = RegNext(s1_fire && !s1_AHint_fire && !s1_CRelease_fire)
+
+  // let Release go through even if MCP2 stall active (not supported by MainPipe)
+  val s1_CRelease_letgo = s1_CRelease_fire
 
-  s2_ready  := !ds_mcp2_stall
+  s2_ready  := !ds_mcp2_stall /*|| s1_CRelease_letgo*/
 
   val task_s2 = RegInit(0.U.asTypeOf(task_s1))
   task_s2.valid := s1_fire

src/main/scala/coupledL2/SinkC.scala

@@ -65,6 +65,8 @@ class SinkC(implicit p: Parameters) extends L2Module {
   val nextPtrReg = RegEnable(nextPtr, 0.U.asTypeOf(nextPtr), io.c.fire && isRelease && first && hasData)
 
   def toTaskBundle(c: TLBundleC): TaskBundle = {
+    require(!c.user.lift(WayKey).isEmpty, "Way field is not present in L2 TLBundleC" 
+      + " (L1 is required to accept and store way info from L2)")
     val task = Wire(new TaskBundle)
     task := 0.U.asTypeOf(new TaskBundle)
     task.channel := "b100".U
@@ -91,7 +93,7 @@ class SinkC(implicit p: Parameters) extends L2Module {
     task.fromL2pft.foreach(_ := false.B)
     task.needHint.foreach(_ := false.B)
     task.dirty := false.B
-    task.way := 0.U(wayBits.W)
+    task.way := c.user.lift(WayKey).getOrElse(0.U)
     task.meta := 0.U.asTypeOf(new MetaEntry)
     task.metaWen := false.B
     task.tagWen := false.B

src/main/scala/coupledL2/debug/Monitor.scala

@@ -18,6 +18,7 @@ class MainpipeMoni(implicit p: Parameters) extends L2Bundle {
   val dirResult_s3 = new DirResult
   val allocMSHR_s3 = ValidIO(UInt(mshrBits.W))
   val metaW_s3 = ValidIO(new MetaWrite)
+  val replaceNestedRelease_s3 = Bool()
 }
 
 class CPL2S3Info(implicit p: Parameters) extends L2Bundle {
@@ -52,6 +53,7 @@ class Monitor(implicit p: Parameters) extends L2Module {
   val mshr_req_s3   = req_s3.mshrTask
   val dirResult_s3  = mp.dirResult_s3
   val meta_s3       = mp.dirResult_s3.meta
+  val replaceHitRelease_s3 = mp.replaceNestedRelease_s3
 
   /* ======== MainPipe Assertions ======== */
   // ! Release w/o data will not trigger nestedWBValid, either
@@ -66,9 +68,9 @@ class Monitor(implicit p: Parameters) extends L2Module {
     meta_s3.state === TRUNK && !meta_s3.clients.orR)),
     "Trunk should have some client hit")
 
-  assert(RegNext(!(s3_valid && req_s3.fromC && dirResult_s3.hit &&
-    !meta_s3.clients.orR)),
-    "Invalid Client should not send Release")
+//assert(RegNext(!(s3_valid && req_s3.fromC && dirResult_s3.hit &&
+//  !meta_s3.clients.orR)),
+//  "Invalid Client should not send Release")
 
   // assertion for set blocking
   // A channel task @s1 never have same-set task @s2/s3

src/main/scala/coupledL2/tl2chi/MSHR.scala

@@ -56,7 +56,7 @@ class MSHR(implicit p: Parameters) extends TL2CHIL2Module with HasCHIOpcodes {
     val alloc = Flipped(ValidIO(new MSHRRequest))
     val tasks = new MSHRTasks()
     val resps = new MSHRResps()
-    val nestedwb = Input(new NestedWriteback)
+    val nestedwb = new NestedWriteback()
     val nestedwbData = Output(Bool())
     val aMergeTask = Flipped(ValidIO(new TaskBundle))
     val replResp = Flipped(ValidIO(new ReplacerResult))
@@ -1366,6 +1366,8 @@ class MSHR(implicit p: Parameters) extends TL2CHIL2Module with HasCHIOpcodes {
     dirResult.set === io.nestedwb.set &&
     dirResult.tag === io.nestedwb.tag
 
+  io.nestedwb.replaceMatch := nestedwb_match
+
   when (nestedwb_match) {
     when (io.nestedwb.c_set_dirty) {
       meta.dirty := true.B

src/main/scala/coupledL2/tl2chi/MSHRCtl.scala

@@ -68,7 +68,7 @@ class MSHRCtl(implicit p: Parameters) extends TL2CHIL2Module with HasCHIOpcodes
     val releaseBufWriteId = Output(UInt(mshrBits.W))
 
     /* nested writeback */
-    val nestedwb = Input(new NestedWriteback)
+    val nestedwb = new NestedWriteback()
     val nestedwbDataId = Output(ValidIO(UInt(mshrBits.W)))
 
     /* MSHR info to Sinks */
@@ -146,13 +146,24 @@ class MSHRCtl(implicit p: Parameters) extends TL2CHIL2Module with HasCHIOpcodes
       m.io.replResp.bits := io.replResp.bits
 
       io.msInfo(i) := m.io.msInfo
-      m.io.nestedwb := io.nestedwb
+
+      m.io.nestedwb.set := io.nestedwb.set
+      m.io.nestedwb.tag := io.nestedwb.tag
+      m.io.nestedwb.c_set_dirty := io.nestedwb.c_set_dirty
+      m.io.nestedwb.c_set_tip := io.nestedwb.c_set_tip
+      m.io.nestedwb.b_inv_dirty := io.nestedwb.b_inv_dirty
+      m.io.nestedwb.b_toB.foreach(_ := io.nestedwb.b_toB.getOrElse(false.B))
+      m.io.nestedwb.b_toN.foreach(_ := io.nestedwb.b_toN.getOrElse(false.B))
+      m.io.nestedwb.b_toClean.foreach(_ := io.nestedwb.b_toClean.getOrElse(false.B))
+
       m.io.aMergeTask.valid := io.aMergeTask.valid && io.aMergeTask.bits.id === i.U
       m.io.aMergeTask.bits := io.aMergeTask.bits.task
 
       io.pCrd(i) <> m.io.pCrd
   }
 
+  io.nestedwb.replaceMatch := ParallelOR(mshrs.map(_.io.nestedwb.replaceMatch))
+
   /* Reserve 1 entry for SinkB */
   io.toReqArb.blockC_s1 := false.B
   io.toReqArb.blockB_s1 := mshrFull   // conflict logic in SinkB