timing(RequestBuffer, MMIOBridge): optimize timing

src/main/scala/coupledL2/Common.scala

@@ -260,6 +260,8 @@ class MSHRInfo(implicit p: Parameters) extends L2Bundle with HasTLChannelBits {
   val mergeA = Bool() // whether the mshr already merge an acquire(avoid alias merge)
 
   val w_grantfirst = Bool()
+  val w_grantlast = Bool()
+  val w_grant = Bool()
   val s_release = Bool()
   val s_refill = Bool()
   val s_cmoresp = Bool()

src/main/scala/coupledL2/RequestBuffer.scala

@@ -151,10 +151,11 @@ class RequestBuffer(flow: Boolean = true, entries: Int = 4)(implicit p: Paramete
   dontTouch(mshrConflictMaskFromA)
 
   // incoming Acquire can be merged with late_pf MSHR block
-  val mergeAMask = VecInit(io.mshrInfo.map(s =>
-    s.valid && s.bits.isPrefetch && sameAddr(in, s.bits) && !s.bits.willFree && !s.bits.dirHit && !s.bits.s_refill &&
+  val mergeAMask = VecInit(io.mshrInfo.map { case s =>
+    val mshrInflight = !(s.bits.w_grantlast && s.bits.w_grant)
+    s.valid && s.bits.isPrefetch && sameAddr(in, s.bits) && !s.bits.dirHit && mshrInflight &&
       in.fromA && (in.opcode === AcquireBlock || in.opcode === AcquirePerm) && !s.bits.mergeA && !(in.param === NtoT && s.bits.param === NtoB)
-  )).asUInt
+  }).asUInt
   val mergeA = mergeAMask.orR
   val mergeAId = OHToUInt(mergeAMask)
   io.aMergeTask.valid := io.in.valid && mergeA
@@ -232,10 +233,7 @@ class RequestBuffer(flow: Boolean = true, entries: Int = 4)(implicit p: Paramete
   /* ======== Issue ======== */
   issueArb.io.in zip buffer foreach {
     case(in, e) =>
-      // when io.out.valid, we temporarily stall all entries of the same set
-      val pipeBlockOut = io.out.valid && sameSet(e.task, io.out.bits)
-
-      in.valid := e.valid && e.rdy && !pipeBlockOut
+      in.valid := e.valid && e.rdy
       in.bits  := e
   }
 
@@ -298,7 +296,10 @@ class RequestBuffer(flow: Boolean = true, entries: Int = 4)(implicit p: Paramete
 
   chosenQ.io.deq.ready := io.out.ready || cancel
   io.out.valid := chosenQValid && !cancel || io.in.valid && canFlow
-  io.out.bits  := Mux(canFlow, io.in.bits, chosenQ.io.deq.bits.bits.task)
+  io.out.bits := {
+    if (!flow) chosenQ.io.deq.bits.bits.task
+    else Mux(chosenQValid, chosenQ.io.deq.bits.bits.task, io.in.bits)
+  }
 
   when(chosenQ.io.deq.fire && !cancel) {
     buffer(chosenQ.io.deq.bits.id).valid := false.B

src/main/scala/coupledL2/tl2chi/MMIOBridge.scala

@@ -89,6 +89,7 @@ class MMIOBridgeEntry(edge: TLEdgeIn)(implicit p: Parameters) extends TL2CHIL2Mo
     val id = Input(UInt())
     val pCrd = new PCrdQueryBundle
     val waitOnReadReceipt = Option.when(needRR)(Output(Bool()))
+    val isRead = Output(Bool()) // only for better timing
   })
 
   val s_txreq = RegInit(true.B)
@@ -115,7 +116,7 @@ class MMIOBridgeEntry(edge: TLEdgeIn)(implicit p: Parameters) extends TL2CHIL2Mo
   val denied = Reg(Bool())
   val corrupt = Reg(Bool())
   val traceTag = Reg(Bool())
-  val isRead = req.opcode === Get
+  val isRead = RegEnable(io.req.bits.opcode === Get, false.B, io.req.fire)
   val isBackTypeMM = req.user.lift(MemBackTypeMM).getOrElse(false.B)
   val isPageTypeNC = req.user.lift(MemPageTypeNC).getOrElse(false.B)
 
@@ -228,11 +229,7 @@ class MMIOBridgeEntry(edge: TLEdgeIn)(implicit p: Parameters) extends TL2CHIL2Mo
   txreq.bits.qos := Fill(QOS_WIDTH, 1.U(1.W)) - 1.U
   txreq.bits.tgtID := SAM(sam).lookup(txreq.bits.addr)
   txreq.bits.txnID := io.id
-  txreq.bits.opcode := ParallelLookUp(req.opcode, Seq(
-    Get -> ReadNoSnp,
-    PutFullData -> WriteNoSnpPtl,
-    PutPartialData -> WriteNoSnpPtl
-  ))
+  txreq.bits.opcode := Mux(isRead, ReadNoSnp, WriteNoSnpPtl)
   txreq.bits.size := req.size
   txreq.bits.addr := req.address
   txreq.bits.ns := enableNS.B
@@ -325,6 +322,7 @@ class MMIOBridgeEntry(edge: TLEdgeIn)(implicit p: Parameters) extends TL2CHIL2Mo
   io.pCrd.query.bits.srcID := srcID
 
   io.waitOnReadReceipt.foreach(_ := !w_readreceipt.get && s_txreq)
+  io.isRead := isRead
 
   /**
     * performance counters
@@ -384,12 +382,14 @@ class MMIOBridgeImp(outer: MMIOBridge) extends LazyModuleImp(outer)
   }
 
   val txreqArb = Module(new RRArbiterInit(chiselTypeOf(io.tx.req.bits), mmioBridgeSize))
-  for ((a, req) <- txreqArb.io.in.zip(entries.map(_.io.chi.tx.req))) {
-    a <> req
-    val isReadNoSnp = req.bits.opcode === ReadNoSnp
-    val block = isReadNoSnp && waitOnReadReceipt
+  for ((a, entry) <- txreqArb.io.in.zip(entries)) {
+    val req = entry.io.chi.tx.req
+    val isRead = entry.io.isRead
+    val block = isRead && waitOnReadReceipt
+    assert(!req.valid || !isRead || req.bits.opcode === ReadNoSnp)
     req.ready := a.ready && !block
     a.valid := req.valid && !block
+    a.bits := req.bits
   }
   io.tx.req <> txreqArb.io.out
   // arb(entries.map(_.io.chi.tx.req), io.tx.req, Some("mmio_txreq"))

src/main/scala/coupledL2/tl2chi/MSHR.scala

@@ -806,15 +806,14 @@ class MSHR(implicit p: Parameters) extends TL2CHIL2Module with HasCHIOpcodes {
     mp_grant.reqSource := 0.U(MemReqSource.reqSourceBits.W)
 
     // Add merge grant task for Acquire and late Prefetch
-    mp_grant.mergeA := mergeA || io.aMergeTask.valid
-    val merge_task_r = RegEnable(io.aMergeTask.bits, 0.U.asTypeOf(new TaskBundle), io.aMergeTask.valid)
-    val merge_task = Mux(io.aMergeTask.valid, io.aMergeTask.bits, merge_task_r)
-    val merge_task_isKeyword = Mux(io.aMergeTask.valid, io.aMergeTask.bits.isKeyword.getOrElse(false.B), merge_task_r.isKeyword.getOrElse(false.B) )
+    mp_grant.mergeA := mergeA
+
+    val merge_task = RegEnable(io.aMergeTask.bits, 0.U.asTypeOf(new TaskBundle), io.aMergeTask.valid)
 
     mp_grant.aMergeTask.off := merge_task.off
     mp_grant.aMergeTask.alias.foreach(_ := merge_task.alias.getOrElse(0.U))
     mp_grant.aMergeTask.vaddr.foreach(_ := merge_task.vaddr.getOrElse(0.U))
-    mp_grant.aMergeTask.isKeyword.foreach(_ := merge_task_isKeyword)
+    mp_grant.aMergeTask.isKeyword.foreach(_ := merge_task.isKeyword.getOrElse(false.B))
     mp_grant.aMergeTask.opcode := odOpGen(merge_task.opcode)
     mp_grant.aMergeTask.param := MuxLookup( // Acquire -> Grant
       merge_task.param,
@@ -1339,6 +1338,8 @@ class MSHR(implicit p: Parameters) extends TL2CHIL2Module with HasCHIOpcodes {
   io.msInfo.bits.param := req.param
   io.msInfo.bits.mergeA := mergeA
   io.msInfo.bits.w_grantfirst := state.w_grantfirst
+  io.msInfo.bits.w_grantlast := state.w_grantlast
+  io.msInfo.bits.w_grant := state.w_grant
   io.msInfo.bits.s_release := state.s_release
   io.msInfo.bits.s_refill := state.s_refill
   io.msInfo.bits.s_cmoresp := state.s_cmoresp

src/main/scala/coupledL2/tl2tl/MSHR.scala

@@ -372,15 +372,14 @@ class MSHR(implicit p: Parameters) extends L2Module {
     mp_grant.corrupt := corrupt
 
     // Add merge grant task for Acquire and late Prefetch
-    mp_grant.mergeA := mergeA || io.aMergeTask.valid
-    val merge_task_r = RegEnable(io.aMergeTask.bits, 0.U.asTypeOf(new TaskBundle), io.aMergeTask.valid)
-    val merge_task = Mux(io.aMergeTask.valid, io.aMergeTask.bits, merge_task_r)
-    val merge_task_isKeyword = Mux(io.aMergeTask.valid, io.aMergeTask.bits.isKeyword.getOrElse(false.B), merge_task_r.isKeyword.getOrElse(false.B) )
+    mp_grant.mergeA := mergeA
+
+    val merge_task = RegEnable(io.aMergeTask.bits, 0.U.asTypeOf(new TaskBundle), io.aMergeTask.valid)
 
     mp_grant.aMergeTask.off := merge_task.off
     mp_grant.aMergeTask.alias.foreach(_ := merge_task.alias.getOrElse(0.U))
     mp_grant.aMergeTask.vaddr.foreach(_ := merge_task.vaddr.getOrElse(0.U))
-    mp_grant.aMergeTask.isKeyword.foreach(_ := merge_task_isKeyword)
+    mp_grant.aMergeTask.isKeyword.foreach(_ := merge_task.isKeyword.getOrElse(false.B))
     mp_grant.aMergeTask.opcode := odOpGen(merge_task.opcode)
     mp_grant.aMergeTask.param := MuxLookup( // Acquire -> Grant
       merge_task.param,
@@ -570,6 +569,8 @@ class MSHR(implicit p: Parameters) extends L2Module {
   io.msInfo.bits.param := req.param
   io.msInfo.bits.mergeA := mergeA
   io.msInfo.bits.w_grantfirst := state.w_grantfirst
+  io.msInfo.bits.w_grantlast := state.w_grantlast
+  io.msInfo.bits.w_grant := state.w_grant
   io.msInfo.bits.s_refill := state.s_refill
   io.msInfo.bits.s_release := state.s_release
   io.msInfo.bits.s_cmoresp := true.B