cpu-o3: stabilize fs-smt shared-memory execution

Change-Id: Iacbf424326a2c391b327b01f49d9795e9a6c5fcb

Author: tastynoob

src/cpu/base.cc

@@ -148,7 +148,20 @@ class BaseCPU : public ClockedObject
         uint8_t data[16] = {};
     };
 
+    struct RecentVisibleSharedStore
+    {
+        bool valid = false;
+        ThreadID tid = InvalidThreadID;
+        Addr addr = 0;
+        size_t size = 0;
+        InstSeqNum seq = 0;
+        uint8_t data[16] = {};
+    };
+
     std::vector<std::deque<RecentCommittedStore>> recentCommittedStores;
+    std::deque<RecentVisibleSharedStore> recentVisibleSharedStores;
+    std::vector<uint8_t> sharedReplayCatchupBudget;
+    std::vector<bool> syncVisibleStoreReplayArmed;
 
     const unsigned IntRegIndexBase = 0;
     const unsigned FPRegIndexBase = 32;
@@ -790,6 +803,21 @@ class BaseCPU : public ClockedObject
     void difftestStep(ThreadID tid, InstSeqNum seq);
 
     void recordCommittedStore(ThreadID tid, const o3::DynInstPtr &inst);
+    void recordVisibleSharedStore(ThreadID tid, Addr addr, size_t size,
+                                  InstSeqNum seq, const uint8_t *data);
+
+    virtual bool findPendingStoreValue(ThreadID tid, Addr addr, size_t size,
+                                       InstSeqNum seq, uint8_t *data) const
+    {
+        return false;
+    }
+
+    virtual bool findPendingLocalStoreValue(ThreadID tid, Addr addr,
+                                            size_t size, InstSeqNum seq,
+                                            uint8_t *data) const
+    {
+        return false;
+    }
 
     inline bool difftestEnabled() const { return enableDifftest; }
 
@@ -824,6 +852,18 @@ class BaseCPU : public ClockedObject
     uint8_t *getGoldenMemPtr() { return goldenMemPtr; }
 
     gem5::GoldenGloablMem *goldenMemManager() { return _goldenMemManager; }
+    void syncVisibleSharedStoreToDiffRefs(Addr paddr, size_t size);
+    void armSharedReplayCatchup(ThreadID tid);
+    void armSyncVisibleStoreReplay(ThreadID tid)
+    {
+        syncVisibleStoreReplayArmed.at(tid) = true;
+    }
+    bool consumeSyncVisibleStoreReplay(ThreadID tid)
+    {
+        bool armed = syncVisibleStoreReplayArmed.at(tid);
+        syncVisibleStoreReplayArmed.at(tid) = false;
+        return armed;
+    }
 
     void checkL1DRefill(Addr paddr, const uint8_t *refill_data, size_t size);
 };

src/cpu/base.hh

@@ -174,9 +174,7 @@ void
 NemuProxy::initState(int coreid, uint8_t *golden_mem)
 {
     if (multiCore) {
-        warn("Setting mhartid to %d\n", coreid);
         setHartId(coreid);
-        warn("Setting gmaddr to %#lx\n", (uint64_t) golden_mem);
         nemuPutGmaddr(golden_mem);
     }
 }

src/cpu/difftest.cc

@@ -1001,6 +1001,33 @@ Commit::commit()
     std::list<ThreadID>::iterator end = activeThreads->end();
 
     int num_squashing_threads = 0;
+    auto hasYoungerInflight = [&](ThreadID tid, InstSeqNum squashed_seq,
+                                  bool include_squash_inst) {
+        const InstSeqNum cutoff =
+            include_squash_inst ? squashed_seq - 1 : squashed_seq;
+
+        if (cutoff <= youngestSeqNum[tid]) {
+            return true;
+        }
+
+        for (const auto &inst : fixedbuffer[tid]) {
+            if (inst && !inst->isSquashed() && inst->seqNum > cutoff) {
+                return true;
+            }
+        }
+
+        for (int i = 0; i < fromRename->size; ++i) {
+            const auto &inst = fromRename->insts[i];
+            if (!inst || inst->threadNumber != tid || inst->isSquashed()) {
+                continue;
+            }
+            if (inst->seqNum > cutoff) {
+                return true;
+            }
+        }
+
+        return false;
+    };
 
     while (threads != end) {
         ThreadID tid = *threads++;
@@ -1030,11 +1057,14 @@ Commit::commit()
         // Squashed sequence number must be older than youngest valid
         // instruction in the ROB. This prevents squashes from younger
         // instructions overriding squashes from older instructions.
+        const bool has_younger_inflight =
+            hasYoungerInflight(tid, fromIEW->squashedSeqNum[tid],
+                               fromIEW->includeSquashInst[tid]);
         DPRINTF(Commit, "fromIEW->squash %d, commitStatus %d, fromIEW->squashedSeqNum %d, youngestSeqNum %d\n",
             fromIEW->squash[tid], commitStatus[tid], fromIEW->squashedSeqNum[tid], youngestSeqNum[tid]);
         if (fromIEW->squash[tid] &&
             commitStatus[tid] != TrapPending &&
-            fromIEW->squashedSeqNum[tid] <= youngestSeqNum[tid]) {
+            has_younger_inflight) {
 
             if (fromIEW->mispredictInst[tid]) {
                 DPRINTF(Commit,
@@ -1212,9 +1242,9 @@ Commit::commitInsts()
             continue;
         }
 
-            while (num_committed < commit_width &&
-                num_committed_per_thread[commit_thread] <
-                    commit_width_per_thread[commit_thread]) {
+        while (num_committed < commit_width &&
+               num_committed_per_thread[commit_thread] <
+                   commit_width_per_thread[commit_thread]) {
             // hardware transactionally memory
             // If executing within a transaction,
             // need to handle interrupts specially
@@ -1453,6 +1483,11 @@ Commit::commitInsts()
 
                     }
 
+                    if (head_inst->isReadBarrier() ||
+                        head_inst->isWriteBarrier()) {
+                        cpu->armSyncVisibleStoreReplay(tid);
+                    }
+
                     if (cpu->difftestEnabled()) {
                         diffInst(tid, head_inst);
                     }
@@ -1651,7 +1686,7 @@ Commit::commitHead(const DynInstPtr &head_inst, unsigned inst_num)
         // race with the TLB invalidation.
         if ((head_inst->isMemRef() || head_inst->isReturn() ||
              head_inst->isReadBarrier() || head_inst->isWriteBarrier()) &&
-            (inst_num > 0 || !iewStage->flushStores(tid))) {
+            (inst_num > 0 || !iewStage->flushStores(tid, head_inst->seqNum))) {
             DPRINTF(Commit,
                     "[tid:%i] [sn:%llu] "
                     "Waiting for all stores to writeback.\n",
@@ -1705,7 +1740,7 @@ Commit::commitHead(const DynInstPtr &head_inst, unsigned inst_num)
 
     if (inst_fault != NoFault) {
         traceLogInstFault(head_inst, inst_fault);
-        if (!iewStage->flushStores(tid) || inst_num > 0) {
+        if (!iewStage->flushStores(tid, head_inst->seqNum) || inst_num > 0) {
             DPRINTF(Commit,
                     "[tid:%i] [sn:%llu] "
                     "Stores outstanding, fault must wait.\n",

src/cpu/o3/commit.cc

@@ -433,6 +433,19 @@ class CPU : public BaseCPU
 
     uint32_t getIQInsts() { return iew.getIQInsts(); }
 
+    bool findPendingStoreValue(ThreadID tid, Addr addr, size_t size,
+                               InstSeqNum seq, uint8_t *data) const override
+    {
+        return iew.findPendingStoreValue(tid, addr, size, seq, data);
+    }
+
+    bool findPendingLocalStoreValue(ThreadID tid, Addr addr, size_t size,
+                                    InstSeqNum seq,
+                                    uint8_t *data) const override
+    {
+        return iew.findPendingLocalStoreValue(tid, addr, size, seq, data);
+    }
+
     /**
      * Return the oldest in-flight instruction sequence number.
      * If there are no in-flight instructions, returns the maximum value

src/cpu/o3/cpu.hh

@@ -255,6 +255,13 @@ Decode::fetchInstsValid()
 void
 Decode::selfSquash(const DynInstPtr &inst, ThreadID tid)
 {
+    if (tid == 1 && inst->seqNum >= 182805 && inst->seqNum <= 182808) {
+        warn("DECODE-SQUASH tid=%d seq=%llu pc=%#lx pred_taken=%d pred_tgt=%#lx "
+             "ftq=%u\n",
+             tid, static_cast<unsigned long long>(inst->seqNum),
+             inst->pcState().instAddr(), inst->readPredTaken(),
+             inst->readPredTarg().instAddr(), inst->getFtqId());
+    }
     DPRINTF(Decode, "[tid:%i] [sn:%llu] Squashing due to incorrect branch "
             "prediction detected at decode.\n", tid, inst->seqNum);
 
@@ -718,29 +725,42 @@ Decode::decodeInsts(ThreadID tid)
                     pred.pc(), pred.npc(), t.pc(), t.npc());
                 inst->setPredTarg(t);
             }
-            if (*target != inst->readPredTarg()) {
+            bool wrong_uncond_direction =
+                inst->isUncondCtrl() && !inst->readPredTaken();
+            bool wrong_target = *target != inst->readPredTarg();
+            if (wrong_uncond_direction || wrong_target) {
                 ++stats.branchMispred;
 
-                RiscvISA::PCState cpTarget = target->clone()->as<RiscvISA::PCState>();
-                RiscvISA::PCState cpPredTarget = inst->readPredTarg().clone()->as<RiscvISA::PCState>();
+                RiscvISA::PCState cpTarget =
+                    target->clone()->as<RiscvISA::PCState>();
+                RiscvISA::PCState cpPredTarget =
+                    inst->readPredTarg().clone()->as<RiscvISA::PCState>();
 
-                if (cpTarget.instAddr() != cpPredTarget.instAddr() && cpTarget.npc() == cpPredTarget.npc()) {
+                if (wrong_target &&
+                    cpTarget.instAddr() != cpPredTarget.instAddr() &&
+                    cpTarget.npc() == cpPredTarget.npc()) {
                     ++stats.mispredictedByPC;
-                } else if (cpTarget.instAddr() == cpPredTarget.instAddr() && cpTarget.npc() != cpPredTarget.npc()) {
+                } else if (wrong_target &&
+                           cpTarget.instAddr() == cpPredTarget.instAddr() &&
+                           cpTarget.npc() != cpPredTarget.npc()) {
                     ++stats.mispredictedByNPC;
                 }
 
                 // Might want to set some sort of boolean and just do
                 // a check at the end
+                if (wrong_uncond_direction) {
+                    inst->setPredTaken(true);
+                }
                 selfSquash(inst, inst->threadNumber);
 
                 decode_stalls.push(StallReason::InstMisPred);
                 breakDecode = StallReason::InstMisPred;
 
                 DPRINTF(Decode,
                         "[tid:%i] [sn:%llu] Updating predictions:"
-                        " Wrong predicted target: %s PredPC: %s\n",
-                        tid, inst->seqNum, inst->readPredTarg(), *target);
+                        " wrong_dir=%d wrong_target=%d PredPC: %s Actual: %s\n",
+                        tid, inst->seqNum, wrong_uncond_direction,
+                        wrong_target, inst->readPredTarg(), *target);
                 //The micro pc after an instruction level branch should be 0
                 inst->setPredTarg(*target);
                 break;

src/cpu/o3/decode.cc

@@ -260,6 +260,23 @@ class IEW
      * the store queue or the store buffer to write back to.
      */
     bool flushStores(ThreadID tid) { return ldstQueue.flushStores(tid); }
+    bool flushStores(ThreadID tid, InstSeqNum seq_num)
+    {
+        return ldstQueue.flushStores(tid, seq_num);
+    }
+
+    bool findPendingStoreValue(ThreadID tid, Addr addr, size_t size,
+                               InstSeqNum seq, uint8_t *data) const
+    {
+        return ldstQueue.findPendingStoreValue(tid, addr, size, seq, data);
+    }
+
+    bool findPendingLocalStoreValue(ThreadID tid, Addr addr, size_t size,
+                                    InstSeqNum seq, uint8_t *data) const
+    {
+        return ldstQueue.findPendingLocalStoreValue(tid, addr, size, seq,
+                                                    data);
+    }
 
     /** Check if we need to squash after a load/store/branch is executed. */
     void SquashCheckAfterExe(DynInstPtr inst);