cpu-o3: fix smt shared sbuffer

Change-Id: Icfcaf46e311e9a1d6e28df53bec101faf5627bf1

Author: tastynoob

src/cpu/base.cc

@@ -139,16 +139,8 @@ struct DiffAllStates
 class BaseCPU : public ClockedObject
 {
   protected:
-    struct RecentCommittedStore
-    {
-        bool valid = false;
-        Addr addr = 0;
-        size_t size = 0;
-        InstSeqNum seq = 0;
-        uint8_t data[16] = {};
-    };
-
-    std::vector<std::deque<RecentCommittedStore>> recentCommittedStores;
+    std::vector<uint8_t> sharedReplayCatchupBudget;
+    std::vector<bool> syncVisibleStoreReplayArmed;
 
     const unsigned IntRegIndexBase = 0;
     const unsigned FPRegIndexBase = 32;
@@ -756,6 +748,8 @@ class BaseCPU : public ClockedObject
         gem5::Addr physEffAddr;
         gem5::Addr effSize;
         uint8_t *goldenValue;
+        uint8_t loadDiffEvidenceKind;
+        uint8_t *loadDiffEvidenceData;
         uint64_t amoOldGoldenValue;
         // Register address causing difftest error
         bool errorRegsValue[diffAllNum];
@@ -789,7 +783,18 @@ class BaseCPU : public ClockedObject
 
     void difftestStep(ThreadID tid, InstSeqNum seq);
 
-    void recordCommittedStore(ThreadID tid, const o3::DynInstPtr &inst);
+    virtual bool findPendingStoreValue(ThreadID tid, Addr addr, size_t size,
+                                       InstSeqNum seq, uint8_t *data) const
+    {
+        return false;
+    }
+
+    virtual bool findPendingLocalStoreValue(ThreadID tid, Addr addr,
+                                            size_t size, InstSeqNum seq,
+                                            uint8_t *data) const
+    {
+        return false;
+    }
 
     inline bool difftestEnabled() const { return enableDifftest; }
 
@@ -824,6 +829,20 @@ class BaseCPU : public ClockedObject
     uint8_t *getGoldenMemPtr() { return goldenMemPtr; }
 
     gem5::GoldenGloablMem *goldenMemManager() { return _goldenMemManager; }
+    void syncVisibleSharedStoreToDiffRefs(Addr paddr, size_t size,
+                                          ThreadID exclude_tid =
+                                              InvalidThreadID);
+    void armSharedReplayCatchup(ThreadID tid);
+    void armSyncVisibleStoreReplay(ThreadID tid)
+    {
+        syncVisibleStoreReplayArmed.at(tid) = true;
+    }
+    bool consumeSyncVisibleStoreReplay(ThreadID tid)
+    {
+        bool armed = syncVisibleStoreReplayArmed.at(tid);
+        syncVisibleStoreReplayArmed.at(tid) = false;
+        return armed;
+    }
 
     void checkL1DRefill(Addr paddr, const uint8_t *refill_data, size_t size);
 };

src/cpu/base.hh

@@ -174,9 +174,7 @@ void
 NemuProxy::initState(int coreid, uint8_t *golden_mem)
 {
     if (multiCore) {
-        warn("Setting mhartid to %d\n", coreid);
         setHartId(coreid);
-        warn("Setting gmaddr to %#lx\n", (uint64_t) golden_mem);
         nemuPutGmaddr(golden_mem);
     }
 }

src/cpu/difftest.cc

@@ -562,6 +562,19 @@ Commit::takeOverFrom()
     rob->takeOverFrom();
 }
 
+void
+Commit::activateThread(ThreadID tid)
+{
+    if (commitPolicy != CommitPolicy::RoundRobin) {
+        return;
+    }
+
+    auto thread_it = std::find(priority_list.begin(), priority_list.end(), tid);
+    if (thread_it == priority_list.end()) {
+        priority_list.push_back(tid);
+    }
+}
+
 void
 Commit::deactivateThread(ThreadID tid)
 {
@@ -1001,7 +1014,6 @@ Commit::commit()
     std::list<ThreadID>::iterator end = activeThreads->end();
 
     int num_squashing_threads = 0;
-
     while (threads != end) {
         ThreadID tid = *threads++;
 
@@ -1212,9 +1224,9 @@ Commit::commitInsts()
             continue;
         }
 
-            while (num_committed < commit_width &&
-                num_committed_per_thread[commit_thread] <
-                    commit_width_per_thread[commit_thread]) {
+        while (num_committed < commit_width &&
+               num_committed_per_thread[commit_thread] <
+                   commit_width_per_thread[commit_thread]) {
             // hardware transactionally memory
             // If executing within a transaction,
             // need to handle interrupts specially
@@ -1453,6 +1465,11 @@ Commit::commitInsts()
 
                     }
 
+                    if (head_inst->isReadBarrier() ||
+                        head_inst->isWriteBarrier()) {
+                        cpu->armSyncVisibleStoreReplay(tid);
+                    }
+
                     if (cpu->difftestEnabled()) {
                         diffInst(tid, head_inst);
                     }
@@ -1562,6 +1579,7 @@ Commit::commitInsts()
                     if (!interrupt && avoidQuiesceLiveLock &&
                         onInstBoundary && cpu->checkInterrupts(0))
                         squashAfter(tid, head_inst);
+
                 } else {
                     DPRINTF(Commit, "Unable to commit head instruction PC:%s "
                             "[tid:%i] [sn:%llu].\n",
@@ -1617,8 +1635,9 @@ Commit::diffInst(ThreadID tid, const DynInstPtr &inst) {
     cpu->diffInfo.physEffAddr = inst->physEffAddr;
     cpu->diffInfo.effSize = inst->effSize;
     cpu->diffInfo.goldenValue = inst->getGolden();
+    cpu->diffInfo.loadDiffEvidenceKind = inst->loadDiffEvidenceKind();
+    cpu->diffInfo.loadDiffEvidenceData = inst->getLoadDiffEvidenceData();
     cpu->diffInfo.amoOldGoldenValue = inst->getAmoOldGoldenValue();
-    cpu->recordCommittedStore(tid, inst);
     cpu->difftestStep(tid, inst->seqNum);
 }
 
@@ -1649,9 +1668,12 @@ Commit::commitHead(const DynInstPtr &head_inst, unsigned inst_num)
         // Memory-ordering instructions such as sfence.vma must not execute
         // until older stores are visible; otherwise page-table updates may
         // race with the TLB invalidation.
-        if ((head_inst->isMemRef() || head_inst->isReturn() ||
-             head_inst->isReadBarrier() || head_inst->isWriteBarrier()) &&
-            (inst_num > 0 || !iewStage->flushStores(tid))) {
+        const bool needs_store_drain =
+            head_inst->isMemRef() || head_inst->isReturn() ||
+            head_inst->isReadBarrier() || head_inst->isWriteBarrier();
+        const bool stores_drained =
+            !needs_store_drain || iewStage->flushStores(tid, head_inst->seqNum);
+        if (needs_store_drain && (inst_num > 0 || !stores_drained)) {
             DPRINTF(Commit,
                     "[tid:%i] [sn:%llu] "
                     "Waiting for all stores to writeback.\n",
@@ -1705,7 +1727,7 @@ Commit::commitHead(const DynInstPtr &head_inst, unsigned inst_num)
 
     if (inst_fault != NoFault) {
         traceLogInstFault(head_inst, inst_fault);
-        if (!iewStage->flushStores(tid) || inst_num > 0) {
+        if (!iewStage->flushStores(tid, head_inst->seqNum) || inst_num > 0) {
             DPRINTF(Commit,
                     "[tid:%i] [sn:%llu] "
                     "Stores outstanding, fault must wait.\n",
@@ -2189,6 +2211,22 @@ Commit::getCommittingThread()
 ThreadID
 Commit::roundRobin()
 {
+    for (auto it = priority_list.begin(); it != priority_list.end();) {
+        if (std::find(activeThreads->begin(), activeThreads->end(), *it) ==
+            activeThreads->end()) {
+            it = priority_list.erase(it);
+        } else {
+            ++it;
+        }
+    }
+
+    for (ThreadID tid : *activeThreads) {
+        if (std::find(priority_list.begin(), priority_list.end(), tid) ==
+            priority_list.end()) {
+            priority_list.push_back(tid);
+        }
+    }
+
     std::list<ThreadID>::iterator pri_iter = priority_list.begin();
     std::list<ThreadID>::iterator end      = priority_list.end();
 

src/cpu/o3/commit.cc

@@ -280,6 +280,9 @@ class Commit
     /** Takes over from another CPU's thread. */
     void takeOverFrom();
 
+    /** Schedules a thread for commit arbitration. */
+    void activateThread(ThreadID tid);
+
     /** Deschedules a thread from scheduling */
     void deactivateThread(ThreadID tid);
 

src/cpu/o3/commit.hh

@@ -657,6 +657,7 @@ CPU::activateThread(ThreadID tid)
         DPRINTF(O3CPU, "[tid:%i] Adding to active threads list\n", tid);
 
         activeThreads.push_back(tid);
+        commit.activateThread(tid);
     }
 }
 
@@ -1561,6 +1562,7 @@ CPU::dumpInsts()
         ++num;
     }
 }
+
 /*
 void
 CPU::wakeDependents(const DynInstPtr &inst)

src/cpu/o3/cpu.cc

@@ -433,6 +433,19 @@ class CPU : public BaseCPU
 
     uint32_t getIQInsts() { return iew.getIQInsts(); }
 
+    bool findPendingStoreValue(ThreadID tid, Addr addr, size_t size,
+                               InstSeqNum seq, uint8_t *data) const override
+    {
+        return iew.findPendingStoreValue(tid, addr, size, seq, data);
+    }
+
+    bool findPendingLocalStoreValue(ThreadID tid, Addr addr, size_t size,
+                                    InstSeqNum seq,
+                                    uint8_t *data) const override
+    {
+        return iew.findPendingLocalStoreValue(tid, addr, size, seq, data);
+    }
+
     /**
      * Return the oldest in-flight instruction sequence number.
      * If there are no in-flight instructions, returns the maximum value

src/cpu/o3/cpu.hh

@@ -255,6 +255,8 @@ class DynInst : public ExecContext, public RefCounted
 
     bool _hasProducerStorePC = false;
     Addr _producerStorePC = 0;
+    uint8_t _loadDiffEvidenceKind = 0;
+    uint8_t _loadDiffEvidenceData[8] = {0};
 
   protected:
     /** The result of the instruction; assumes an instruction can have many
@@ -403,6 +405,25 @@ class DynInst : public ExecContext, public RefCounted
         _producerStorePC = 0;
     }
 
+    void
+    clearLoadDiffEvidence()
+    {
+        _loadDiffEvidenceKind = 0;
+        memset(_loadDiffEvidenceData, 0, sizeof(_loadDiffEvidenceData));
+    }
+
+    void
+    setLoadDiffEvidence(uint8_t kind, const uint8_t *data)
+    {
+        panic_if(effSize > sizeof(_loadDiffEvidenceData),
+                 "Unexpected load diff evidence size %u\n", effSize);
+        _loadDiffEvidenceKind = kind;
+        memcpy(_loadDiffEvidenceData, data, effSize);
+    }
+
+    uint8_t loadDiffEvidenceKind() const { return _loadDiffEvidenceKind; }
+    uint8_t *getLoadDiffEvidenceData() { return _loadDiffEvidenceData; }
+
     /** The thread this instruction is from. */
     ThreadID threadNumber = 0;
 

src/cpu/o3/dyn_inst.hh

@@ -260,6 +260,23 @@ class IEW
      * the store queue or the store buffer to write back to.
      */
     bool flushStores(ThreadID tid) { return ldstQueue.flushStores(tid); }
+    bool flushStores(ThreadID tid, InstSeqNum seq_num)
+    {
+        return ldstQueue.flushStores(tid, seq_num);
+    }
+
+    bool findPendingStoreValue(ThreadID tid, Addr addr, size_t size,
+                               InstSeqNum seq, uint8_t *data) const
+    {
+        return ldstQueue.findPendingStoreValue(tid, addr, size, seq, data);
+    }
+
+    bool findPendingLocalStoreValue(ThreadID tid, Addr addr, size_t size,
+                                    InstSeqNum seq, uint8_t *data) const
+    {
+        return ldstQueue.findPendingLocalStoreValue(tid, addr, size, seq,
+                                                    data);
+    }
 
     /** Check if we need to squash after a load/store/branch is executed. */
     void SquashCheckAfterExe(DynInstPtr inst);