Unrelated stability issue fix

Author: WildFireFlum

bftengine/src/bftengine/DbCheckpointManager.cpp

@@ -54,19 +54,15 @@ Status DbCheckpointManager::createDbCheckpoint(const CheckpointId& checkPointId,
       }
       auto end_ckpnt = Clock::now();
       auto duration_chpnt_ms = std::chrono::duration_cast<std::chrono::milliseconds>(end_ckpnt - start);
-      LOG_INFO(getLogger(), "just checkpoint duation: " << KVLOG(duration_chpnt_ms.count()));
+      LOG_INFO(getLogger(), "just checkpoint duration: " << KVLOG(duration_chpnt_ms.count()));
       prepareCheckpointCb_(lastBlockId, dbClient_->getPathForCheckpoint(checkPointId));
       auto end = Clock::now();
 
       auto duration_ms = std::chrono::duration_cast<std::chrono::milliseconds>(end - start);
-      LOG_INFO(getLogger(), "checkpoint with cb duation: " << KVLOG(duration_ms.count()));
+      LOG_INFO(getLogger(), "checkpoint with cb duration: " << KVLOG(duration_ms.count()));
       lastDbCheckpointBlockId_.Get().Set(lastBlockId);
-      numOfDbCheckpointsCreated_++;
       auto maxSoFar = maxDbCheckpointCreationTimeMsec_.Get().Get();
       maxDbCheckpointCreationTimeMsec_.Get().Set(std::max(maxSoFar, static_cast<uint64_t>(duration_ms.count())));
-      auto count = numOfDbCheckpointsCreated_.Get().Get();
-      (void)count;
-      metrics_.UpdateAggregator();
       LOG_INFO(getLogger(), "rocksdb checkpoint created: " << KVLOG(checkPointId, duration_ms.count(), seqNum));
       lastCheckpointSeqNum_ = seqNum;
       {
@@ -84,6 +80,8 @@ Status DbCheckpointManager::createDbCheckpoint(const CheckpointId& checkPointId,
       if (cb) cb(seqNum);
     }
     checkpointInProcessCb_(false, lastBlockId);
+    numOfDbCheckpointsCreated_++;
+    metrics_.UpdateAggregator();
   }
   updateMetrics();
   return Status::OK();

tests/apollo/test_skvbc_dbsnapshot.py

@@ -993,14 +993,14 @@ async def verify_db_size_on_disk(self, bft_network):
             for checkPtId,db_size in res[0].response.mapCheckpointIdDbSize:
                 db_dir = os.path.join(
                     bft_network.testdir, DB_FILE_PREFIX + str(replica_id))
-                if checkPtId != 0 :
+                if checkPtId != 0:
                     db_dir = os.path.join(
                         bft_network.testdir, DB_SNAPSHOT_PREFIX + str(replica_id) + "/" + str(checkPtId))
                 size = 0
                 for element in os.scandir(db_dir):
                     if element.is_file():
                         size += os.path.getsize(element)
-                assert size == db_size
+                assert size == db_size, f"replica: {replica_id}, expected_size:{db_size}, actual_size: {size}"
 
     async def validate_epoch_number(self, bft_network, epoch_number, replicas):
         with trio.fail_after(seconds=70):

tests/apollo/test_skvbc_view_change.py

@@ -20,7 +20,7 @@
 from util.test_base import ApolloTest
 from util import bft_network_partitioning as net
 from util import skvbc as kvbc
-from util.bft import with_trio, with_bft_network, KEY_FILE_PREFIX, skip_for_tls
+from util.bft import with_trio, with_bft_network, KEY_FILE_PREFIX, skip_for_tls, ConsensusPathType
 from util.skvbc_history_tracker import verify_linearizability
 from util import eliot_logging as log
 
@@ -442,6 +442,8 @@ async def test_multiple_vc_slow_path(self, bft_network, tracker):
         c = bft_network.config.c
         current_primary = 0
         view = 0
+        messages_to_send = 5
+        accumulated_slow_paths = 0
         for _ in range(2):
             self.assertEqual(len(bft_network.procs), n,
                              "Make sure all replicas are up initially.")
@@ -465,10 +467,11 @@ async def test_multiple_vc_slow_path(self, bft_network, tracker):
             await bft_network.wait_for_view_with_threshold(view)
             current_primary = view
 
-            # Main key publications on startup might have been executed in fast path
-            await bft_network.assert_slow_path_prevalent(
-                nb_fast_paths_to_ignore=n, nb_slow_paths_to_ignore=0,
-                replica_id=random.choice(bft_network.all_replicas(without=crashed_replicas)))
+            accumulated_slow_paths += messages_to_send
+            await bft_network.wait_for_consensus_path(path_type=ConsensusPathType.SLOW,
+                                                      replica_id=current_primary,
+                                                      run_ops=lambda: skvbc.send_n_kvs_sequentially(messages_to_send),
+                                                      threshold=accumulated_slow_paths)
 
             for i in crashed_replicas:
                 bft_network.start_replica(i)