Update pruning metrics when pruning is done after state transfer (#2643)

Problem Overview
When pruning is done after state transfer, we use a code path that doesn't update the pruning metrics
This PR is for fixing this bug
Testing Done
CI tests + enabling buck the test_pruning_with_failures_test and update it to also test that the lagged replica updates its metrics

Author: Yehonatan Buchnik

kvbc/src/categorization/kv_blockchain.cpp

@@ -967,9 +967,10 @@ void KeyValueBlockchain::pruneOnSTLink(const RawBlock& block) {
   auto key_it = internal_kvs.find(keyTypes::genesis_block_key);
   if (key_it != internal_kvs.cend()) {
     const auto block_genesis_id = concordUtils::fromBigEndianBuffer<BlockId>(key_it->second.data.data());
-    while (getGenesisBlockId() >= INITIAL_GENESIS_BLOCK_ID && getGenesisBlockId() < getLastReachableBlockId() &&
-           block_genesis_id > getGenesisBlockId()) {
-      deleteGenesisBlock();
+    if (getGenesisBlockId() >= INITIAL_GENESIS_BLOCK_ID && getGenesisBlockId() < getLastReachableBlockId()) {
+      for (auto i = getGenesisBlockId(); i < block_genesis_id; i++) {
+        ConcordAssert(deleteBlock(i));
+      }
     }
   }
 }

tests/apollo/test_skvbc_reconfiguration.py

@@ -1061,7 +1061,6 @@ async def test_pruning_command(self, bft_network):
             log.log_message(message_type=f"pruned_block {pruned_block}")
             assert pruned_block <= 97
 
-    @unittest.skip("Unstable test - BC-19253")
     @with_trio
     @with_bft_network(start_replica_cmd, selected_configs=lambda n, f, c: n == 7, publish_master_keys=True)
     async def test_pruning_command_with_failures(self, bft_network):
@@ -1121,7 +1120,26 @@ async def test_pruning_command_with_failures(self, bft_network):
                             num_replies += 1
                     if num_replies == bft_network.config.n:
                         break
+            # Validate the crashed replica has managed to updates its metrics
+            with trio.fail_after(60):
+                while True:
+                    with trio.move_on_after(seconds=.5):
+                        try:
+                            key = ['kv_blockchain_deletes', 'Counters', 'numOfVersionedKeysDeleted']
+                            deletes = await bft_network.metrics.get(crashed_replica, *key)
+
+                            key = ['kv_blockchain_deletes', 'Counters', 'numOfImmutableKeysDeleted']
+                            deletes += await bft_network.metrics.get(crashed_replica, *key)
 
+                            key = ['kv_blockchain_deletes', 'Counters', 'numOfMerkleKeysDeleted']
+                            deletes += await bft_network.metrics.get(crashed_replica, *key)
+                            
+                        except KeyError:
+                            continue
+                        else:
+                            # success!
+                            if deletes >= 0:
+                                break
             # Now, crash the same replica again.
             crashed_replica = 3
             bft_network.stop_replica(crashed_replica)