Changes for ST in fullNode

Author: Tarunkumar Banda

bftengine/CMakeLists.txt

@@ -11,6 +11,7 @@ set(corebft_source_files
     src/bftengine/DebugStatistics.cpp
     src/bftengine/SeqNumInfo.cpp
     src/bftengine/ReadOnlyReplica.cpp
+    src/bftengine/FullNodeReplica.cpp
     src/bftengine/ReplicaBase.cpp
     src/bftengine/ReplicaForStateTransfer.cpp
     src/bftengine/ReplicaImp.cpp

bftengine/include/bcstatetransfer/SimpleBCStateTransfer.hpp

@@ -150,8 +150,10 @@ struct Config {
   uint16_t cVal = 0;
   uint16_t numReplicas = 0;  // number of consensus replicas
   uint16_t numRoReplicas = 0;
+  uint16_t numFnReplicas = 0;
   bool pedanticChecks = false;
   bool isReadOnly = false;
+  bool isFullNode = false;
 
   // sizes
   uint32_t maxChunkSize = 0;
@@ -219,6 +221,8 @@ inline std::ostream &operator<<(std::ostream &os, const Config &c) {
               c.enableSourceBlocksPreFetch,
               c.enableSourceSelectorPrimaryAwareness,
               c.enableStoreRvbDataDuringCheckpointing);
+  os << ",";
+  os << KVLOG(c.numFnReplicas, c.isFullNode);
   return os;
 }
 // creates an instance of the state transfer module.

bftengine/include/bftengine/ReplicaConfig.hpp

@@ -46,8 +46,10 @@ class ReplicaConfig : public concord::serialize::SerializableFactory<ReplicaConf
   }
 
   CONFIG_PARAM(isReadOnly, bool, false, "Am I a read-only replica?");
+  CONFIG_PARAM(isFullNode, bool, false, "Am I a fullNode replica?");
   CONFIG_PARAM(numReplicas, uint16_t, 0, "number of regular replicas");
   CONFIG_PARAM(numRoReplicas, uint16_t, 0, "number of read-only replicas");
+  CONFIG_PARAM(numFnReplicas, uint16_t, 0, "number of full-node replicas");
   CONFIG_PARAM(fVal, uint16_t, 0, "F value - max number of faulty/malicious replicas. fVal >= 1");
   CONFIG_PARAM(cVal, uint16_t, 0, "C value. cVal >=0");
   CONFIG_PARAM(replicaId,
@@ -327,8 +329,10 @@ class ReplicaConfig : public concord::serialize::SerializableFactory<ReplicaConf
 
   void serializeDataMembers(std::ostream& outStream) const {
     serialize(outStream, isReadOnly);
+    serialize(outStream, isFullNode);
     serialize(outStream, numReplicas);
     serialize(outStream, numRoReplicas);
+    serialize(outStream, numFnReplicas);
     serialize(outStream, fVal);
     serialize(outStream, cVal);
     serialize(outStream, replicaId);
@@ -427,8 +431,10 @@ class ReplicaConfig : public concord::serialize::SerializableFactory<ReplicaConf
   }
   void deserializeDataMembers(std::istream& inStream) {
     deserialize(inStream, isReadOnly);
+    deserialize(inStream, isFullNode);
     deserialize(inStream, numReplicas);
     deserialize(inStream, numRoReplicas);
+    deserialize(inStream, numFnReplicas);
     deserialize(inStream, fVal);
     deserialize(inStream, cVal);
     deserialize(inStream, replicaId);
@@ -605,7 +611,8 @@ inline std::ostream& operator<<(std::ostream& os, const ReplicaConfig& rc) {
               rc.maxNumberOfDbCheckpoints,
               rc.dbCheckPointWindowSize,
               rc.dbCheckpointDirPath,
-              rc.dbSnapshotIntervalSeconds.count());
+              rc.dbSnapshotIntervalSeconds.count(),
+              rc.isFullNode);
   os << ",";
   const auto replicaMsgSignAlgo =
       (concord::crypto::SignatureAlgorithm::EdDSA == rc.replicaMsgSigningAlgo) ? "eddsa" : "undefined";

bftengine/include/bftengine/ReplicaFactory.hpp

@@ -15,6 +15,7 @@
 #include "Replica.hpp"
 #include "ReplicaImp.hpp"
 #include "ReadOnlyReplica.hpp"
+#include "FullNodeReplica.hpp"
 #include "ReplicaLoader.hpp"
 
 namespace preprocessor {
@@ -48,6 +49,12 @@ class ReplicaFactory {
                                      bft::communication::ICommunication *,
                                      MetadataStorage *);
 
+  static IReplicaPtr createFullNodeReplica(const ReplicaConfig &,
+                                           std::shared_ptr<IRequestsHandler>,
+                                           IStateTransfer *,
+                                           bft::communication::ICommunication *,
+                                           MetadataStorage *);
+
   static void setAggregator(const std::shared_ptr<concordMetrics::Aggregator> &aggregator);
   static logging::Logger logger_;
 

bftengine/src/bcstatetransfer/AsyncStateTransferCRE.cpp

@@ -148,7 +148,9 @@ std::shared_ptr<ClientReconfigurationEngine> CreFactory::create(std::shared_ptr<
   for (uint16_t i = 0; i < repConfig.numReplicas; i++) {
     bftClientConf.all_replicas.emplace(bft::client::ReplicaId{i});
   }
-  for (uint16_t i = repConfig.numReplicas; i < repConfig.numReplicas + repConfig.numRoReplicas; i++) {
+  for (uint16_t i = repConfig.numReplicas;
+       i < repConfig.numReplicas + repConfig.numRoReplicas + repConfig.numFnReplicas;
+       i++) {
     bftClientConf.ro_replicas.emplace(bft::client::ReplicaId{i});
   }
   bftClientConf.replicas_master_key_folder_path = std::nullopt;

bftengine/src/bcstatetransfer/BCStateTran.cpp

@@ -308,7 +308,7 @@ BCStateTran::BCStateTran(const Config &config, IAppState *const stateApi, DataSt
   // Validate input parameters and some of the configuration
   ConcordAssertNE(stateApi, nullptr);
   ConcordAssertGE(replicas_.size(), 3U * config_.fVal + 1U);
-  ConcordAssert(replicas_.count(config_.myReplicaId) == 1 || config.isReadOnly);
+  ConcordAssert(replicas_.count(config_.myReplicaId) == 1 || config.isReadOnly || config.isFullNode);
   ConcordAssertLT(finalizePutblockTimeoutMilli_, config_.refreshTimerMs);
   ConcordAssertEQ(RejectFetchingMsg::reasonMessages.size(), RejectFetchingMsg::Reason::LAST - 1);
   if (config_.sourceSessionExpiryDurationMs > 0) {
@@ -1159,7 +1159,7 @@ void BCStateTran::handleStateTransferMessageImpl(char *msg,
     time_in_incoming_events_queue_rec_.end();
     histograms_.incoming_events_queue_size->record(incomingEventsQ_->size());
   }
-  bool invalidSender = (senderId >= (config_.numReplicas + config_.numRoReplicas));
+  bool invalidSender = (senderId >= (config_.numReplicas + config_.numRoReplicas + config_.numFnReplicas));
   bool sentFromSelf = senderId == config_.myReplicaId;
   bool msgSizeTooSmall = msgLen < sizeof(BCStateTranBaseMsg);
   if (msgSizeTooSmall || sentFromSelf || invalidSender) {
@@ -3971,13 +3971,13 @@ void BCStateTran::computeDigestOfPage(
   if (checkpointNumber > 0) {
     digestGenerator.update(page, pageSize);
   }
-  digestGenerator.writeDigest(outDigest.getForUpdate());
+  digestGenerator.writeDigest(reinterpret_cast<char *>(&outDigest));
 }
 
 void BCStateTran::computeDigestOfPagesDescriptor(const DataStore::ResPagesDescriptor *pagesDesc, Digest &outDigest) {
   DigestGenerator digestGenerator;
   digestGenerator.update(reinterpret_cast<const char *>(pagesDesc), pagesDesc->size());
-  digestGenerator.writeDigest(outDigest.getForUpdate());
+  digestGenerator.writeDigest(reinterpret_cast<char *>(&outDigest));
 }
 
 void BCStateTran::computeDigestOfBlockImpl(const uint64_t blockNum,
@@ -3996,7 +3996,7 @@ void BCStateTran::computeDigestOfBlock(const uint64_t blockNum,
                                        const char *block,
                                        const uint32_t blockSize,
                                        Digest *outDigest) {
-  computeDigestOfBlockImpl(blockNum, block, blockSize, outDigest->getForUpdate());
+  computeDigestOfBlockImpl(blockNum, block, blockSize, reinterpret_cast<char *>(outDigest));
 }
 
 BlockDigest BCStateTran::computeDigestOfBlock(const uint64_t blockNum, const char *block, const uint32_t blockSize) {

bftengine/src/bftengine/ClientsManager.cpp

@@ -217,7 +217,9 @@ ClientsManager::ClientsManager(const std::set<NodeIdType>& proxyClients,
   ConcordAssert(maxNumOfReqsPerClient_ > 0);
   reservedPagesPerRequest_ = reservedPagesPerRequest(sizeOfReservedPage(), maxReplySize_);
   reservedPagesPerClient_ = reservedPagesPerClient(sizeOfReservedPage(), maxReplySize_, maxNumOfReqsPerClient_);
-  for (NodeIdType i = 0; i < ReplicaConfig::instance().numReplicas + ReplicaConfig::instance().numRoReplicas; i++) {
+  for (NodeIdType i = 0; i < ReplicaConfig::instance().numReplicas + ReplicaConfig::instance().numRoReplicas +
+                                 ReplicaConfig::instance().numFnReplicas;
+       i++) {
     clientIds_.insert(i);
   }
   clientIds_.insert(proxyClients_.begin(), proxyClients_.end());