Add apollo case for tls exchange with single signature scheme

WildFireFlum · WildFireFlum · commit 03192af1e773 · 2023-04-09T16:31:42.000+03:00
diff --git a/client/bftclient/include/bftclient/config.h b/client/bftclient/include/bftclient/config.h
@@ -70,7 +70,7 @@ struct ClientConfig {
   bool use_unified_certs = false;
   std::optional<std::string> transaction_signing_private_key_file_path = std::nullopt;
   std::optional<concord::secretsmanager::SecretData> secrets_manager_config = std::nullopt;
-  std::optional<std::string> replicas_master_key_folder_path = "./replicas_rsa_keys";
+  std::optional<std::string> replicas_master_key_folder_path = "./replicas_main_keys";
   concord::crypto::SignatureAlgorithm message_sigs_algorithm = concord::crypto::SignatureAlgorithm::EdDSA;
 };
 
diff --git a/tests/apollo/test_skvbc_reconfiguration.py b/tests/apollo/test_skvbc_reconfiguration.py
@@ -487,6 +487,36 @@ async def test_tls_exchange_replicas_replicas_and_replicas_client(self, bft_netw
                 nb_fast_path = await bft_network.get_metric(r, bft_network, "Counters", "totalFastPaths")
                 self.assertGreater(nb_fast_path, fast_paths[r])
 
+    @with_trio
+    @with_bft_network(
+        lambda builddir, replica_id: start_replica_cmd(builddir, replica_id) + ["--key-exchange-on-start", "True"],
+        bft_configs=[BFTConfig(n=4, f=1, c=0)],
+        with_cre=True, publish_master_keys=True, rotate_keys=True)
+    async def test_tls_exchange_single_signature_scheme(self, bft_network):
+        """
+            Ensure that progress can be made after tls exchange when the replicas'
+            main key and the consensus key are identical
+        """
+        bft_network.start_all_replicas()
+        bft_network.start_cre()
+        await bft_network.check_initial_master_key_publication(bft_network.all_replicas())
+        await self.wait_until_cre_gets_replicas_master_keys(bft_network, bft_network.all_replicas())
+        skvbc = kvbc.SimpleKVBCProtocol(bft_network)
+        ops_for_exchange = (CHECKPOINT_SEQUENCES * 3) + 1
+        await bft_network.wait_for_consensus_path(
+            path_type=ConsensusPathType.OPTIMISTIC_FAST,
+            run_ops=lambda: skvbc.send_n_kvs_sequentially(ops_for_exchange, description='Trigger stale key removal'),
+            threshold=ops_for_exchange)
+
+        await self.run_replica_tls_key_exchange_cycle(bft_network, bft_network.all_replicas(), list(bft_network.all_replicas()) + [bft_network.cre_id])
+        bft_network.copy_certs_from_server_to_clients(src=bft_network.config.n - 1)
+        bft_network.restart_clients(False, False)
+        ops_after_exchange = 100
+        await bft_network.wait_for_consensus_path(
+            path_type=ConsensusPathType.OPTIMISTIC_FAST,
+            run_ops=lambda: skvbc.send_n_kvs_sequentially(ops_after_exchange, description='Make progress after tls rotation'),
+            threshold=ops_after_exchange)
+
     @with_trio
     @with_bft_network(start_replica_cmd, selected_configs=lambda n, f, c: n == 7, with_cre=True, publish_master_keys=True)
     async def test_tls_exchange_replicas_replicas_and_replicas_clients_with_st(self, bft_network):
@@ -563,28 +593,31 @@ async def run_replica_tls_key_exchange_cycle(self, bft_network, replicas, affect
         reps_data = {}
         if len(affected_replicas) == 0:
             affected_replicas = bft_network.all_replicas()
-        for rep in affected_replicas:
-            reps_data[rep] = {}
-            for r in replicas:
-                val = []
-                cert_path = os.path.join(bft_network.certdir + "/" + str(rep), str(r))
-                if bft_network.use_unified_certs:
-                    cert_path = os.path.join(cert_path, "node.cert")
-                else:
-                    cert_path = os.path.join(cert_path, "server", "server.cert")
-
-                with open(cert_path) as orig_key:
-                    cert_text = orig_key.readlines()
-                    val += [cert_text]
-                cert_path = os.path.join(bft_network.certdir + "/" + str(rep), str(r))
-                if bft_network.use_unified_certs:
-                    cert_path = os.path.join(cert_path, "node.cert")
-                else:
-                    cert_path = os.path.join(cert_path, "client", "client.cert")
-                with open(cert_path) as orig_key:
-                    cert_text = orig_key.readlines()
-                    val += [cert_text]
-                reps_data[rep][r] = val
+
+        with log.start_action(action_type="read_tls_certificates_before_rotation",
+                              participants_to_update=affected_replicas, participants_with_new_certs=replicas):
+            for rep in affected_replicas:
+                reps_data[rep] = {}
+                for r in replicas:
+                    val = []
+                    cert_path = os.path.join(bft_network.certdir + "/" + str(rep), str(r))
+                    if bft_network.use_unified_certs:
+                        cert_path = os.path.join(cert_path, "node.cert")
+                    else:
+                        cert_path = os.path.join(cert_path, "server", "server.cert")
+
+                    with open(cert_path) as orig_key:
+                        cert_text = orig_key.readlines()
+                        val += [cert_text]
+                    cert_path = os.path.join(bft_network.certdir + "/" + str(rep), str(r))
+                    if bft_network.use_unified_certs:
+                        cert_path = os.path.join(cert_path, "node.cert")
+                    else:
+                        cert_path = os.path.join(cert_path, "client", "client.cert")
+                    with open(cert_path) as orig_key:
+                        cert_text = orig_key.readlines()
+                        val += [cert_text]
+                    reps_data[rep][r] = val
         client = bft_network.random_client()
         op = operator.Operator(bft_network.config, client, bft_network.builddir)
 
@@ -593,7 +626,8 @@ async def run_replica_tls_key_exchange_cycle(self, bft_network, replicas, affect
         except trio.TooSlowError:
             # As we rotate the TLS keys immediately, the call may return with timeout
             pass
-        with trio.fail_after(60):
+        with log.start_action(action_type="wait_for_new_tls_certificates",
+                              participants_to_update=affected_replicas, participants_with_new_certs=replicas) as action, trio.fail_after(60):
             succ = False
             while not succ:
                 await trio.sleep(1)
@@ -610,6 +644,7 @@ async def run_replica_tls_key_exchange_cycle(self, bft_network, replicas, affect
                             diff = difflib.unified_diff(reps_data[rep][r][0], cert_text, fromfile="new", tofile="old", lineterm='')
                             lines = sum(1 for l in diff)
                             if lines > 0:
+                                action.log(message_type=f"Participant id: {rep} got a new tls certificate for participant id: {r}")
                                 continue
                         cert_path = os.path.join(bft_network.certdir + "/" + str(rep), str(r))
                         if bft_network.use_unified_certs:
@@ -621,6 +656,7 @@ async def run_replica_tls_key_exchange_cycle(self, bft_network, replicas, affect
                             diff = difflib.unified_diff(reps_data[rep][r][1], cert_text, fromfile="new", tofile="old", lineterm='')
                             lines = sum(1 for l in diff)
                             if lines > 0:
+                                action.log(message_type=f"Participant id: {rep} got a new tls certificate for participant id: {r}")
                                 continue
                         succ = False
                         break
@@ -665,7 +701,7 @@ async def test_key_exchange_no_single_signature_scheme(self, bft_network):
         selected_configs=lambda n, f, c: n == 7, publish_master_keys=True)
     async def test_single_signature_scheme_to_no_single_signature_scheme(self, bft_network):
         """
-        Ensure that the network can
+        Ensures that the network can make progress after switching signature schemes
         """
         bft_network.start_all_replicas()
         skvbc = kvbc.SimpleKVBCProtocol(bft_network)
@@ -2178,7 +2214,7 @@ async def wait_until_cre_gets_replicas_master_keys(self, bft_network, replicas):
             while succ is False:
                 succ = True
                 for r in replicas:
-                    master_key_path = os.path.join(bft_network.testdir, "replicas_rsa_keys", str(r), "pub_key")
+                    master_key_path = os.path.join(bft_network.testdir, "replicas_main_keys", str(r), "pub_key")
                     if os.path.isfile(master_key_path) is False:
                         succ = False
                         break
diff --git a/tests/apollo/util/bft.py b/tests/apollo/util/bft.py
@@ -12,6 +12,7 @@
 
 # Add the pyclient directory to $PYTHONPATH
 import hashlib
+import pathlib
 from logging import setLogRecordFactory
 import sys
 import signal
@@ -33,6 +34,7 @@
 from typing import Coroutine, Sequence, Callable, Optional, Tuple, Dict, TextIO
 import re
 import trio
+import json
 
 from util.test_base import repeat_test, ApolloTest
 from util.consts import CHECKPOINT_SEQUENCES
@@ -46,7 +48,8 @@
 import inspect
 from enum import Enum
 from util import bft_metrics, eliot_logging as log
-from util.eliot_logging import log_call, logdir_timestamp, log_message
+from util.eliot_logging import log_call, logdir_timestamp, log_message, add_destinations, remove_destination, \
+    format_eliot_message
 from util import skvbc as kvbc
 from util.bft_test_exceptions import AlreadyRunningError, AlreadyStoppedError, KeyExchangeError, CreError
 from bft_config import BFTConfig
@@ -217,8 +220,9 @@ async def wrapper(*args, **kwargs):
                         async with trio.open_nursery() as background_nursery:
                             @repeat_test(num_repeats, break_on_first_failure, break_on_first_success, test_name)
                             async def test_with_bft_network():
-                                with BftTestNetwork.new(config, background_nursery, with_cre=with_cre, use_unified_certs=use_unified_certs) as bft_network:
-                                    bft_network.current_test = test_name
+                                with BftTestNetwork.new(
+                                        config, background_nursery, with_cre=with_cre,
+                                        use_unified_certs=use_unified_certs, case_name=test_name) as bft_network:
                                     seed = test_instance.test_seed
                                     with log.start_task(action_type=f"{async_fn.__name__}",
                                                         bft_config=f'{bft_config}_clients={config.num_clients}',
@@ -256,8 +260,14 @@ async def test_with_bft_network():
 class BftTestNetwork:
     """Encapsulates a BFT network instance for testing purposes"""
 
+    def write_message(self, message):
+        self._eliot_log_file.write(f"{json.dumps(message)}\n")
+
     def __enter__(self):
         """context manager method for 'with' statements"""
+        if not self._eliot_log_file:
+            self._eliot_log_file = (self.current_test_case_path / 'eliot.log').open('a+')
+        add_destinations(self.write_message)
         return self
 
     def __exit__(self, etype, value, tb):
@@ -293,9 +303,14 @@ def __exit__(self, etype, value, tb):
                 self.perf_proc.wait()
 
             self.check_error_logs()
+            remove_destination(self.write_message)
+            if self._eliot_log_file:
+                self._eliot_log_file.close()
+                self._eliot_log_file = None
 
     def __init__(self, is_existing, origdir, config, testdir, certdir, builddir, toolsdir,
-                 procs, replicas, clients, metrics, client_factory, background_nursery, ro_replicas=[]):
+                 procs, replicas, clients, metrics, client_factory, background_nursery, ro_replicas=[],
+                 case_name=""):
         self.is_existing = is_existing
         # An existing deployment might pass some of the folders paths as empty so we skip the next assertion.
         if not is_existing:
@@ -322,9 +337,8 @@ def __init__(self, is_existing, origdir, config, testdir, certdir, builddir, too
         else:
             self.client_factory = partial(self._create_new_client, BFT_CLIENT_TYPE)
         self.open_fds = {}
-        self.current_test = ""
+        self.current_test = case_name
         self.background_nursery = background_nursery
-        self.current_test_case_path = None
         self.test_start_time = None
         self.perf_proc = None
         self.ro_replicas = ro_replicas
@@ -335,13 +349,25 @@ def __init__(self, is_existing, origdir, config, testdir, certdir, builddir, too
         self.cre_fds = None
         self.cre_id = self.config.n + self.config.num_ro_replicas + self.config.num_clients + RESERVED_CLIENTS_QUOTA
         self._logs_dir = Path(self.builddir) / "tests" / "apollo" / "logs" / logdir_timestamp()
+        self._eliot_log_file = None
+        self._suite_name = os.environ.get('TEST_NAME', None)
+        if not self._suite_name:
+            now = datetime.now().strftime("%y-%m-%d_%H:%M:%S")
+            self._suite_name = f"{now}_{self.current_test}"
+
+        if os.environ.get('BLOCKCHAIN_VERSION', default="1").lower() == "4":
+            self._suite_name = self._suite_name + "_v4"
+
+        self.current_test_case_path = self._logs_dir / self._suite_name / self.current_test
+        self.current_test_case_path.mkdir(parents=True, exist_ok=True)
 
         # Setup transaction signing parameters
         self.setup_txn_signing()
         self._generate_operator_keys()
 
     @classmethod
-    def new(cls, config, background_nursery, client_factory=None, with_cre=False, use_unified_certs=False):
+    def new(cls, config, background_nursery, client_factory=None, with_cre=False, use_unified_certs=False,
+            case_name=None):
         builddir = os.path.abspath("../../build")
         toolsdir = os.path.join(builddir, "tools")
         testdir = tempfile.mkdtemp(prefix='testdir_')
@@ -367,7 +393,8 @@ def new(cls, config, background_nursery, client_factory=None, with_cre=False, us
             ro_replicas=[bft_config.Replica(i, "127.0.0.1",
                                             bft_config.bft_msg_port_from_node_id(i),
                                             bft_config.metrics_port_from_node_id(i))
-                         for i in range(config.n, config.n + config.num_ro_replicas)]
+                         for i in range(config.n, config.n + config.num_ro_replicas)],
+            case_name=case_name
         )
         bft_network.with_cre = with_cre
         bft_network.use_unified_certs = use_unified_certs
@@ -427,13 +454,6 @@ def start_cre(self):
             stderr_file = None
 
             if os.environ.get('KEEP_APOLLO_LOGS', "").lower() in ["true", "on"]:
-                test_name = os.environ.get('TEST_NAME')
-
-                if not test_name:
-                    now = datetime.now().strftime("%y-%m-%d_%H:%M:%S")
-                    test_name = f"{now}_{self.current_test}"
-
-                self.current_test_case_path = self._logs_dir / test_name / self.current_test
                 test_log = f"{self.current_test_case_path / 'stdout_cre.log' }"
 
                 Path(self.current_test_case_path).mkdir(parents=True, exist_ok=True)
@@ -590,7 +610,7 @@ def generate_tls_certs(self, num_to_generate, start_index=0, use_unified_certs=F
 
     def copy_certs_from_server_to_clients(self, src):
         src_cert = self.certdir + "/" + str(src)
-        for c in range(self.num_total_replicas() , self.num_total_replicas() + self.config.num_clients + RESERVED_CLIENTS_QUOTA):
+        for c in range(self.num_total_replicas(), self.num_total_replicas() + self.config.num_clients + RESERVED_CLIENTS_QUOTA):
             comp_cert_dir = self.certdir + "/" + str(c)
             shutil.rmtree(comp_cert_dir, ignore_errors=True)
             shutil.copytree(src_cert, comp_cert_dir)
@@ -615,7 +635,7 @@ def _create_reserved_clients(self):
 
     def new_reserved_client(self):
         if len(self.reserved_client_ids_in_use) == RESERVED_CLIENTS_QUOTA:
-            raise  NotImplemented("You must increase RESERVED_CLIENTS_QUOTA, see comment above")
+            raise NotImplemented("You must increase RESERVED_CLIENTS_QUOTA, see comment above")
         start_id = self.num_total_replicas() + self.config.num_clients
         reserved_client_ids = [ id for id in range(start_id, start_id + RESERVED_CLIENTS_QUOTA) ]
         free_reserved_client_ids = set(reserved_client_ids) - set(self.reserved_client_ids_in_use)
@@ -849,13 +869,7 @@ def start_replica(self, replica_id):
         keep_logs = os.environ.get('KEEP_APOLLO_LOGS', "").lower() in ["true", "on"]
 
         if keep_logs:
-            test_name = os.environ.get('TEST_NAME')
-            if os.environ.get('BLOCKCHAIN_VERSION', default="1") == "4" :
-                test_name = test_name + "_v4"
-
-            test_name = test_name if test_name else f"{self.current_test}"
-
-            self.current_test_case_path = self._logs_dir / test_name / self.current_test
+            suite_name = self._suite_name
             replica_test_log_path = self.current_test_case_path / f"stdout_{replica_id}.log"
 
             Path(self.current_test_case_path).mkdir(parents=True, exist_ok=True)
@@ -876,7 +890,7 @@ def start_replica(self, replica_id):
         """
         if os.environ.get('CODECOVERAGE', "").lower() in ["true", "on"] and \
                 os.environ.get('GRACEFUL_SHUTDOWN',"").lower() in ["true", "on"]:
-            self.coverage_dir = f"{self.builddir}/tests/apollo/codecoverage/{test_name}/{self.current_test}/"
+            self.coverage_dir = f"{self.builddir}/tests/apollo/codecoverage/{suite_name}/{self.current_test}/"
             Path(self.coverage_dir).mkdir(parents=True, exist_ok=True)
             profraw_file_name = f"coverage_{replica_id}_%9m.profraw"
             profraw_file_path = os.path.join(
diff --git a/tests/apollo/util/eliot_logging.py b/tests/apollo/util/eliot_logging.py
@@ -1,5 +1,5 @@
 from functools import lru_cache
-from eliot import start_action, start_task, to_file, add_destinations, log_call, log_message
+from eliot import start_action, start_task, to_file, add_destinations, log_call, log_message, remove_destination
 from datetime import datetime
 import os
 import sys
@@ -60,29 +60,35 @@ def set_file_destination():
     to_file(open(test_log, "a+"))
 
 
-# Set logs to the console
-def stdout(message):
+def format_eliot_message(message):
     if message.get("action_status", "") == "succeeded":
         return
 
     additional_fields = [(key, val) for key, val in message.items() if key not in
-                       ("action_type", "action_status", "result", "task_uuid",
-                        "timestamp", "task_level", "message_type")]
+                         ("action_type", "action_status", "result", "task_uuid",
+                          "timestamp", "task_level", "message_type")]
 
     fields = [datetime.fromtimestamp(message["timestamp"]).strftime("%d/%m/%Y %H:%M:%S.%f"),
-     message.get("message_type", None),
-     message.get("action_type", None),
-     message.get("action_status", None),
-     message.get("result", None),
-     str(additional_fields).replace('\\n', '\n'),
-     message["task_uuid"],
-     ]
+              message.get("message_type", None),
+              message.get("action_type", None),
+              message.get("action_status", None),
+              message.get("result", None),
+              str(additional_fields).replace('\\n', '\n'),
+              message["task_uuid"],
+              ]
+
+    return f' - '.join([field for field in fields if field])
 
-    print(f' - '.join([field for field in fields if field]))
+
+# Set logs to the console
+def eliot_stdout(message):
+    formatted = format_eliot_message(message)
+    if formatted:
+        print(formatted)
 
 
 if os.getenv('APOLLO_LOG_STDOUT', False):
-    add_destinations(stdout)
+    add_destinations(eliot_stdout)
 
 if os.environ.get('KEEP_APOLLO_LOGS', "").lower() in ["true", "on"]:
     # Uncomment to see logs in console
diff --git a/tests/simpleKVBC/TesterCRE/main.cpp b/tests/simpleKVBC/TesterCRE/main.cpp
@@ -53,7 +53,7 @@ creParams setupCreParams(int argc, char** argv) {
                                         {"use-unified-certs", optional_argument, 0, 'U'},
                                         {0, 0, 0, 0}};
   creParams cre_param;
-  cre_param.replicasKeysFolder = "./replicas_rsa_keys";
+  cre_param.replicasKeysFolder = "./replicas_main_keys";
   ClientConfig& client_config = cre_param.bftConfig;
   int o = 0;
   int optionIndex = 0;
