Merge pull request #6559 from IntersectMBO/testnet-specify-node-bin-per-node

cardano-testnet: Add `--nodes` flag for per-node binary configuration

Author: palas

cardano-node-chairman/test/Spec/Chairman/Cardano.hs

@@ -19,7 +19,7 @@ hprop_chairman :: H.Property
 hprop_chairman = integrationRetryWorkspace 2 "cardano-chairman" $ \tempAbsPath' -> H.runWithDefaultWatchdog_ $ do
   conf <- mkConf tempAbsPath'
 
-  let creationOptions = def{ creationNodes = cardanoDefaultTestnetNodeOptions }
+  let creationOptions = def{ creationNodes = cardanoDefaultTestnetNodesWithOptions }
   allNodes <- testnetNodes <$> createAndRunTestnet creationOptions def conf
 
   chairmanOver 120 50 conf allNodes

cardano-node/src/Cardano/Node/Testnet/Paths.hs

@@ -15,6 +15,7 @@ module Cardano.Node.Testnet.Paths
   , defaultSocketPath
   , defaultConfigFile
   , defaultPortFile
+  , defaultNodeEnvFile
   ) where
 
 import           System.FilePath ((</>))
@@ -62,3 +63,7 @@ defaultConfigFile = "configuration.yaml"
 -- | Relative path to a node's port file: @defaultNodeDataDir n </> "port"@
 defaultPortFile :: Int -> FilePath
 defaultPortFile n = defaultNodeDataDir n </> "port"
+
+-- | Relative path to a node's env file: @defaultNodeDataDir n </> "env"@
+defaultNodeEnvFile :: Int -> FilePath
+defaultNodeEnvFile n = defaultNodeDataDir n </> "env"

cardano-testnet/cardano-testnet.cabal

@@ -84,6 +84,7 @@ library
                       , network
                       , network-mux
                       , optparse-applicative-fork
+                      , parsec
                       , ouroboros-network:{api, framework, ouroboros-network} ^>= 1.1
                       , cardano-diffusion:{api, cardano-diffusion} ^>= 1.0
                       , prettyprinter
@@ -135,7 +136,8 @@ library
                         Testnet.TestQueryCmds
                         Testnet.Types
 
-  other-modules:        Parsers.Cardano
+  exposed-modules:      Parsers.Cardano
+  other-modules:
                         Parsers.Help
                         Parsers.Version
                         Testnet.TestEnumGenerator
@@ -240,6 +242,7 @@ test-suite cardano-testnet-test
                         Cardano.Testnet.Test.Rpc.Transaction
                         Cardano.Testnet.Test.Misc
                         Cardano.Testnet.Test.Node.Shutdown
+                        Cardano.Testnet.Test.Parser
                         Cardano.Testnet.Test.MainnetParams
                         Cardano.Testnet.Test.SanityCheck
                         Cardano.Testnet.Test.RunTestnet
@@ -287,6 +290,7 @@ test-suite cardano-testnet-test
                       , regex-compat
                       , rio
                       , tasty ^>= 1.5
+                      , tasty-hedgehog
                       , text
                       , time
                       , transformers

cardano-testnet/changelog.d/20260506_035740_palas_testnet_specify_node_bin_per_node.md

@@ -0,0 +1,10 @@
+### Added
+
+- Added `--nodes` flag to specify node roles (SPO/relay) and custom `cardano-node` binaries per node.
+  Example: `--nodes spo,spo:node-bin=/path/to/bin,relay,relay`.
+
+### Changed
+
+- Renamed `NodeOptions` to `NodeWithOptions` and `TestnetNodeOptions` to `TestnetNodesWithOptions`
+  (exported from `Testnet.Start.Types` and `Cardano.Testnet`). The new types include a `nodeBin` field for
+  specifying a per-node `cardano-node` binary.

cardano-testnet/src/Cardano/Testnet.hs

@@ -14,9 +14,9 @@ module Cardano.Testnet (
   TestnetRuntimeOptions(..),
   TestnetEnvOptions(..),
   RpcSupport(..),
-  TestnetNodeOptions(..),
-  NodeOptions(..),
-  cardanoDefaultTestnetNodeOptions,
+  TestnetNodesWithOptions(..),
+  NodeWithOptions(..),
+  cardanoDefaultTestnetNodesWithOptions,
   getDefaultAlonzoGenesis,
   getDefaultShelleyGenesis,
 

cardano-testnet/src/Parsers/Cardano.hs

@@ -3,6 +3,7 @@
 module Parsers.Cardano
   ( cmdCardano
   , cmdCreateEnv
+  , parseNodeSpecs
   ) where
 
 import           Cardano.Api (AnyShelleyBasedEra (..))
@@ -13,6 +14,7 @@ import           Cardano.Prelude (readMaybe)
 import           Prelude
 
 import           Control.Applicative (optional, (<|>))
+import           Control.Monad (unless)
 import           Data.Default.Class (def)
 import qualified Data.List as L
 import           Data.List.NonEmpty (NonEmpty ((:|)))
@@ -21,6 +23,10 @@ import           Data.Word (Word64)
 import           Options.Applicative (CommandFields, Mod, Parser)
 import qualified Options.Applicative as OA
 import           Options.Applicative.Types (readerAsk)
+import           Text.Parsec (char, many1, noneOf,
+                     sepBy1, string, try, (<?>), parse, eof, notFollowedBy)
+import qualified Text.Parsec as Parsec
+import qualified Text.Parsec.String as Parsec
 
 import           Testnet.Defaults (defaultEra)
 import           Testnet.Start.Cardano
@@ -55,7 +61,7 @@ pFromEnv = TestnetEnvOptions
 
 pCreationOptions :: Parser TestnetCreationOptions
 pCreationOptions = TestnetCreationOptions
-  <$> pTestnetNodeOptions
+  <$> pTestnetNodesWithOptions
   <*> pure (AnyShelleyBasedEra defaultEra)
   <*> pMaxLovelaceSupply
   <*> pNumDReps
@@ -105,28 +111,99 @@ pKesSource = OA.flag UseKesKeyFile UseKesSocket
   <>  OA.showDefault
   )
 
-pTestnetNodeOptions :: Parser TestnetNodeOptions
-pTestnetNodeOptions =
-  fmap (maybe cardanoDefaultTestnetNodeOptions mkPoolNodes) <$>
-    optional $ OA.option ensureAtLeastOne
-      (   OA.long "num-pool-nodes"
-      <>  OA.help "Number of pool nodes. Note this uses a default node configuration for all nodes."
-      <>  OA.metavar "COUNT"
-      )
+pTestnetNodesWithOptions :: Parser TestnetNodesWithOptions
+pTestnetNodesWithOptions =
+  pNodes <|> pNumPoolNodes <|> pure cardanoDefaultTestnetNodesWithOptions
   where
-    defaultSpoOption = NodeOptions []
-
-    mkPoolNodes num = TestnetNodeOptions
-      { optSpoNodes = defaultSpoOption :| L.replicate (num - 1) defaultSpoOption
-      , optRelayNodes = []
-      }
+    pNumPoolNodes :: Parser TestnetNodesWithOptions
+    pNumPoolNodes =
+      (\num -> TestnetNodesWithOptions { optSpoNodes = defaultSpoOption :| L.replicate (num - 1) defaultSpoOption, optRelayNodes = [] }) <$>
+        OA.option ensureAtLeastOne
+          (   OA.long "num-pool-nodes"
+          <>  OA.help "Number of pool nodes. Note this uses a default node configuration for all nodes."
+          <>  OA.metavar "COUNT"
+          )
+    defaultSpoOption = NodeWithOptions Nothing []
 
     ensureAtLeastOne :: OA.ReadM Int
     ensureAtLeastOne = readerAsk >>= \arg ->
       case readMaybe arg of
         Just n | n >= 1 -> pure n
         _ -> fail "Need at least one SPO node to produce blocks, but got none."
 
+    pNodes :: Parser TestnetNodesWithOptions
+    pNodes = OA.option readNodeSpecs
+      (   OA.long "nodes"
+      <>  OA.help "Comma-separated node specifications. SPO nodes must come before relay nodes. \
+                   \Each spec is a role (spo or relay) optionally followed by :node-bin=<path>. \
+                   \If the path contains commas, colons, double quotes, or backslashes, wrap it \
+                   \in double quotes and escape any literal double quotes as \\\" and backslashes \
+                   \as \\\\ within. To prevent bash from consuming the double quotes, enclose the \
+                   \whole argument in single quotes. \
+                   \Examples: --nodes spo,spo:node-bin=/path/to/bin,relay,relay | \
+                   \--nodes 'spo:node-bin=\"/path,with:commas\",relay'"
+      <>  OA.metavar "SPEC[,SPEC...]"
+      )
+
+    readNodeSpecs :: OA.ReadM TestnetNodesWithOptions
+    readNodeSpecs = readerAsk >>= either (fail . show) pure . parseNodeSpecs
+
+-- | Parse a @--nodes@ argument string into 'TestnetNodesWithOptions'.
+--
+-- SPO nodes are required to appear before relay nodes because:
+--
+-- 1. The testnet configuration assigns node directories by position (node-spo1,
+--    node-spo2, …, relay1, relay2, …). Allowing arbitrary ordering would require
+--    maintaining a separate mapping between pool indices and node positions, which
+--    much of the existing code does not expect.
+--
+-- 2. Silently reordering (e.g. turning @relay,spo@ into @spo,relay@) would
+--    violate the user's expectations about which node gets which configuration.
+--
+-- 3. Requiring SPOs first lets us represent the result directly as a 'NonEmpty'
+--    list of SPO nodes (guaranteeing at least one) plus a plain list of relays,
+--    so the type itself makes an invalid configuration unrepresentable.
+parseNodeSpecs :: String -> Either Parsec.ParseError TestnetNodesWithOptions
+parseNodeSpecs = parse (nodeSpecsParser <* eof) "Error parsing node specifications"
+  where
+    nodeSpecsParser :: Parsec.Parser TestnetNodesWithOptions
+    nodeSpecsParser = do
+      specs <- nodeSpec `sepBy1` char ','
+      let (spos, relays) = span (\(role, _) -> role == Spo) specs
+      unless (all (\(role, _) -> role == Relay) relays) $
+        fail "SPO nodes must come before relay nodes. Example: --nodes spo,spo,relay,relay"
+      case map snd spos of
+        [] -> fail "Need at least one SPO node to produce blocks."
+        (s:ss) -> pure $ TestnetNodesWithOptions
+          { optSpoNodes = s :| ss
+          , optRelayNodes = map snd relays
+          }
+
+    nodeSpec :: Parsec.Parser (NodeRole, NodeWithOptions)
+    nodeSpec = do
+      role <- nodeRole
+      bin <- optional $ char ':' *> nodeBinKV
+      pure (role, NodeWithOptions bin [])
+
+    nodeRole :: Parsec.Parser NodeRole
+    nodeRole =
+          Spo <$ try (string "spo" <* notFollowedBy (noneOf ",:\"\\"))
+      <|> Relay <$ try (string "relay" <* notFollowedBy (noneOf ",:\"\\"))
+      <?> "node role (\"spo\" or \"relay\")"
+
+    nodeBinKV :: Parsec.Parser FilePath
+    nodeBinKV = string "node-bin=" *> (quotedPath <|> unquotedPath) <?> "\"node-bin=<path>\", where <path> is the path to the node binary, optionally quoted if it contains special characters"
+
+    quotedPath :: Parsec.Parser FilePath
+    quotedPath = char '"' *> Parsec.many quotedChar <* char '"'
+      where
+        quotedChar = try (char '\\' *> (char '"' <|> char '\\')) <|> noneOf "\""
+
+    unquotedPath :: Parsec.Parser FilePath
+    unquotedPath = many1 (noneOf ",:\"\\")
+
+data NodeRole = Spo | Relay deriving Eq
+
 pOnChainParams :: Parser TestnetOnChainParams
 pOnChainParams = fmap (fromMaybe DefaultParams) <$> optional $
   pCustomParamsFile <|> pMainnetParams

cardano-testnet/src/Parsers/Run.hs

@@ -91,7 +91,7 @@ runCardanoOptions = \case
     let dirName = envPath fromEnvOptions
     unlessM (doesDirectoryExist dirName) $ error $ "The provided path does not exist or is not a directory: " <> dirName
     conf <- mkConfigAbs dirName
-    nodes <- readNodeOptionsFromEnv (unTmpAbsPath (tempAbsPath conf))
+    nodes <- readNodesWithOptionsFromEnv (unTmpAbsPath (tempAbsPath conf))
     runSimpleApp . runResourceT $ do
       logInfo $ "Starting testnet in environment: " <> display (tempAbsPath conf)
       void $ cardanoTestnet nodes fromEnvRuntimeOptions

cardano-testnet/src/Testnet/Process/RunIO.hs

@@ -12,6 +12,7 @@ module Testnet.Process.RunIO
   , procNode
   , procKesAgent
   , execKesAgentControl_
+  , procCustom
   , procFlex
   , liftIOAnnotated
   ) where
@@ -144,8 +145,7 @@ execFlexAny' execConfig pkgBin envBin arguments = GHC.withFrozenCallStack $ do
   cp <- procFlex' execConfig pkgBin envBin arguments
   liftIOAnnotated $ IO.readCreateProcessWithExitCode cp ""
 
-
-
+-- | Like 'procFlex', but takes an explicit 'ExecConfig' instead of using 'defaultExecConfig'.
 procFlex'
   :: HasCallStack
   => MonadIO m
@@ -160,7 +160,20 @@ procFlex'
   -- ^ Captured stdout
 procFlex' execConfig pkg binaryEnv arguments = GHC.withFrozenCallStack $ do
   bin <- binFlex pkg binaryEnv
-  return (IO.proc bin arguments)
+  procCustom' execConfig bin arguments
+
+-- | Build a 'CreateProcess' from an already-resolved binary path, arguments, and 'ExecConfig'.
+procCustom'
+  :: (HasCallStack)
+  => MonadIO m
+  => ExecConfig
+  -> FilePath
+  -- ^ Path to the binary
+  -> [String]
+  -- ^ Arguments to the CLI command
+  -> m CreateProcess
+procCustom' execConfig bin arguments = GHC.withFrozenCallStack $
+  pure (IO.proc bin arguments)
     { IO.env = getLast $ execConfigEnv execConfig
     , IO.cwd = getLast $ execConfigCwd execConfig
     -- this allows sending signals to the created processes, without killing the test-suite process
@@ -311,6 +324,17 @@ procFlex
   -- ^ Captured stdout
 procFlex = procFlex' defaultExecConfig
 
+-- | Like 'procFlex', but takes an explicit binary path instead of resolving
+-- via package name and environment variable.
+procCustom
+  :: (HasCallStack)
+  => FilePath
+  -- ^ Path to the binary
+  -> [String]
+  -- ^ Arguments to the CLI command
+  -> RIO env CreateProcess
+procCustom = procCustom' defaultExecConfig
+
 -- This will also catch async exceptions as well.
 liftIOAnnotated :: (HasCallStack, MonadIO m) => IO a -> m a
 liftIOAnnotated action = GHC.withFrozenCallStack $

cardano-testnet/src/Testnet/Runtime.hs

@@ -57,7 +57,7 @@ import           Cardano.Node.Testnet.Paths (defaultSocketName)
 import qualified Testnet.Ping as Ping
 import           Testnet.Process.Run (ProcessError (..), initiateProcess)
 import           Testnet.Process.RunIO (execCli_, execKesAgentControl_, liftIOAnnotated,
-                   procKesAgent, procNode)
+                   procCustom, procKesAgent, procNode)
 import           Testnet.Types (TestnetKesAgent (..), TestnetNode (..),
                    TestnetRuntime (configurationFile), showIpv4Address, testnetSprockets)
 
@@ -121,11 +121,13 @@ startNode
   -- ^ Node port
   -> Int
   -- ^ Testnet magic
+  -> Maybe FilePath
+  -- ^ Optional custom node binary. 'Nothing' uses the default resolution.
   -> [String]
   -- ^ The command to execute to start the node.
   -- @--socket-path@, @--port@, and @--host-addr@ gets added automatically.
   -> ExceptT NodeStartFailure m TestnetNode
-startNode tp node ipv4 port _testnetMagic nodeCmd = GHC.withFrozenCallStack $ do
+startNode tp node ipv4 port _testnetMagic mNodeBin nodeCmd = GHC.withFrozenCallStack $ do
   let tempBaseAbsPath = makeTmpBaseAbsPath tp
       socketDir = makeSocketDir tp
       logDir = makeLogDir tp
@@ -156,7 +158,10 @@ startNode tp node ipv4 port _testnetMagic nodeCmd = GHC.withFrozenCallStack $ do
                              , "--port", show port
                              , "--host-addr", showIpv4Address ipv4
                              ]
-    nodeProcess <- newExceptT . fmap (first ExecutableRelatedFailure) . try $ runRIO () $ procNode completeNodeCmd
+    nodeProcess <- newExceptT . fmap (first ExecutableRelatedFailure) . try $ runRIO () $
+      case mNodeBin of
+        Nothing -> procNode completeNodeCmd
+        Just bin -> procCustom bin completeNodeCmd
 
     -- The port number if it is obtained using 'H.randomPort', it is firstly bound to and then closed. The closing
     -- and release in the operating system is done asynchronously and can be slow. Here we wait until the port