feat(multinode): teardown old containers on remote nodes and improve node management

- Fix old containers not being cleaned up on worker nodes during
  redeployment by routing teardown through RemoteNodeDeployer when
  container has a node_id
- Add encryption_service to MarkDeploymentCompleteJob and
  DeploymentService for decrypting node tokens during remote teardown
- Improve node health checks, proxy routing, and CLI node management

Author: dviejokfs

CHANGELOG.md

@@ -48,6 +48,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Docker container names are now used instead of Docker network aliases for cross-node environment variable rewriting, fixing service connectivity on remote worker nodes
 - Deployment "marking complete" step could hang for the full 60-second timeout when the job queue was busy: the DB poll fallback (which confirms the route table update via database query) was only checked when the queue receiver timed out, but a steady stream of unrelated queue events prevented the timeout from ever firing; the poll now runs on every loop iteration regardless of queue activity
 - Remote environment variables are no longer built when no active worker nodes exist, avoiding unnecessary work in single-node deployments
+- **Phantom deployments on node drain/failover**: drain and failover previously called `trigger_pipeline` with no branch/tag/commit, creating broken "preview" deployments with empty git context; now uses smart drain logic that retires containers on the draining node when healthy replicas exist on other nodes, and only triggers a full redeploy (with correct git context from the latest successful deployment) when all replicas are on the affected node
 
 ### Added
 - Automatic `CRON_SECRET` injection into deployed containers: the deployment token is now set as `CRON_SECRET` in the container environment on every deployment, and the cron scheduler sends `Authorization: Bearer <CRON_SECRET>` when invoking endpoints — no manual configuration needed

crates/temps-cli/src/commands/join.rs

@@ -131,7 +131,7 @@ impl JoinCommand {
             "name": node_name,
             "token": agent_token,
             "join_token": self.token,
-            "address": format!("http://{}:{}", private_address, self.agent_address.split(':').next_back().unwrap_or("3100")),
+            "address": format!("http://{}:{}", private_address.trim(), self.agent_address.split(':').next_back().unwrap_or("3100").trim()),
             "private_address": private_address,
             "labels": labels,
         });
@@ -266,7 +266,12 @@ impl JoinCommand {
             relay_response.control_plane_url
         );
 
-        let agent_port = self.agent_address.split(':').next_back().unwrap_or("3100");
+        let agent_port = self
+            .agent_address
+            .split(':')
+            .next_back()
+            .unwrap_or("3100")
+            .trim();
 
         let register_body = serde_json::json!({
             "name": node_name,

crates/temps-cli/src/commands/node.rs

@@ -23,6 +23,8 @@ pub enum NodeSubcommand {
     Show(NodeShowCommand),
     /// Drain a node: stop scheduling new containers and redeploy existing ones
     Drain(NodeDrainCommand),
+    /// Undrain a node: reactivate it so it can accept new deployments again
+    Undrain(NodeUndrainCommand),
     /// Remove a node from the cluster (must be drained first)
     #[command(alias = "rm")]
     Remove(NodeRemoveCommand),
@@ -68,6 +70,18 @@ pub struct NodeDrainCommand {
     pub timeout: u64,
 }
 
+#[derive(Args)]
+pub struct NodeUndrainCommand {
+    /// Node ID to undrain
+    pub node_id: i32,
+    /// API base URL
+    #[arg(long, env = "TEMPS_API_URL")]
+    pub api_url: String,
+    /// API authentication token
+    #[arg(long, env = "TEMPS_API_TOKEN")]
+    pub api_token: String,
+}
+
 #[derive(Args)]
 pub struct NodeRemoveCommand {
     /// Node ID to remove
@@ -203,6 +217,7 @@ impl NodeCommand {
                 NodeSubcommand::List(cmd) => execute_list(cmd).await,
                 NodeSubcommand::Show(cmd) => execute_show(cmd).await,
                 NodeSubcommand::Drain(cmd) => execute_drain(cmd).await,
+                NodeSubcommand::Undrain(cmd) => execute_undrain(cmd).await,
                 NodeSubcommand::Remove(cmd) => execute_remove(cmd).await,
             }
         })
@@ -458,6 +473,47 @@ async fn execute_drain(cmd: NodeDrainCommand) -> anyhow::Result<()> {
     Ok(())
 }
 
+#[derive(Debug, Deserialize)]
+struct UndrainNodeResponse {
+    name: String,
+    status: String,
+    message: String,
+}
+
+async fn execute_undrain(cmd: NodeUndrainCommand) -> anyhow::Result<()> {
+    let client = make_client();
+    let url = api_url(&cmd.api_url, &format!("/nodes/{}/drain", cmd.node_id));
+
+    println!(
+        "  {} Undraining node {}...",
+        "⏳".bright_yellow(),
+        cmd.node_id
+    );
+
+    let response = client
+        .delete(&url)
+        .header("Authorization", format!("Bearer {}", cmd.api_token))
+        .send()
+        .await
+        .map_err(|e| anyhow::anyhow!("Failed to connect to API: {}", e))?;
+
+    if !response.status().is_success() {
+        return Err(handle_api_error(response).await);
+    }
+
+    let data: UndrainNodeResponse = response.json().await?;
+
+    println!(
+        "  {} Node '{}' is now {}",
+        "✓".bright_green(),
+        data.name.bright_cyan(),
+        data.status.bright_green()
+    );
+    println!("  {}", data.message);
+
+    Ok(())
+}
+
 async fn execute_remove(cmd: NodeRemoveCommand) -> anyhow::Result<()> {
     if !cmd.yes {
         // Show node info first

crates/temps-deployments/src/handlers/deployments.rs

@@ -2581,22 +2581,23 @@ mod tests {
                 "temps-test".to_string(),
             ));
 
+        let encryption_service = Arc::new(
+            temps_core::EncryptionService::new(
+                "0000000000000000000000000000000000000000000000000000000000000000",
+            )
+            .expect("Failed to create test encryption service"),
+        );
+
         let deployment_service = Arc::new(crate::services::services::DeploymentService::new(
             db.clone(),
             log_service.clone(),
             config_service.clone(),
             queue_service.clone(),
             docker_log_service,
             deployer,
+            encryption_service.clone(),
         ));
 
-        let encryption_service = Arc::new(
-            temps_core::EncryptionService::new(
-                "0000000000000000000000000000000000000000000000000000000000000000",
-            )
-            .expect("Failed to create test encryption service"),
-        );
-
         let deployment_token_service = Arc::new(
             crate::services::deployment_token_service::DeploymentTokenService::new(
                 db.clone(),
@@ -2701,6 +2702,9 @@ mod tests {
             image_builder: Arc::new(MockImageBuilder) as Arc<dyn temps_deployer::ImageBuilder>,
             audit_service: Arc::new(MockAuditLogger) as Arc<dyn temps_core::AuditLogger>,
             node_service: Arc::new(crate::services::NodeService::new(db.clone())),
+            encryption_service: Arc::new(
+                temps_core::EncryptionService::new("01234567890123456789012345678901").unwrap(),
+            ),
         })
     }