Fix multi-node kill bug due to memberlist not resolving hostnames; add admin API for inspecting node discovery state

Signed-off-by: tinswzy <zhenyuan.wei@zilliz.com>

Author: tinswzy

cmd/main.go

@@ -20,8 +20,6 @@ import (
 	"context"
 	"flag"
 	"fmt"
-	"github.com/zilliztech/woodpecker/common/logger"
-	"github.com/zilliztech/woodpecker/common/tracer"
 	"log"
 	"net"
 	"os"
@@ -32,7 +30,9 @@ import (
 	"github.com/zilliztech/woodpecker/cmd/external"
 	"github.com/zilliztech/woodpecker/common/config"
 	commonhttp "github.com/zilliztech/woodpecker/common/http"
+	"github.com/zilliztech/woodpecker/common/logger"
 	"github.com/zilliztech/woodpecker/common/membership"
+	"github.com/zilliztech/woodpecker/common/tracer"
 	"github.com/zilliztech/woodpecker/server"
 )
 
@@ -54,43 +54,6 @@ func parseAdvertiseAddr(addrPort string) (string, int, error) {
 
 	return host, port, nil
 }
-
-// resolveAdvertiseAddr resolves hostname to IP address if needed
-func resolveAdvertiseAddr(addr string) string {
-	if addr == "" {
-		return ""
-	}
-
-	// Check if it's already an IP address
-	if ip := net.ParseIP(addr); ip != nil {
-		return addr
-	}
-
-	// Try to resolve hostname to IP
-	ips, err := net.LookupIP(addr)
-	if err != nil {
-		log.Printf("Warning: Failed to resolve hostname '%s' to IP: %v. Using as-is.", addr, err)
-		return addr
-	}
-
-	// Prefer IPv4 address
-	for _, ip := range ips {
-		if ipv4 := ip.To4(); ipv4 != nil {
-			log.Printf("Resolved hostname '%s' to IPv4: %s", addr, ipv4.String())
-			return ipv4.String()
-		}
-	}
-
-	// Fallback to first IP (could be IPv6)
-	if len(ips) > 0 {
-		log.Printf("Resolved hostname '%s' to IP: %s", addr, ips[0].String())
-		return ips[0].String()
-	}
-
-	log.Printf("Warning: No IP found for hostname '%s'. Using as-is.", addr)
-	return addr
-}
-
 func main() {
 	var (
 		servicePort               = flag.Int("service-port", 18080, "service port")
@@ -180,7 +143,6 @@ func main() {
 	} else {
 		advertisePort = *gossipPort
 	}
-	resourceAdvertiseGossipAddrStr := resolveAdvertiseAddr(advertiseAddrStr)
 
 	if *advertiseServiceAddr != "" {
 		addr, port, err := parseAdvertiseAddr(*advertiseServiceAddr)
@@ -198,10 +160,10 @@ func main() {
 		NodeID:               *nodeName,
 		BindPort:             *gossipPort,
 		ServicePort:          *servicePort,
-		AdvertiseAddr:        resourceAdvertiseGossipAddrStr, // Gossip advertise address (IP only)
-		AdvertisePort:        advertisePort,                  // Gossip advertise port
-		AdvertiseServiceAddr: advertiseServiceAddrStr,        // Service advertise address (hostname only)
-		AdvertiseServicePort: advertiseServicePort,           // Service advertise port
+		AdvertiseAddr:        advertiseAddrStr,        // Gossip advertise address (IP only)
+		AdvertisePort:        advertisePort,           // Gossip advertise port
+		AdvertiseServiceAddr: advertiseServiceAddrStr, // Service advertise address (hostname only)
+		AdvertiseServicePort: advertiseServicePort,    // Service advertise port
 		ResourceGroup:        *resourceGroup,
 		AZ:                   *availabilityZone,
 		Tags:                 map[string]string{"role": "logstore"},
@@ -217,10 +179,10 @@ func main() {
 	}
 
 	// Start HTTP server for metrics, health check, and pprof
-	if err := commonhttp.Start(cfg); err != nil {
+	if err := commonhttp.Start(cfg, srv.GetServerNodeMemberlistStatus); err != nil {
 		log.Fatalf("Failed to start HTTP server: %v", err)
 	}
-	log.Printf("HTTP server started on port %s (metrics, health, pprof)", commonhttp.DefaultListenPort)
+	log.Printf("HTTP server started on port %s (metrics, health, pprof, admin)", commonhttp.DefaultListenPort)
 
 	log.Printf("Starting Woodpecker Server:")
 	log.Printf("  Node Name: %s", *nodeName)

common/http/router.go

@@ -25,6 +25,9 @@ const MetricsRouterPath = "/metrics"
 // LogLevelRouterPath is path for Get and Update log level at runtime.
 const LogLevelRouterPath = "/log/level"
 
+// AdminMemberlistPath is path for memberlist status.
+const AdminMemberlistPath = "/admin/memberlist"
+
 // Pprof paths are automatically registered by importing net/http/pprof
 // Available paths:
 // - /debug/pprof/

common/http/server.go

@@ -102,10 +102,18 @@ func registerDefaults(cfg *config.Configuration) {
 }
 
 // Start initializes and starts the HTTP server
-func Start(cfg *config.Configuration) error {
+func Start(cfg *config.Configuration, GetServerNodeMemberlistStatus func() string) error {
 	// Register default handlers
 	registerDefaults(cfg)
 
+	// Register admin handler for memberlist status
+	Register(&Handler{
+		Path: AdminMemberlistPath,
+		HandlerFunc: func(writer http.ResponseWriter, request *http.Request) {
+			fmt.Fprintf(writer, GetServerNodeMemberlistStatus())
+		},
+	})
+
 	// Get listen port from environment or use default
 	port := os.Getenv(ListenPortEnvKey)
 	if port == "" {

common/membership/server_node.go

@@ -189,3 +189,22 @@ func (n *ServerNode) GetServerCfg() *ServerConfig {
 func (n *ServerNode) GetMeta() *proto.NodeMeta {
 	return n.meta
 }
+
+// GetMemberlistStatus returns a formatted string of all memberlist members and their addresses
+func (n *ServerNode) GetMemberlistStatus() string {
+	members := n.memberlist.Members()
+	if len(members) == 0 {
+		return "No members in memberlist"
+	}
+
+	result := fmt.Sprintf("Total Members: %d\n\n", len(members))
+	for i, member := range members {
+		result += fmt.Sprintf("[%d] Name: %s\n", i+1, member.Name)
+		result += fmt.Sprintf("    Addr: %s:%d\n", member.Addr.String(), member.Port)
+		result += fmt.Sprintf("    State: %d\n", member.State)
+		if i < len(members)-1 {
+			result += "\n"
+		}
+	}
+	return result
+}

common/net/addr.go

@@ -77,3 +77,35 @@ func GetValidLocalIP(addrs []net.Addr) string {
 	}
 	return ""
 }
+
+// ResolveAdvertiseAddr resolves hostname to IP address if needed
+func ResolveAdvertiseAddr(addr string) net.IP {
+	if addr == "" {
+		return nil
+	}
+
+	// Check if it's already an IP address
+	if ip := net.ParseIP(addr); ip != nil {
+		return ip
+	}
+
+	// Try to resolve hostname to IP
+	ips, err := net.LookupIP(addr)
+	if err != nil {
+		return nil
+	}
+
+	// Prefer IPv4 address
+	for _, ip := range ips {
+		if ipv4 := ip.To4(); ipv4 != nil {
+			return ipv4
+		}
+	}
+
+	// Fallback to first IP (could be IPv6)
+	if len(ips) > 0 {
+		return ips[0]
+	}
+
+	return nil
+}

go.mod

@@ -13,6 +13,7 @@ require (
 	github.com/google/gops v0.3.28
 	github.com/google/uuid v1.6.0
 	github.com/grafana/pyroscope-go/godeltaprof v0.1.8
+	github.com/hashicorp/golang-lru/v2 v2.0.7
 	github.com/hashicorp/memberlist v0.5.3
 	github.com/json-iterator/go v1.1.12
 	github.com/klauspost/compress v1.17.9
@@ -29,6 +30,7 @@ require (
 	github.com/uber/jaeger-client-go v2.30.0+incompatible
 	go.etcd.io/etcd/client/v3 v3.5.5
 	go.etcd.io/etcd/server/v3 v3.5.5
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0
 	go.opentelemetry.io/otel v1.28.0
 	go.opentelemetry.io/otel/exporters/jaeger v1.13.0
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0
@@ -45,6 +47,8 @@ require (
 	gopkg.in/yaml.v3 v3.0.1
 )
 
+replace github.com/hashicorp/memberlist => github.com/tinswzy/memberlist v0.5.3-hotfix-20251119
+
 require (
 	cloud.google.com/go/compute/metadata v0.3.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.8.0 // indirect
@@ -81,12 +85,10 @@ require (
 	github.com/hashicorp/errwrap v1.0.0 // indirect
 	github.com/hashicorp/go-immutable-radix v1.0.0 // indirect
 	github.com/hashicorp/go-metrics v0.5.4 // indirect
-	github.com/hashicorp/go-msgpack v0.5.3 // indirect
 	github.com/hashicorp/go-msgpack/v2 v2.1.1 // indirect
 	github.com/hashicorp/go-multierror v1.0.0 // indirect
 	github.com/hashicorp/go-sockaddr v1.0.0 // indirect
 	github.com/hashicorp/golang-lru v0.5.1 // indirect
-	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
 	github.com/jonboulle/clockwork v0.2.2 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.8 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
@@ -129,7 +131,6 @@ require (
 	go.etcd.io/etcd/client/v2 v2.305.5 // indirect
 	go.etcd.io/etcd/pkg/v3 v3.5.5 // indirect
 	go.etcd.io/etcd/raft/v3 v3.5.5 // indirect
-	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.20.0 // indirect
 	go.opentelemetry.io/otel/metric v1.28.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.0.0 // indirect