Creates exemplary/representative pod fuzzer tool

Author: seans3

fuzzer/README.md

@@ -0,0 +1,93 @@
+# Exemplary Pod Fuzzer
+
+The Exemplary Pod Fuzzer is a high-fidelity synthetic pod generator designed to stress-test the Kubernetes control plane. It uses a **"Sanitize & Clone"** model: it takes a real production pod manifest as a "base", scrubs all PII/sensitive data, and preserves the exact structural complexity (volume mounts, env vars, managed fields) for benchmarking.
+
+## Key Features
+
+- **Direct Sanitization**: Automatically scrubs Names, Namespaces, UIDs, and OwnerReferences from a real pod.
+- **Spec Randomization**: Randomizes all environment variable keys and values while maintaining the original count and structure.
+- **ManagedFields Cloning**: Clones the exact history and field ownership of the base pod, but randomizes the internal JSON paths (supporting both `f:` and `k:` prefixes).
+- **Safety by Default**: Forces pods to be unschedulable (Pending state) using non-existent nodeSelectors and schedulerNames.
+- **Performance Optimized**: Uses high-performance `client-go` settings (500 QPS / 1000 Burst) and precomputes "Fuzzed Prototypes" to ensure string interning memory optimizations are correctly triggered.
+
+## Benchmarking with Kind
+
+To perform a realistic 50,000 pod stress test on a local `kind` cluster, follow these steps:
+
+### 1. Create a Large-Scale Kind Cluster
+Standard `kind` clusters have a default `etcd` quota that is too small for 50k heavy pods. Use this configuration to increase the quota to 8GB:
+
+```bash
+cat <<EOF > kind-config.yaml
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+- role: control-plane
+  kubeadmConfigPatches:
+  - |
+    kind: ClusterConfiguration
+    etcd:
+      local:
+        extraArgs:
+          quota-backend-bytes: "8589934592"
+EOF
+
+kind create cluster --config kind-config.yaml --name fuzzer-test
+```
+
+### 2. Prepare the Namespace and Dependencies
+Real production pods (like the templates in this repo) often have dependencies like specific **Namespaces** or **ServiceAccounts**. The fuzzer preserves these references. You must create them before injecting pods:
+
+```bash
+# Example: Using the complex-daemonset.yaml base
+kubectl create namespace fuzz-test
+kubectl create serviceaccount cilium -n fuzz-test
+```
+
+### 3. Run the Fuzzer
+Inject the pods using high concurrency.
+
+```bash
+go run test/utils/fuzzer/cmd/main.go \
+  --base-pod test/utils/fuzzer/templates/complex-daemonset.yaml \
+  --namespace fuzz-test \
+  --name-prefix representative-pod \
+  --count 50000 \
+  --concurrency 100
+```
+
+## Usage (General)
+
+### Generate Pod Manifests to Disk
+To generate fuzzed manifests for manual inspection:
+```bash
+go run test/utils/fuzzer/cmd/main.go \
+  --base-pod path/to/real-pod.yaml \
+  --name-prefix representative-pod \
+  --namespace my-test-ns \
+  --count 1000 \
+  --out-dir ./generated-pods
+```
+
+## Flags
+
+- `--base-pod`: Path to the real `v1.Pod` YAML manifest used as a structural source.
+- `--name-prefix`: Prefix for the generated pod names (default: `fuzzed-pod`).
+- `--namespace`: Target namespace for the generated pods (default: `fuzz-test`).
+- `--count`: Number of pods to generate.
+- `--offset`: Starting index for naming (useful for incremental runs).
+- `--concurrency`: Number of concurrent workers (default: 50).
+- `--kubeconfig`: Path to the kubeconfig file. Defaults to `$HOME/.kube/config`.
+- `--out-dir`: If specified, write YAMLs to this directory instead of injecting into a cluster.
+
+## Verification & Metrics
+
+To quickly verify the number of pods in the API server storage without timing out:
+```bash
+kubectl get --raw /metrics | grep 'apiserver_storage_objects{resource="pods"}'
+```
+
+To measure control plane memory usage:
+```bash
+docker exec fuzzer-test-control-plane ps aux | grep -E "kube-apiserver|etcd|kube-scheduler"
+```

fuzzer/cmd/main.go

@@ -0,0 +1,104 @@
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"time"
+
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/kubernetes/test/utils/fuzzer"
+	"sigs.k8s.io/yaml"
+)
+
+func main() {
+	count := flag.Int("count", 1000, "Number of pods to generate")
+	offset := flag.Int("offset", 0, "Starting index for pod naming")
+	basePodPath := flag.String("base-pod", "test/utils/fuzzer/templates/complex-daemonset.yaml", "Path to the real pod YAML to sanitize and clone")
+	namespace := flag.String("namespace", "fuzz-test", "Target namespace for fuzzed pods")
+	namePrefix := flag.String("name-prefix", "fuzzed-pod", "Prefix for generated pod names")
+	outDir := flag.String("out-dir", "", "Directory to write YAMLs (if specified, no cluster injection)")
+	concurrency := flag.Int("concurrency", 50, "Number of concurrent workers")
+	kubeconfig := flag.String("kubeconfig", "", "Path to the kubeconfig file for direct injection")
+
+	flag.Parse()
+
+	// Load Base Pod
+	basePodData, err := os.ReadFile(*basePodPath)
+	if err != nil {
+		log.Fatalf("Failed to read base pod file: %v", err)
+	}
+	var basePod v1.Pod
+	if err := yaml.Unmarshal(basePodData, &basePod); err != nil {
+		log.Fatalf("Failed to unmarshal base pod: %v", err)
+	}
+
+	var clientset *kubernetes.Clientset
+	if *outDir == "" {
+		loadingRules := clientcmd.NewDefaultClientConfigLoadingRules()
+		if *kubeconfig != "" {
+			loadingRules.ExplicitPath = *kubeconfig
+		}
+		config, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, &clientcmd.ConfigOverrides{}).ClientConfig()
+		if err != nil {
+			log.Fatalf("Failed to load kubeconfig: %v", err)
+		}
+
+		// Increase QPS and Burst for high-performance injection
+		config.QPS = 500
+		config.Burst = 1000
+		clientset, err = kubernetes.NewForConfig(config)
+		if err != nil {
+			log.Fatalf("Failed to create clientset: %v", err)
+		}
+	}
+
+	creator := fuzzer.NewExemplaryPodCreator(clientset, time.Now().UnixNano(), *namePrefix, *namespace)
+
+	progress := func(current, total int) {
+		fmt.Printf("\rProgress: %d/%d pods (%.1f%%)", current, total, float64(current)/float64(total)*100)
+		if current == total {
+			fmt.Println()
+		}
+	}
+
+	start := time.Now()
+	if *outDir != "" {
+		fmt.Printf("Writing %d fuzzed pod manifests to %s (base: %s)...\n", *count, *outDir, *basePodPath)
+		dir, err := creator.WriteExemplaryPodsToDir(context.Background(), &basePod, *count, *offset, *concurrency, *outDir, progress)
+		if err != nil {
+			log.Fatalf("\nFailed to write pods: %v", err)
+		}
+		fmt.Printf("Successfully created %d pod manifests in: %s\n", *count, dir)
+	} else {
+		fmt.Printf("Injecting %d fuzzed pods into cluster (base: %s)...\n", *count, *basePodPath)
+		err := creator.CreateExemplaryPods(context.Background(), &basePod, *count, *offset, *concurrency, progress)
+		if err != nil {
+			log.Fatalf("\nFailed to inject pods: %v", err)
+		}
+		fmt.Printf("Successfully injected %d pods.\n", *count)
+	}
+
+	duration := time.Since(start)
+	fmt.Printf("Time taken: %v\n", duration)
+}