feat(resmonitor): Allow custom-endpoint metrics via appendpoint config (Issue #1220)

- Extend ResourceMonitor CRD (deploy/kubeplus-chart/crds/kubeplus-crds.yaml) to accept `appendpoint` section for custom nodePort endpoints and metrics
- Update Go API (platform-operator/pkg/apis/workflowcontroller/v1alpha1/types.go) to include new struct fields for appendpoint
- Enhance plugin (plugins/crmetrics.py) with _get_custom_metrics() to detect and fetch metrics from the configured endpoint
- Add example managed-service "custom-hello-world" under examples/managed-service/appmetrics to demonstrate custom metrics use case
- Expand tests (tests/tests.py) to cover both default metrics (test_metrics()) and custom-endpoint metrics (test_metrics_custom())

Signed-off-by: Tony Nguyen <tonynguyenhuy@gmail.com>

Author: tonyn10

deploy/kubeplus-chart/crds/kubeplus-crds.yaml

@@ -54,6 +54,17 @@ spec:
                       type: string
                 monitorRelationships:
                   type: string
+                appEndpoints:
+                  type: object
+                  properties:
+                    label:
+                      type: string
+                    endpoint:
+                      type: string
+                    metrics:
+                      type: array
+                      items:
+                        type: string
   names:
     kind: ResourceMonitor
     plural: resourcemonitors
@@ -184,6 +195,17 @@ spec:
                               type: string
                         monitorRelationships:
                           type: string
+                        appEndpoints:
+                          type: object
+                          properties:
+                            label:
+                              type: string
+                            endpoint:
+                              type: string
+                            metrics:
+                              type: array
+                              items:
+                                type: string
   names:
     kind: ResourceComposition 
     plural: resourcecompositions 

examples/managed-service/appmetrics/custom-hello-world/Dockerfile

@@ -0,0 +1,5 @@
+FROM ubuntu:22.04
+RUN apt-get update -y && apt-get install -y python-setuptools python3-pip
+ADD . /src
+RUN pip install -r /src/requirements.txt
+CMD ["python3", "/src/app.py"]

examples/managed-service/appmetrics/custom-hello-world/app.py

@@ -0,0 +1,28 @@
+# Custom Hello World app
+
+from flask import Flask
+from prometheus_client import Counter, generate_latest, CONTENT_TYPE_LATEST
+
+app = Flask(__name__)
+
+# Prometheus metrics (counts requests to "/" and "/bye")
+HELLO_REQUEST_COUNT = Counter("hello_requests_total", "Total requests to hello endpoint")
+BYE_REQUEST_COUNT = Counter("bye_requests_total", "Total requests to bye endpoint")
+
+@app.route("/")
+def hello():
+    HELLO_REQUEST_COUNT.inc()  # increment counter every time / is hit
+    return "Hello World, from Kubernetes!<br>"
+
+@app.route("/bye")
+def bye():
+    BYE_REQUEST_COUNT.inc()  # increment counter every time /bye is hit
+    return "Bye, from Kubernetes!<br>"
+
+# Prometheus metrics endpoint
+@app.route("/metrics")
+def metrics():
+    return generate_latest(), 200, {"Content-Type": CONTENT_TYPE_LATEST}
+
+if __name__ == "__main__":
+    app.run(host="0.0.0.0", port=5000)

examples/managed-service/appmetrics/custom-hello-world/build.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+IMAGE_NAME="custom-hello-world-app:latest"
+
+eval $(minikube docker-env)
+docker build -t "$IMAGE_NAME" $KUBEPLUS_HOME/examples/managed-service/appmetrics/custom-hello-world/

examples/managed-service/appmetrics/custom-hello-world/custom-hello-chart-0.0.1.tgz

@@ -0,0 +1,49 @@
+apiVersion: workflows.kubeplus/v1alpha1
+kind: ResourceComposition
+metadata:
+  name: custom-hello-world-app-composition
+spec:
+  # newResource defines the new CRD to be installed define a workflow.
+  newResource:
+    resource:
+      kind: CustomHelloWorldApp
+      group: platformapi.kubeplus
+      version: v1alpha1
+      plural: customhelloworldapps 
+    # URL of the Helm chart that contains Kubernetes resources that represent a workflow.
+    chartURL: file:///custom-hello-chart-0.0.1.tgz
+    chartName: custom-hello-chart
+  # respolicy defines the resource policy to be applied to instances of the specified custom resource.
+  respolicy:
+    apiVersion: workflows.kubeplus/v1alpha1
+    kind: ResourcePolicy 
+    metadata:
+      name: custom-hello-world-app-policy
+    spec:
+      resource:
+        kind: CustomHelloWorldApp 
+        group: platformapi.kubeplus
+        version: v1alpha1
+  # resmonitor identifies the resource instances that should be monitored for CPU/Memory/Storage.
+  # All the Pods that are related to the resource instance through either ownerReference relationship, or all the relationships
+  # (ownerReference, label, annotation, spec properties) are considered in calculating the statistics. 
+  # The generated output is in Prometheus format.
+  resmonitor:
+    apiVersion: workflows.kubeplus/v1alpha1
+    kind: ResourceMonitor
+    metadata:
+      name: custom-hello-world-app-monitor
+    spec:
+      resource:
+        kind: CustomHelloWorldApp 
+        group: platformapi.kubeplus
+        version: v1alpha1
+      # This attribute indicates that Pods that are reachable through all the relationships should be used
+      # as part of calculating the monitoring statistics.
+      monitorRelationships: all
+
+      # Define endpoint for where to pull application specific metrics
+      appEndpoints:
+        label: "app=customhelloworld"
+        endpoint: "metrics"
+        metrics: ["hello_requests_total", "bye_requests_total"]

examples/managed-service/appmetrics/custom-hello-world/custom-hello-world-app-composition.yaml

@@ -0,0 +1,7 @@
+# defines an instance of the HelloWorldService kind
+apiVersion: platformapi.kubeplus/v1alpha1
+kind: CustomHelloWorldApp
+metadata:
+  name: custom-hs1
+spec:
+  # no specific specs

examples/managed-service/appmetrics/custom-hello-world/custom-hs1.yaml

@@ -0,0 +1,2 @@
+flask
+prometheus_client

examples/managed-service/appmetrics/custom-hello-world/requirements.txt

@@ -170,7 +170,14 @@ type ResourceMonitorStatus struct {
 type ResourceMonitorSpec struct {
 	Resource Res `json:"resource"`
 	//MonitoringPolicy Mon `json:"monitoringpolicy"`
-	MonitorRelationships string `json:"monitorRelationships"`
+	MonitorRelationships string                      `json:"monitorRelationships"`
+	AppEndpoints         ResourceMonitorAppEndpoints `json:"appEndpoints"`
+}
+
+type ResourceMonitorAppEndpoints struct {
+	Label	 string   `json:"label"`
+	Endpoint string   `json:"endpoint"`
+	Metrics  []string `json:"metrics"` 
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

platform-operator/pkg/apis/workflowcontroller/v1alpha1/types.go

@@ -8,6 +8,7 @@
 import time
 import yaml
 import utils
+import requests
 
 class CRBase(object):
 
@@ -993,12 +994,111 @@ def _get_metrics_creator_account_with_connections(self, account):
 		print("    Number of Pods: " + str(len(pod_list_for_metrics)))
 		print("        Number of Containers: " + str(num_of_containers))
 		print("        Number of Nodes: " + str(num_of_hosts))
-		print("Underlying Physical Resoures consumed:")
+		print("Underlying Physical Resources consumed:")
 		print("    Total CPU(cores): " + str(cpu) + "m")
 		print("    Total MEMORY(bytes): " + str(mem) + "Mi")
 		print("    Total Storage(bytes): " + str(storage) + "Gi")
 		print("---------------------------------------------------------- ")
 
+
+	def _get_custom_metrics(self, custom_resource, custom_res_instance):
+		metrics_data = {}
+		metrics_descriptions = {}
+
+		# Get all ResourceCompositions
+		rc_list_raw, err = self.run_command("kubectl get resourcecompositions -o json")
+		if not rc_list_raw:
+			return metrics_data, metrics_descriptions
+
+		rc_list = json.loads(rc_list_raw)
+		for item in rc_list["items"]:
+			rc_name = item["metadata"]["name"]
+
+			# Get full ResourceComposition
+			rc_json_raw, err = self.run_command(f"kubectl get resourcecomposition {rc_name} -o json")
+			if not rc_json_raw:
+				continue
+			resource_composition = json.loads(rc_json_raw)
+
+			# Check if this ResourceComposition defines our custom_resource
+			kind_in_rc = resource_composition["spec"]["newResource"]["resource"]["kind"]
+			if kind_in_rc != custom_resource:
+				continue
+			
+			# Get appEndpoints from resmonitor
+			try:
+				app_endpoint = resource_composition["spec"]["resmonitor"]["spec"]["appEndpoints"]
+				label_selector = app_endpoint["label"]
+				endpoint_path = app_endpoint["endpoint"]
+				metrics_names = app_endpoint["metrics"]	# list of metric names to filter
+			except KeyError:
+				continue  # No endpoints defined
+
+			# label_selector, endpoint_path, and metrics_names must be non-empty
+			if not label_selector or not endpoint_path or not metrics_names:
+				continue
+
+			# Find pods matching the label
+			pods_raw, err = self.run_command(f"kubectl get pods -n {custom_res_instance} -l {label_selector} -o json")
+			if not pods_raw:
+				print(err)
+				continue
+
+			pods = json.loads(pods_raw)
+			for pod in pods["items"]:
+				pod_name = pod["metadata"]["name"]
+				host_ip = pod["status"]["hostIP"]
+				if not host_ip:
+					continue
+
+				svc_raw, err = self.run_command(f"kubectl get svc -n {custom_res_instance} -l {label_selector} -o json")
+				svc_json = json.loads(svc_raw)
+				try:
+					node_port = svc_json["items"][0]["spec"]["ports"][0]["nodePort"]
+				except KeyError:
+					continue
+
+				# Query metrics endpoint using the host_ip and node_port
+				try:
+					url = f"http://{host_ip}:{node_port}/{endpoint_path}"
+					resp = requests.get(url)
+					resp.raise_for_status()
+					metrics_string = resp.text
+				except Exception as e:
+					print(f"Failed to query metrics for pod {pod_name} at {url}: {e}")
+					continue
+
+				# Filter metrics
+				for line in metrics_string.splitlines():
+					line = line.strip()
+					if not line or line.startswith("# TYPE"):
+						continue
+
+					# Extract the description of our desired metric (for pretty format)
+					if line.startswith("# HELP"):
+						parts = line.split(' ', 3)
+						_, _, metric_name, description = parts
+						if metric_name in metrics_names:
+							metrics_descriptions[metric_name] = description
+						continue
+
+					metric_part, value = line.rsplit(' ', 1)
+					
+					# Extract base metric name (strip "{...}" if present)
+					base_name = ''
+					if '{' in metric_part:
+						base_name = metric_part.split('{', 1)[0]
+					else:
+						base_name = metric_part
+					
+					# Store the value for the given metric
+					if base_name in metrics_names:
+						metrics_data[base_name] = value
+
+
+		return metrics_data, metrics_descriptions
+
+
 	def get_metrics_cr(self, custom_resource, custom_res_instance, opformat, kubeconfig):
 		namespace = self.get_kubeplus_namespace(kubeconfig)
 		accountidentity = self._get_identity(custom_resource, custom_res_instance, namespace)
@@ -1013,6 +1113,7 @@ def get_metrics_cr(self, custom_resource, custom_res_instance, opformat, kubecon
 		num_of_hosts_conn = self._parse_number_of_hosts(pod_list, kubecfg=kubeconfig)
 		cpu_conn, memory_conn, individual_pod_metrics = self._get_cpu_memory_usage_kubelet(pod_list, kubecfg=kubeconfig)
 		networkReceiveBytesTotal, networkTransmitBytesTotal, oom_events = self._get_cadvisor_metrics(pod_list, kubecfg=kubeconfig)
+		custom_metrics_data, custom_metrics_descriptions = self._get_custom_metrics(custom_resource, custom_res_instance)
 
 		num_of_not_running_pods = self._num_of_not_running_pods(pod_list, kubecfg=kubeconfig)
 
@@ -1037,6 +1138,11 @@ def get_metrics_cr(self, custom_resource, custom_res_instance, opformat, kubecon
 			op['networkTransmitBytes'] = str(networkTransmitBytesTotal) + " bytes"
 			op['notRunningPods'] = str(num_of_not_running_pods)
 			op['oom_events'] = str(oom_events)
+
+			# Append custom metrics
+			for metric, value in custom_metrics_data.items():
+				op[f'{metric}'] = str(value)
+
 			json_op = json.dumps(op)
 			print(json_op)
 		elif opformat == 'prometheus':
@@ -1066,6 +1172,11 @@ def get_metrics_cr(self, custom_resource, custom_res_instance, opformat, kubecon
 				podMetrics = podMetrics + pod_cpu_mem
 
 			metricsToReturn = cpuMetrics + "\n" + memoryMetrics + "\n" + storageMetrics + "\n" + numOfPods + "\n" + numOfContainers + "\n" + networkReceiveBytes + "\n" + networkTransmitBytes + "\n" + numOfNotRunningPods + "\n" + oomEvents + "\n" + podMetrics
+
+			# Append custom metrics
+			for metric, value in custom_metrics_data.items():
+				metricsToReturn += metric + '{custom_resource="' + fq_instance + '"} ' + str(value) + ' ' + timeInMillis + "\n"
+
 			print(metricsToReturn)
 		elif opformat == 'pretty':
 			print("---------------------------------------------------------- ")
@@ -1081,6 +1192,10 @@ def get_metrics_cr(self, custom_resource, custom_res_instance, opformat, kubecon
 			print("    Total Storage(bytes): " + str(total_storage) + "Gi")
 			print("    Total Network bytes received: " + str(networkReceiveBytesTotal))
 			print("    Total Network bytes transferred: " + str(networkTransmitBytesTotal))
+			print("Custom application metrics:")
+			for metric, description in custom_metrics_descriptions.items():
+				print("    " + description + ": " + str(custom_metrics_data[metric]))
+
 			print("---------------------------------------------------------- ")
 		else:
 			print("Unknown output format specified. Accepted values: pretty, json, prometheus")