observability: conditionally publish Go-runtime Prometheus metrics

* at startup: register lightweight Go runtime collector wrapper
* introduce Enable-Go-Runtime-Metrics feature flag:
  - scope: cluster
  - when enabled: call Prometheus Go collector
----
* part two, prev. commit: 7e08a84 "register subset of go runtime metrics"

Signed-off-by: Alex Aizman <alex.aizman@gmail.com>

Author: alex-aizman

cmd/cli/cli/feat.go

@@ -50,6 +50,7 @@ var clusterFeatDesc = [...]string{
 	"do not delete unrecognized/invalid FQNs during space cleanup ('ais space-cleanup')",
 	"when bucket is n-way mirrored read object replica from the least-utilized mountpath",
 	"count GET(object) 404 as errors (default: don't)",
+	"publish selected Go runtime metrics via Prometheus",
 
 	// apc.ResetToken ("none") ===========
 }
@@ -81,6 +82,7 @@ var featTags = map[string]string{
 	"Keep-Unknown-FQN":                     "integrity?,ops",
 	"Load-Balance-GET":                     "perf",
 	"Count-Object-NotFound-Stats":          "telemetry,ops",
+	"Enable-Go-Runtime-Metrics":            "telemetry,ops,overhead",
 }
 
 // common (cluster, bucket) feature-flags (set, show) helper

cmd/cli/go.mod

@@ -3,7 +3,7 @@ module github.com/NVIDIA/aistore/cmd/cli
 go 1.26
 
 require (
-	github.com/NVIDIA/aistore v1.4.6
+	github.com/NVIDIA/aistore v1.4.7-0.20260603160611-79a7e658c646
 	github.com/fatih/color v1.19.0
 	github.com/json-iterator/go v1.1.12
 	github.com/onsi/ginkgo/v2 v2.28.1

cmd/cli/go.sum

@@ -1,8 +1,8 @@
 github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
 github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0=
 github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
-github.com/NVIDIA/aistore v1.4.6 h1:k7W8NTDdp2/LREYIO1hxgJw3Enyj+tZ8IVSyCwIG/Vk=
-github.com/NVIDIA/aistore v1.4.6/go.mod h1:gQVg+bfLzebJFLhHTQ3cqyLI1EdVm32LzF1f1ETDYgc=
+github.com/NVIDIA/aistore v1.4.7-0.20260603160611-79a7e658c646 h1:qaUyyh2bERa1925tWOGkUkTl+FKdAPDKHH9qYYNTcW4=
+github.com/NVIDIA/aistore v1.4.7-0.20260603160611-79a7e658c646/go.mod h1:gQVg+bfLzebJFLhHTQ3cqyLI1EdVm32LzF1f1ETDYgc=
 github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8=
 github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q=
 github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=

cmn/feat/feat.go

@@ -50,6 +50,7 @@ const (
 	KeepUnknownFQN            // do not delete unrecognized/invalid FQNs during space cleanup ('ais space-cleanup')
 	LoadBalanceGET            // when bucket is n-way mirrored read object replica from the least-utilized mountpath
 	CountObjectNotFoundStats  // count GET(object) 404 (not-found) as errors (default: don't); TODO: add Prometheus to count HEAD(object) errors
+	EnableGoRuntimeMetrics    // publish selected Go runtime metrics via Prometheus
 )
 
 var Cluster = [...]string{
@@ -78,6 +79,7 @@ var Cluster = [...]string{
 	"Keep-Unknown-FQN",
 	"Load-Balance-GET",
 	"Count-Object-NotFound-Stats",
+	"Enable-Go-Runtime-Metrics",
 
 	// apc.ResetToken ("none") ===========
 }

stats/api.go

@@ -37,10 +37,9 @@ const (
 	KindThroughput = "bw"      // ditto (MB/s)
 )
 
-// static labels
+// common static labels (see also: stats/go_runtime)
 const (
-	ConstlabNode      = "node_id"
-	ConstlabComponent = "component"
+	ConstlabNode = "node_id"
 )
 
 // variable labels

stats/common.go

@@ -188,8 +188,6 @@ func (r *runner) RegExtMetric(snode *meta.Snode, name, kind string, extra *Extra
 func (r *runner) regCommon(snode *meta.Snode) {
 	initProm(snode)
 
-	regGoRuntime(snode.Type())
-
 	// basic counters
 	r.reg(snode, GetCount, KindCounter,
 		&Extra{

stats/common_prom.go

@@ -7,7 +7,6 @@ package stats
 
 import (
 	"net/http"
-	"regexp"
 	"strings"
 	ratomic "sync/atomic"
 	"time"
@@ -17,7 +16,6 @@ import (
 	"github.com/NVIDIA/aistore/memsys"
 
 	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/collectors"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
@@ -47,38 +45,14 @@ var (
 	staticLabs = prometheus.Labels{ConstlabNode: ""}
 )
 
+// 1) initialize Prometheus registry without default Go/process collectors
+// 2) register Go runtime collector - a no-op unless feat.EnableGoRuntimeMetrics
 func initProm(snode *meta.Snode) {
 	promRegistry = prometheus.NewRegistry()
 
 	staticLabs[ConstlabNode] = strings.ReplaceAll(snode.ID(), ".", "_")
-}
 
-// Expose a low-cardinality subset of Go runtime metrics on the AIS Prometheus registry:
-//   - base collector (unconditional):
-//     go_goroutines, go_threads, go_info, go_gc_duration_seconds, go_memstats_last_gc_time_seconds
-//   - default runtime/metrics matcher:
-//     go_gc_gogc_percent, go_gc_gomemlimit_bytes, go_sched_gomaxprocs_threads
-//   - explicit picks via WithGoCollectorRuntimeMetrics below (one series each)
-func regGoRuntime(daeType string) {
-	nodeLabs := prometheus.Labels{
-		ConstlabNode:      staticLabs[ConstlabNode],
-		ConstlabComponent: daeType,
-	}
-	reg := prometheus.WrapRegistererWith(nodeLabs, promRegistry)
-	reg.MustRegister(collectors.NewGoCollector(
-		// skip legacy go_memstats_* block (21 series, mostly duplicative); use runtime/metrics names instead
-		collectors.WithGoCollectorMemStatsMetricsDisabled(),
-		collectors.WithGoCollectorRuntimeMetrics(
-			collectors.GoRuntimeMetricsRule{Matcher: regexp.MustCompile(
-				`^/memory/classes/heap/objects:bytes$` + // heap in use
-					`|^/memory/classes/total:bytes$` + // total from OS
-					`|^/gc/heap/goal:bytes$` + // next GC target
-					`|^/gc/heap/allocs:bytes$` + // cumulative alloc bytes (counter)
-					`|^/cpu/classes/gc/total:cpu-seconds$` + // cumulative GC CPU time
-					`|^/gc/cycles/total:gc-cycles$`, // cumulative GC cycle count
-			)},
-		),
-	))
+	regGoRuntime(snode.Type())
 }
 
 // usage: log resulting `copyValue` numbers:

stats/go_runtime.go

@@ -0,0 +1,66 @@
+// Package stats provides methods and functionality to register, track, log,
+// and export metrics that, for the most part, include "counter" and "latency" kinds.
+/*
+ * Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
+ */
+package stats
+
+import (
+	"regexp"
+
+	"github.com/NVIDIA/aistore/cmn"
+	"github.com/NVIDIA/aistore/cmn/feat"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/collectors"
+)
+
+// additional static label for Go runtime
+const (
+	ConstlabNodeType = "node_type"
+)
+
+type goRuntimeCollector struct {
+	collector prometheus.Collector
+}
+
+var regexGoMetrics = regexp.MustCompile(
+	`^/memory/classes/heap/objects:bytes$` + // heap in use
+		`|^/memory/classes/total:bytes$` + // total from OS
+		`|^/gc/heap/goal:bytes$` + // next GC target
+		`|^/gc/heap/allocs:bytes$` + // cumulative alloc bytes
+		`|^/cpu/classes/gc/total:cpu-seconds$`, // cumulative GC CPU time
+)
+
+func regGoRuntime(nodeType string) {
+	reg := prometheus.WrapRegistererWith(
+		prometheus.Labels{
+			ConstlabNode:     staticLabs[ConstlabNode],
+			ConstlabNodeType: nodeType,
+		},
+		promRegistry,
+	)
+	reg.MustRegister(newGoRuntimeCollector())
+}
+
+func newGoRuntimeCollector() *goRuntimeCollector {
+	return &goRuntimeCollector{
+		collector: collectors.NewGoCollector(
+			// skip go_memstats_* block; expose selected runtime/metrics instead.
+			collectors.WithGoCollectorMemStatsMetricsDisabled(),
+			collectors.WithGoCollectorRuntimeMetrics(
+				collectors.GoRuntimeMetricsRule{Matcher: regexGoMetrics},
+			),
+		),
+	}
+}
+
+func (c *goRuntimeCollector) Describe(ch chan<- *prometheus.Desc) {
+	c.collector.Describe(ch)
+}
+
+func (c *goRuntimeCollector) Collect(ch chan<- prometheus.Metric) {
+	if cmn.Rom.Features().IsSet(feat.EnableGoRuntimeMetrics) {
+		c.collector.Collect(ch)
+	}
+}