diff --git a/go.mod b/go.mod index 6118436..0ca3f2f 100644 --- a/go.mod +++ b/go.mod @@ -25,6 +25,7 @@ require ( github.com/kcp-dev/sdk v0.31.0 github.com/kcp-dev/virtual-workspace-framework v0.31.0 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 + github.com/prometheus/client_golang v1.23.2 github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.10 github.com/tidwall/gjson v1.18.0 @@ -95,7 +96,6 @@ require ( github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/onsi/gomega v1.39.1 // indirect - github.com/prometheus/client_golang v1.23.2 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.67.5 // indirect github.com/prometheus/procfs v0.20.1 // indirect diff --git a/internal/controller/apiexport/controller.go b/internal/controller/apiexport/controller.go index 4372cd7..cfea93e 100644 --- a/internal/controller/apiexport/controller.go +++ b/internal/controller/apiexport/controller.go @@ -26,6 +26,7 @@ import ( predicateutil "github.com/kcp-dev/api-syncagent/internal/controllerutil/predicate" "github.com/kcp-dev/api-syncagent/internal/discovery" "github.com/kcp-dev/api-syncagent/internal/kcp" + "github.com/kcp-dev/api-syncagent/internal/metrics" "github.com/kcp-dev/api-syncagent/internal/projection" "github.com/kcp-dev/api-syncagent/internal/resources/reconciling" syncagentv1alpha1 "github.com/kcp-dev/api-syncagent/sdk/apis/syncagent/v1alpha1" @@ -144,6 +145,8 @@ func (r *Reconciler) reconcile(ctx context.Context, apiExport *kcpapisv1alpha1.A return fmt.Errorf("failed to list PublishedResources: %w", err) } + metrics.PublishedResourcesManaged.Set(float64(len(pubResources.Items))) + // Create two lists of schema names: ready schemas are those already processed by the // apiresourceschema controller, the other list includes all possible schema names. // For calculating the required permission claims we need _all_ schemas, but we actually diff --git a/internal/controller/apiexport/controller_test.go b/internal/controller/apiexport/controller_test.go new file mode 100644 index 0000000..b6cbef4 --- /dev/null +++ b/internal/controller/apiexport/controller_test.go @@ -0,0 +1,137 @@ +/* +Copyright 2026 The KCP Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apiexport + +import ( + "context" + "testing" + + "github.com/prometheus/client_golang/prometheus/testutil" + "go.uber.org/zap" + + "github.com/kcp-dev/api-syncagent/internal/metrics" + syncagentv1alpha1 "github.com/kcp-dev/api-syncagent/sdk/apis/syncagent/v1alpha1" + + kcpapisv1alpha1 "github.com/kcp-dev/sdk/apis/apis/v1alpha1" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/client-go/tools/record" + ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" + fakectrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +const testAPIExportName = "test-export" +const testAgentName = "test-agent" + +func TestReconcileSetsPublishedResourceMetric(t *testing.T) { + tests := []struct { + name string + pubResources []ctrlruntimeclient.Object + expectedMetric float64 + }{ + { + name: "no PublishedResources", + pubResources: nil, + expectedMetric: 0, + }, + { + name: "single PublishedResource", + pubResources: []ctrlruntimeclient.Object{ + newPublishedResource("test-pr-1", "v1.widgets.example.com"), + }, + expectedMetric: 1, + }, + { + name: "multiple PublishedResources", + pubResources: []ctrlruntimeclient.Object{ + newPublishedResource("test-pr-1", "v1.widgets.example.com"), + newPublishedResource("test-pr-2", "v1.gadgets.example.com"), + newPublishedResource("test-pr-3", "v1.things.example.com"), + }, + expectedMetric: 3, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + scheme := runtime.NewScheme() + utilruntime.Must(syncagentv1alpha1.AddToScheme(scheme)) + utilruntime.Must(kcpapisv1alpha1.AddToScheme(scheme)) + + localClient := fakectrlruntimeclient.NewClientBuilder(). + WithScheme(scheme). + WithObjects(tt.pubResources...). + Build() + + apiExport := &kcpapisv1alpha1.APIExport{ + ObjectMeta: metav1.ObjectMeta{ + Name: testAPIExportName, + }, + } + kcpClient := fakectrlruntimeclient.NewClientBuilder(). + WithScheme(scheme). + WithObjects(apiExport). + WithStatusSubresource(apiExport). + Build() + + r := &Reconciler{ + localClient: localClient, + kcpClient: kcpClient, + log: zap.NewNop().Sugar(), + recorder: record.NewFakeRecorder(99), + apiExportName: testAPIExportName, + agentName: testAgentName, + prFilter: labels.Everything(), // the same filter we use by default in the controller + } + + _, err := r.Reconcile(context.Background(), reconcile.Request{ + NamespacedName: types.NamespacedName{Name: testAPIExportName}, + }) + if err != nil { + t.Fatalf("unexpected reconcile error: %v", err) + } + + got := testutil.ToFloat64(metrics.PublishedResourcesManaged) + if got != tt.expectedMetric { + t.Errorf("expected metric value %v, got %v", tt.expectedMetric, got) + } + }) + } +} + +func newPublishedResource(name, schemaName string) *syncagentv1alpha1.PublishedResource { + return &syncagentv1alpha1.PublishedResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Spec: syncagentv1alpha1.PublishedResourceSpec{ + Resource: syncagentv1alpha1.SourceResourceDescriptor{ + APIGroup: "example.com", + Version: "v1", + Kind: name, + }, + }, + Status: syncagentv1alpha1.PublishedResourceStatus{ + ResourceSchemaName: schemaName, + }, + } +} diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go new file mode 100644 index 0000000..bbc820d --- /dev/null +++ b/internal/metrics/metrics.go @@ -0,0 +1,32 @@ +/* +Copyright 2026 The KCP Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + + "sigs.k8s.io/controller-runtime/pkg/metrics" +) + +var PublishedResourcesManaged = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "sync_agent_published_resource", + Help: "Number of PublishedResources currently managed by this syncagent instance.", +}) + +func init() { + metrics.Registry.MustRegister(PublishedResourcesManaged) +} diff --git a/test/e2e/metrics/metrics_test.go b/test/e2e/metrics/metrics_test.go new file mode 100644 index 0000000..60fed59 --- /dev/null +++ b/test/e2e/metrics/metrics_test.go @@ -0,0 +1,158 @@ +//go:build e2e + +/* +Copyright 2026 The KCP Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "context" + "fmt" + "io" + "net" + "net/http" + "strings" + "testing" + "time" + + "github.com/go-logr/logr" + + syncagentv1alpha1 "github.com/kcp-dev/api-syncagent/sdk/apis/syncagent/v1alpha1" + "github.com/kcp-dev/api-syncagent/test/utils" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + ctrlruntime "sigs.k8s.io/controller-runtime" +) + +func TestPublishedResourceMetric(t *testing.T) { + const ( + apiExportName = "kcp.example.com" + ) + + metricsAddr := freeAddr(t) + + ctx := t.Context() + ctrlruntime.SetLogger(logr.Discard()) + + // setup a test environment in kcp + orgKubeconfig := utils.CreateOrganization(t, ctx, "metrics-test", apiExportName) + + // start a service cluster + envtestKubeconfig, envtestClient, _ := utils.RunEnvtest(t, []string{ + "test/crds/crontab.yaml", + "test/crds/backup.yaml", + }) + + // publish CronTabs + prCrontabs := &syncagentv1alpha1.PublishedResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: "publish-crontabs", + }, + Spec: syncagentv1alpha1.PublishedResourceSpec{ + Resource: syncagentv1alpha1.SourceResourceDescriptor{ + APIGroup: "example.com", + Version: "v1", + Kind: "CronTab", + }, + }, + } + + if err := envtestClient.Create(ctx, prCrontabs); err != nil { + t.Fatalf("Failed to create PublishedResource: %v", err) + } + + // start agent with a known metrics address + utils.RunAgentWithMetrics(ctx, t, "bob", orgKubeconfig, envtestKubeconfig, apiExportName, "", metricsAddr) + + // wait for the metric to appear and equal 1 + assertMetricValue(t, ctx, metricsAddr, "sync_agent_published_resource", 1) + + // add a second PublishedResource + prBackups := &syncagentv1alpha1.PublishedResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: "publish-backups", + }, + Spec: syncagentv1alpha1.PublishedResourceSpec{ + Resource: syncagentv1alpha1.SourceResourceDescriptor{ + APIGroup: "eksempel.no", + Version: "v1", + Kind: "Backup", + }, + }, + } + + if err := envtestClient.Create(ctx, prBackups); err != nil { + t.Fatalf("Failed to create PublishedResource: %v", err) + } + + // wait for metric to update to 2 + assertMetricValue(t, ctx, metricsAddr, "sync_agent_published_resource", 2) + + // delete first PublishedResource + if err := envtestClient.Delete(ctx, prCrontabs); err != nil { + t.Fatalf("Failed to delete PublishedResource: %v", err) + } + + // wait for metric to drop back to 1 + assertMetricValue(t, ctx, metricsAddr, "sync_agent_published_resource", 1) +} + +func assertMetricValue(t *testing.T, ctx context.Context, addr, metricName string, expected int) { + t.Helper() + + url := fmt.Sprintf("http://%s/metrics", addr) + expectedLine := fmt.Sprintf("%s %d", metricName, expected) + + err := wait.PollUntilContextTimeout(ctx, 1*time.Second, 1*time.Minute, false, func(ctx context.Context) (bool, error) { + resp, err := http.Get(url) //nolint:gosec // test-only, fixed local address + if err != nil { + return false, nil // retry on connection errors + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return false, nil + } + + for _, line := range strings.Split(string(body), "\n") { + if strings.HasPrefix(line, metricName+" ") { + return line == expectedLine, nil + } + } + + return false, nil + }) + if err != nil { + t.Fatalf("Metric %s did not reach expected value %d within timeout: %v", metricName, expected, err) + } + + t.Logf("✓ %s = %d", metricName, expected) +} + +func freeAddr(t *testing.T) string { + t.Helper() + + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("Failed to find free port: %v", err) + } + addr := l.Addr().String() + l.Close() + + return addr +} diff --git a/test/utils/process.go b/test/utils/process.go index e9cbbab..6e2f006 100644 --- a/test/utils/process.go +++ b/test/utils/process.go @@ -76,6 +76,19 @@ func RunAgent( localKubeconfig string, apiExportEndpointSlice string, labelSelector string, +) context.CancelFunc { + return RunAgentWithMetrics(ctx, t, name, kcpKubeconfig, localKubeconfig, apiExportEndpointSlice, labelSelector, "0") +} + +func RunAgentWithMetrics( + ctx context.Context, + t *testing.T, + name string, + kcpKubeconfig string, + localKubeconfig string, + apiExportEndpointSlice string, + labelSelector string, + metricsAddr string, ) context.CancelFunc { t.Helper() @@ -91,7 +104,7 @@ func RunAgent( "--log-format", "Console", "--log-debug=true", "--health-address", "0", - "--metrics-address", "0", + "--metrics-address", metricsAddr, } if labelSelector != "" {