Automate HMAC secret management

Automatically generate and manage HMAC secrets when SessionManagementV2
is enabled in VirtualMCPServer resources, eliminating manual secret
creation.

Also add e2e and integration tests to validate the functionality

Related-to: #3867

Author: taskbot

cmd/thv-operator/controllers/virtualmcpserver_controller.go

@@ -7,6 +7,8 @@ package controllers
 
 import (
 	"context"
+	"crypto/rand"
+	"encoding/base64"
 	"fmt"
 	"maps"
 	"reflect"
@@ -23,6 +25,7 @@ import (
 	"k8s.io/client-go/tools/record"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	"sigs.k8s.io/controller-runtime/pkg/handler"
 	"sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
@@ -100,7 +103,7 @@ type VirtualMCPServerReconciler struct {
 // +kubebuilder:rbac:groups="rbac.authorization.k8s.io",resources=roles,verbs=create;delete;get;list;patch;update;watch
 // +kubebuilder:rbac:groups="rbac.authorization.k8s.io",resources=rolebindings,verbs=create;delete;get;list;patch;update;watch
 // +kubebuilder:rbac:groups="",resources=events,verbs=create;patch
-// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch
+// +kubebuilder:rbac:groups="",resources=secrets,verbs=create;get;list;watch
 // +kubebuilder:rbac:groups=apps,resources=deployments,verbs=create;delete;get;list;patch;update;watch
 // +kubebuilder:rbac:groups="",resources=serviceaccounts,verbs=create;delete;get;list;patch;update;watch
 // +kubebuilder:rbac:groups=toolhive.stacklok.dev,resources=embeddingservers,verbs=get;list;watch
@@ -592,6 +595,12 @@ func (r *VirtualMCPServerReconciler) ensureAllResources(
 		return ctrl.Result{}, err
 	}
 
+	// Ensure HMAC secret for session token binding (Session Management V2)
+	if err := r.ensureHMACSecret(ctx, vmcp); err != nil {
+		ctxLogger.Error(err, "Failed to ensure HMAC secret")
+		return ctrl.Result{}, err
+	}
+
 	// Ensure vmcp Config ConfigMap
 	if err := r.ensureVmcpConfigConfigMap(ctx, vmcp, workloadNames, statusManager); err != nil {
 		ctxLogger.Error(err, "Failed to ensure vmcp Config ConfigMap")
@@ -660,6 +669,161 @@ func (r *VirtualMCPServerReconciler) ensureRBACResources(
 	return err
 }
 
+// ensureHMACSecret ensures the HMAC secret exists for session token binding.
+// This secret is required when Session Management V2 is enabled.
+// The secret is automatically generated with a cryptographically secure random value.
+//
+// The secret follows this naming pattern: {vmcp-name}-hmac-secret
+// and contains a single key: hmac-secret with a 32-byte base64-encoded random value.
+func (r *VirtualMCPServerReconciler) ensureHMACSecret(
+	ctx context.Context,
+	vmcp *mcpv1alpha1.VirtualMCPServer,
+) error {
+	ctxLogger := log.FromContext(ctx)
+
+	// Only ensure HMAC secret if Session Management V2 is enabled
+	if vmcp.Spec.Config.Operational == nil || !vmcp.Spec.Config.Operational.SessionManagementV2 {
+		return nil
+	}
+
+	secretName := fmt.Sprintf("%s-hmac-secret", vmcp.Name)
+	secret := &corev1.Secret{}
+	err := r.Get(ctx, types.NamespacedName{Name: secretName, Namespace: vmcp.Namespace}, secret)
+
+	if errors.IsNotFound(err) {
+		// Generate a cryptographically secure 32-byte HMAC secret
+		hmacSecret, err := generateHMACSecret()
+		if err != nil {
+			ctxLogger.Error(err, "Failed to generate HMAC secret")
+			if r.Recorder != nil {
+				r.Recorder.Eventf(vmcp, corev1.EventTypeWarning, "HMACSecretGenerationFailed",
+					"Failed to generate HMAC secret: %v", err)
+			}
+			return fmt.Errorf("failed to generate HMAC secret: %w", err)
+		}
+
+		newSecret := &corev1.Secret{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      secretName,
+				Namespace: vmcp.Namespace,
+				Labels: map[string]string{
+					"app.kubernetes.io/name":       "virtualmcpserver",
+					"app.kubernetes.io/instance":   vmcp.Name,
+					"app.kubernetes.io/component":  "session-security",
+					"app.kubernetes.io/managed-by": "toolhive-operator",
+				},
+				Annotations: map[string]string{
+					"toolhive.stacklok.dev/purpose": "hmac-secret-for-session-token-binding",
+				},
+			},
+			Type: corev1.SecretTypeOpaque,
+			Data: map[string][]byte{
+				"hmac-secret": []byte(hmacSecret),
+			},
+		}
+
+		// Set VirtualMCPServer as owner so secret is automatically deleted when VMCP is deleted
+		if err := controllerutil.SetControllerReference(vmcp, newSecret, r.Scheme); err != nil {
+			ctxLogger.Error(err, "Failed to set controller reference for HMAC secret")
+			return fmt.Errorf("failed to set controller reference: %w", err)
+		}
+
+		ctxLogger.Info("Creating HMAC secret for session token binding", "Secret.Name", secretName)
+		if err := r.Create(ctx, newSecret); err != nil {
+			ctxLogger.Error(err, "Failed to create HMAC secret")
+			if r.Recorder != nil {
+				r.Recorder.Eventf(vmcp, corev1.EventTypeWarning, "HMACSecretCreationFailed",
+					"Failed to create HMAC secret: %v", err)
+			}
+			return fmt.Errorf("failed to create HMAC secret: %w", err)
+		}
+
+		// Record success event
+		if r.Recorder != nil {
+			r.Recorder.Event(vmcp, corev1.EventTypeNormal, "HMACSecretCreated",
+				"HMAC secret created for session token binding")
+		}
+		return nil
+	} else if err != nil {
+		ctxLogger.Error(err, "Failed to get HMAC secret")
+		return fmt.Errorf("failed to get HMAC secret: %w", err)
+	}
+
+	// Secret exists - validate ownership and structure before accepting it
+	if err := r.validateHMACSecret(ctx, vmcp, secret); err != nil {
+		ctxLogger.Error(err, "Existing HMAC secret is invalid", "Secret.Name", secretName)
+		if r.Recorder != nil {
+			r.Recorder.Eventf(vmcp, corev1.EventTypeWarning, "HMACSecretValidationFailed",
+				"Existing HMAC secret validation failed: %v", err)
+		}
+		return fmt.Errorf("existing HMAC secret validation failed: %w", err)
+	}
+
+	return nil
+}
+
+// validateHMACSecret validates that an existing HMAC secret has the correct ownership,
+// structure, and content. This prevents accepting stale, malformed, or attacker-controlled
+// secrets that could weaken session token signing or cause pod startup failures.
+func (*VirtualMCPServerReconciler) validateHMACSecret(
+	ctx context.Context,
+	vmcp *mcpv1alpha1.VirtualMCPServer,
+	secret *corev1.Secret,
+) error {
+	ctxLogger := log.FromContext(ctx)
+
+	// Verify the secret is owned by this VirtualMCPServer
+	// This prevents accepting secrets created by other actors
+	isOwned := false
+	for _, ownerRef := range secret.OwnerReferences {
+		if ownerRef.UID == vmcp.UID &&
+			ownerRef.Kind == "VirtualMCPServer" &&
+			ownerRef.Name == vmcp.Name {
+			isOwned = true
+			break
+		}
+	}
+	if !isOwned {
+		return fmt.Errorf("secret is not owned by VirtualMCPServer %s/%s", vmcp.Namespace, vmcp.Name)
+	}
+
+	// Verify the hmac-secret key exists
+	hmacSecretData, exists := secret.Data["hmac-secret"]
+	if !exists {
+		return fmt.Errorf("secret missing required 'hmac-secret' key")
+	}
+
+	// Verify it's valid base64 and decodes to exactly 32 bytes
+	hmacSecretBase64 := string(hmacSecretData)
+	if hmacSecretBase64 == "" {
+		return fmt.Errorf("hmac-secret is empty")
+	}
+
+	decoded, err := base64.StdEncoding.DecodeString(hmacSecretBase64)
+	if err != nil {
+		return fmt.Errorf("hmac-secret is not valid base64: %w", err)
+	}
+
+	if len(decoded) != 32 {
+		return fmt.Errorf("hmac-secret must be exactly 32 bytes, got %d bytes", len(decoded))
+	}
+
+	// Verify it's not all zeros (would indicate a weak/predictable key)
+	allZeros := true
+	for _, b := range decoded {
+		if b != 0 {
+			allZeros = false
+			break
+		}
+	}
+	if allZeros {
+		return fmt.Errorf("hmac-secret is all zeros (weak key)")
+	}
+
+	ctxLogger.V(1).Info("HMAC secret validation passed", "Secret.Name", secret.Name)
+	return nil
+}
+
 // getVmcpConfigChecksum fetches the vmcp Config ConfigMap checksum annotation.
 // This is used to trigger deployment rollouts when the configuration changes.
 //
@@ -2368,3 +2532,18 @@ func setAuthConfigConditions(
 	// auth config errors are non-fatal. The system can continue operating with
 	// the auth configs that are valid.
 }
+
+// generateHMACSecret generates a cryptographically secure 32-byte HMAC secret
+// encoded as base64. This secret is used for session token binding in Session Management V2.
+//
+// Returns a base64-encoded string suitable for use as VMCP_SESSION_HMAC_SECRET.
+func generateHMACSecret() (string, error) {
+	// Generate 32 bytes of cryptographically secure random data
+	secret := make([]byte, 32)
+	if _, err := rand.Read(secret); err != nil {
+		return "", fmt.Errorf("failed to generate random bytes: %w", err)
+	}
+
+	// Encode as base64 for safe storage and environment variable use
+	return base64.StdEncoding.EncodeToString(secret), nil
+}

cmd/thv-operator/controllers/virtualmcpserver_deployment.go

@@ -277,6 +277,11 @@ func (r *VirtualMCPServerReconciler) buildEnvVarsForVmcp(
 	// Mount outgoing auth secrets
 	env = append(env, r.buildOutgoingAuthEnvVars(ctx, vmcp, typedWorkloads)...)
 
+	// Mount HMAC secret for session token binding (Session Management V2)
+	if vmcp.Spec.Config.Operational != nil && vmcp.Spec.Config.Operational.SessionManagementV2 {
+		env = append(env, r.buildHMACSecretEnvVar(vmcp))
+	}
+
 	// Note: Other secrets (Redis passwords, service account credentials) may be added here in the future
 	// following the same pattern of mounting from Kubernetes Secrets as environment variables.
 
@@ -325,6 +330,25 @@ func (*VirtualMCPServerReconciler) buildOIDCEnvVars(vmcp *mcpv1alpha1.VirtualMCP
 	return env
 }
 
+// buildHMACSecretEnvVar builds environment variable for HMAC secret mounting.
+// This secret is used for session token binding in Session Management V2.
+// The operator automatically generates and manages this secret if it doesn't exist.
+func (*VirtualMCPServerReconciler) buildHMACSecretEnvVar(vmcp *mcpv1alpha1.VirtualMCPServer) corev1.EnvVar {
+	secretName := fmt.Sprintf("%s-hmac-secret", vmcp.Name)
+
+	return corev1.EnvVar{
+		Name: "VMCP_SESSION_HMAC_SECRET",
+		ValueFrom: &corev1.EnvVarSource{
+			SecretKeyRef: &corev1.SecretKeySelector{
+				LocalObjectReference: corev1.LocalObjectReference{
+					Name: secretName,
+				},
+				Key: "hmac-secret",
+			},
+		},
+	}
+}
+
 // buildOutgoingAuthEnvVars builds environment variables for outgoing auth secrets.
 func (r *VirtualMCPServerReconciler) buildOutgoingAuthEnvVars(
 	ctx context.Context,

cmd/thv-operator/controllers/virtualmcpserver_hmac_secret_test.go

@@ -0,0 +1,79 @@
+// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+package controllers
+
+import (
+	"encoding/base64"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestGenerateHMACSecret tests the HMAC secret generation function.
+func TestGenerateHMACSecret(t *testing.T) {
+	t.Parallel()
+
+	t.Run("generates valid base64 encoded secret", func(t *testing.T) {
+		t.Parallel()
+
+		secret, err := generateHMACSecret()
+		require.NoError(t, err)
+		require.NotEmpty(t, secret)
+
+		// Verify it's valid base64
+		decoded, err := base64.StdEncoding.DecodeString(secret)
+		require.NoError(t, err)
+		assert.Len(t, decoded, 32, "decoded secret should be exactly 32 bytes")
+	})
+
+	t.Run("generates unique secrets", func(t *testing.T) {
+		t.Parallel()
+
+		secret1, err := generateHMACSecret()
+		require.NoError(t, err)
+
+		secret2, err := generateHMACSecret()
+		require.NoError(t, err)
+
+		// Two generated secrets should be different
+		assert.NotEqual(t, secret1, secret2, "consecutive secrets should be unique")
+	})
+
+	t.Run("generates cryptographically secure random data", func(t *testing.T) {
+		t.Parallel()
+
+		secret, err := generateHMACSecret()
+		require.NoError(t, err)
+
+		decoded, err := base64.StdEncoding.DecodeString(secret)
+		require.NoError(t, err)
+
+		// Check that it's not all zeros (would indicate failure of crypto/rand)
+		allZeros := make([]byte, 32)
+		assert.NotEqual(t, allZeros, decoded, "secret should not be all zeros")
+	})
+
+	t.Run("generates multiple valid secrets", func(t *testing.T) {
+		t.Parallel()
+
+		// Generate 100 secrets to ensure consistency
+		secrets := make(map[string]bool)
+		for i := 0; i < 100; i++ {
+			secret, err := generateHMACSecret()
+			require.NoError(t, err)
+
+			// Verify base64 decoding
+			decoded, err := base64.StdEncoding.DecodeString(secret)
+			require.NoError(t, err)
+			assert.Len(t, decoded, 32)
+
+			// Track uniqueness
+			secrets[secret] = true
+		}
+
+		// All secrets should be unique
+		assert.Len(t, secrets, 100, "all generated secrets should be unique")
+	})
+}

cmd/vmcp/README.md

@@ -237,7 +237,35 @@ spec:
               key: hmac-secret
 ```
 
-**Note**: Kubernetes deployments **require** `VMCP_SESSION_HMAC_SECRET` to be set (the server will fail to start without it). For non-Kubernetes environments (local development/testing), a default insecure secret is used as a fallback, but this is **NOT recommended for production**.
+**Note**: When **Session Management V2 is enabled**, Kubernetes deployments **require** `VMCP_SESSION_HMAC_SECRET` to be set (the server will fail to start without it). For non-Kubernetes environments (local development/testing), a default insecure secret is used as a fallback, but this is **NOT recommended for production**. If Session Management V2 is disabled, this environment variable is not required.
+
+### Automatic Secret Management (ToolHive Operator)
+
+When deploying vMCP via the **ToolHive operator** with Session Management V2 enabled, the HMAC secret is **automatically generated and managed** for you:
+
+```yaml
+apiVersion: toolhive.stacklok.dev/v1alpha1
+kind: VirtualMCPServer
+metadata:
+  name: my-vmcp
+spec:
+  config:
+    operational:
+      sessionManagementV2: true  # Enables automatic HMAC secret creation
+    group: my-group
+```
+
+The operator will:
+
+- ✅ Automatically generate a cryptographically secure 32-byte HMAC secret
+- ✅ Store it in a Kubernetes Secret named `{vmcp-name}-hmac-secret`
+- ✅ Inject it into the vMCP deployment as `VMCP_SESSION_HMAC_SECRET`
+- ✅ Validate existing secrets (ownership, structure, and content)
+- ✅ Automatically delete the secret when the VirtualMCPServer is removed
+
+**No manual secret generation or management required!** The operator handles all of this automatically when you enable Session Management V2.
+
+> **Note**: The secret is generated once at creation time and persists for the lifetime of the VirtualMCPServer. Secret rotation is not currently supported but may be added in a future release.
 
 ## Tool Aggregation & Conflict Resolution