Stabilize performance test SLAs with tolerance thresholds (#16371)

* Stabilize performance test SLAs with tolerance thresholds

Fixes #14793

This change adds tolerance thresholds to SLA checks that previously
used strict equality for request counts/rates. Due to timing variations
in test execution, actual counts can vary slightly from expected values,
causing false SLA failures.

Changes:
- scale-from-zero SLA 3: Allow 1 request tolerance
- dataplane-probe SLA 2: Allow 0.1% tolerance (min 1 request)
- rollout-probe SLA 2: Allow 1% rate tolerance

This approach follows the pattern already established in load-test.

Signed-off-by: aviralgarg05 <gargaviral99@gmail.com>

* Fix performance SLA edge cases

* Fix style check failures

---------

Signed-off-by: aviralgarg05 <gargaviral99@gmail.com>

Author: aviralgarg05

test/performance/benchmarks/dataplane-probe/main.go

@@ -175,11 +175,24 @@ func checkSLA(results *vegeta.Metrics, slaMin time.Duration, slaMax time.Duratio
 		return fmt.Errorf("SLA 1 failed. P95 latency is not in %d-%dms time range: %s", slaMin, slaMax, results.Latencies.P95)
 	}
 
-	// SLA 2: making sure the defined total request is met
-	if results.Requests == uint64(rate.Rate(time.Second)*duration.Seconds()) {
-		log.Printf("SLA 2 passed. vegeta total request is %d", results.Requests)
+	// SLA 2: making sure the defined total request is met (within 0.1% threshold)
+	expectedRequests := uint64(rate.Rate(time.Second) * duration.Seconds())
+	threshold := expectedRequests / 1000 // 0.1% tolerance
+	if threshold == 0 {
+		threshold = 1 // Minimum tolerance of 1 request
+	}
+
+	var difference uint64
+	if results.Requests >= expectedRequests {
+		difference = results.Requests - expectedRequests
+	} else {
+		difference = expectedRequests - results.Requests
+	}
+
+	if difference <= threshold {
+		log.Printf("SLA 2 passed. vegeta total request is %d, expected requests is %d (tolerance: %d)", results.Requests, expectedRequests, threshold)
 	} else {
-		return fmt.Errorf("SLA 2 failed. vegeta total request is %d, expected total request is %f", results.Requests, rate.Rate(time.Second)*duration.Seconds())
+		return fmt.Errorf("SLA 2 failed. vegeta total request is %d, expected requests is %d (tolerance: %d)", results.Requests, expectedRequests, threshold)
 	}
 
 	return nil

test/performance/benchmarks/dataplane-probe/main_test.go

@@ -0,0 +1,55 @@
+/*
+Copyright 2026 The Knative Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"testing"
+	"time"
+
+	vegeta "github.com/tsenart/vegeta/v12/lib"
+)
+
+func TestCheckSLATotalRequestsTolerance(t *testing.T) {
+	rate := vegeta.Rate{Freq: 1000, Per: time.Second}
+
+	tests := []struct {
+		name     string
+		requests uint64
+		wantErr  bool
+	}{
+		{name: "within tolerance below expected", requests: 999},
+		{name: "within tolerance above expected", requests: 1001},
+		{name: "outside tolerance below expected", requests: 998, wantErr: true},
+		{name: "outside tolerance above expected", requests: 1002, wantErr: true},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			results := &vegeta.Metrics{
+				Requests: tt.requests,
+				Latencies: vegeta.LatencyMetrics{
+					P95: 10 * time.Millisecond,
+				},
+			}
+
+			err := checkSLA(results, 0, 20*time.Millisecond, rate, time.Second)
+			if (err != nil) != tt.wantErr {
+				t.Fatalf("checkSLA() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}

test/performance/benchmarks/rollout-probe/main.go

@@ -228,11 +228,13 @@ func checkSLA(results *vegeta.Metrics, rate vegeta.ConstantPacer) error {
 		return fmt.Errorf("SLA 1 failed. P95 latency is not in 100-110ms time range: %s", results.Latencies.P95)
 	}
 
-	// SLA 2: making sure the defined vegeta rates is met
-	if math.Round(results.Rate) == rate.Rate(time.Second) {
-		log.Printf("SLA 2 passed. vegeta rate is %f", rate.Rate(time.Second))
+	// SLA 2: making sure the defined vegeta rates is met (within 1% tolerance)
+	expectedRate := rate.Rate(time.Second)
+	tolerance := expectedRate * 0.01 // 1% tolerance
+	if math.Abs(results.Rate-expectedRate) <= tolerance {
+		log.Printf("SLA 2 passed. vegeta rate is %f, expected rate is %f (tolerance: 1%%)", results.Rate, expectedRate)
 	} else {
-		return fmt.Errorf("SLA 2 failed. vegeta rate is %f, expected Rate is %f", results.Rate, rate.Rate(time.Second))
+		return fmt.Errorf("SLA 2 failed. vegeta rate is %f, expected rate is %f (tolerance: 1%%)", results.Rate, expectedRate)
 	}
 
 	return nil

test/performance/benchmarks/scale-from-zero/main.go

@@ -360,10 +360,16 @@ func checkSLA(results *vegeta.Metrics, p95min time.Duration, p95max time.Duratio
 	}
 
 	// SLA 3: making sure the defined vegeta total requests is met, the defined vegeta total requests should equal to the count of ksvcs we want to run scale-from-zero in parallel
-	if results.Requests == uint64(parallel) {
-		log.Printf("SLA 3 passed. total requests is %d", results.Requests)
+	// Allow a tolerance of 1 request to account for timing variations
+	expectedRequests := uint64(parallel)
+	lowerBound := expectedRequests
+	if lowerBound > 0 {
+		lowerBound--
+	}
+	if results.Requests >= lowerBound && results.Requests <= expectedRequests {
+		log.Printf("SLA 3 passed. total requests is %d, expected requests is %d (tolerance: 1)", results.Requests, expectedRequests)
 	} else {
-		return fmt.Errorf("SLA 3 failed. total requests is %d, expected total requests is %d", results.Requests, uint64(parallel))
+		return fmt.Errorf("SLA 3 failed. total requests is %d, expected requests is %d (tolerance: 1)", results.Requests, expectedRequests)
 	}
 
 	return nil

test/performance/benchmarks/scale-from-zero/main_test.go

@@ -0,0 +1,38 @@
+/*
+Copyright 2026 The Knative Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"testing"
+	"time"
+
+	vegeta "github.com/tsenart/vegeta/v12/lib"
+)
+
+func TestCheckSLAZeroParallelDoesNotUnderflow(t *testing.T) {
+	results := &vegeta.Metrics{
+		Requests: 0,
+		Latencies: vegeta.LatencyMetrics{
+			P95: 0,
+			Max: 0,
+		},
+	}
+
+	if err := checkSLA(results, 0, time.Millisecond, time.Millisecond, 0); err != nil {
+		t.Fatalf("checkSLA() error = %v, want nil", err)
+	}
+}