Add support for otel.java.metrics.cardinality.limit system property (#10541)

Add support for 'otel.java.metrics.cardinality.limit' system property

Add OTel/DD cardinality limit config keys

Co-authored-by: stuart.mcculloch <stuart.mcculloch@datadoghq.com>

Author: mcculls

dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/data/OtelMetricStorage.java

@@ -23,7 +23,7 @@ public final class OtelMetricStorage {
   private static final RatelimitedLogger RATELIMITED_LOGGER =
       new RatelimitedLogger(LOGGER, 5, TimeUnit.MINUTES);
 
-  private static final int DEFAULT_MAX_CARDINALITY = 2_000;
+  private static final int CARDINALITY_LIMIT = Config.get().getMetricsOtelCardinalityLimit();
 
   private static final Attributes CARDINALITY_OVERFLOW =
       Attributes.builder().put("otel.metric.overflow", true).build();
@@ -102,11 +102,11 @@ private OtelAggregator aggregator(
     if (null != aggregator) {
       return aggregator;
     }
-    if (aggregators.size() >= DEFAULT_MAX_CARDINALITY) {
+    if (aggregators.size() >= CARDINALITY_LIMIT) {
       RATELIMITED_LOGGER.warn(
           "Instrument {} has exceeded the maximum allowed cardinality ({}).",
           descriptor.getName(),
-          DEFAULT_MAX_CARDINALITY);
+          CARDINALITY_LIMIT);
       attributes = CARDINALITY_OVERFLOW; // write data to overflow
     }
     return aggregators.computeIfAbsent(attributes, aggregatorSupplier);
@@ -153,7 +153,7 @@ private void doCollectAndReset(OtelInstrumentVisitor visitor) {
     Map<Attributes, OtelAggregator> aggregators = recording.aggregators;
 
     // avoid churn: only remove empty aggregators if we're over cardinality
-    if (aggregators.size() >= DEFAULT_MAX_CARDINALITY) {
+    if (aggregators.size() >= CARDINALITY_LIMIT) {
       aggregators.values().removeIf(OtelAggregator::isEmpty);
     }
 

dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java

@@ -104,9 +104,10 @@ public final class ConfigDefaults {
 
   public static final boolean DEFAULT_METRICS_OTEL_ENABLED = false;
   // Default recommended by Datadog; it differs from Otel’s default of 60000 (60s)
-  static final int DEFAULT_METRICS_OTEL_INTERVAL = 10000; // ms
+  static final int DEFAULT_METRICS_OTEL_INTERVAL = 10_000; // ms
   // Default recommended by Datadog; it differs from Otel’s  default of 30000 (30s)
-  static final int DEFAULT_METRICS_OTEL_TIMEOUT = 7500; // ms
+  static final int DEFAULT_METRICS_OTEL_TIMEOUT = 7_500; // ms
+  static final int DEFAULT_METRICS_OTEL_CARDINALITY_LIMIT = 2_000;
 
   static final String DEFAULT_OTLP_HTTP_METRIC_ENDPOINT = "v1/metrics";
   static final String DEFAULT_OTLP_HTTP_PORT = "4318";

dd-trace-api/src/main/java/datadog/trace/api/config/OtlpConfig.java

@@ -5,6 +5,7 @@ public final class OtlpConfig {
   public static final String METRICS_OTEL_ENABLED = "metrics.otel.enabled";
   public static final String METRICS_OTEL_INTERVAL = "metrics.otel.interval";
   public static final String METRICS_OTEL_TIMEOUT = "metrics.otel.timeout";
+  public static final String METRICS_OTEL_CARDINALITY_LIMIT = "metrics.otel.cardinality.limit";
 
   public static final String OTLP_METRICS_ENDPOINT = "otlp.metrics.endpoint";
   public static final String OTLP_METRICS_HEADERS = "otlp.metrics.headers";

internal-api/src/main/java/datadog/trace/api/Config.java

@@ -111,6 +111,7 @@
 import static datadog.trace.api.ConfigDefaults.DEFAULT_JMX_FETCH_MULTIPLE_RUNTIME_SERVICES_LIMIT;
 import static datadog.trace.api.ConfigDefaults.DEFAULT_LLM_OBS_AGENTLESS_ENABLED;
 import static datadog.trace.api.ConfigDefaults.DEFAULT_LOGS_INJECTION_ENABLED;
+import static datadog.trace.api.ConfigDefaults.DEFAULT_METRICS_OTEL_CARDINALITY_LIMIT;
 import static datadog.trace.api.ConfigDefaults.DEFAULT_METRICS_OTEL_ENABLED;
 import static datadog.trace.api.ConfigDefaults.DEFAULT_METRICS_OTEL_INTERVAL;
 import static datadog.trace.api.ConfigDefaults.DEFAULT_METRICS_OTEL_TIMEOUT;
@@ -444,6 +445,7 @@
 import static datadog.trace.api.config.JmxFetchConfig.JMX_TAGS;
 import static datadog.trace.api.config.LlmObsConfig.LLMOBS_AGENTLESS_ENABLED;
 import static datadog.trace.api.config.LlmObsConfig.LLMOBS_ML_APP;
+import static datadog.trace.api.config.OtlpConfig.METRICS_OTEL_CARDINALITY_LIMIT;
 import static datadog.trace.api.config.OtlpConfig.METRICS_OTEL_ENABLED;
 import static datadog.trace.api.config.OtlpConfig.METRICS_OTEL_INTERVAL;
 import static datadog.trace.api.config.OtlpConfig.METRICS_OTEL_TIMEOUT;
@@ -913,6 +915,7 @@ public static String getHostName() {
   private final boolean metricsOtelEnabled;
   private final int metricsOtelInterval;
   private final int metricsOtelTimeout;
+  private final int metricsOtelCardinalityLimit;
   private final String otlpMetricsEndpoint;
   private final Map<String, String> otlpMetricsHeaders;
   private final OtlpConfig.Protocol otlpMetricsProtocol;
@@ -1887,6 +1890,17 @@ private Config(final ConfigProvider configProvider, final InstrumenterConfig ins
     metricsOtelEnabled =
         configProvider.getBoolean(METRICS_OTEL_ENABLED, DEFAULT_METRICS_OTEL_ENABLED);
 
+    int cardinalityLimit =
+        configProvider.getInteger(
+            METRICS_OTEL_CARDINALITY_LIMIT, DEFAULT_METRICS_OTEL_CARDINALITY_LIMIT);
+    if (cardinalityLimit < 0) {
+      log.warn(
+          "Invalid OTel metrics cardinality limit: {}. The value must be positive",
+          cardinalityLimit);
+      cardinalityLimit = DEFAULT_METRICS_OTEL_CARDINALITY_LIMIT;
+    }
+    metricsOtelCardinalityLimit = cardinalityLimit;
+
     int otelInterval =
         configProvider.getInteger(METRICS_OTEL_INTERVAL, DEFAULT_METRICS_OTEL_INTERVAL);
     if (otelInterval < 0) {
@@ -5189,6 +5203,10 @@ public boolean isMetricsOtelEnabled() {
     return metricsOtelEnabled;
   }
 
+  public int getMetricsOtelCardinalityLimit() {
+    return metricsOtelCardinalityLimit;
+  }
+
   public int getMetricsOtelInterval() {
     return metricsOtelInterval;
   }
@@ -6232,6 +6250,8 @@ public String toString() {
         + metricsOtelInterval
         + ", metricsOtelTimeout="
         + metricsOtelTimeout
+        + ", metricsOtelCardinalityLimit="
+        + metricsOtelCardinalityLimit
         + ", otlpMetricsEndpoint="
         + otlpMetricsEndpoint
         + ", otlpMetricsHeaders="

metadata/supported-configurations.json

@@ -2265,6 +2265,14 @@
         "aliases": []
       }
     ],
+    "DD_METRICS_OTEL_CARDINALITY_LIMIT": [
+      {
+        "version": "A",
+        "type": "int",
+        "default": "2000",
+        "aliases": []
+      }
+    ],
     "DD_OBFUSCATION_QUERY_STRING_REGEXP": [
       {
         "version": "A",
@@ -11096,6 +11104,14 @@
         "default": null,
         "aliases": []
       }
+    ],
+    "OTEL_JAVA_METRICS_CARDINALITY_LIMIT": [
+      {
+        "version": "A",
+        "type": "int",
+        "default": "2000",
+        "aliases": []
+      }
     ]
   },
   "deprecations": {}

utils/config-utils/src/main/java/datadog/trace/bootstrap/config/provider/OtelEnvironmentConfigSource.java

@@ -8,6 +8,7 @@
 import static datadog.trace.api.config.GeneralConfig.SERVICE_NAME;
 import static datadog.trace.api.config.GeneralConfig.TAGS;
 import static datadog.trace.api.config.GeneralConfig.VERSION;
+import static datadog.trace.api.config.OtlpConfig.METRICS_OTEL_CARDINALITY_LIMIT;
 import static datadog.trace.api.config.OtlpConfig.METRICS_OTEL_ENABLED;
 import static datadog.trace.api.config.OtlpConfig.METRICS_OTEL_INTERVAL;
 import static datadog.trace.api.config.OtlpConfig.METRICS_OTEL_TIMEOUT;
@@ -144,6 +145,10 @@ private void setupMetricsOtelEnvironment() {
     capture(
         METRICS_OTEL_TIMEOUT,
         getOtelProperty("otel.metric.export.timeout", "dd." + METRICS_OTEL_TIMEOUT));
+    capture(
+        METRICS_OTEL_CARDINALITY_LIMIT,
+        getOtelProperty(
+            "otel.java.metrics.cardinality.limit", "dd." + METRICS_OTEL_CARDINALITY_LIMIT));
 
     String exporter = getOtelProperty("otel.metrics.exporter");
     if (exporter == null || "otlp".equalsIgnoreCase(exporter)) {