Adds append_dimensions to collectd, supporting custom metric dimensions

translator/cmdutil/translatorutil_test.go

@@ -156,6 +156,10 @@ func TestEthtoolConfig(t *testing.T) {
 	checkIfSchemaValidateAsExpected(t, "../../translator/config/sampleSchema/validEthtoolConfig.json", true, map[string]int{})
 }
 
+func TestCollectdConfig(t *testing.T) {
+	checkIfSchemaValidateAsExpected(t, "../../translator/config/sampleSchema/validCollectdConfig.json", true, map[string]int{})
+}
+
 func TestNvidiaGpuConfig(t *testing.T) {
 	checkIfSchemaValidateAsExpected(t, "../../translator/config/sampleSchema/validNvidiaGpuConfig.json", true, map[string]int{})
 }

translator/config/sampleSchema/validCollectdConfig.json

@@ -0,0 +1,18 @@
+{
+  "metrics": {
+    "metrics_collected": {
+      "collectd": {
+        "service_address": "udp://127.0.0.1:25826",
+        "name_prefix": "collectd_",
+        "collectd_auth_file": "/etc/collectd/auth_file",
+        "collectd_security_level": "encrypt",
+        "metrics_aggregation_interval": 60,
+        "append_dimensions": {
+          "InstanceId": "${aws:InstanceId}",
+          "Component": "MyService",
+          "Environment": "Production"
+        }
+      }
+    }
+  }
+}

translator/config/schema.json

@@ -286,6 +286,9 @@
               "type": "string",
               "minLength": 1,
               "maxLength": 4096
+            },
+            "append_dimensions": {
+              "$ref": "#/definitions/generalAppendDimensionsDefinition"
             }
           },
           "additionalProperties": false

translator/tocwconfig/sampleConfig/collectd_append_dimensions_linux.conf

@@ -0,0 +1,33 @@
+[agent]
+  collection_jitter = "0s"
+  debug = false
+  flush_interval = "1s"
+  flush_jitter = "0s"
+  hostname = ""
+  interval = "60s"
+  logfile = "/opt/aws/amazon-cloudwatch-agent/logs/amazon-cloudwatch-agent.log"
+  logtarget = "lumberjack"
+  metric_batch_size = 1000
+  metric_buffer_limit = 10000
+  omit_hostname = false
+  precision = ""
+  quiet = false
+  round_interval = false
+
+[inputs]
+
+  [[inputs.socket_listener]]
+    collectd_auth_file = "/etc/collectd/auth_file"
+    collectd_security_level = "encrypt"
+    collectd_typesdb = ["/usr/share/collectd/types.db"]
+    data_format = "collectd"
+    name_prefix = "collectd_"
+    service_address = "udp://127.0.0.1:25826"
+    [inputs.socket_listener.tags]
+      "aws:AggregationInterval" = "60s"
+      d1 = "foo"
+      d2 = "bar"
+
+[outputs]
+
+  [[outputs.cloudwatch]]

translator/tocwconfig/sampleConfig/collectd_append_dimensions_linux.json

@@ -0,0 +1,18 @@
+{
+  "metrics": {
+    "metrics_collected": {
+      "collectd": {
+        "service_address": "udp://127.0.0.1:25826",
+        "name_prefix": "collectd_",
+        "collectd_auth_file": "/etc/collectd/auth_file",
+        "collectd_security_level": "encrypt",
+        "collectd_typesdb": ["/usr/share/collectd/types.db"],
+        "metrics_aggregation_interval": 60,
+        "append_dimensions": {
+          "d1": "foo",
+          "d2": "bar"
+        }
+      }
+    }
+  }
+}

translator/tocwconfig/sampleConfig/collectd_append_dimensions_linux.yaml

@@ -0,0 +1,67 @@
+exporters:
+    awscloudwatch:
+        force_flush_interval: 1m0s
+        max_datums_per_call: 1000
+        max_values_per_datum: 150
+        middleware: agenthealth/metrics
+        namespace: CWAgent
+        region: us-west-2
+        resource_to_telemetry_conversion:
+            enabled: true
+extensions:
+    agenthealth/metrics:
+        is_usage_data_enabled: true
+        stats:
+            operations:
+                - PutMetricData
+            usage_flags:
+                mode: EC2
+                region_type: ACJ
+    agenthealth/statuscode:
+        is_status_code_enabled: true
+        is_usage_data_enabled: true
+        stats:
+            usage_flags:
+                mode: EC2
+                region_type: ACJ
+    entitystore:
+        mode: ec2
+        region: us-west-2
+processors:
+    awsentity/service/telegraf:
+        entity_type: Service
+        platform: ec2
+        scrape_datapoint_attribute: true
+receivers:
+    telegraf_socket_listener:
+        collection_interval: 1m0s
+        initial_delay: 1s
+        timeout: 0s
+service:
+    extensions:
+        - agenthealth/metrics
+        - agenthealth/statuscode
+        - entitystore
+    pipelines:
+        metrics/hostCustomMetrics:
+            exporters:
+                - awscloudwatch
+            processors:
+                - awsentity/service/telegraf
+            receivers:
+                - telegraf_socket_listener
+    telemetry:
+        logs:
+            encoding: console
+            level: info
+            output_paths:
+                - /opt/aws/amazon-cloudwatch-agent/logs/amazon-cloudwatch-agent.log
+            sampling:
+                enabled: true
+                initial: 2
+                thereafter: 500
+                tick: 10s
+        metrics:
+            level: None
+        traces:
+            level: None

translator/tocwconfig/tocwconfig_test.go

@@ -415,6 +415,15 @@ func TestCollectDConfig(t *testing.T) {
 	checkTranslation(t, "collectd_config_linux", "darwin", nil, "")
 }
 
+// CollectD with append_dimensions
+func TestCollectDAppendDimensionsConfig(t *testing.T) {
+	resetContext(t)
+	context.CurrentContext().SetMode(config.ModeEC2)
+	expectedEnvVars := map[string]string{}
+	checkTranslation(t, "collectd_append_dimensions_linux", "linux", expectedEnvVars, "")
+	checkTranslation(t, "collectd_append_dimensions_linux", "darwin", nil, "")
+}
+
 // diskio
 func TestDiskIOTelegrafConfig(t *testing.T) {
 	resetContext(t)

translator/translate/metrics/metrics_collect/collectd/collectd.go

@@ -6,6 +6,7 @@ package collected
 import (
 	"github.com/aws/amazon-cloudwatch-agent/translator"
 	parent "github.com/aws/amazon-cloudwatch-agent/translator/translate/metrics/metrics_collect"
+	"github.com/aws/amazon-cloudwatch-agent/translator/translate/metrics/util"
 )
 
 //
@@ -50,7 +51,7 @@ func (obj *CollectD) ApplyRule(input interface{}) (returnKey string, returnVal i
 		returnVal = ""
 	} else {
 		//If exists, process it
-		//Check if there are some config entry with rules applied
+		util.ProcessAppendDimensions(m[SectionKey].(map[string]interface{}), SectionKey, result)
 		result = translator.ProcessRuleToMergeAndApply(m[SectionKey], ChildRule, result)
 		resArray = append(resArray, result)
 		returnKey = SectionMappedKey

translator/translate/metrics/metrics_collect/collectd/collectd_test.go

@@ -4,26 +4,23 @@
 package collected
 
 import (
-	"encoding/json"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+
+	"github.com/aws/amazon-cloudwatch-agent/translator/translate/metrics/testutil"
+	"github.com/aws/amazon-cloudwatch-agent/translator/translate/util"
 )
 
 func TestCollectD_HappyCase(t *testing.T) {
-	obj := new(CollectD)
-	var input interface{}
-	err := json.Unmarshal([]byte(`{"collectd": {
+	_, actual := testutil.UnmarshalAndApplyRule(t, `{"collectd": {
 		"service_address": "udp://127.0.0.1:123",
 		"name_prefix": "collectd_prefix_",
 		"collectd_auth_file": "/etc/collectd/_auth_file",
 		"collectd_security_level": "none",
 		"collectd_typesdb": ["/usr/share/collectd/types.db", "/custom_location/types.db"],
 		"metrics_aggregation_interval": 30
-	}}`), &input)
-	assert.NoError(t, err)
-
-	_, actual := obj.ApplyRule(input)
+	}}`, new(CollectD))
 
 	expect := []interface{}{
 		map[string]interface{}{
@@ -41,12 +38,7 @@ func TestCollectD_HappyCase(t *testing.T) {
 }
 
 func TestCollectD_MinimumConfig(t *testing.T) {
-	obj := new(CollectD)
-	var input interface{}
-	err := json.Unmarshal([]byte(`{"collectd": {}}`), &input)
-	assert.NoError(t, err)
-
-	_, actual := obj.ApplyRule(input)
+	_, actual := testutil.UnmarshalAndApplyRule(t, `{"collectd": {}}`, new(CollectD))
 
 	expect := []interface{}{
 		map[string]interface{}{
@@ -62,3 +54,100 @@ func TestCollectD_MinimumConfig(t *testing.T) {
 
 	assert.Equal(t, expect, actual)
 }
+
+func TestCollectD_WithAppendDimensions(t *testing.T) {
+	cleanup := testutil.MockEC2Metadata(&util.Metadata{InstanceID: "i-1234567890abcdef0", InstanceType: "t3.medium"})
+	defer cleanup()
+
+	_, actual := testutil.UnmarshalAndApplyRule(t, `{"collectd": {
+		"service_address": "udp://127.0.0.1:123",
+		"append_dimensions": {
+			"InstanceId": "${aws:InstanceId}",
+			"CustomDimension": "CustomValue"
+		}
+	}}`, new(CollectD))
+
+	testutil.AssertDimensionsEqual(t, actual, map[string]interface{}{
+		"aws:AggregationInterval": "60s",
+		"CustomDimension":         "CustomValue",
+		"InstanceId":              "i-1234567890abcdef0",
+	})
+}
+
+func TestCollectD_WithAppendDimensionsAndAggregationInterval(t *testing.T) {
+	_, actual := testutil.UnmarshalAndApplyRule(t, `{"collectd": {
+		"metrics_aggregation_interval": 30,
+		"append_dimensions": {
+			"Environment": "Production",
+			"Team": "Infrastructure"
+		}
+	}}`, new(CollectD))
+
+	expect := []interface{}{
+		map[string]interface{}{
+			"data_format":             "collectd",
+			"service_address":         "udp://127.0.0.1:25826",
+			"name_prefix":             "collectd_",
+			"collectd_auth_file":      "/etc/collectd/auth_file",
+			"collectd_security_level": "encrypt",
+			"collectd_typesdb":        []interface{}{"/usr/share/collectd/types.db"},
+			"tags": map[string]interface{}{
+				"aws:AggregationInterval": "30s",
+				"Environment":             "Production",
+				"Team":                    "Infrastructure",
+			},
+		},
+	}
+
+	assert.Equal(t, expect, actual)
+}
+
+func TestCollectD_WithFullConfigAndAppendDimensions(t *testing.T) {
+	cleanup := testutil.MockEC2Metadata(&util.Metadata{InstanceID: "i-1234567890abcdef0", InstanceType: "t3.large", ImageID: "ami-12345678"})
+	defer cleanup()
+
+	_, actual := testutil.UnmarshalAndApplyRule(t, `{"collectd": {
+		"service_address": "udp://127.0.0.1:123",
+		"name_prefix": "collectd_prefix_",
+		"collectd_auth_file": "/etc/collectd/_auth_file",
+		"collectd_security_level": "none",
+		"collectd_typesdb": ["/usr/share/collectd/types.db", "/custom_location/types.db"],
+		"metrics_aggregation_interval": 30,
+		"append_dimensions": {
+			"InstanceId": "${aws:InstanceId}",
+			"InstanceType": "${aws:InstanceType}",
+			"ImageId": "${aws:ImageId}",
+			"CustomTag": "MyValue"
+		}
+	}}`, new(CollectD))
+
+	testutil.AssertDimensionsEqual(t, actual, map[string]interface{}{
+		"aws:AggregationInterval": "30s",
+		"CustomTag":               "MyValue",
+		"InstanceId":              "i-1234567890abcdef0",
+		"InstanceType":            "t3.large",
+		"ImageId":                 "ami-12345678",
+	})
+}
+
+func TestCollectD_EmptyAppendDimensions(t *testing.T) {
+	_, actual := testutil.UnmarshalAndApplyRule(t, `{"collectd": {
+		"append_dimensions": {}
+	}}`, new(CollectD))
+
+	expect := []interface{}{
+		map[string]interface{}{
+			"data_format":             "collectd",
+			"service_address":         "udp://127.0.0.1:25826",
+			"name_prefix":             "collectd_",
+			"collectd_auth_file":      "/etc/collectd/auth_file",
+			"collectd_security_level": "encrypt",
+			"collectd_typesdb":        []interface{}{"/usr/share/collectd/types.db"},
+			"tags": map[string]interface{}{
+				"aws:AggregationInterval": "60s",
+			},
+		},
+	}
+
+	assert.Equal(t, expect, actual)
+}

translator/translate/metrics/metrics_collect/ethtool/ethtool_test.go

@@ -8,15 +8,12 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+
+	"github.com/aws/amazon-cloudwatch-agent/translator/translate/metrics/testutil"
 )
 
 func TestDefaultConfig(t *testing.T) {
-	d := new(Ethtool)
-	var input interface{}
-	err := json.Unmarshal([]byte(`{"ethtool": {
-					}}`), &input)
-	assert.NoError(t, err)
-	_, actual := d.ApplyRule(input)
+	_, actual := testutil.UnmarshalAndApplyRule(t, `{"ethtool": {}}`, new(Ethtool))
 
 	expected := []interface{}{map[string]interface{}{
 		"interface_include": []string{"*"},
@@ -26,9 +23,7 @@ func TestDefaultConfig(t *testing.T) {
 }
 
 func TestFullConfig(t *testing.T) {
-	d := new(Ethtool)
-	var input interface{}
-	err := json.Unmarshal([]byte(`{"ethtool": {
+	_, actual := testutil.UnmarshalAndApplyRule(t, `{"ethtool": {
 					"interface_include": [
 						"eth0"
 					],
@@ -41,9 +36,7 @@ func TestFullConfig(t *testing.T) {
 					"append_dimensions":{
 						"name":"sampleName"
 					}
-					}}`), &input)
-	assert.NoError(t, err)
-	_, actual := d.ApplyRule(input)
+					}}`, new(Ethtool))
 
 	expected := []interface{}{map[string]interface{}{
 		"interface_include": []string{"eth0"},