Skip to content

Commit a2dfe56

Browse files
TravisStarkTravisStark
committed
Adds append_dimensions to collectd, supporting custom metric dimensions
1 parent bd03a3d commit a2dfe56

File tree

13 files changed

+355
-67
lines changed

13 files changed

+355
-67
lines changed

‎translator/cmdutil/translatorutil_test.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -156,6 +156,10 @@ func TestEthtoolConfig(t *testing.T) {
156156
checkIfSchemaValidateAsExpected(t, "../../translator/config/sampleSchema/validEthtoolConfig.json", true, map[string]int{})
157157
}
158158

159+
func TestCollectdConfig(t *testing.T) {
160+
checkIfSchemaValidateAsExpected(t, "../../translator/config/sampleSchema/validCollectdConfig.json", true, map[string]int{})
161+
}
162+
159163
func TestNvidiaGpuConfig(t *testing.T) {
160164
checkIfSchemaValidateAsExpected(t, "../../translator/config/sampleSchema/validNvidiaGpuConfig.json", true, map[string]int{})
161165
}

‎translator/config/sampleSchema/validCollectdConfig.json‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
{
2+
"metrics": {
3+
"metrics_collected": {
4+
"collectd": {
5+
"service_address": "udp://127.0.0.1:25826",
6+
"name_prefix": "collectd_",
7+
"collectd_auth_file": "/etc/collectd/auth_file",
8+
"collectd_security_level": "encrypt",
9+
"metrics_aggregation_interval": 60,
10+
"append_dimensions": {
11+
"InstanceId": "${aws:InstanceId}",
12+
"Component": "MyService",
13+
"Environment": "Production"
14+
}
15+
}
16+
}
17+
}
18+
}

‎translator/config/schema.json‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -286,6 +286,9 @@
286286
"type": "string",
287287
"minLength": 1,
288288
"maxLength": 4096
289+
},
290+
"append_dimensions": {
291+
"$ref": "#/definitions/generalAppendDimensionsDefinition"
289292
}
290293
},
291294
"additionalProperties": false

‎translator/tocwconfig/sampleConfig/collectd_append_dimensions_linux.conf‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
[agent]
2+
collection_jitter = "0s"
3+
debug = false
4+
flush_interval = "1s"
5+
flush_jitter = "0s"
6+
hostname = ""
7+
interval = "60s"
8+
logfile = "/opt/aws/amazon-cloudwatch-agent/logs/amazon-cloudwatch-agent.log"
9+
logtarget = "lumberjack"
10+
metric_batch_size = 1000
11+
metric_buffer_limit = 10000
12+
omit_hostname = false
13+
precision = ""
14+
quiet = false
15+
round_interval = false
16+
17+
[inputs]
18+
19+
[[inputs.socket_listener]]
20+
collectd_auth_file = "/etc/collectd/auth_file"
21+
collectd_security_level = "encrypt"
22+
collectd_typesdb = ["/usr/share/collectd/types.db"]
23+
data_format = "collectd"
24+
name_prefix = "collectd_"
25+
service_address = "udp://127.0.0.1:25826"
26+
[inputs.socket_listener.tags]
27+
"aws:AggregationInterval" = "60s"
28+
d1 = "foo"
29+
d2 = "bar"
30+
31+
[outputs]
32+
33+
[[outputs.cloudwatch]]

‎translator/tocwconfig/sampleConfig/collectd_append_dimensions_linux.json‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
{
2+
"metrics": {
3+
"metrics_collected": {
4+
"collectd": {
5+
"service_address": "udp://127.0.0.1:25826",
6+
"name_prefix": "collectd_",
7+
"collectd_auth_file": "/etc/collectd/auth_file",
8+
"collectd_security_level": "encrypt",
9+
"collectd_typesdb": ["/usr/share/collectd/types.db"],
10+
"metrics_aggregation_interval": 60,
11+
"append_dimensions": {
12+
"d1": "foo",
13+
"d2": "bar"
14+
}
15+
}
16+
}
17+
}
18+
}

‎translator/tocwconfig/sampleConfig/collectd_append_dimensions_linux.yaml‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
exporters:
2+
awscloudwatch:
3+
force_flush_interval: 1m0s
4+
max_datums_per_call: 1000
5+
max_values_per_datum: 150
6+
middleware: agenthealth/metrics
7+
namespace: CWAgent
8+
region: us-west-2
9+
resource_to_telemetry_conversion:
10+
enabled: true
11+
extensions:
12+
agenthealth/metrics:
13+
is_usage_data_enabled: true
14+
stats:
15+
operations:
16+
- PutMetricData
17+
usage_flags:
18+
mode: EC2
19+
region_type: ACJ
20+
agenthealth/statuscode:
21+
is_status_code_enabled: true
22+
is_usage_data_enabled: true
23+
stats:
24+
usage_flags:
25+
mode: EC2
26+
region_type: ACJ
27+
entitystore:
28+
mode: ec2
29+
region: us-west-2
30+
processors:
31+
awsentity/service/telegraf:
32+
entity_type: Service
33+
platform: ec2
34+
scrape_datapoint_attribute: true
35+
receivers:
36+
telegraf_socket_listener:
37+
collection_interval: 1m0s
38+
initial_delay: 1s
39+
timeout: 0s
40+
service:
41+
extensions:
42+
- agenthealth/metrics
43+
- agenthealth/statuscode
44+
- entitystore
45+
pipelines:
46+
metrics/hostCustomMetrics:
47+
exporters:
48+
- awscloudwatch
49+
processors:
50+
- awsentity/service/telegraf
51+
receivers:
52+
- telegraf_socket_listener
53+
telemetry:
54+
logs:
55+
encoding: console
56+
level: info
57+
output_paths:
58+
- /opt/aws/amazon-cloudwatch-agent/logs/amazon-cloudwatch-agent.log
59+
sampling:
60+
enabled: true
61+
initial: 2
62+
thereafter: 500
63+
tick: 10s
64+
metrics:
65+
level: None
66+
traces:
67+
level: None

‎translator/tocwconfig/tocwconfig_test.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -415,6 +415,15 @@ func TestCollectDConfig(t *testing.T) {
415415
checkTranslation(t, "collectd_config_linux", "darwin", nil, "")
416416
}
417417

418+
// CollectD with append_dimensions
419+
func TestCollectDAppendDimensionsConfig(t *testing.T) {
420+
resetContext(t)
421+
context.CurrentContext().SetMode(config.ModeEC2)
422+
expectedEnvVars := map[string]string{}
423+
checkTranslation(t, "collectd_append_dimensions_linux", "linux", expectedEnvVars, "")
424+
checkTranslation(t, "collectd_append_dimensions_linux", "darwin", nil, "")
425+
}
426+
418427
// diskio
419428
func TestDiskIOTelegrafConfig(t *testing.T) {
420429
resetContext(t)

‎translator/translate/metrics/metrics_collect/collectd/collectd.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ package collected
66
import (
77
"github.com/aws/amazon-cloudwatch-agent/translator"
88
parent "github.com/aws/amazon-cloudwatch-agent/translator/translate/metrics/metrics_collect"
9+
"github.com/aws/amazon-cloudwatch-agent/translator/translate/metrics/util"
910
)
1011

1112
//
@@ -50,7 +51,7 @@ func (obj *CollectD) ApplyRule(input interface{}) (returnKey string, returnVal i
5051
returnVal = ""
5152
} else {
5253
//If exists, process it
53-
//Check if there are some config entry with rules applied
54+
util.ProcessAppendDimensions(m[SectionKey].(map[string]interface{}), SectionKey, result)
5455
result = translator.ProcessRuleToMergeAndApply(m[SectionKey], ChildRule, result)
5556
resArray = append(resArray, result)
5657
returnKey = SectionMappedKey

‎translator/translate/metrics/metrics_collect/collectd/collectd_test.go‎

Lines changed: 103 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -4,26 +4,23 @@
44
package collected
55

66
import (
7-
"encoding/json"
87
"testing"
98

109
"github.com/stretchr/testify/assert"
10+
11+
"github.com/aws/amazon-cloudwatch-agent/translator/translate/metrics/testutil"
12+
"github.com/aws/amazon-cloudwatch-agent/translator/translate/util"
1113
)
1214

1315
func TestCollectD_HappyCase(t *testing.T) {
14-
obj := new(CollectD)
15-
var input interface{}
16-
err := json.Unmarshal([]byte(`{"collectd": {
16+
_, actual := testutil.UnmarshalAndApplyRule(t, `{"collectd": {
1717
"service_address": "udp://127.0.0.1:123",
1818
"name_prefix": "collectd_prefix_",
1919
"collectd_auth_file": "/etc/collectd/_auth_file",
2020
"collectd_security_level": "none",
2121
"collectd_typesdb": ["/usr/share/collectd/types.db", "/custom_location/types.db"],
2222
"metrics_aggregation_interval": 30
23-
}}`), &input)
24-
assert.NoError(t, err)
25-
26-
_, actual := obj.ApplyRule(input)
23+
}}`, new(CollectD))
2724

2825
expect := []interface{}{
2926
map[string]interface{}{
@@ -41,12 +38,7 @@ func TestCollectD_HappyCase(t *testing.T) {
4138
}
4239

4340
func TestCollectD_MinimumConfig(t *testing.T) {
44-
obj := new(CollectD)
45-
var input interface{}
46-
err := json.Unmarshal([]byte(`{"collectd": {}}`), &input)
47-
assert.NoError(t, err)
48-
49-
_, actual := obj.ApplyRule(input)
41+
_, actual := testutil.UnmarshalAndApplyRule(t, `{"collectd": {}}`, new(CollectD))
5042

5143
expect := []interface{}{
5244
map[string]interface{}{
@@ -62,3 +54,100 @@ func TestCollectD_MinimumConfig(t *testing.T) {
6254

6355
assert.Equal(t, expect, actual)
6456
}
57+
58+
func TestCollectD_WithAppendDimensions(t *testing.T) {
59+
cleanup := testutil.MockEC2Metadata(&util.Metadata{InstanceID: "i-1234567890abcdef0", InstanceType: "t3.medium"})
60+
defer cleanup()
61+
62+
_, actual := testutil.UnmarshalAndApplyRule(t, `{"collectd": {
63+
"service_address": "udp://127.0.0.1:123",
64+
"append_dimensions": {
65+
"InstanceId": "${aws:InstanceId}",
66+
"CustomDimension": "CustomValue"
67+
}
68+
}}`, new(CollectD))
69+
70+
testutil.AssertDimensionsEqual(t, actual, map[string]interface{}{
71+
"aws:AggregationInterval": "60s",
72+
"CustomDimension": "CustomValue",
73+
"InstanceId": "i-1234567890abcdef0",
74+
})
75+
}
76+
77+
func TestCollectD_WithAppendDimensionsAndAggregationInterval(t *testing.T) {
78+
_, actual := testutil.UnmarshalAndApplyRule(t, `{"collectd": {
79+
"metrics_aggregation_interval": 30,
80+
"append_dimensions": {
81+
"Environment": "Production",
82+
"Team": "Infrastructure"
83+
}
84+
}}`, new(CollectD))
85+
86+
expect := []interface{}{
87+
map[string]interface{}{
88+
"data_format": "collectd",
89+
"service_address": "udp://127.0.0.1:25826",
90+
"name_prefix": "collectd_",
91+
"collectd_auth_file": "/etc/collectd/auth_file",
92+
"collectd_security_level": "encrypt",
93+
"collectd_typesdb": []interface{}{"/usr/share/collectd/types.db"},
94+
"tags": map[string]interface{}{
95+
"aws:AggregationInterval": "30s",
96+
"Environment": "Production",
97+
"Team": "Infrastructure",
98+
},
99+
},
100+
}
101+
102+
assert.Equal(t, expect, actual)
103+
}
104+
105+
func TestCollectD_WithFullConfigAndAppendDimensions(t *testing.T) {
106+
cleanup := testutil.MockEC2Metadata(&util.Metadata{InstanceID: "i-1234567890abcdef0", InstanceType: "t3.large", ImageID: "ami-12345678"})
107+
defer cleanup()
108+
109+
_, actual := testutil.UnmarshalAndApplyRule(t, `{"collectd": {
110+
"service_address": "udp://127.0.0.1:123",
111+
"name_prefix": "collectd_prefix_",
112+
"collectd_auth_file": "/etc/collectd/_auth_file",
113+
"collectd_security_level": "none",
114+
"collectd_typesdb": ["/usr/share/collectd/types.db", "/custom_location/types.db"],
115+
"metrics_aggregation_interval": 30,
116+
"append_dimensions": {
117+
"InstanceId": "${aws:InstanceId}",
118+
"InstanceType": "${aws:InstanceType}",
119+
"ImageId": "${aws:ImageId}",
120+
"CustomTag": "MyValue"
121+
}
122+
}}`, new(CollectD))
123+
124+
testutil.AssertDimensionsEqual(t, actual, map[string]interface{}{
125+
"aws:AggregationInterval": "30s",
126+
"CustomTag": "MyValue",
127+
"InstanceId": "i-1234567890abcdef0",
128+
"InstanceType": "t3.large",
129+
"ImageId": "ami-12345678",
130+
})
131+
}
132+
133+
func TestCollectD_EmptyAppendDimensions(t *testing.T) {
134+
_, actual := testutil.UnmarshalAndApplyRule(t, `{"collectd": {
135+
"append_dimensions": {}
136+
}}`, new(CollectD))
137+
138+
expect := []interface{}{
139+
map[string]interface{}{
140+
"data_format": "collectd",
141+
"service_address": "udp://127.0.0.1:25826",
142+
"name_prefix": "collectd_",
143+
"collectd_auth_file": "/etc/collectd/auth_file",
144+
"collectd_security_level": "encrypt",
145+
"collectd_typesdb": []interface{}{"/usr/share/collectd/types.db"},
146+
"tags": map[string]interface{}{
147+
"aws:AggregationInterval": "60s",
148+
},
149+
},
150+
}
151+
152+
assert.Equal(t, expect, actual)
153+
}

‎translator/translate/metrics/metrics_collect/ethtool/ethtool_test.go‎

Lines changed: 5 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -8,15 +8,12 @@ import (
88
"testing"
99

1010
"github.com/stretchr/testify/assert"
11+
12+
"github.com/aws/amazon-cloudwatch-agent/translator/translate/metrics/testutil"
1113
)
1214

1315
func TestDefaultConfig(t *testing.T) {
14-
d := new(Ethtool)
15-
var input interface{}
16-
err := json.Unmarshal([]byte(`{"ethtool": {
17-
}}`), &input)
18-
assert.NoError(t, err)
19-
_, actual := d.ApplyRule(input)
16+
_, actual := testutil.UnmarshalAndApplyRule(t, `{"ethtool": {}}`, new(Ethtool))
2017

2118
expected := []interface{}{map[string]interface{}{
2219
"interface_include": []string{"*"},
@@ -26,9 +23,7 @@ func TestDefaultConfig(t *testing.T) {
2623
}
2724

2825
func TestFullConfig(t *testing.T) {
29-
d := new(Ethtool)
30-
var input interface{}
31-
err := json.Unmarshal([]byte(`{"ethtool": {
26+
_, actual := testutil.UnmarshalAndApplyRule(t, `{"ethtool": {
3227
"interface_include": [
3328
"eth0"
3429
],
@@ -41,9 +36,7 @@ func TestFullConfig(t *testing.T) {
4136
"append_dimensions":{
4237
"name":"sampleName"
4338
}
44-
}}`), &input)
45-
assert.NoError(t, err)
46-
_, actual := d.ApplyRule(input)
39+
}}`, new(Ethtool))
4740

4841
expected := []interface{}{map[string]interface{}{
4942
"interface_include": []string{"eth0"},

0 commit comments

Comments
 (0)