Skip to content

Commit 7f58ca5

Browse files
jensendwtruthbk
jensendw
authored and
truthbk
committed
added kafka integration (#343)
* added kafka integration * fix parameter description
1 parent eb6d773 commit 7f58ca5

3 files changed

Lines changed: 227 additions & 69 deletions

File tree

‎manifests/integrations/kafka.pp‎

Lines changed: 45 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,47 +1,65 @@
11
# Class: datadog_agent::integrations::kafka
22
#
3-
# This class will install the necessary configuration for the kafka
4-
# integration
3+
# This class will install the necessary configuration for the kafka integration
54
#
65
# Parameters:
76
# $host:
8-
# The hostname kafka is running on
9-
# $port:
10-
# The port to connect on
11-
# $user
12-
# The user for the datadog user
7+
# The host kafka is running on. Defaults to 'localhost'
8+
# $username
9+
# Optionally specify username for connection
1310
# $password
14-
# The password for the datadog user
11+
# Optionally specify password for connection
12+
# $port
13+
# The port kafka is running on. Defaults to 9999
14+
# $name
15+
# Name given to kafka instance
16+
# $process_name_regex
17+
# Instead of specifying a host, and port. The agent can connect using the attach api.
18+
# $tools_jar_path
19+
# Path to tools jar needs to be set when process_name_regex is set
20+
# $java_bin_path
21+
# Path to java binary, should be set if agent cant find your java executable
22+
# $trust_store_path
23+
# Path to the trust store, should be set if ssl is enabled
24+
# $trust_store_password
25+
# Password to the trust store
1526
# $tags
1627
# Optional array of tags
1728
#
29+
#
1830
# Sample Usage:
1931
#
2032
# class { 'datadog_agent::integrations::kafka' :
21-
# host => 'localhost',
22-
# tags => {
23-
# environment => "production",
24-
# },
33+
# servers => [
34+
# {
35+
# 'host' => 'localhost',
36+
# 'username' => 'kafka_username',
37+
# 'password' => 'kafka_password',
38+
# 'port' => '9999',
39+
# 'name' => 'kafka_instance',
40+
# 'process_name_regex' => '.*process_name.*',
41+
# 'tools_jar_path' => '/usr/lib/jvm/java-7-openjdk-amd64/lib/tools.jar',
42+
# 'java_bin_path' => '/path/to/java',
43+
# 'trust_store_path' => '/path/to/trustStore.jks',
44+
# 'trust_store_password' => 'password',
45+
# 'tags' => ['env: test', 'sometag: someinfo'],
46+
# },
47+
# {
48+
# 'host' => 'localhost',
49+
# 'port' => '9999',
50+
# 'tags' => [],
51+
# },
52+
# ]
2553
# }
2654
#
27-
#
2855
class datadog_agent::integrations::kafka(
29-
$host = 'localhost',
30-
$port = 9999,
31-
$user = undef,
32-
$password = undef,
33-
$tags = { kafka => broker },
56+
$servers = [{'host' => 'localhost', 'port' => '9999'}]
3457
) inherits datadog_agent::params {
35-
require ::datadog_agent
36-
validate_hash($tags)
58+
include datadog_agent
3759

38-
if !$::datadog_agent::agent5_enable {
39-
$dst = "${datadog_agent::conf6_dir}/kafka.yaml"
40-
} else {
41-
$dst = "${datadog_agent::conf_dir}/kafka.yaml"
42-
}
60+
validate_array($servers)
4361

44-
file { $dst:
62+
file { "${datadog_agent::params::conf_dir}/kafka.yaml":
4563
ensure => file,
4664
owner => $datadog_agent::params::dd_user,
4765
group => $datadog_agent::params::dd_group,
@@ -50,4 +68,4 @@
5068
require => Package[$datadog_agent::params::package_name],
5169
notify => Service[$datadog_agent::params::service_name],
5270
}
53-
}
71+
}

‎spec/classes/datadog_agent_integrations_kafka_spec.rb‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
require 'spec_helper'
2+
3+
describe 'datadog_agent::integrations::kafka' do
4+
let(:facts) {{
5+
operatingsystem: 'Ubuntu',
6+
}}
7+
let(:conf_dir) { '/etc/dd-agent/conf.d' }
8+
let(:dd_user) { 'dd-agent' }
9+
let(:dd_group) { 'root' }
10+
let(:dd_package) { 'datadog-agent' }
11+
let(:dd_service) { 'datadog-agent' }
12+
let(:conf_file) { "#{conf_dir}/kafka.yaml" }
13+
14+
it { should compile.with_all_deps }
15+
it { should contain_file(conf_file).with(
16+
owner: dd_user,
17+
group: dd_group,
18+
mode: '0600',
19+
)}
20+
it { should contain_file(conf_file).that_requires("Package[#{dd_package}]") }
21+
it { should contain_file(conf_file).that_notifies("Service[#{dd_service}]") }
22+
23+
context 'with default parameters' do
24+
it { should contain_file(conf_file).with_content(%r{- host: localhost\s+port: 9999}) }
25+
it { should contain_file(conf_file).without_content(%r{tags:}) }
26+
end
27+
28+
29+
context 'with one kafka broker' do
30+
let(:params) {{
31+
servers: [
32+
{
33+
'host' => 'localhost',
34+
'port' => '9997',
35+
'tags' => %w{ kafka:broker tag1:value1 },
36+
}
37+
]
38+
}}
39+
40+
it { should contain_file(conf_file).with_content(%r{host: localhost\s+port: 9997\s+tags:\s+- kafka:broker\s+- tag1:value1}) }
41+
end
42+
43+
context 'with two kafka brokers' do
44+
let(:params) {{
45+
servers: [
46+
{
47+
'host' => 'localhost',
48+
'port' => '9997',
49+
'tags' => %w{ kafka:broker tag1:value1 },
50+
},
51+
{
52+
'host' => 'remotehost',
53+
'port' => '9998',
54+
'tags' => %w{ kafka:remote tag2:value2 },
55+
}
56+
]
57+
}}
58+
59+
it { should contain_file(conf_file).with_content(%r{host: localhost\s+port: 9997\s+tags:\s+- kafka:broker\s+- tag1:value1}) }
60+
it { should contain_file(conf_file).with_content(%r{host: remotehost\s+port: 9998\s+tags:\s+- kafka:remote\s+- tag2:value2}) }
61+
62+
end
63+
64+
context 'one kafka broker all options' do
65+
let(:params) {{
66+
servers: [
67+
{
68+
'host' => 'localhost',
69+
'port' => '9997',
70+
'tags' => %w{ kafka:broker tag1:value1 },
71+
'username' => 'username',
72+
'password' => 'password',
73+
'process_name_regex' => 'regex',
74+
'tools_jar_path' => 'tools.jar',
75+
'name' => 'kafka_instance',
76+
'java_bin_path' => '/path/to/java',
77+
'trust_store_path' => '/path/to/trustStore.jks',
78+
'trust_store_password' => 'password'
79+
}
80+
]
81+
}}
82+
83+
it { should contain_file(conf_file).with_content(%r{host: localhost\s+port: 9997\s+tags:\s+- kafka:broker\s+- tag1:value1\s+user: username\s+password: password\s+process_name_regex: regex\s+tools_jar_path: tools.jar\s+name: kafka_instance\s+java_bin_path: /path/to/java\s+trust_store_path: /path/to/trustStore.jks\s+trust_store_password: password}) }
84+
end
85+
=begin
86+
context 'with multiple mongos' do
87+
let(:params) {{
88+
servers: [
89+
{
90+
'host' => '127.0.0.1',
91+
'port' => '34567',
92+
'tags' => %w{foo bar},
93+
},
94+
{
95+
'host' => '127.0.0.2',
96+
'port' => '45678',
97+
'tags' => %w{baz bat},
98+
}
99+
]
100+
}}
101+
it { should contain_file(conf_file).with_content(%r{server: mongodb://127.0.0.1:34567/\s+tags:\s+- foo\s+- bar}) }
102+
it { should contain_file(conf_file).with_content(%r{server: mongodb://127.0.0.2:45678/\s+tags:\s+- baz\s+- bat}) }
103+
it { should contain_file(conf_file).with_content(%r{server:.*127.0.0.1.*server:.*127.0.0.2}m) }
104+
end
105+
106+
context 'without tags' do
107+
let(:params) {{
108+
servers: [
109+
{
110+
'host' => '127.0.0.1',
111+
'port' => '12345',
112+
}
113+
]
114+
}}
115+
116+
end
117+
118+
context 'weird tags' do
119+
let(:params) {{
120+
servers: [
121+
{
122+
'host' => '127.0.0.1',
123+
'port' => '56789',
124+
'tags' => 'word'
125+
}
126+
]
127+
}}
128+
it { should_not compile }
129+
end
130+
=end
131+
end

‎templates/agent-conf.d/kafka.yaml.erb‎

Lines changed: 51 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -1,51 +1,57 @@
1-
### MANAGED BY PUPPET
1+
#
2+
# MANAGED BY PUPPET
3+
#
24

35
##########
46
# WARNING
57
##########
68
# This sample works only for Kafka >= 0.8.2.
79
# If you are running a version older than that, you can refer to agent 5.2.x released
810
# sample files, https://raw.githubusercontent.com/DataDog/dd-agent/5.2.1/conf.d/kafka.yaml.example
9-
1011
instances:
11-
- host: <%= @host %>
12-
port: <%= @port %> # This is the JMX port on which Kafka exposes its metrics (usually 9999)
13-
<% if @tags.length > 0 -%>
12+
<% @servers.each do |server| -%>
13+
- host: <%= server['host'] %>
14+
port: <%= server['port'] %>
15+
<%- if !server['tags'].nil? && server['tags'].any? -%>
1416
tags:
15-
<%- @tags.each do |tag_name, tag_value| -%>
16-
<%= tag_name %>: <%= tag_value %>
17-
<% end -%>
18-
<% end -%>
19-
<% if @user -%>
20-
user: <%= @user %>
21-
password: <%= @password %>
17+
<%- server['tags'].each do |tag| -%>
18+
- <%= tag %>
19+
<%- end -%>
20+
<%- end -%>
21+
<%- if !server['username'].nil? -%>
22+
user: <%= server['username'] %>
23+
<%- end -%>
24+
<%- if !server['password'].nil? -%>
25+
password: <%= server['password'] %>
26+
<%- end -%>
27+
<%- if !server['process_name_regex'].nil? -%>
28+
process_name_regex: <%= server['process_name_regex'] %>
29+
<%- end -%>
30+
<%- if !server['tools_jar_path'].nil? -%>
31+
tools_jar_path: <%= server['tools_jar_path'] %>
32+
<%- end -%>
33+
<%- if !server['name'].nil? -%>
34+
name: <%= server['name'] %>
35+
<%- end -%>
36+
<%- if !server['java_bin_path'].nil? -%>
37+
java_bin_path: <%= server['java_bin_path'] %>
38+
<%- end -%>
39+
<%- if !server['trust_store_path'].nil? -%>
40+
trust_store_path: <%= server['trust_store_path'] %>
41+
<%- end -%>
42+
<%- if !server['trust_store_password'].nil? -%>
43+
trust_store_password: <%= server['trust_store_password'] %>
44+
<%- end -%>
2245
<% end -%>
23-
# process_name_regex: .*process_name.* # Instead of specifying a host, and port. The agent can connect using the attach api.
24-
# # This requires the JDK to be installed and the path to tools.jar to be set below.
25-
# tools_jar_path: /usr/lib/jvm/java-7-openjdk-amd64/lib/tools.jar # To be set when process_name_regex is set
26-
# name: kafka_instance
27-
# java_bin_path: /path/to/java # Optional, should be set if the agent cannot find your java executable
28-
# trust_store_path: /path/to/trustStore.jks # Optional, should be set if ssl is enabled
29-
# trust_store_password: password
30-
# - host: remotehost
31-
# port: 9998 # Producer
32-
# tags:
33-
# kafka: producer0
34-
# env: stage
35-
# newTag: test
36-
# - host: remotehost
37-
# port: 9997 # Consumer
38-
# tags:
39-
# kafka: consumer0
40-
# env: stage
41-
# newTag: test
4246

4347
init_config:
4448
is_jmx: true
4549

4650
# Metrics collected by this check. You should not have to modify this.
4751
conf:
48-
# v0.8.2.x Producers
52+
#
53+
# Producers (only v0.8.2.x)
54+
#
4955
- include:
5056
domain: 'kafka.producer'
5157
bean_regex: 'kafka\.producer:type=ProducerRequestMetrics,name=ProducerRequestRateAndTimeMs,clientId=.*'
@@ -75,8 +81,9 @@ init_config:
7581
metric_type: rate
7682
alias: kafka.producer.message_rate
7783

78-
79-
# v0.9.0.x Producers
84+
#
85+
# Producers (v0.9.0.x to v0.10.2.x)
86+
#
8087
- include:
8188
domain: 'kafka.producer'
8289
bean_regex: 'kafka\.producer:type=producer-metrics,client-id=.*'
@@ -113,8 +120,9 @@ init_config:
113120
metric_type: gauge
114121
alias: kafka.producer.io_wait
115122

116-
117-
# v0.8.2.x Consumers
123+
#
124+
# Consumers (only v0.8.2.x)
125+
#
118126
- include:
119127
domain: 'kafka.consumer'
120128
bean_regex: 'kafka\.consumer:type=ConsumerFetcherManager,name=MaxLag,clientId=.*'
@@ -143,25 +151,26 @@ init_config:
143151
Count:
144152
metric_type: rate
145153
alias: kafka.consumer.messages_in
146-
147-
# Offsets committed to ZooKeeper
148154
- include:
155+
# Offsets committed to ZooKeeper
149156
domain: 'kafka.consumer'
150157
bean_regex: 'kafka\.consumer:type=ZookeeperConsumerConnector,name=ZooKeeperCommitsPerSec,clientId=.*'
151158
attribute:
152159
Count:
153160
metric_type: rate
154161
alias: kafka.consumer.zookeeper_commits
155-
# Offsets committed to Kafka
156162
- include:
163+
# Offsets committed to Kafka
157164
domain: 'kafka.consumer'
158165
bean_regex: 'kafka\.consumer:type=ZookeeperConsumerConnector,name=KafkaCommitsPerSec,clientId=.*'
159166
attribute:
160167
Count:
161168
metric_type: rate
162169
alias: kafka.consumer.kafka_commits
163170

164-
# v0.9.0.x Consumers
171+
#
172+
# Consumers (v0.9.0.x to v0.10.2.x)
173+
#
165174
- include:
166175
domain: 'kafka.consumer'
167176
bean_regex: 'kafka\.consumer:type=consumer-fetch-manager-metrics,client-id=.*'
@@ -311,8 +320,8 @@ init_config:
311320
domain: 'kafka.server'
312321
bean: 'kafka.server:type=KafkaRequestHandlerPool,name=RequestHandlerAvgIdlePercent'
313322
attribute:
314-
Count:
315-
metric_type: rate
323+
OneMinuteRate:
324+
metric_type: gauge
316325
alias: kafka.request.handler.avg.idle.pct.rate
317326
- include:
318327
domain: 'kafka.server'

0 commit comments

Comments
 (0)
⚡