Skip to content

Commit a7830b7

Browse files
dpricha89truthbk
dpricha89
authored and
truthbk
committed
Updated kafka integration to include all stats (DataDog#498)
* Updated kafka integration to include all stats https://github.com/DataDog/integrations-core/blob/master/kafka/datadog_checks/kafka/data/conf.yaml.example * Updated comment * Tags can no longer be used for not being present Tags are part of the static portion of the template so let's look for another conditional parameter.
1 parent 7de3c7f commit a7830b7

2 files changed

Lines changed: 249 additions & 1 deletion

File tree

‎spec/classes/datadog_agent_integrations_kafka_spec.rb‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@
3535

3636
context 'with default parameters' do
3737
it { should contain_file(conf_file).with_content(%r{- host: localhost\s+port: 9999}) }
38-
it { should contain_file(conf_file).without_content(%r{tags:}) }
38+
it { should contain_file(conf_file).without_content(%r{user:}) }
3939
end
4040

4141
context 'with one kafka broker' do

‎templates/agent-conf.d/kafka.yaml.erb‎

Lines changed: 248 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,131 @@ init_config:
118118
metric_type: gauge
119119
alias: kafka.producer.io_wait
120120

121+
#
122+
# Producers (v0.11.x)
123+
#
124+
- include:
125+
domain: 'kafka.producer'
126+
bean_regex: 'kafka.producer:type=producer-metrics,client-id=([-.\w]+)'
127+
attribute:
128+
- waiting-threads:
129+
metric_type: gauge
130+
alias: kafka.producer.waiting_threads
131+
- buffer-total-bytes:
132+
metric_type: gauge
133+
alias: kafka.producer.buffer_bytes_total
134+
- buffer-available-bytes:
135+
metric_type: gauge
136+
alias: kafka.producer.available_buffer_bytes
137+
- bufferpool-wait-time:
138+
metric_type: gauge
139+
alias: kafka.producer.bufferpool_wait_time
140+
- batch-size-avg:
141+
metric_type: gauge
142+
alias: kafka.producer.batch_size_avg
143+
- batch-size-max:
144+
metric_type: gauge
145+
alias: kafka.producer.batch_size_max
146+
- compression-rate-avg:
147+
metric_type: rate
148+
alias: kafka.producer.compression_rate_avg
149+
- record-queue-time-avg:
150+
metric_type: gauge
151+
alias: kafka.producer.record_queue_time_avg
152+
- record-queue-time-max:
153+
metric_type: gauge
154+
alias: kafka.producer.record_queue_time_max
155+
- request-latency-avg:
156+
metric_type: gauge
157+
alias: kafka.producer.request_latency_avg
158+
- request-latency-max:
159+
metric_type: gauge
160+
alias: kafka.producer.request_latency_max
161+
- record-send-rate:
162+
metric_type: gauge
163+
alias: kafka.producer.records_send_rate
164+
- records-per-request-avg:
165+
metric_type: gauge
166+
alias: kafka.producer.records_per_request
167+
- record-retry-rate:
168+
metric_type: gauge
169+
alias: kafka.producer.record_retry_rate
170+
- record-error-rate:
171+
metric_type: gauge
172+
alias: kafka.producer.record_error_rate
173+
- record-size-max:
174+
metric_type: gauge
175+
alias: kafka.producer.record_size_max
176+
- record-size-avg:
177+
metric_type: gauge
178+
alias: kafka.producer.record_size_avg
179+
- requests-in-flight:
180+
metric_type: gauge
181+
alias: kafka.producer.requests_in_flight
182+
- metadata-age:
183+
metric_type: gauge
184+
alias: kafka.producer.metadata_age
185+
- produce-throttle-time-max:
186+
metric_type: gauge
187+
alias: kafka.producer.throttle_time_max
188+
- produce-throttle-time-avg:
189+
metric_type: gauge
190+
alias: kafka.producer.throttle_time_avg
191+
192+
#
193+
# Producers: Per Topic Metrics
194+
#
195+
- include:
196+
domain: 'kafka.producer'
197+
bean_regex: 'kafka\.producer:type=producer-topic-metrics,client-id=(.*?),topic=(.*?)(?:,|$)'
198+
attribute:
199+
byte-rate:
200+
metric_type: gauge
201+
alias: kafka.producer.bytes_out
202+
tags:
203+
client: $1
204+
topic: $2
205+
- include:
206+
domain: 'kafka.producer'
207+
bean_regex: 'kafka\.producer:type=producer-topic-metrics,client-id=(.*?),topic=(.*?)(?:,|$)'
208+
attribute:
209+
record-send-rate:
210+
metric_type: gauge
211+
alias: kafka.producer.record_send_rate
212+
tags:
213+
client: $1
214+
topic: $2
215+
- include:
216+
domain: 'kafka.producer'
217+
bean_regex: 'kafka\.producer:type=producer-topic-metrics,client-id=(.*?),topic=(.*?)(?:,|$)'
218+
attribute:
219+
compression-rate:
220+
metric_type: gauge
221+
alias: kafka.producer.compression_rate
222+
tags:
223+
client: $1
224+
topic: $2
225+
- include:
226+
domain: 'kafka.producer'
227+
bean_regex: 'kafka\.producer:type=producer-topic-metrics,client-id=(.*?),topic=(.*?)(?:,|$)'
228+
attribute:
229+
record-retry-rate:
230+
metric_type: gauge
231+
alias: kafka.producer.record_retry_rate
232+
tags:
233+
client: $1
234+
topic: $2
235+
- include:
236+
domain: 'kafka.producer'
237+
bean_regex: 'kafka\.producer:type=producer-topic-metrics,client-id=(.*?),topic=(.*?)(?:,|$)'
238+
attribute:
239+
record-error-rate:
240+
metric_type: gauge
241+
alias: kafka.producer.record_error_rate
242+
tags:
243+
client: $1
244+
topic: $2
245+
121246
#
122247
# Consumers (only v0.8.2.x)
123248
#
@@ -184,6 +309,61 @@ init_config:
184309
metric_type: gauge
185310
alias: kafka.consumer.messages_in
186311

312+
#
313+
# Consumers: Per Topic Metrics
314+
#
315+
316+
- include:
317+
domain: 'kafka.consumer'
318+
bean_regex: 'kafka\.consumer:type=consumer-fetch-manager-metrics,client-id=(.*?),topic=(.*?)(?:,|$)'
319+
attribute:
320+
fetch-size-avg:
321+
metric_type: gauge
322+
alias: kafka.consumer.fetch_size_avg
323+
tags:
324+
client: $1
325+
topic: $2
326+
- include:
327+
domain: 'kafka.consumer'
328+
bean_regex: 'kafka\.consumer:type=consumer-fetch-manager-metrics,client-id=(.*?),topic=(.*?)(?:,|$)'
329+
attribute:
330+
fetch-size-max:
331+
metric_type: gauge
332+
alias: kafka.consumer.fetch_size_max
333+
tags:
334+
client: $1
335+
topic: $2
336+
- include:
337+
domain: 'kafka.consumer'
338+
bean_regex: 'kafka\.consumer:type=consumer-fetch-manager-metrics,client-id=(.*?),topic=(.*?)(?:,|$)'
339+
attribute:
340+
bytes-consumed-rate:
341+
metric_type: gauge
342+
alias: kafka.consumer.bytes_consumed
343+
tags:
344+
client: $1
345+
topic: $2
346+
- include:
347+
domain: 'kafka.consumer'
348+
bean_regex: 'kafka\.consumer:type=consumer-fetch-manager-metrics,client-id=(.*?),topic=(.*?)(?:,|$)'
349+
attribute:
350+
records-per-request-avg:
351+
metric_type: gauge
352+
alias: kafka.consumer.records_per_request_avg
353+
tags:
354+
client: $1
355+
topic: $2
356+
- include:
357+
domain: 'kafka.consumer'
358+
bean_regex: 'kafka\.consumer:type=consumer-fetch-manager-metrics,client-id=(.*?),topic=(.*?)(?:,|$)'
359+
attribute:
360+
records-consumed-rate:
361+
metric_type: gauge
362+
alias: kafka.consumer.records_consumed
363+
tags:
364+
client: $1
365+
topic: $2
366+
187367
#
188368
# Aggregate cluster stats
189369
#
@@ -216,6 +396,38 @@ init_config:
216396
metric_type: rate
217397
alias: kafka.net.bytes_rejected.rate
218398

399+
#
400+
# Brokers: Per Topic Metrics
401+
#
402+
- include:
403+
domain: 'kafka.server'
404+
bean_regex: 'kafka\.server:type=BrokerTopicMetrics,name=BytesOutPerSec,topic=.*'
405+
attribute:
406+
Count:
407+
metric_type: rate
408+
alias: kafka.topics.net.bytes_out.rate
409+
- include:
410+
domain: 'kafka.server'
411+
bean_regex: 'kafka\.server:type=BrokerTopicMetrics,name=BytesInPerSec,topic=.*'
412+
attribute:
413+
Count:
414+
metric_type: rate
415+
alias: kafka.topics.net.bytes_in.rate
416+
- include:
417+
domain: 'kafka.server'
418+
bean_regex: 'kafka\.server:type=BrokerTopicMetrics,name=MessagesInPerSec,topic=.*'
419+
attribute:
420+
Count:
421+
metric_type: rate
422+
alias: kafka.topics.messages_in.rate
423+
- include:
424+
domain: 'kafka.server'
425+
bean_regex: 'kafka\.server:type=BrokerTopicMetrics,name=BytesRejectedPerSec,topic=.*'
426+
attribute:
427+
Count:
428+
metric_type: rate
429+
alias: kafka.topics.net.bytes_rejected.rate
430+
219431
#
220432
# Request timings
221433
#
@@ -314,13 +526,31 @@ init_config:
314526
99thPercentile:
315527
metric_type: gauge
316528
alias: kafka.request.offsets.time.99percentile
529+
- include:
530+
domain: 'kafka.network'
531+
bean: 'kafka.network:type=RequestChannel,name=RequestQueueSize'
532+
attribute:
533+
Value:
534+
metric_type: gauge
535+
alias: kafka.request.channel.queue.size
536+
- include:
537+
domain: 'kafka.network'
538+
bean: 'kafka.network:type=SocketServer,name=NetworkProcessorAvgIdlePercent'
539+
attribute:
540+
Value:
541+
metric_type: gauge
542+
alias: kafka.net.handler.avg.idle.pct.rate
317543
- include:
318544
domain: 'kafka.server'
319545
bean: 'kafka.server:type=KafkaRequestHandlerPool,name=RequestHandlerAvgIdlePercent'
320546
attribute:
321547
OneMinuteRate:
322548
metric_type: gauge
323549
alias: kafka.request.handler.avg.idle.pct.rate
550+
551+
#
552+
# Request Purgatory (only v0.8.2.x)
553+
#
324554
- include:
325555
domain: 'kafka.server'
326556
bean: 'kafka.server:type=ProducerRequestPurgatory,name=PurgatorySize'
@@ -336,6 +566,24 @@ init_config:
336566
metric_type: gauge
337567
alias: kafka.request.fetch_request_purgatory.size
338568

569+
#
570+
# Request Purgatory (v0.9.0.x onwards)
571+
#
572+
- include:
573+
domain: 'kafka.server'
574+
bean: 'kafka.server:type=DelayedOperationPurgatory,name=PurgatorySize,delayedOperation=Produce'
575+
attribute:
576+
Value:
577+
metric_type: gauge
578+
alias: kafka.request.producer_request_purgatory.size
579+
- include:
580+
domain: 'kafka.server'
581+
bean: 'kafka.server:type=DelayedOperationPurgatory,name=PurgatorySize,delayedOperation=Fetch'
582+
attribute:
583+
Value:
584+
metric_type: gauge
585+
alias: kafka.request.fetch_request_purgatory.size
586+
339587
#
340588
# Replication stats
341589
#

0 commit comments

Comments
 (0)
⚡