Kafka, Linux_proc_extras, Directory, and many other small fixes or additions

manifests/init.pp

@@ -256,6 +256,7 @@
   $package_name = $datadog_agent::params::package_name,
   $dd_user = $datadog_agent::params::dd_user,
   $dd_group = $datadog_agent::params::dd_group,
+  $dd_groups = $datadog_agent::params::dd_groups,
   $apm_enabled = false,
   $apm_env = '',
 ) inherits datadog_agent::params {
@@ -385,15 +386,30 @@
     default: { fail("Class[datadog_agent]: Unsupported operatingsystem: ${::operatingsystem}") }
   }
 
-  # required by reports even in agent5 scenario
-  file { '/etc/datadog-agent':
+  if ($dd_groups) {
+    user { "$dd_user":
+      groups  => $dd_groups,
+      notify  => Service[$datadog_agent::params::service_name],
+    }
+  }
+
+  file { '/etc/dd-agent':
     ensure  => directory,
     owner   => $dd_user,
     group   => $dd_group,
     mode    => '0755',
     require => Package['datadog-agent'],
   }
 
+  file { $conf_dir:
+    ensure  => directory,
+    purge   => $conf_dir_purge,
+    recurse => true,
+    force   => $conf_dir_purge,
+    owner   => $dd_user,
+    group   => $dd_group,
+    notify  => Service[$datadog_agent::params::service_name],
+  }
 
   if !$agent6_enable {
     file { '/etc/dd-agent':

manifests/integrations/directory.pp

@@ -1,35 +1,72 @@
 # Class: datadog_agent::integrations::directory
 #
-# This class will install the necessary config to hook the directory check
+# This class will install the necessary config to hook the directory in the agent
 #
 # Parameters:
-#   $directory
-#       The directory to gather stats for
-#   $tag_name
-#       The name used to tag the metrics (directory alias)
-#   $pattern
-#       The `fnmatch` pattern to use when reading the "directory"'s files. default "*"
-#   $recursive
-#       Boolean, when true the stats will recurse into directories
+#   directory
+#       (Required) - string, the directory path to monitor
+#       This will be included as a tag: name:<name>.
+#
+#   name
+#       (Optional) - string, tag metrics with specified name. defaults to the "directory"
+#
+#   dirtagname
+#       (Optional) - string, the name of the key for the tag used for the directory, the value will be the value of "name" (see above). The resulting tag will be "<dirtagname>:<name>". defaults to "name"
+#
+#   filetagname
+#       (Optional) - string, the name of the key for the tag used for each file, the value will be the filename. The resulting tag will be "<filetagname>:<filename>". defaults to "filename"
+#
+#   filegauges
+#       (Optional) - boolean, when true stats will be an individual gauge per file (max. 20 files!) and not a histogram of the whole directory. default False
+#
+#   pattern
+#       (Optional) - string, the `fnmatch` pattern to use when reading the "directory"'s files. The pattern will be matched against the files' absolute paths and relative paths in "directory". default "*"
+#
+#   recursive
+#       (Optional) - boolean, when true the stats will recurse into directories. default False
+#
+#   countonly
+#       (Optional) - boolean, when true the stats will only count the number of files matching the pattern. Useful for very large directories.
+#
 #
 # Sample Usage:
 #
-#  class { 'datadog_agent::integrations::directory' :
-#      directory     => '/mnt/media',
-#      tag_name      => 'name',
-#      pattern       => '*',
-#      recursive     => true,
-#  }
+# Add a class for each check instance:
+#
+# class { 'datadog_agent::integrations::directory':
+#   directory => '/opt/ftp_data',
+#   recursive => true,
+#   countonly => true,
+# }
+#
+# Add multiple instances in one class declaration:
+#
+# class { 'datadog_agent::integrations::directory':
+#   instances => [{
+#     'directory' => '/opt/ftp_data',
+#     'recursive' => true,
+#     'countonly' => true,
+#     },
+#     'directory' => '/opt/ftp_data-staging',
+#     'recursive' => true,
+#     'countonly' => true,
+#     },
+#   ]
+# }
+
 class datadog_agent::integrations::directory (
-  $directory     = undef,
-  $tag_name      = '',
-  $pattern       = '*',
-  $recursive     = false
+  Array[Struct[{
+    directory   => String,
+    name        => Optional[String],
+    dirtagname  => Optional[String],
+    filetagname => Optional[String],
+    filegauges  => Optional[Boolean],
+    pattern     => Optional[String],
+    recursive   => Optional[Boolean],
+    countonly   => Optional[Boolean],
+  }]] $instances = [],
 ) inherits datadog_agent::params {
-
-  if $directory == undef {
-    fail('you must specify a directory path within the datadog_agent::integrations::directory class')
-  }
+  include datadog_agent
 
   if $::datadog_agent::agent6_enable {
     $dst = "${datadog_agent::conf6_dir}/directory.yaml"

manifests/integrations/disk.pp

@@ -32,7 +32,7 @@
 #      excluded_disk_re     => '/dev/sd[e-z]*'
 #  }
 class datadog_agent::integrations::disk (
-  $use_mount              = 'no',
+  $use_mount              = false,
   $excluded_filesystems   = undef,
   $excluded_disks         = undef,
   $excluded_disk_re       = undef,
@@ -42,9 +42,10 @@
 ) inherits datadog_agent::params {
   include datadog_agent
 
-  validate_re($use_mount, '^(no|yes)$', "use_mount should be either 'yes' or 'no'")
+  validate_bool($use_mount)
+
   if $all_partitions {
-    validate_re($all_partitions, '^(no|yes)$', "all_partitions should be either 'yes' or 'no'")
+    validate_bool($all_partitions)
   }
 
   if $::datadog_agent::agent6_enable {

manifests/integrations/http_check.pp

@@ -88,6 +88,10 @@
 #       but may be problematic with some HTTP servers
 #       (See: https://code.google.com/p/httplib2/issues/detail?id=169)
 #
+#   allow_redirects
+#       The (optional) allow_redirects parameter can enable redirection.
+#       Defaults to True.
+#
 #   contact
 #       For service-specific notifications, you can optionally specify
 #       a list of users to notify within the service configuration.
@@ -156,10 +160,11 @@
   $disable_ssl_validation = false,
   $skip_event = true,
   $no_proxy  = false,
-  $check_certificate_expiration = undef,
+  $check_certificate_expiration = true,
   $days_warning = undef,
   $days_critical = undef,
   $headers   = [],
+  $allow_redirects = true,
   $tags      = [],
   $contact   = [],
   $instances  = undef,
@@ -186,6 +191,7 @@
       'days_warning'                 => $days_warning,
       'days_critical'                => $days_critical,
       'headers'                      => $headers,
+      'allow_redirects'              => $allow_redirects,
       'tags'                         => $tags,
       'contact'                      => $contact,
     }]

manifests/integrations/kafka.pp

@@ -0,0 +1,47 @@
+# Class: datadog_agent::integrations::kafka
+#
+# This class will install the necessary configuration for the kafka
+# integration
+#
+# Parameters:
+#   $host:
+#       The hostname kafka is running on
+#   $port:
+#       The port to connect on
+#   $user
+#       The user for the datadog user
+#   $password
+#       The password for the datadog user
+#   $tags
+#       Optional array of tags
+#
+# Sample Usage:
+#
+#  class { 'datadog_agent::integrations::kafka' :
+#    host     => 'localhost',
+#    tags     => {
+#      environment => "production",
+#    },
+#  }
+#
+#
+class datadog_agent::integrations::kafka(
+  $host = 'localhost',
+  $port = 9999,
+  $user = undef,
+  $password = undef,
+  $tags = { kafka => broker },
+) inherits datadog_agent::params {
+  require ::datadog_agent
+  validate_hash($tags)
+
+  file { "${datadog_agent::params::conf_dir}/kafka.yaml":
+    ensure  => file,
+    owner   => $datadog_agent::params::dd_user,
+    group   => $datadog_agent::params::dd_group,
+    mode    => '0600',
+    content => template('datadog_agent/agent-conf.d/kafka.yaml.erb'),
+    require => Package[$datadog_agent::params::package_name],
+    notify  => Service[$datadog_agent::params::service_name],
+  }
+}

manifests/integrations/linux_proc_extras.pp

@@ -0,0 +1,31 @@
+# Class: datadog_agent::integrations::system_core
+#
+# This class will install the necessary configuration for the CPU cores integration
+#
+# Parameters:
+#
+#   tags
+#       The (optional) tags to add to the check instance.
+#
+# Sample Usage:
+#
+#  class { 'datadog_agent::integrations::linux_proc_extras':
+#      tags => [ 'env:production' ],
+#  }
+
+class datadog_agent::integrations::linux_proc_extras(
+  $tags = [],
+) inherits datadog_agent::params {
+  include datadog_agent
+
+  file { "${datadog_agent::params::conf_dir}/linux_proc_extras.yaml":
+    ensure  => file,
+    owner   => $datadog_agent::params::dd_user,
+    group   => $datadog_agent::params::dd_group,
+    mode    => '0600',
+    content => template('datadog_agent/agent-conf.d/linux_proc_extras.yaml.erb'),
+    require => Package[$datadog_agent::params::package_name],
+    notify  => Service[$datadog_agent::params::service_name]
+  }
+
+}

manifests/integrations/process.pp

@@ -5,6 +5,8 @@
 # Parameters:
 #   $processes:
 #       Array of process hashes. See example
+#   $hiera_processes:
+#       Boolean to grab processes from hiera to allow merging
 #
 # Process hash keys:
 #   search_strings
@@ -39,12 +41,20 @@
 #
 #
 class datadog_agent::integrations::process(
+  $hiera_processes = false,
   $processes = [],
-) inherits datadog_agent::params {
+  ) inherits datadog_agent::params {
   include datadog_agent
 
+  validate_bool( $hiera_processes )
   validate_array( $processes )
 
+  if $hiera_processes {
+    $local_processes = hiera_array('datadog_agent::integrations::process::processes')
+  } else {
+    $local_processes = $processes
+  }
+
   if $::datadog_agent::agent6_enable {
     $dst = "${datadog_agent::conf6_dir}/process.yaml"
   } else {

manifests/params.pp

@@ -20,6 +20,7 @@
   $conf6_dir      = '/etc/datadog-agent/conf.d'
   $dd_user        = 'dd-agent'
   $dd_group       = 'root'
+  $dd_groups      = undef
   $package_name   = 'datadog-agent'
   $service_name   = 'datadog-agent'
   $dogapi_version = 'installed'

templates/agent-conf.d/directory.yaml.erb

@@ -1,7 +1,27 @@
-init_config:
+### MANAGED BY PUPPET
+<%
+require 'yaml'
 
-instances:
-    -   directory: "<%= @directory %>"
-        name: "<%= @tag_name %>"
-        pattern: "<%= @pattern %>"
-        recursive: <%= @recursive %>
+if RUBY_VERSION < "1.9"
+# Hack to avoid ruby 1.8 always reordering the hashes and
+# changing this template resource at every run
+# See https://github.com/DataDog/puppet-datadog-agent/issues/43
+# Taken from http://www.dzone.com/snippets/generating-yaml-hashes-sorted
+class Hash
+  # Replacing the to_yaml function so it'll serialize hashes sorted (by their keys)
+  #
+  # Original function is in /usr/lib/ruby/1.8/yaml/rubytypes.rb
+  def to_yaml( opts = {} )
+    YAML::quick_emit( object_id, opts ) do |out|
+      out.map( taguri, to_yaml_style ) do |map|
+        sort.each do |k, v|   # <-- here's my addition (the 'sort')
+          map.add( k, v )
+        end
+      end
+    end
+  end
+end
+end
+%>
+
+<%= {'init_config'=>{}, 'instances'=>@instances}.to_yaml %>

templates/agent-conf.d/disk.yaml.erb

@@ -1,3 +1,43 @@
+### MANAGED BY PUPPET
+
+# This file is overwritten upon Agent upgrade.
+# To make modifications to the check configuration, please copy this file
+# to `disk.yaml` and make your changes on that file.
+
+# init_config:
+
+# instances:
+  # The use_mount parameter will instruct the check to collect disk
+  # and fs metrics using mount points instead of volumes
+  # - use_mount: no
+    # The (optional) excluded_filesystems parameter will instruct the check to
+    # ignore disks using these filesystems
+    # excluded_filesystems:
+    #   - tmpfs
+
+    # The (optional) excluded_disks parameter will instruct the check to
+    # ignore this list of disks
+    # excluded_disks:
+    #   - /dev/sda1
+    #   - /dev/sda2
+    #
+    # The (optional) excluded_disk_re parameter will instruct the check to
+    # ignore all disks matching this regex
+    # excluded_disk_re: /dev/sde.*
+    #
+    # The (optional) tag_by_filesystem parameter will instruct the check to
+    # tag all disks with their filesystem (for ex: filesystem:nfs)
+    # tag_by_filesystem: no
+    #
+    # The (optional) excluded_mountpoint_re parameter will instruct the check to
+    # ignore all mountpoints matching this regex
+    # excluded_mountpoint_re: /mnt/somebody-elses-problem.*
+    #
+    # The (optional) all_partitions parameter will instruct the check to
+    # get metrics for all partitions. use_mount should be set to yes (to avoid
+    # collecting empty device names) when using this option.
+    # all_partitions: no
+
 init_config:
 
 instances: