Add disk label (#3953)

* Add disk label

* Fix CI

* Address reviews

* Add label to rates

* Fix plateform tests

* Add tag_by_label option

* Use regex

* Fix tests with undefined behaviour

* Add os information

* Update tagging test

* Address reviews

Author: coignetp

disk/datadog_checks/disk/data/conf.yaml.default

@@ -114,6 +114,12 @@ instances:
     #
     # tag_by_filesystem: false
 
+    ## @param tag_by_label - boolean - optional - default: true
+    ## Instruct the check to tag all the metrics with disk label if there is one.
+    ## Works on Linux only.
+    #
+    # tag_by_label: true
+
     ## @param device_tag_re - list of regex:tags string - optional
     ## Instruct the check to apply additional tags to matching
     ## devices (or mount points if `use_mount` is true).

disk/datadog_checks/disk/disk.py

@@ -11,7 +11,7 @@
 
 from datadog_checks.base import AgentCheck, ConfigurationError, is_affirmative
 from datadog_checks.base.utils.platform import Platform
-from datadog_checks.base.utils.subprocess_output import get_subprocess_output
+from datadog_checks.base.utils.subprocess_output import SubprocessOutputEmptyError, get_subprocess_output
 from datadog_checks.base.utils.timeout import TimeoutException, timeout
 
 try:
@@ -52,6 +52,7 @@ def __init__(self, name, init_config, agentConfig, instances=None):
         self._mount_point_whitelist = instance.get('mount_point_whitelist', [])
         self._mount_point_blacklist = instance.get('mount_point_blacklist', [])
         self._tag_by_filesystem = is_affirmative(instance.get('tag_by_filesystem', False))
+        self._tag_by_label = is_affirmative(instance.get('tag_by_label', True))
         self._device_tag_re = instance.get('device_tag_re', {})
         self._custom_tags = instance.get('tags', [])
         self._service_check_rw = is_affirmative(instance.get('service_check_rw', False))
@@ -69,6 +70,9 @@ def __init__(self, name, init_config, agentConfig, instances=None):
             )
         self._compile_pattern_filters(instance)
         self._compile_tag_re()
+        self._blkid_label_re = re.compile('LABEL=\"(.*?)\"', re.I)
+
+        self.devices_label = {}
 
     def _load_legacy_option(self, instance, option, default, legacy_name=None, operation=lambda l: l):
         value = instance.get(option, default)
@@ -85,6 +89,8 @@ def _load_legacy_option(self, instance, option, default, legacy_name=None, opera
 
     def check(self, instance):
         """Get disk space/inode stats"""
+        if self._tag_by_label and Platform.is_linux():
+            self.devices_label = self._get_devices_label()
         # Windows and Mac will always have psutil
         # (we have packaged for both of them)
         if self._psutil():
@@ -134,6 +140,9 @@ def collect_metrics_psutil(self):
                 if regex.match(device_name):
                     tags.extend(device_tags)
 
+            if self.devices_label.get(device_name):
+                tags.append(self.devices_label.get(device_name))
+
             # legacy check names c: vs psutil name C:\\
             if Platform.is_win32():
                 device_name = device_name.strip('\\').lower()
@@ -284,6 +293,8 @@ def collect_latency_metrics(self):
                 write_time_pct = disk.write_time * 100 / 1000
                 metric_tags = [] if self._custom_tags is None else self._custom_tags[:]
                 metric_tags.append('device:{}'.format(disk_name))
+                if self.devices_label.get(disk_name):
+                    metric_tags.append(self.devices_label.get(disk_name))
                 self.rate(self.METRIC_DISK.format('read_time_pct'), read_time_pct, tags=metric_tags)
                 self.rate(self.METRIC_DISK.format('write_time_pct'), write_time_pct, tags=metric_tags)
             except AttributeError as e:
@@ -308,6 +319,10 @@ def collect_metrics_manually(self):
                 if regex.match(device_name):
                     tags += device_tags
             tags.append('device:{}'.format(device_name))
+
+            if self.devices_label.get(device_name):
+                tags.append(self.devices_label.get(device_name))
+
             for metric_name, value in iteritems(self._collect_metrics_manually(device)):
                 self.gauge(metric_name, value, tags=tags)
 
@@ -455,3 +470,24 @@ def _compile_tag_re(self):
             except TypeError:
                 self.log.warning('{} is not a valid regular expression and will be ignored'.format(regex_str))
         self._device_tag_re = device_tag_list
+
+    def _get_devices_label(self):
+        """
+        Get every label to create tags
+        """
+        devices_label = {}
+        try:
+            blkid_out, _, _ = get_subprocess_output(['blkid'], self.log)
+            all_devices = [l.split(':', 1) for l in blkid_out.splitlines()]
+
+            for d in all_devices:
+                # Line sample
+                # /dev/sda1: LABEL="MYLABEL" UUID="5eea373d-db36-4ce2-8c71-12ce544e8559" TYPE="ext4"
+                labels = self._blkid_label_re.findall(d[1])
+                if labels:
+                    devices_label[d[0]] = 'label:{}'.format(labels[0])
+
+        except SubprocessOutputEmptyError:
+            self.log.debug("Couldn't use blkid to have device labels")
+
+        return devices_label

disk/tests/conftest.py

@@ -30,12 +30,12 @@ def dd_environment(instance_basic_volume):
 
 @pytest.fixture(scope='session')
 def instance_basic_volume():
-    return {'use_mount': 'false'}
+    return {'use_mount': 'false', 'tag_by_label': False}
 
 
 @pytest.fixture(scope='session')
 def instance_basic_mount():
-    return {'use_mount': 'true'}
+    return {'use_mount': 'true', 'tag_by_label': False}
 
 
 @pytest.fixture(scope='session')

disk/tests/fixtures/blkid

@@ -0,0 +1,3 @@
+/dev/sda1: UUID="9RbyZC-LtBG-J6RS-EH7g-wYMT-wQP2-wftnVi" TYPE="LVM2_member" PARTUUID="1e4b9125-01"
+/dev/mapper/vagrant--vg-root: LABEL="DATA" UUID="5eea373d-db36-4ce2-8c71-12ce544e8559" TYPE="ext4"
+/dev/mapper/vagrant--vg-swap_1: UUID="1678c43d-0654-4c4a-87f1-90dffa21ed34" TYPE="swap"

disk/tests/mocks.py

@@ -15,6 +15,15 @@ def mock_df_output(fname):
         return f.read(), '', ''
 
 
+def mock_blkid_output():
+    """
+    Load fixtures from tests/fixtures/ folder and return a tuple matching the
+    return value of `get_subprocess_output`
+    """
+    with open(os.path.join(HERE, 'fixtures', 'blkid')) as f:
+        return f.read(), '', ''
+
+
 class MockPart(object):
     def __init__(
         self, device=DEFAULT_DEVICE_NAME, fstype=DEFAULT_FILE_SYSTEM, mountpoint=DEFAULT_MOUNT_POINT, opts='ro'

disk/tests/test_filter.py

@@ -34,12 +34,12 @@ def test_bad_config_string_regex():
     }
     c = Disk('disk', None, {}, [instance])
 
-    assert c._file_system_whitelist == re.compile('test', re.I)
-    assert c._file_system_blacklist == re.compile('test|iso9660$', re.I)
-    assert c._device_whitelist == re.compile('test', IGNORE_CASE)
-    assert c._device_blacklist == re.compile('test', IGNORE_CASE)
-    assert c._mount_point_whitelist == re.compile('test', IGNORE_CASE)
-    assert c._mount_point_blacklist == re.compile('test', IGNORE_CASE)
+    assert c._file_system_whitelist.pattern == 'test'
+    assert c._file_system_blacklist.pattern == 'test|iso9660$'
+    assert c._device_whitelist.pattern == 'test'
+    assert c._device_blacklist.pattern == 'test'
+    assert c._mount_point_whitelist.pattern == 'test'
+    assert c._mount_point_blacklist.pattern == 'test'
 
 
 def test_ignore_empty_regex():
@@ -53,12 +53,12 @@ def test_ignore_empty_regex():
     }
     c = Disk('disk', None, {}, [instance])
 
-    assert c._file_system_whitelist == re.compile('test', re.I)
-    assert c._file_system_blacklist == re.compile('test|iso9660$', re.I)
-    assert c._device_whitelist == re.compile('test', IGNORE_CASE)
-    assert c._device_blacklist == re.compile('test', IGNORE_CASE)
-    assert c._mount_point_whitelist == re.compile('test', IGNORE_CASE)
-    assert c._mount_point_blacklist == re.compile('test', IGNORE_CASE)
+    assert c._file_system_whitelist.pattern == 'test'
+    assert c._file_system_blacklist.pattern == 'test|iso9660$'
+    assert c._device_whitelist.pattern == 'test'
+    assert c._device_blacklist.pattern == 'test'
+    assert c._mount_point_whitelist.pattern == 'test'
+    assert c._mount_point_blacklist.pattern == 'test'
 
 
 def test_exclude_bad_devices():

disk/tests/test_unit.py

@@ -10,7 +10,7 @@
 from datadog_checks.disk import Disk
 
 from .common import DEFAULT_DEVICE_NAME, DEFAULT_FILE_SYSTEM, DEFAULT_MOUNT_POINT
-from .mocks import MockInodesMetrics, mock_df_output
+from .mocks import MockInodesMetrics, mock_blkid_output, mock_df_output
 from .utils import requires_unix
 
 
@@ -56,7 +56,7 @@ def test_psutil(aggregator, gauge_metrics, rate_metrics):
     Mock psutil and run the check
     """
     for tag_by in ['true', 'false']:
-        instance = {'tag_by_filesystem': tag_by}
+        instance = {'tag_by_filesystem': tag_by, 'tag_by_label': False}
         c = Disk('disk', None, {}, [instance])
         c.check(instance)
 
@@ -83,7 +83,7 @@ def test_psutil_rw(aggregator):
     """
     Check for 'ro' option in the mounts
     """
-    instance = {'service_check_rw': 'yes'}
+    instance = {'service_check_rw': 'yes', 'tag_by_label': False}
     c = Disk('disk', None, {}, [instance])
     c.check(instance)
 
@@ -113,24 +113,33 @@ def test_device_tagging(aggregator, gauge_metrics, rate_metrics):
         'use_mount': 'no',
         'device_tag_re': {'{}.*'.format(DEFAULT_DEVICE_NAME[:-1]): 'type:dev,tag:two'},
         'tags': ['optional:tags1'],
+        'tag_by_label': False,
     }
     c = Disk('disk', None, {}, [instance])
-    c.check(instance)
+
+    with mock.patch('datadog_checks.disk.disk.Disk._get_devices_label'):
+        # _get_devices_label is only called on linux, so devices_label is manually filled
+        # to make the test run on everything
+        c.devices_label = {DEFAULT_DEVICE_NAME: 'label:mylab'}
+        c.check(instance)
 
     # Assert metrics
-    tags = ['type:dev', 'tag:two', 'device:{}'.format(DEFAULT_DEVICE_NAME), 'optional:tags1']
+    tags = ['type:dev', 'tag:two', 'device:{}'.format(DEFAULT_DEVICE_NAME), 'optional:tags1', 'label:mylab']
+
     for name, value in iteritems(gauge_metrics):
         aggregator.assert_metric(name, value=value, tags=tags)
 
     for name, value in iteritems(rate_metrics):
-        aggregator.assert_metric(name, value=value, tags=['device:{}'.format(DEFAULT_DEVICE_NAME), 'optional:tags1'])
+        aggregator.assert_metric(
+            name, value=value, tags=['device:{}'.format(DEFAULT_DEVICE_NAME), 'optional:tags1', 'label:mylab']
+        )
 
     aggregator.assert_all_metrics_covered()
 
 
 @requires_unix
 def test_no_psutil_debian(aggregator, gauge_metrics):
-    instance = {'use_mount': 'no', 'excluded_filesystems': ['tmpfs']}
+    instance = {'use_mount': 'no', 'excluded_filesystems': ['tmpfs'], 'tag_by_label': False}
     c = Disk('disk', None, {}, [instance])
     # disable psutil
     c._psutil = lambda: False
@@ -155,7 +164,12 @@ def test_no_psutil_debian(aggregator, gauge_metrics):
 
 @requires_unix
 def test_no_psutil_freebsd(aggregator, gauge_metrics):
-    instance = {'use_mount': 'no', 'excluded_filesystems': ['devfs'], 'excluded_disk_re': 'zroot/.+'}
+    instance = {
+        'use_mount': 'no',
+        'excluded_filesystems': ['devfs'],
+        'excluded_disk_re': 'zroot/.+',
+        'tag_by_label': False,
+    }
     c = Disk('disk', None, {}, [instance])
     # disable psutil
     c._psutil = lambda: False
@@ -178,7 +192,12 @@ def test_no_psutil_freebsd(aggregator, gauge_metrics):
 
 @requires_unix
 def test_no_psutil_centos(aggregator, gauge_metrics):
-    instance = {'use_mount': 'no', 'excluded_filesystems': ['devfs', 'tmpfs'], 'excluded_disks': ['/dev/sda1']}
+    instance = {
+        'use_mount': 'no',
+        'excluded_filesystems': ['devfs', 'tmpfs'],
+        'excluded_disks': ['/dev/sda1'],
+        'tag_by_label': False,
+    }
     c = Disk('disk', None, {}, [instance])
     # disable psutil
     c._psutil = lambda: False
@@ -198,3 +217,15 @@ def test_no_psutil_centos(aggregator, gauge_metrics):
             aggregator.assert_metric(name, tags=['device:{}'.format(device)])
 
     aggregator.assert_all_metrics_covered()
+
+
+def test_get_devices_label():
+    c = Disk('disk', None, {}, [{}])
+
+    with mock.patch(
+        "datadog_checks.disk.disk.get_subprocess_output",
+        return_value=mock_blkid_output(),
+        __name__='get_subprocess_output',
+    ):
+        labels = c._get_devices_label()
+        assert labels.get("/dev/mapper/vagrant--vg-root") == "label:DATA"