Skip to content

Commit ef53636

Browse files
mx-psimx-psi
committed
[disk] Add option to exclude special file systems on Linux
Improve log messages
1 parent f9fac0c commit ef53636

5 files changed

Lines changed: 124 additions & 4 deletions

File tree

‎disk/assets/configuration/spec.yaml‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,9 @@ files:
4242
start matching from the beginning and therefore to match anywhere you
4343
must prepend `.*`. For exact matches append `$`.
4444
45+
Some common Linux special file systems can be excluded by enabling the
46+
`exclude_special_file_systems` option.
47+
4548
When conflicts arise, this will override `file_system_whitelist`.
4649
value:
4750
example:
@@ -51,6 +54,30 @@ files:
5154
type: array
5255
items:
5356
type: string
57+
- name: exclude_special_file_systems
58+
description: |
59+
Instruct the check to not collect from some common special file systems.
60+
Works on Linux only.
61+
62+
The special file systems excluded with this option are:
63+
- binfmt_misc
64+
- configfs
65+
- debugfs
66+
- devtmpfs
67+
- overlay
68+
- proc
69+
- rootfs
70+
- securityfs
71+
- sysfs
72+
- tmpfs
73+
- tracefs
74+
75+
You may cease to ignore a specific special file system by adding it to
76+
the `file_system_whitelist`.
77+
value:
78+
example: true
79+
type: boolean
80+
default: false
5481
- name: device_whitelist
5582
description: |
5683
Instruct the check to only collect from matching devices.

‎disk/datadog_checks/disk/data/conf.yaml.default‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,13 +44,38 @@ instances:
4444
## start matching from the beginning and therefore to match anywhere you
4545
## must prepend `.*`. For exact matches append `$`.
4646
##
47+
## Some common Linux special file systems can be excluded by enabling the
48+
## `exclude_special_file_systems` option.
49+
##
4750
## When conflicts arise, this will override `file_system_whitelist`.
4851
#
4952
# file_system_blacklist:
5053
# - tmpfs$
5154
# - rootfs$
5255
# - autofs$
5356

57+
## @param exclude_special_file_systems - boolean - optional - default: False
58+
## Instruct the check to not collect from some common special file systems.
59+
## Works on Linux only.
60+
##
61+
## The special file systems excluded with this option are:
62+
## - binfmt_misc
63+
## - configfs
64+
## - debugfs
65+
## - devtmpfs
66+
## - overlay
67+
## - proc
68+
## - rootfs
69+
## - securityfs
70+
## - sysfs
71+
## - tmpfs
72+
## - tracefs
73+
##
74+
## You may cease to ignore a specific special file system by adding it to
75+
## the `file_system_whitelist`.
76+
#
77+
# exclude_special_file_systems: true
78+
5479
## @param device_whitelist - list of strings - optional
5580
## Instruct the check to only collect from matching devices.
5681
##

‎disk/datadog_checks/disk/disk.py‎

Lines changed: 60 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,13 +20,42 @@
2020
# See: https://github.com/DataDog/integrations-core/pull/1109#discussion_r167133580
2121
IGNORE_CASE = re.I
2222

23+
KNOWN_SPECIAL_FILE_SYSTEMS = set()
24+
2325
def _base_device_name(device):
2426
return device.strip('\\').lower()
2527

2628

29+
elif platform.system() == "Linux":
30+
IGNORE_CASE = 0
31+
32+
# Remember to update the configuration specification
33+
# if modifying the list of special file systems.
34+
KNOWN_SPECIAL_FILE_SYSTEMS = set(
35+
[
36+
'binfmt_msc',
37+
'configfs',
38+
'debugfs',
39+
'devtmpfs',
40+
'overlay',
41+
'proc',
42+
'rootfs',
43+
'securityfs',
44+
'sysfs',
45+
'tmpfs',
46+
'tracefs',
47+
]
48+
)
49+
50+
def _base_device_name(device):
51+
return os.path.basename(device)
52+
53+
2754
else:
2855
IGNORE_CASE = 0
2956

57+
KNOWN_SPECIAL_FILE_SYSTEMS = set()
58+
3059
def _base_device_name(device):
3160
return os.path.basename(device)
3261

@@ -47,6 +76,8 @@ def __init__(self, name, init_config, instances):
4776
self._all_partitions = is_affirmative(instance.get('all_partitions', False))
4877
self._file_system_whitelist = instance.get('file_system_whitelist', [])
4978
self._file_system_blacklist = instance.get('file_system_blacklist', [])
79+
# FIXME (8.X): Exclude special file systems by default
80+
self._exclude_special_file_systems = instance.get('exclude_special_file_systems', False)
5081
self._device_whitelist = instance.get('device_whitelist', [])
5182
self._device_blacklist = instance.get('device_blacklist', [])
5283
self._mount_point_whitelist = instance.get('mount_point_whitelist', [])
@@ -88,7 +119,21 @@ def check(self, instance):
88119
)
89120
continue
90121
except Exception as e:
91-
self.log.warning('Unable to get disk metrics for %s: %s', part.mountpoint, e)
122+
if not self._exclude_special_file_systems and part.fstype in KNOWN_SPECIAL_FILE_SYSTEMS:
123+
self.log.warning(
124+
u'Unable to get disk metrics for %s with special file system %s: %s. '
125+
u'Enable `exclude_special_file_systems` to ignore common special file systems.',
126+
part.mountpoint,
127+
part.fstype,
128+
e,
129+
)
130+
else:
131+
self.log.warning(
132+
u'Unable to get disk metrics for %s: %s. '
133+
u'You can exclude this mountpoint in the settings if it is invalid.',
134+
part.mountpoint,
135+
e,
136+
)
92137
continue
93138

94139
# Exclude disks with size less than min_disk_size
@@ -147,7 +192,6 @@ def _exclude_disk(self, device, file_system, mount_point):
147192
"""
148193
Return True for disks we don't want or that match regex in the config file
149194
"""
150-
self.log.debug('_exclude_disk: %s, %s, %s', device, file_system, mount_point)
151195

152196
if not device or device == 'none':
153197
device = None
@@ -190,6 +234,14 @@ def _file_system_blacklisted(self, file_system):
190234
if self._file_system_blacklist is None:
191235
return False
192236

237+
# Check if it is a known special file system
238+
if (
239+
self._exclude_special_file_systems
240+
and file_system in KNOWN_SPECIAL_FILE_SYSTEMS
241+
and not self._file_system_whitelisted(file_system)
242+
):
243+
return True
244+
193245
return not not self._file_system_blacklist.match(file_system)
194246

195247
def _device_whitelisted(self, device):
@@ -245,7 +297,12 @@ def _collect_inodes_metrics(self, mountpoint):
245297
)
246298
return metrics
247299
except Exception as e:
248-
self.log.warning('Unable to get disk metrics for %s: %s', mountpoint, e)
300+
self.log.warning(
301+
u'Unable to get disk metrics for %s: %s. '
302+
u'You can exclude this mountpoint in the settings if it is invalid.',
303+
mountpoint,
304+
e,
305+
)
249306
return metrics
250307

251308
if inodes.f_files != 0:

‎disk/log/.lock‎

Whitespace-only changes.

‎disk/tests/test_filter.py‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
from datadog_checks.disk.disk import IGNORE_CASE, Disk
88

99
from .mocks import MockPart
10-
from .utils import assert_regex_equal, requires_windows
10+
from .utils import assert_regex_equal, requires_linux, requires_windows
1111

1212

1313
def test_default_casing():
@@ -104,6 +104,17 @@ def test_file_system_whitelist_blacklist():
104104
assert c.exclude_disk(MockPart(fstype='NTFS')) is True
105105

106106

107+
@requires_linux
108+
def test_exclude_special_file_system():
109+
instance = {'file_system_whitelist': ['ext[2-4]', 'tracefs$'], 'exclude_special_file_systems': True}
110+
c = Disk('disk', {}, [instance])
111+
112+
assert c.exclude_disk(MockPart(fstype='debugfs')) is True
113+
assert c.exclude_disk(MockPart(fstype='tempfs')) is True
114+
assert c.exclude_disk(MockPart(fstype='ext4')) is False
115+
assert c.exclude_disk(MockPart(fstype='tracefs')) is False
116+
117+
107118
def test_device_whitelist():
108119
instance = {'device_whitelist': ['/dev/sda[1-3]', 'c:']}
109120
c = Disk('disk', {}, [instance])

0 commit comments

Comments
 (0)
⚡