Add Disk I/O Latency stats #1070

This commit is contained in:
nicolargo 2025-08-10 19:08:02 +02:00
parent 5f3e1de01e
commit 1daac0e6d0
12 changed files with 6315 additions and 6385 deletions

View File

@ -300,15 +300,32 @@ hide_zero=False
#show=sda.*
# Alias for sda1 and sdb1
#alias=sda1:SystemDisk,sdb1:DataDisk
# Set thresholds (in bytes per second) for a given disk name (rx = read / tx = write)
# Default latency thresholds (in ms) (rx = read / tx = write)
rx_latency_careful=10
rx_latency_warning=20
rx_latency_critical=50
tx_latency_careful=10
tx_latency_warning=20
tx_latency_critical=50
# Set latency thresholds (latency in ms) for a given disk name (rx = read / tx = write)
# dm-0_rx_latency_careful=10
# dm-0_rx_latency_warning=20
# dm-0_rx_latency_critical=50
# dm-0_rx_latency_log=False
# dm-0_tx_latency_careful=10
# dm-0_tx_latency_warning=20
# dm-0_tx_latency_critical=50
# dm-0_tx_latency_log=False
# There is no default bitrate thresholds for disk (because it is not possible to know the disk speed)
# Set bitrate thresholds (in bytes per second) for a given disk name (rx = read / tx = write)
#dm-0_rx_careful=4000000000
#dm-0_rx_warning=5000000000
#dm-0_rx_critical=6000000000
#dm-0_rx_log=True
#dm-0_rx_log=False
#dm-0_tx_careful=700000000
#dm-0_tx_warning=900000000
#dm-0_tx_critical=1000000000
#dm-0_tx_log=True
#dm-0_tx_log=False
[fs]
disable=False

View File

@ -300,15 +300,32 @@ hide_zero=False
#show=sda.*
# Alias for sda1 and sdb1
#alias=sda1:SystemDisk,sdb1:DataDisk
# Set thresholds (in bytes per second) for a given disk name (rx = read / tx = write)
# Default latency thresholds (in ms) (rx = read / tx = write)
rx_latency_careful=10
rx_latency_warning=20
rx_latency_critical=50
tx_latency_careful=10
tx_latency_warning=20
tx_latency_critical=50
# Set latency thresholds (latency in ms) for a given disk name (rx = read / tx = write)
# dm-0_rx_latency_careful=10
# dm-0_rx_latency_warning=20
# dm-0_rx_latency_critical=50
# dm-0_rx_latency_log=False
# dm-0_tx_latency_careful=10
# dm-0_tx_latency_warning=20
# dm-0_tx_latency_critical=50
# dm-0_tx_latency_log=False
# There is no default bitrate thresholds for disk (because it is not possible to know the disk speed)
# Set bitrate thresholds (in bytes per second) for a given disk name (rx = read / tx = write)
#dm-0_rx_careful=4000000000
#dm-0_rx_warning=5000000000
#dm-0_rx_critical=6000000000
#dm-0_rx_log=True
#dm-0_rx_log=False
#dm-0_tx_careful=700000000
#dm-0_tx_warning=900000000
#dm-0_tx_critical=1000000000
#dm-0_tx_log=True
#dm-0_tx_log=False
[fs]
disable=False

View File

@ -5,17 +5,12 @@ Disk I/O
.. image:: ../_static/diskio.png
Glances displays the disk I/O throughput. The unit is adapted
dynamically.
You can display:
Glances displays the disk I/O throughput, count and mean latency:
- bytes per second (default behavior / Bytes/s, KBytes/s, MBytes/s, etc)
- requests per second (using --diskio-iops option or *B* hotkey)
- mean latency (using --diskio-latency option or *L* hotkey)
There is no alert on this information.
It's possible to define:
It's also possible to define:
- a list of disk to show (white list)
- a list of disks to hide
@ -42,13 +37,20 @@ Filtering is based on regular expression. Please be sure that your regular
expression works as expected. You can use an online tool like `regex101`_ in
order to test your regular expression.
It is also possible to define thesholds for bytes read and write per second:
It is also possible to define thesholds for latency and bytes read and write per second:
.. code-block:: ini
[diskio]
# Alias for sda1 and sdb1
#alias=sda1:SystemDisk,sdb1:DataDisk
# Default latency thresholds (in ms) (rx = read / tx = write)
rx_latency_careful=10
rx_latency_warning=20
rx_latency_critical=50
tx_latency_careful=10
tx_latency_warning=20
tx_latency_critical=50
# Set thresholds (in bytes per second) for a given disk name (rx = read / tx = write)
dm-0_rx_careful=4000000000
dm-0_rx_warning=5000000000

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -28,7 +28,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
..
.TH "GLANCES" "1" "Aug 01, 2025" "4.4.0_dev1" "Glances"
.TH "GLANCES" "1" "Aug 10, 2025" "4.4.0_dev1" "Glances"
.SH NAME
glances \- An eye on your system
.SH SYNOPSIS

View File

@ -593,6 +593,13 @@ Examples of use:
dest='diskio_iops',
help='show IO per second in the DiskIO plugin',
)
parser.add_argument(
'--diskio-latency',
action='store_true',
default=False,
dest='diskio_latency',
help='show IO latency in the DiskIO plugin',
)
parser.add_argument(
'--fahrenheit',
action='store_true',

View File

@ -51,7 +51,7 @@ class _GlancesCurses:
'a': {'sort_key': 'auto'},
'A': {'switch': 'disable_amps'},
'b': {'switch': 'byte'},
'B': {'switch': 'diskio_iops'},
'B': {'handler': '_handle_diskio_iops'},
'c': {'sort_key': 'cpu_percent'},
'C': {'switch': 'disable_cloud'},
'd': {'switch': 'disable_diskio'},
@ -69,6 +69,7 @@ class _GlancesCurses:
# 'k' > Kill selected process
'K': {'switch': 'disable_connections'},
'l': {'switch': 'disable_alert'},
'L': {'handler': '_handle_diskio_latency'},
'm': {'sort_key': 'memory_percent'},
'M': {'switch': 'reset_minmax_tag'},
'n': {'switch': 'disable_network'},
@ -363,6 +364,18 @@ class _GlancesCurses:
else:
glances_processes.enable()
def _handle_diskio_iops(self):
"""Switch between bytes/s and IOPS for Disk IO."""
self.args.diskio_iops = not self.args.diskio_iops
if self.args.diskio_iops:
self.args.diskio_latency = False
def _handle_diskio_latency(self):
"""Switch between bytes/s and latency for Disk IO."""
self.args.diskio_latency = not self.args.diskio_latency
if self.args.diskio_latency:
self.args.diskio_iops = False
def _handle_sort_left(self):
next_sort = (self.loop_position() - 1) % len(self._sort_loop)
glances_processes.set_sort_key(self._sort_loop[next_sort], False)

View File

@ -316,6 +316,17 @@ export default {
// 'B' => Switch between bit/s and IO/s for Disk IO
hotkeys('shift+B', () => {
this.store.args.diskio_iops = !this.store.args.diskio_iops;
if (this.store.args.diskio_iops) {
this.store.args.diskio_latency = false;
}
});
// 'L' => Switch to latency for Disk IO
hotkeys('shift+L', () => {
this.store.args.diskio_latency = !this.store.args.diskio_latency;
if (this.store.args.diskio_latency) {
this.store.args.diskio_iops = false;
}
});
// l => Show/hide alert logs

View File

@ -4,8 +4,10 @@
<thead>
<tr>
<th scope="col">DISK I/O</th>
<th v-show="!args.diskio_iops" scope="col" class="text-end w-25">Rps</th>
<th v-show="!args.diskio_iops" scope="col" class="text-end w-25">Wps</th>
<th v-show="!args.diskio_iops && !args.diskio_latency" scope="col" class="text-end w-25">Rps</th>
<th v-show="!args.diskio_iops && !args.diskio_latency" scope="col" class="text-end w-25">Wps</th>
<th v-show="args.diskio_latency" scope="col" class="text-end w-25">ms/opR</th>
<th v-show="args.diskio_latency" scope="col" class="text-end w-25">ms/opW</th>
<th v-show="args.diskio_iops" scope="col" class="text-end w-25">IORps</th>
<th v-show="args.diskio_iops" scope="col" class="text-end w-25">IOWps</th>
</tr>
@ -15,16 +17,22 @@
<td scope="row" class="text-truncate">
{{ $filters.minSize(disk.alias ? disk.alias : disk.name, 16) }}
</td>
<td
v-show="!args.diskio_iops" class="text-end w-25"
<td v-show="!args.diskio_iops && !args.diskio_latency" class="text-end w-25"
:class="getDecoration(disk.name, 'write_bytes_rate_per_sec')">
{{ disk.bitrate.txps }}
</td>
<td
v-show="!args.diskio_iops" class="text-end w-25"
<td v-show="!args.diskio_iops && !args.diskio_latency" class="text-end w-25"
:class="getDecoration(disk.name, 'read_bytes_rate_per_sec')">
{{ disk.bitrate.rxps }}
</td>
<td v-show="args.diskio_latency" class="text-end w-25"
:class="getDecoration(disk.name, 'write_latency')">
{{ disk.latency.txps }}
</td>
<td v-show="args.diskio_latency" class="text-end w-25"
:class="getDecoration(disk.name, 'read_latency')">
{{ disk.latency.rxps }}
</td>
<td v-show="args.diskio_iops" class="text-end w-25">
{{ disk.count.txps }}
</td>
@ -75,6 +83,10 @@ export default {
count: {
txps: bytes(diskioData['read_count_rate_per_sec']),
rxps: bytes(diskioData['write_count_rate_per_sec'])
},
latency: {
txps: bytes(diskioData['read_latency']),
rxps: bytes(diskioData['write_latency'])
}
};
}).filter(disk => {
@ -91,7 +103,11 @@ export default {
methods: {
getDecoration(diskName, field) {
if (this.view[diskName][field] == undefined) {
return;
if (this.view[field] == undefined) {
return;
} else {
return this.view[field].decoration.toLowerCase();
}
}
return this.view[diskName][field].decoration.toLowerCase();
}

File diff suppressed because one or more lines are too long

View File

@ -42,6 +42,24 @@ fields_description = {
'rate': True,
'unit': 'byte',
},
'read_time': {
'description': 'Time spent reading.',
'rate': True,
'unit': 'millisecond',
},
'write_time': {
'description': 'Time spent writing.',
'rate': True,
'unit': 'millisecond',
},
'read_latency': {
'description': 'Mean time spent reading per operation.',
'unit': 'millisecond',
},
'write_latency': {
'description': 'Mean time spent writing per operation.',
'unit': 'millisecond',
},
}
# Define the history items list
@ -94,6 +112,9 @@ class DiskioPlugin(GlancesPluginModel):
# Update the stats
if self.input_method == 'local':
stats = self.update_local()
# Compute latency (need rate stats, so should be done after decorator)
stats = self.update_latency(stats)
else:
stats = self.get_init_value()
@ -102,6 +123,23 @@ class DiskioPlugin(GlancesPluginModel):
return self.stats
def update_latency(self, stats):
"""Update the latency stats."""
# Compute read/write latency if we have the rate stats
for stat in stats:
# Compute read/write latency if we have the rate stats
if stat.get("read_count_rate_per_sec", 0) > 0:
stat["read_latency"] = int(stat["read_time_rate_per_sec"] / stat["read_count_rate_per_sec"])
else:
stat["read_latency"] = 0
if stat.get("write_count_rate_per_sec", 0) > 0:
stat["write_latency"] = int(stat["write_time_rate_per_sec"] / stat["write_count_rate_per_sec"])
else:
stat["write_latency"] = 0
return stats
@GlancesPluginModel._manage_rate
def update_local(self):
stats = self.get_init_value()
@ -143,22 +181,31 @@ class DiskioPlugin(GlancesPluginModel):
# Call the father's method
super().update_views()
# Add specifics information
# Alert
for i in self.get_raw():
disk_real_name = i['disk_name']
# Skip alert if no timespan to measure
if not i.get('read_bytes_rate_per_sec') or not i.get('write_bytes_rate_per_sec'):
continue
# # Skip alert if no timespan to measure
# if not i.get('read_bytes_rate_per_sec') or not i.get('write_bytes_rate_per_sec'):
# continue
# Decorate the bitrate with the configuration file
alert_rx = self.get_alert(i['read_bytes'], header=disk_real_name + '_rx')
alert_tx = self.get_alert(i['write_bytes'], header=disk_real_name + '_tx')
self.views[i[self.get_key()]]['read_bytes']['decoration'] = alert_rx
self.views[i[self.get_key()]]['read_bytes_rate_per_sec']['decoration'] = alert_rx
self.views[i[self.get_key()]]['write_bytes']['decoration'] = alert_tx
self.views[i[self.get_key()]]['write_bytes_rate_per_sec']['decoration'] = alert_tx
# Decorate the latency with the configuration file
# Try to get the read/write latency for the current disk
alert_latency_rx = self.get_alert(i['read_latency'], header=disk_real_name + '_rx_latency')
alert_latency_tx = self.get_alert(i['write_latency'], header=disk_real_name + '_tx_latency')
# If the alert is not defined, use the default one
if alert_latency_rx == 'DEFAULT':
alert_latency_rx = self.get_alert(i['read_latency'], header='rx_latency')
if alert_latency_tx == 'DEFAULT':
alert_latency_tx = self.get_alert(i['write_latency'], header='tx_latency')
self.views[i[self.get_key()]]['read_latency']['decoration'] = alert_latency_rx
self.views[i[self.get_key()]]['write_latency']['decoration'] = alert_latency_tx
def msg_curse(self, args=None, max_width=None):
"""Return the dict to display in the curse interface."""
@ -185,6 +232,11 @@ class DiskioPlugin(GlancesPluginModel):
ret.append(self.curse_add_line(msg))
msg = '{:>7}'.format('IOW/s')
ret.append(self.curse_add_line(msg))
elif args.diskio_latency:
msg = '{:>8}'.format('ms/opR')
ret.append(self.curse_add_line(msg))
msg = '{:>7}'.format('ms/opW')
ret.append(self.curse_add_line(msg))
else:
msg = '{:>8}'.format('R/s')
ret.append(self.curse_add_line(msg))
@ -220,6 +272,22 @@ class DiskioPlugin(GlancesPluginModel):
msg, self.get_views(item=i[self.get_key()], key='write_count', option='decoration')
)
)
elif args.diskio_latency:
# latency (mean time spent reading/writing per operation)
txps = self.auto_unit(i.get('read_latency', None), low_precision=True)
rxps = self.auto_unit(i.get('write_latency', None), low_precision=True)
msg = f'{txps:>7}'
ret.append(
self.curse_add_line(
msg, self.get_views(item=i[self.get_key()], key='read_latency', option='decoration')
)
)
msg = f'{rxps:>7}'
ret.append(
self.curse_add_line(
msg, self.get_views(item=i[self.get_key()], key='write_latency', option='decoration')
)
)
else:
# Bitrate
txps = self.auto_unit(i.get('read_bytes_rate_per_sec', None))