Add Disk I/O Latency stats #1070

conf/glances.conf

@@ -300,15 +300,32 @@ hide_zero=False
 #show=sda.*
 # Alias for sda1 and sdb1
 #alias=sda1:SystemDisk,sdb1:DataDisk
-# Set thresholds (in bytes per second) for a given disk name (rx = read / tx = write)
+# Default latency thresholds (in ms) (rx = read / tx = write)
+rx_latency_careful=10
+rx_latency_warning=20
+rx_latency_critical=50
+tx_latency_careful=10
+tx_latency_warning=20
+tx_latency_critical=50
+# Set latency thresholds (latency in ms) for a given disk name (rx = read / tx = write)
+# dm-0_rx_latency_careful=10
+# dm-0_rx_latency_warning=20
+# dm-0_rx_latency_critical=50
+# dm-0_rx_latency_log=False
+# dm-0_tx_latency_careful=10
+# dm-0_tx_latency_warning=20
+# dm-0_tx_latency_critical=50
+# dm-0_tx_latency_log=False
+# There is no default bitrate thresholds for disk (because it is not possible to know the disk speed)
+# Set bitrate thresholds (in bytes per second) for a given disk name (rx = read / tx = write)
 #dm-0_rx_careful=4000000000
 #dm-0_rx_warning=5000000000
 #dm-0_rx_critical=6000000000
-#dm-0_rx_log=True
+#dm-0_rx_log=False
 #dm-0_tx_careful=700000000
 #dm-0_tx_warning=900000000
 #dm-0_tx_critical=1000000000
-#dm-0_tx_log=True
+#dm-0_tx_log=False
 
 [fs]
 disable=False

docker-compose/glances.conf

@@ -300,15 +300,32 @@ hide_zero=False
 #show=sda.*
 # Alias for sda1 and sdb1
 #alias=sda1:SystemDisk,sdb1:DataDisk
-# Set thresholds (in bytes per second) for a given disk name (rx = read / tx = write)
+# Default latency thresholds (in ms) (rx = read / tx = write)
+rx_latency_careful=10
+rx_latency_warning=20
+rx_latency_critical=50
+tx_latency_careful=10
+tx_latency_warning=20
+tx_latency_critical=50
+# Set latency thresholds (latency in ms) for a given disk name (rx = read / tx = write)
+# dm-0_rx_latency_careful=10
+# dm-0_rx_latency_warning=20
+# dm-0_rx_latency_critical=50
+# dm-0_rx_latency_log=False
+# dm-0_tx_latency_careful=10
+# dm-0_tx_latency_warning=20
+# dm-0_tx_latency_critical=50
+# dm-0_tx_latency_log=False
+# There is no default bitrate thresholds for disk (because it is not possible to know the disk speed)
+# Set bitrate thresholds (in bytes per second) for a given disk name (rx = read / tx = write)
 #dm-0_rx_careful=4000000000
 #dm-0_rx_warning=5000000000
 #dm-0_rx_critical=6000000000
-#dm-0_rx_log=True
+#dm-0_rx_log=False
 #dm-0_tx_careful=700000000
 #dm-0_tx_warning=900000000
 #dm-0_tx_critical=1000000000
-#dm-0_tx_log=True
+#dm-0_tx_log=False
 
 [fs]
 disable=False

docs/aoa/diskio.rst

@@ -5,17 +5,12 @@ Disk I/O
 
 .. image:: ../_static/diskio.png
 
-Glances displays the disk I/O throughput. The unit is adapted
+Glances displays the disk I/O throughput, count and mean latency:
-dynamically.
-
-You can display:
-
 - bytes per second (default behavior / Bytes/s, KBytes/s, MBytes/s, etc)
 - requests per second (using --diskio-iops option or *B* hotkey)
+- mean latency (using --diskio-latency option or *L* hotkey)
 
-There is no alert on this information.
+It's also possible to define:
-
-It's possible to define:
 
 - a list of disk to show (white list)
 - a list of disks to hide
@@ -42,13 +37,20 @@ Filtering is based on regular expression. Please be sure that your regular
 expression works as expected. You can use an online tool like `regex101`_ in
 order to test your regular expression.
 
-It is also possible to define thesholds for bytes read and write per second:
+It is also possible to define thesholds for latency and bytes read and write per second:
 
 .. code-block:: ini
 
     [diskio]
     # Alias for sda1 and sdb1
     #alias=sda1:SystemDisk,sdb1:DataDisk
+    # Default latency thresholds (in ms) (rx = read / tx = write)
+    rx_latency_careful=10
+    rx_latency_warning=20
+    rx_latency_critical=50
+    tx_latency_careful=10
+    tx_latency_warning=20
+    tx_latency_critical=50
     # Set thresholds (in bytes per second) for a given disk name (rx = read / tx = write)
     dm-0_rx_careful=4000000000
     dm-0_rx_warning=5000000000

docs/man/glances.1

@@ -28,7 +28,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .in \\n[rst2man-indent\\n[rst2man-indent-level]]u
 ..
-.TH "GLANCES" "1" "Aug 01, 2025" "4.4.0_dev1" "Glances"
+.TH "GLANCES" "1" "Aug 10, 2025" "4.4.0_dev1" "Glances"
 .SH NAME
 glances \- An eye on your system
 .SH SYNOPSIS

glances/main.py

@@ -593,6 +593,13 @@ Examples of use:
             dest='diskio_iops',
             help='show IO per second in the DiskIO plugin',
         )
+        parser.add_argument(
+            '--diskio-latency',
+            action='store_true',
+            default=False,
+            dest='diskio_latency',
+            help='show IO latency in the DiskIO plugin',
+        )
         parser.add_argument(
             '--fahrenheit',
             action='store_true',

glances/outputs/glances_curses.py

@@ -51,7 +51,7 @@ class _GlancesCurses:
         'a': {'sort_key': 'auto'},
         'A': {'switch': 'disable_amps'},
         'b': {'switch': 'byte'},
-        'B': {'switch': 'diskio_iops'},
+        'B': {'handler': '_handle_diskio_iops'},
         'c': {'sort_key': 'cpu_percent'},
         'C': {'switch': 'disable_cloud'},
         'd': {'switch': 'disable_diskio'},
@@ -69,6 +69,7 @@ class _GlancesCurses:
         # 'k' > Kill selected process
         'K': {'switch': 'disable_connections'},
         'l': {'switch': 'disable_alert'},
+        'L': {'handler': '_handle_diskio_latency'},
         'm': {'sort_key': 'memory_percent'},
         'M': {'switch': 'reset_minmax_tag'},
         'n': {'switch': 'disable_network'},
@@ -363,6 +364,18 @@ class _GlancesCurses:
         else:
             glances_processes.enable()
 
+    def _handle_diskio_iops(self):
+        """Switch between bytes/s and IOPS for Disk IO."""
+        self.args.diskio_iops = not self.args.diskio_iops
+        if self.args.diskio_iops:
+            self.args.diskio_latency = False
+
+    def _handle_diskio_latency(self):
+        """Switch between bytes/s and latency for Disk IO."""
+        self.args.diskio_latency = not self.args.diskio_latency
+        if self.args.diskio_latency:
+            self.args.diskio_iops = False
+
     def _handle_sort_left(self):
         next_sort = (self.loop_position() - 1) % len(self._sort_loop)
         glances_processes.set_sort_key(self._sort_loop[next_sort], False)

glances/outputs/static/js/App.vue

@@ -316,6 +316,17 @@ export default {
             // 'B' => Switch between bit/s and IO/s for Disk IO
             hotkeys('shift+B', () => {
                 this.store.args.diskio_iops = !this.store.args.diskio_iops;
+                if (this.store.args.diskio_iops) {
+                    this.store.args.diskio_latency = false;
+                }
+            });
+
+            // 'L' => Switch to latency for Disk IO
+            hotkeys('shift+L', () => {
+                this.store.args.diskio_latency = !this.store.args.diskio_latency;
+                if (this.store.args.diskio_latency) {
+                    this.store.args.diskio_iops = false;
+                }
             });
 
             // l => Show/hide alert logs

glances/outputs/static/js/components/plugin-diskio.vue

@@ -4,8 +4,10 @@
             <thead>
                 <tr>
                     <th scope="col">DISK I/O</th>
-                    <th v-show="!args.diskio_iops" scope="col" class="text-end w-25">Rps</th>
+                    <th v-show="!args.diskio_iops && !args.diskio_latency" scope="col" class="text-end w-25">Rps</th>
-                    <th v-show="!args.diskio_iops" scope="col" class="text-end w-25">Wps</th>
+                    <th v-show="!args.diskio_iops && !args.diskio_latency" scope="col" class="text-end w-25">Wps</th>
+                    <th v-show="args.diskio_latency" scope="col" class="text-end w-25">ms/opR</th>
+                    <th v-show="args.diskio_latency" scope="col" class="text-end w-25">ms/opW</th>
                     <th v-show="args.diskio_iops" scope="col" class="text-end w-25">IORps</th>
                     <th v-show="args.diskio_iops" scope="col" class="text-end w-25">IOWps</th>
                 </tr>
@@ -15,16 +17,22 @@
                     <td scope="row" class="text-truncate">
                         {{ $filters.minSize(disk.alias ? disk.alias : disk.name, 16) }}
                     </td>
-                    <td
+                    <td v-show="!args.diskio_iops && !args.diskio_latency" class="text-end w-25"
-v-show="!args.diskio_iops" class="text-end w-25"
                         :class="getDecoration(disk.name, 'write_bytes_rate_per_sec')">
                         {{ disk.bitrate.txps }}
                     </td>
-                    <td
+                    <td v-show="!args.diskio_iops && !args.diskio_latency" class="text-end w-25"
-v-show="!args.diskio_iops" class="text-end w-25"
                         :class="getDecoration(disk.name, 'read_bytes_rate_per_sec')">
                         {{ disk.bitrate.rxps }}
                     </td>
+                    <td v-show="args.diskio_latency" class="text-end w-25"
+                        :class="getDecoration(disk.name, 'write_latency')">
+                        {{ disk.latency.txps }}
+                    </td>
+                    <td v-show="args.diskio_latency" class="text-end w-25"
+                        :class="getDecoration(disk.name, 'read_latency')">
+                        {{ disk.latency.rxps }}
+                    </td>
                     <td v-show="args.diskio_iops" class="text-end w-25">
                         {{ disk.count.txps }}
                     </td>
@@ -75,6 +83,10 @@ export default {
                     count: {
                         txps: bytes(diskioData['read_count_rate_per_sec']),
                         rxps: bytes(diskioData['write_count_rate_per_sec'])
+                    },
+                    latency: {
+                        txps: bytes(diskioData['read_latency']),
+                        rxps: bytes(diskioData['write_latency'])
                     }
                 };
             }).filter(disk => {
@@ -91,7 +103,11 @@ export default {
     methods: {
         getDecoration(diskName, field) {
             if (this.view[diskName][field] == undefined) {
-                return;
+                if (this.view[field] == undefined) {
+                    return;
+                } else {
+                    return this.view[field].decoration.toLowerCase();
+                }
             }
             return this.view[diskName][field].decoration.toLowerCase();
         }

glances/plugins/diskio/__init__.py

@@ -42,6 +42,24 @@ fields_description = {
         'rate': True,
         'unit': 'byte',
     },
+    'read_time': {
+        'description': 'Time spent reading.',
+        'rate': True,
+        'unit': 'millisecond',
+    },
+    'write_time': {
+        'description': 'Time spent writing.',
+        'rate': True,
+        'unit': 'millisecond',
+    },
+    'read_latency': {
+        'description': 'Mean time spent reading per operation.',
+        'unit': 'millisecond',
+    },
+    'write_latency': {
+        'description': 'Mean time spent writing per operation.',
+        'unit': 'millisecond',
+    },
 }
 
 # Define the history items list
@@ -94,6 +112,9 @@ class DiskioPlugin(GlancesPluginModel):
         # Update the stats
         if self.input_method == 'local':
             stats = self.update_local()
+
+            # Compute latency (need rate stats, so should be done after decorator)
+            stats = self.update_latency(stats)
         else:
             stats = self.get_init_value()
 
@@ -102,6 +123,23 @@ class DiskioPlugin(GlancesPluginModel):
 
         return self.stats
 
+    def update_latency(self, stats):
+        """Update the latency stats."""
+        # Compute read/write latency if we have the rate stats
+        for stat in stats:
+            # Compute read/write latency if we have the rate stats
+            if stat.get("read_count_rate_per_sec", 0) > 0:
+                stat["read_latency"] = int(stat["read_time_rate_per_sec"] / stat["read_count_rate_per_sec"])
+            else:
+                stat["read_latency"] = 0
+
+            if stat.get("write_count_rate_per_sec", 0) > 0:
+                stat["write_latency"] = int(stat["write_time_rate_per_sec"] / stat["write_count_rate_per_sec"])
+            else:
+                stat["write_latency"] = 0
+
+        return stats
+
     @GlancesPluginModel._manage_rate
     def update_local(self):
         stats = self.get_init_value()
@@ -143,22 +181,31 @@ class DiskioPlugin(GlancesPluginModel):
         # Call the father's method
         super().update_views()
 
-        # Add specifics information
         # Alert
         for i in self.get_raw():
             disk_real_name = i['disk_name']
 
-            # Skip alert if no timespan to measure
+            # # Skip alert if no timespan to measure
-            if not i.get('read_bytes_rate_per_sec') or not i.get('write_bytes_rate_per_sec'):
+            # if not i.get('read_bytes_rate_per_sec') or not i.get('write_bytes_rate_per_sec'):
-                continue
+            #     continue
 
             # Decorate the bitrate with the configuration file
             alert_rx = self.get_alert(i['read_bytes'], header=disk_real_name + '_rx')
             alert_tx = self.get_alert(i['write_bytes'], header=disk_real_name + '_tx')
             self.views[i[self.get_key()]]['read_bytes']['decoration'] = alert_rx
-            self.views[i[self.get_key()]]['read_bytes_rate_per_sec']['decoration'] = alert_rx
             self.views[i[self.get_key()]]['write_bytes']['decoration'] = alert_tx
-            self.views[i[self.get_key()]]['write_bytes_rate_per_sec']['decoration'] = alert_tx
+
+            # Decorate the latency with the configuration file
+            # Try to get the read/write latency for the current disk
+            alert_latency_rx = self.get_alert(i['read_latency'], header=disk_real_name + '_rx_latency')
+            alert_latency_tx = self.get_alert(i['write_latency'], header=disk_real_name + '_tx_latency')
+            # If the alert is not defined, use the default one
+            if alert_latency_rx == 'DEFAULT':
+                alert_latency_rx = self.get_alert(i['read_latency'], header='rx_latency')
+            if alert_latency_tx == 'DEFAULT':
+                alert_latency_tx = self.get_alert(i['write_latency'], header='tx_latency')
+            self.views[i[self.get_key()]]['read_latency']['decoration'] = alert_latency_rx
+            self.views[i[self.get_key()]]['write_latency']['decoration'] = alert_latency_tx
 
     def msg_curse(self, args=None, max_width=None):
         """Return the dict to display in the curse interface."""
@@ -185,6 +232,11 @@ class DiskioPlugin(GlancesPluginModel):
             ret.append(self.curse_add_line(msg))
             msg = '{:>7}'.format('IOW/s')
             ret.append(self.curse_add_line(msg))
+        elif args.diskio_latency:
+            msg = '{:>8}'.format('ms/opR')
+            ret.append(self.curse_add_line(msg))
+            msg = '{:>7}'.format('ms/opW')
+            ret.append(self.curse_add_line(msg))
         else:
             msg = '{:>8}'.format('R/s')
             ret.append(self.curse_add_line(msg))
@@ -220,6 +272,22 @@ class DiskioPlugin(GlancesPluginModel):
                         msg, self.get_views(item=i[self.get_key()], key='write_count', option='decoration')
                     )
                 )
+            elif args.diskio_latency:
+                # latency (mean time spent reading/writing per operation)
+                txps = self.auto_unit(i.get('read_latency', None), low_precision=True)
+                rxps = self.auto_unit(i.get('write_latency', None), low_precision=True)
+                msg = f'{txps:>7}'
+                ret.append(
+                    self.curse_add_line(
+                        msg, self.get_views(item=i[self.get_key()], key='read_latency', option='decoration')
+                    )
+                )
+                msg = f'{rxps:>7}'
+                ret.append(
+                    self.curse_add_line(
+                        msg, self.get_views(item=i[self.get_key()], key='write_latency', option='decoration')
+                    )
+                )
             else:
                 # Bitrate
                 txps = self.auto_unit(i.get('read_bytes_rate_per_sec', None))