Docker memory usage is incorrect

This commit is contained in:
Nicolas Hennion 2024-01-02 17:16:30 +00:00 committed by Bharath Vignesh J K
parent de69635d41
commit 8edf6e70b5
5 changed files with 36 additions and 21 deletions

View File

@ -14,6 +14,8 @@ You can install this dependency using:
.. image:: ../_static/containers.png
Note: Memory usage is compute as following "display memory usage = memory usage - inactive_file"
It is possible to define limits and actions from the configuration file
under the ``[containers]`` section:

View File

@ -54,7 +54,7 @@
{{ $filters.number(container.cpu_percent, 1) }}
</div>
<div class="table-cell">
{{ $filters.bytes(container.memory_usage) }}
{{ $filters.bytes(container.memory_usage_no_cache) }}
</div>
<div class="table-cell">
{{ $filters.bytes(container.limit) }}
@ -112,14 +112,21 @@ export default {
const { sorter } = this;
const containers = (this.stats || []).map(
(containerData) => {
// prettier-ignore
let memory_usage_no_cache = '?'
if (containerData.memory.usage != undefined) {
memory_usage_no_cache = containerData.memory.usage;
if (containerData.memory.inactive_file != undefined) {
memory_usage_no_cache = memory_usage_no_cache - containerData.memory.inactive_file;
}
}
return {
'id': containerData.id,
'name': containerData.name,
'status': containerData.status,
'uptime': containerData.uptime,
'cpu_percent': containerData.cpu.total,
'memory_usage': containerData.memory.usage != undefined ? containerData.memory.usage : '?',
'memory_usage_no_cache': memory_usage_no_cache,
'limit': containerData.memory.limit != undefined ? containerData.memory.limit : '?',
'io_rx': containerData.io_rx != undefined ? containerData.io_rx : '?',
'io_wx': containerData.io_wx != undefined ? containerData.io_wx : '?',

View File

@ -256,6 +256,10 @@ class PluginModel(GlancesPluginModel):
"""Return the user ticks by reading the environment variable."""
return os.sysconf(os.sysconf_names['SC_CLK_TCK'])
def memory_usage_no_cache(self, mem):
"""Return the 'real' memory usage by removing inactive_file to usage"""
return mem['usage'] - (mem['inactive_file'] if 'inactive_file' in mem else 0)
def update_views(self):
"""Update stats views."""
# Call the father's method
@ -281,11 +285,17 @@ class PluginModel(GlancesPluginModel):
if 'memory' in i and 'usage' in i['memory']:
# Looking for specific MEM container threshold in the conf file
alert = self.get_alert(
i['memory']['usage'], maximum=i['memory']['limit'], header=i['name'] + '_mem', action_key=i['name']
self.memory_usage_no_cache(i['memory']),
maximum=i['memory']['limit'],
header=i['name'] + '_mem', action_key=i['name']
)
if alert == 'DEFAULT':
# Not found ? Get back to default MEM threshold value
alert = self.get_alert(i['memory']['usage'], maximum=i['memory']['limit'], header='mem')
alert = self.get_alert(
self.memory_usage_no_cache(i['memory']),
maximum=i['memory']['limit'],
header='mem'
)
self.views[i[self.get_key()]]['mem']['decoration'] = alert
# Display Engine and Pod name ?
@ -383,8 +393,8 @@ class PluginModel(GlancesPluginModel):
ret.append(self.curse_add_line(msg, self.get_views(item=container['name'], key='cpu', option='decoration')))
# MEM
try:
msg = '{:>7}'.format(self.auto_unit(container['memory']['usage']))
except (KeyError, TypeError):
msg = '{:>7}'.format(self.auto_unit(self.memory_usage_no_cache(container['memory'])))
except KeyError:
msg = '{:>7}'.format('_')
ret.append(self.curse_add_line(msg, self.get_views(item=container['name'], key='mem', option='decoration')))
try:

View File

@ -2,7 +2,7 @@
#
# This file is part of Glances.
#
# SPDX-FileCopyrightText: 2022 Nicolas Hennion <nicolas@nicolargo.com>
# SPDX-FileCopyrightText: 2023 Nicolas Hennion <nicolas@nicolargo.com>
#
# SPDX-License-Identifier: LGPL-3.0-only
#
@ -29,7 +29,7 @@ else:
class DockerStatsFetcher:
MANDATORY_MEMORY_FIELDS = ["usage", 'limit']
MANDATORY_MEMORY_FIELDS = ['usage', 'limit']
def __init__(self, container):
self._container = container
@ -120,7 +120,7 @@ class DockerStatsFetcher:
def _get_memory_stats(self):
"""Return the container MEMORY.
Output: a dict {'rss': 1015808, 'cache': 356352, 'usage': ..., 'max_usage': ...}
Output: a dict {'usage': ..., 'limit': ..., 'max_usage': ...}
"""
memory_stats = self._streamer.stats.get('memory_stats')
@ -130,15 +130,11 @@ class DockerStatsFetcher:
return None
stats = {field: memory_stats[field] for field in self.MANDATORY_MEMORY_FIELDS}
try:
# Issue #1857 - Some stats are not always available in ['memory_stats']['stats']
detailed_stats = memory_stats['stats']
stats['rss'] = detailed_stats.get('rss') or detailed_stats.get('total_rss')
stats['max_usage'] = detailed_stats.get('max_usage')
stats['cache'] = detailed_stats.get('cache')
except (KeyError, TypeError) as e:
self._log_debug("Can't grab MEM usage", e) # stats do not have MEM information
return None
# Optional field
stats['inactive_file'] = 0
if 'stats' in memory_stats:
stats['inactive_file'] = memory_stats['stats'].get('inactive_file', 0)
# Return the stats
return stats

View File

@ -139,7 +139,7 @@ class PodmanPodStatsFetcher:
def _get_memory_stats(self, stats):
"""Return the container MEMORY.
Output: a dict {'rss': 1015808, 'cache': 356352, 'usage': ..., 'max_usage': ...}
Output: a dict {'usage': ..., 'limit': ...}
"""
if "MemUsage" not in stats or "/" not in stats["MemUsage"]:
self._log_debug("Missing MEM usage fields")
@ -155,7 +155,7 @@ class PodmanPodStatsFetcher:
self._log_debug("Compute MEM usage failed", e)
return None
return {"usage": usage, "limit": limit}
return {'usage': usage, 'limit': limit, 'inactive_file': 0}
def _get_network_stats(self, stats):
"""Return the container network usage using the Docker API (v1.0 or higher).