Merge develop

Makefile

@@ -78,6 +78,13 @@ show-issue: venv
 profiling: venv venv-dev
 	@echo "Please complete and run: sudo ./venv/bin/py-spy record -o ./docs/_static/glances-flame.svg -d 60 -s --pid <GLANCES PID>"
 
+trace-malloc: venv
+	@echo "Malloc test is running, please wait ~30 secondes..."
+	./venv/bin/python -m glances -C ./conf/glances.conf --trace-malloc --stop-after 15 --quiet
+
+memory-leak: venv
+	./venv/bin/python -m glances -C ./conf/glances.conf --memory-leak
+
 release-note:
 	git --no-pager log $(LASTTAG)..HEAD --first-parent --pretty=format:"* %s"
 	@echo "\n"

glances/__init__.py

@@ -42,6 +42,7 @@ except ImportError:
 
 # Import Glances libs
 # Note: others Glances libs will be imported optionally
+from glances.compat import PY3
 from glances.logger import logger
 from glances.main import GlancesMain
 from glances.timer import Counter
@@ -64,6 +65,9 @@ if psutil_version_info < psutil_min_version:
     print('psutil 5.3.0 or higher is needed. Glances cannot start.')
     sys.exit(1)
 
+# Trac malloc is only available on Python 3.4 or higher
+if PY3:
+    import tracemalloc
 
 def __signal_handler(signal, frame):
     """Callback for CTRL-C."""
@@ -81,6 +85,8 @@ def end():
 
     logger.info("Glances stopped (key pressed: CTRL-C)")
 
+
+
     # The end...
     sys.exit(0)
 
@@ -91,6 +97,9 @@ def start(config, args):
     # Load mode
     global mode
 
+    if args.trace_malloc or args.memory_leak:
+        tracemalloc.start()
+
     start_duration = Counter()
 
     if core.is_standalone():
@@ -111,6 +120,18 @@ def start(config, args):
 
     # Start the main loop
     logger.debug("Glances started in {} seconds".format(start_duration.get()))
+    if args.stop_after:
+        logger.info('Glances will be stopped in ~{} seconds'.format(
+            args.stop_after * args.time * args.memory_leak * 2))
+
+    if args.memory_leak:
+        print('Memory leak detection, please wait ~{} seconds...'.format(
+            args.stop_after * args.time * args.memory_leak * 2))
+        # First run without dump to fill the memory
+        mode.serve_n(args.stop_after)
+        # Then start the memory-leak loop
+        snapshot_begin = tracemalloc.take_snapshot()
+
     if args.stdout_issue or args.stdout_apidoc:
         # Serve once for issue/test mode
         mode.serve_issue()
@@ -118,6 +139,23 @@ def start(config, args):
         # Serve forever
         mode.serve_forever()
 
+    if args.memory_leak:
+        snapshot_end = tracemalloc.take_snapshot()
+        snapshot_diff = snapshot_end.compare_to(snapshot_begin, 'filename')
+        memory_leak = sum([s.size_diff for s in snapshot_diff])
+        print("Memory comsumption: {0:.1f}KB (see log for details)".format(
+            memory_leak / 1000))
+        logger.info("Memory consumption (top 5):")
+        for stat in snapshot_diff[:5]:
+            logger.info(stat)
+    elif args.trace_malloc:
+        # See more options here: https://docs.python.org/3/library/tracemalloc.html
+        snapshot = tracemalloc.take_snapshot()
+        top_stats = snapshot.statistics("filename")
+        print("[ Trace malloc - Top 10 ]")
+        for stat in top_stats[:10]:
+            print(stat)
+
     # Shutdown
     mode.end()
 

glances/main.py

@@ -122,10 +122,16 @@ Examples of use:
             help='display modules (plugins & exports) list and exit',
         )
         parser.add_argument(
-            '--disable-plugin', '--disable-plugins', dest='disable_plugin', help='disable plugin (comma separed list)'
+            '--disable-plugin',
+            '--disable-plugins',
+            dest='disable_plugin',
+            help='disable plugin (comma separed list)'
         )
         parser.add_argument(
-            '--enable-plugin', '--enable-plugins', dest='enable_plugin', help='enable plugin (comma separed list)'
+            '--enable-plugin',
+            '--enable-plugins',
+            dest='enable_plugin',
+            help='enable plugin (comma separed list)'
         )
         parser.add_argument(
             '--disable-process',
@@ -218,7 +224,11 @@ Examples of use:
             help='disable background colors in the terminal',
         )
         parser.add_argument(
-            '--enable-irq', action='store_true', default=False, dest='enable_irq', help='enable IRQ module'
+            '--enable-irq',
+            action='store_true',
+            default=False,
+            dest='enable_irq',
+            help='enable IRQ module'
         ),
         parser.add_argument(
             '--enable-process-extended',
@@ -227,6 +237,14 @@ Examples of use:
             dest='enable_process_extended',
             help='enable extended stats on top process',
         )
+        parser.add_argument(
+            '--separator',
+            '--enable-separator',
+            action='store_true',
+            default=False,
+            dest='enable_separator',
+            help='enable separator in the UI'
+        ),
         # Sort processes list
         parser.add_argument(
             '--sort-processes',
@@ -339,6 +357,13 @@ Examples of use:
             dest='cached_time',
             help='set the server cache time [default: {} sec]'.format(self.cached_time),
         )
+        parser.add_argument(
+            '--stop-after',
+            default=None,
+            type=int,
+            dest='stop_after',
+            help='stop Glances after n refresh',
+        )
         parser.add_argument(
             '--open-web-browser',
             action='store_true',
@@ -396,6 +421,20 @@ Examples of use:
             dest='stdout_issue',
             help='test all plugins and exit (please copy/paste the output if you open an issue)',
         )
+        parser.add_argument(
+            '--trace-malloc',
+            default=False,
+            action='store_true',
+            dest='trace_malloc',
+            help='trace memory allocation and display it at the end of the process (python 3.4 or higher needed)',
+        )
+        parser.add_argument(
+            '--memory-leak',
+            default=False,
+            action='store_true',
+            dest='memory_leak',
+            help='test memory leak (python 3.4 or higher needed)',
+        )
         parser.add_argument(
             '--api-doc', default=None, action='store_true', dest='stdout_apidoc', help='display fields descriptions'
         )
@@ -648,10 +687,25 @@ Examples of use:
             sys.exit(2)
 
         # Disable HDDTemp if sensors are disabled
-        if getattr(args, 'disable_sensors', False):
-            disable(args, 'hddtemp')
+        if getattr(self.args, 'disable_sensors', False):
+            disable(self.args, 'hddtemp')
             logger.debug("Sensors and HDDTemp are disabled")
 
+        if getattr(self.args, 'trace_malloc', True) and not self.is_standalone():
+            logger.critical("Option --trace-malloc is only available in the terminal mode")
+            sys.exit(2)
+
+        if getattr(self.args, 'memory_leak', True) and not self.is_standalone():
+            logger.critical("Option --memory-leak is only available in the terminal mode")
+            sys.exit(2)
+        else:
+            logger.info('Memory leak detection enabled')
+            self.args.quiet = True
+            if not self.args.stop_after:
+                self.args.stop_after = 60
+            self.args.time = 1
+            self.args.disable_history = True
+
         # Let the plugins known the Glances mode
         self.args.is_standalone = self.is_standalone()
         self.args.is_client = self.is_client()

glances/outputs/glances_curses.py

@@ -26,6 +26,7 @@ from glances.globals import MACOS, WINDOWS, nativestr, u, itervalues, enable, di
 from glances.logger import logger
 from glances.events import glances_events
 from glances.processes import glances_processes, sort_processes_key_list
+from glances.outputs.glances_unicode import unicode_message
 from glances.timer import Timer
 
 # Import curses library for "normal" operating system
@@ -524,7 +525,7 @@ class _GlancesCurses(object):
         self.column = 0
         self.next_column = 0
 
-    def new_line(self):
+    def new_line(self, separator=False):
         """New line in the curses interface."""
         self.line = self.next_line
 
@@ -532,6 +533,18 @@ class _GlancesCurses(object):
         """New column in the curses interface."""
         self.column = self.next_column
 
+    def separator_line(self, color='TITLE'):
+        """New separator line in the curses interface."""
+        if not self.args.enable_separator:
+            return
+        self.new_line()
+        self.line -= 1
+        line_width = self.term_window.getmaxyx()[1] - self.column
+        self.term_window.addnstr(self.line, self.column,
+                                    unicode_message('MEDIUM_LINE', self.args) * line_width,
+                                    line_width,
+                                    self.colors_list[color])
+
     def __get_stat_display(self, stats, layer):
         """Return a dict of dict with all the stats display.
         # TODO: Drop extra parameter
@@ -631,11 +644,14 @@ class _GlancesCurses(object):
         # Optionally: Cloud on second line
         # =====================================
         self.__display_header(__stat_display)
+        self.separator_line()
 
         # ==============================================================
         # Display second line (<SUMMARY>+CPU|PERCPU+<GPU>+LOAD+MEM+SWAP)
         # ==============================================================
         self.__display_top(__stat_display, stats)
+        self.init_column()
+        self.separator_line()
 
         # ==================================================================
         # Display left sidebar (NETWORK+PORTS+DISKIO+FS+SENSORS+Current time)
@@ -724,12 +740,14 @@ class _GlancesCurses(object):
             self.display_plugin(stat_display["ip"])
         self.new_column()
         self.display_plugin(
-            stat_display["uptime"], add_space=-(self.get_stats_display_width(stat_display["cloud"]) != 0)
+            stat_display["uptime"],
+            add_space=-(self.get_stats_display_width(stat_display["cloud"]) != 0)
         )
-        # Second line (optional)
         self.init_column()
-        self.new_line()
-        self.display_plugin(stat_display["cloud"])
+        if self.get_stats_display_width(stat_display["cloud"]) != 0:
+            # Second line (optional)
+            self.new_line()
+            self.display_plugin(stat_display["cloud"])
 
     def __display_top(self, stat_display, stats):
         """Display the second line in the Curses interface.

glances/outputs/glances_unicode.py

@@ -26,9 +26,10 @@ _unicode_message = {
     'ARROW_DOWN': [u'\u2193', u'v'],
     'CHECK': [u'\u2713', u''],
     'PROCESS_SELECTOR': [u'>', u'>'],
+    'MEDIUM_LINE': [u'\u23AF', u'-'],
+    'LOW_LINE': [u'\u2581', u'_'],
 }
 
-
 def unicode_message(key, args=None):
     """Return the unicode message for the given key."""
     if args and hasattr(args, 'disable_unicode') and args.disable_unicode:

glances/outputs/static/js/components/plugin-gpu/view.html

@@ -21,7 +21,7 @@
             <div class="table-cell text-left">temperature::</div>
             <div class="table-cell" ng-class="vm.getMeanDecoration('temperature')" ng-if="vm.mean.temperature != null">{{ vm.mean.temperature | number
                 :
-                0 }}%
+                0 }}°
             </div>
             <div class="table-cell" ng-if="vm.mean.temperature == null">N/A</div>
         </div>

glances/outputs/static/public/glances.js

@@ -60226,7 +60226,7 @@ module.exports = path;
 /***/ ((module) => {
 
 // Module
-var code = "<section id=\"gpu\" class=\"plugin\">\n    <div class=\"gpu-name title\">\n        {{ vm.name }}\n    </div>\n    <div class=\"table\">\n        <div class=\"table-row\" ng-if=\"arguments.meangpu || vm.gpus.length === 1\">\n            <div class=\"table-cell text-left\">proc:</div>\n            <div class=\"table-cell\" ng-class=\"vm.getMeanDecoration('proc')\" ng-if=\"vm.mean.proc != null\">{{ vm.mean.proc |\n                number : 0 }}%\n            </div>\n            <div class=\"table-cell\" ng-if=\"vm.mean.proc == null\">N/A</div>\n        </div>\n        <div class=\"table-row\" ng-if=\"arguments.meangpu || vm.gpus.length === 1\">\n            <div class=\"table-cell text-left\">mem:</div>\n            <div class=\"table-cell\" ng-class=\"vm.getMeanDecoration('mem')\" ng-if=\"vm.mean.mem != null\">{{ vm.mean.mem | number :\n                0 }}%\n            </div>\n            <div class=\"table-cell\" ng-if=\"vm.mean.mem == null\">N/A</div>\n        </div>\n        <div class=\"table-row\" ng-if=\"arguments.meangpu || vm.gpus.length === 1\">\n            <div class=\"table-cell text-left\">temperature::</div>\n            <div class=\"table-cell\" ng-class=\"vm.getMeanDecoration('temperature')\" ng-if=\"vm.mean.temperature != null\">{{ vm.mean.temperature | number\n                :\n                0 }}%\n            </div>\n            <div class=\"table-cell\" ng-if=\"vm.mean.temperature == null\">N/A</div>\n        </div>\n        <div class=\"table-row\" ng-if=\"!arguments.meangpu && vm.gpus.length > 1\" ng-repeat=\"gpu in vm.gpus\">\n            <div class=\"table-cell text-left\">\n                {{ gpu.gpu_id }}:\n                <span ng-class=\"vm.getDecoration(gpu.gpu_id, 'proc')\" ng-if=\"gpu.proc != null\">{{ gpu.proc | number : 0 }}%</span>\n                <span ng-if=\"gpu.proc == null\">N/A</span>\n                mem:\n                <span ng-class=\"vm.getDecoration(gpu.gpu_id, 'mem')\" ng-if=\"gpu.mem != null\">{{ gpu.mem | number : 0 }}%</span>\n                <span ng-if=\"gpu.mem == null\">N/A</span>\n                temp:\n                <span ng-class=\"vm.getDecoration(gpu.gpu_id, 'temperature')\" ng-if=\"gpu.temperature != null\">{{ gpu.temperature | number : 0 }}C</span>\n                <span ng-if=\"gpu.temperature == null\">N/A</span>\n            </div>\n        </div>\n    </div>\n</section>\n";
+var code = "<section id=\"gpu\" class=\"plugin\">\n    <div class=\"gpu-name title\">\n        {{ vm.name }}\n    </div>\n    <div class=\"table\">\n        <div class=\"table-row\" ng-if=\"arguments.meangpu || vm.gpus.length === 1\">\n            <div class=\"table-cell text-left\">proc:</div>\n            <div class=\"table-cell\" ng-class=\"vm.getMeanDecoration('proc')\" ng-if=\"vm.mean.proc != null\">{{ vm.mean.proc |\n                number : 0 }}%\n            </div>\n            <div class=\"table-cell\" ng-if=\"vm.mean.proc == null\">N/A</div>\n        </div>\n        <div class=\"table-row\" ng-if=\"arguments.meangpu || vm.gpus.length === 1\">\n            <div class=\"table-cell text-left\">mem:</div>\n            <div class=\"table-cell\" ng-class=\"vm.getMeanDecoration('mem')\" ng-if=\"vm.mean.mem != null\">{{ vm.mean.mem | number :\n                0 }}%\n            </div>\n            <div class=\"table-cell\" ng-if=\"vm.mean.mem == null\">N/A</div>\n        </div>\n        <div class=\"table-row\" ng-if=\"arguments.meangpu || vm.gpus.length === 1\">\n            <div class=\"table-cell text-left\">temperature::</div>\n            <div class=\"table-cell\" ng-class=\"vm.getMeanDecoration('temperature')\" ng-if=\"vm.mean.temperature != null\">{{ vm.mean.temperature | number\n                :\n                0 }}°\n            </div>\n            <div class=\"table-cell\" ng-if=\"vm.mean.temperature == null\">N/A</div>\n        </div>\n        <div class=\"table-row\" ng-if=\"!arguments.meangpu && vm.gpus.length > 1\" ng-repeat=\"gpu in vm.gpus\">\n            <div class=\"table-cell text-left\">\n                {{ gpu.gpu_id }}:\n                <span ng-class=\"vm.getDecoration(gpu.gpu_id, 'proc')\" ng-if=\"gpu.proc != null\">{{ gpu.proc | number : 0 }}%</span>\n                <span ng-if=\"gpu.proc == null\">N/A</span>\n                mem:\n                <span ng-class=\"vm.getDecoration(gpu.gpu_id, 'mem')\" ng-if=\"gpu.mem != null\">{{ gpu.mem | number : 0 }}%</span>\n                <span ng-if=\"gpu.mem == null\">N/A</span>\n                temp:\n                <span ng-class=\"vm.getDecoration(gpu.gpu_id, 'temperature')\" ng-if=\"gpu.temperature != null\">{{ gpu.temperature | number : 0 }}C</span>\n                <span ng-if=\"gpu.temperature == null\">N/A</span>\n            </div>\n        </div>\n    </div>\n</section>\n";
 // Exports
 var _module_exports =code;;
 var path = '/home/nicolargo/dev/glances/glances/outputs/static/js/components/plugin-gpu/view.html';

glances/plugins/ports/model.py

@@ -334,6 +334,8 @@ class ThreadScanner(threading.Thread):
         except Exception as e:
             logger.debug("{}: Error while pinging host {} ({})".format(self.plugin_name, port['host'], e))
 
+        fnull.close()
+
         return ret
 
     def _port_scan_tcp(self, port):

glances/standalone.py

@@ -165,12 +165,21 @@ class GlancesStandalone(object):
 
     def serve_forever(self):
         """Wrapper to the serve_forever function."""
-        # loop = True
-        # while loop:
-        #     loop = self.__serve_once()
-        while self.__serve_once():
-            pass
-        self.end()
+        if self.args.stop_after:
+            for _ in range(self.args.stop_after):
+                if not self.__serve_once():
+                    break
+        else:
+            while self.__serve_once():
+                pass
+        # self.end()
+
+    def serve_n(self, n=1):
+        """Serve n time."""
+        for _ in range(n):
+            if not self.__serve_once():
+                break
+        # self.end()
 
     def end(self):
         """End of the standalone CLI."""

unitest.py

@@ -21,6 +21,7 @@
 """Glances unitary tests suite."""
 
 import time
+from tracemalloc import Snapshot
 import unittest
 import sys
 
@@ -41,16 +42,19 @@ from glances.thresholds import GlancesThresholdCritical
 from glances.thresholds import GlancesThresholds
 from glances.plugins.plugin.model import GlancesPluginModel
 from glances.secure import secure_popen
+from glances.compat import PY3
 
 # Global variables
 # =================
 
 # Init Glances core
 core = GlancesMain()
+test_config = core.get_config()
+test_args = core.get_args()
 
 # Init Glances stats
-stats = GlancesStats(config=core.get_config(),
-                     args=core.get_args())
+stats = GlancesStats(config=test_config,
+                     args=test_args)
 
 # Unitest class
 # ==============
@@ -388,6 +392,49 @@ class TestGlances(unittest.TestCase):
         self.assertEqual(secure_popen('echo FOO | grep FOO'), 'FOO\n')
         self.assertEqual(secure_popen('echo -n TEST1 && echo -n TEST2'), 'TEST1TEST2')
 
+    def test_200_memory_leak(self):
+        """Memory leak check"""
+        # Only available in PY3
+        if not PY3:
+            return
+        import tracemalloc
+        print('INFO: [TEST_200] Memory leak check')
+        tracemalloc.start()
+        # 3 iterations just to init the stats and fill the memory
+        for _ in range(3):
+            stats.update()
+
+        # Start the memory leak check
+        snapshot_begin = tracemalloc.take_snapshot()
+        for _ in range(3):
+            stats.update()
+        snapshot_end = tracemalloc.take_snapshot()
+        snapshot_diff = snapshot_end.compare_to(snapshot_begin, 'filename')
+        memory_leak = sum([s.size_diff for s in snapshot_diff])
+        print('INFO: Memory leak: {} bytes'.format(memory_leak))
+
+        # snapshot_begin = tracemalloc.take_snapshot()
+        for _ in range(30):
+            stats.update()
+        snapshot_end = tracemalloc.take_snapshot()
+        snapshot_diff = snapshot_end.compare_to(snapshot_begin, 'filename')
+        memory_leak = sum([s.size_diff for s in snapshot_diff])
+        print('INFO: Memory leak: {} bytes'.format(memory_leak))
+
+        # snapshot_begin = tracemalloc.take_snapshot()
+        for _ in range(300):
+            stats.update()
+        snapshot_end = tracemalloc.take_snapshot()
+        snapshot_diff = snapshot_end.compare_to(snapshot_begin, 'filename')
+        memory_leak = sum([s.size_diff for s in snapshot_diff])
+        print('INFO: Memory leak: {} bytes'.format(memory_leak))
+        snapshot_top = snapshot_end.compare_to(snapshot_begin, 'traceback')
+        print("Memory consumption (top 5):")
+        for stat in snapshot_top[:5]:
+            print(stat)
+            for line in stat.traceback.format():
+                print(line)
+
     def test_999_the_end(self):
         """Free all the stats"""
         print('INFO: [TEST_999] Free the stats')