(postgre)SQL export support / TimeScaleDB #2814

Makefile

@@ -126,7 +126,10 @@ test-export-influxdb-v1: ## Run interface tests with InfluxDB version 1 (Legacy)
 test-export-influxdb-v3: ## Run interface tests with InfluxDB version 3 (Core)
 	/bin/bash ./tests/test_export_influxdb_v3.sh
 
-test-export: test-export-csv test-export-json test-export-influxdb-v1 test-export-influxdb-v3 ## Tests all exports
+test-export-timescaledb: ## Run interface tests with TimescaleDB
+	/bin/bash ./tests/test_export_timescaledb.sh
+
+test-export: test-export-csv test-export-json test-export-influxdb-v1 test-export-influxdb-v3 test-export-timescaledb## Tests all exports
 
 # ===================================================================
 # Linters, profilers and cyber security

conf/glances.conf

@@ -839,6 +839,18 @@ prefix=glances
 # By default, system_name = FQDN
 #system_name=mycomputer
 
+[timescaledb]
+# Configuration for the --export timescaledb option
+# https://www.timescale.com/
+host=localhost
+port=5432
+db=glances
+user=postgres
+password=password
+# Overwrite device name (default is the FQDN)
+# Most of the time, you should not overwrite this value
+#hostname=mycomputer
+
 ##############################################################################
 # AMPS
 # * enable: Enable (true) or disable (false) the AMP

docker-compose/glances.conf

@@ -651,7 +651,28 @@ port=8086
 protocol=http
 org=nicolargo
 bucket=glances
-token=EjFUTWe8U-MIseEAkaVIgVnej_TrnbdvEcRkaB1imstW7gapSqy6_6-8XD-yd51V0zUUpDy-kAdVD1purDLuxA==
+token=PUT_YOUR_INFLUXDB2_TOKEN_HERE
+# Set the interval between two exports (in seconds)
+# If the interval is set to 0, the Glances refresh time is used (default behavor)
+#interval=0
+# Prefix will be added for all measurement name
+# Ex: prefix=foo
+#     => foo.cpu
+#     => foo.mem
+# You can also use dynamic values
+#prefix=foo
+# Following tags will be added for all measurements
+# You can also use dynamic values.
+# Note: hostname and name (for process) are always added as a tag
+#tags=foo:bar,spam:eggs,domain:`domainname`
+
+[influxdb3]
+# Configuration for the --export influxdb3 option
+# https://influxdb.com/
+host=http://localhost:8181
+org=nicolargo
+database=glances
+token=PUT_YOUR_INFLUXDB3_TOKEN_HERE
 # Set the interval between two exports (in seconds)
 # If the interval is set to 0, the Glances refresh time is used (default behavor)
 #interval=0
@@ -817,6 +838,18 @@ prefix=glances
 # By default, system_name = FQDN
 #system_name=mycomputer
 
+[timescaledb]
+# Configuration for the --export timescaledb option
+# https://www.timescale.com/
+host=localhost
+port=5432
+db=glances
+user=postgres
+password=password
+# Overwrite device name (default is the FQDN)
+# Most of the time, you should not overwrite this value
+#hostname=mycomputer
+
 ##############################################################################
 # AMPS
 # * enable: Enable (true) or disable (false) the AMP

docs/gw/index.rst

@@ -25,4 +25,5 @@ to providing stats to multiple services (see list below).
    restful
    riemann
    statsd
+   timescaledb
    zeromq

docs/gw/timescaledb.rst

@@ -0,0 +1,48 @@
+.. _timescale:
+
+TimeScaleDB
+===========
+
+TimescaleDB is a time-series database built on top of PostgreSQL.
+
+You can export statistics to a ``TimescaleDB`` server.
+
+The connection should be defined in the Glances configuration file as
+following:
+
+.. code-block:: ini
+
+    [timescaledb]
+    host=localhost
+    port=5432
+    db=glances
+    user=postgres
+    password=password
+
+and run Glances with:
+
+.. code-block:: console
+
+    $ glances --export timescaledb
+
+Data model
+-----------
+
+Each plugin will create an `hypertable`_ in the TimescaleDB database.
+
+Tables are partitionned by time (using the ``time`` column).
+
+Tables are segmented by hostname (in order to have multiple host stored in the Glances database).
+
+For plugin with a key (example network where the key is the interface name), the key will
+be added as a column in the table (named key_id) and added to the timescaledb.segmentby option.
+
+Current limitations
+-------------------
+
+Sensors and Fs plugins are not supported by the TimescaleDB exporter.
+
+In the cpu plugin, the user field is exported as user_cpu (user_percpu in the percpu plugin)
+because user is a reserved keyword in PostgreSQL.
+
+.. _hypertable: https://docs.tigerdata.com/use-timescale/latest/hypertables/

docs/man/glances.1

@@ -28,7 +28,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .in \\n[rst2man-indent\\n[rst2man-indent-level]]u
 ..
-.TH "GLANCES" "1" "May 27, 2025" "4.3.2_dev01" "Glances"
+.TH "GLANCES" "1" "Jun 22, 2025" "4.3.2_dev05" "Glances"
 .SH NAME
 glances \- An eye on your system
 .SH SYNOPSIS

glances/__init__.py

@@ -19,7 +19,7 @@ import tracemalloc
 # Global name
 # Version should start and end with a numerical char
 # See https://packaging.python.org/specifications/core-metadata/#version
-__version__ = "4.3.2_dev04"
+__version__ = "4.3.2_dev05"
 __apiversion__ = '4'
 __author__ = 'Nicolas Hennion <nicolas@nicolargo.com>'
 __license__ = 'LGPLv3'

glances/exports/export.py

@@ -225,7 +225,7 @@ class GlancesExport:
 
         The method builds two lists: names and values and calls the export method to export the stats.
 
-        Note: this class can be overwritten (for example in CSV and Graph).
+        Note: if needed this class can be overwritten.
         """
         if not self.export_enable:
             return False
@@ -245,6 +245,8 @@ class GlancesExport:
                 # TypeError: string indices must be integers (Network plugin) #1054
                 for i in all_stats[plugin]:
                     i.update(all_limits[plugin])
+                    # Remove the <plugin>_disable field
+                    i.pop(f"{plugin}_disable", None)
             else:
                 continue
             export_names, export_values = self.build_export(all_stats[plugin])
@@ -253,7 +255,11 @@ class GlancesExport:
         return True
 
     def build_export(self, stats):
-        """Build the export lists."""
+        """Build the export lists.
+        This method builds two lists: names and values.
+        """
+
+        # Initialize export lists
         export_names = []
         export_values = []
 
@@ -278,6 +284,7 @@ class GlancesExport:
                     export_names += item_names
                     export_values += item_values
                 else:
+                    # We are on a simple value
                     export_names.append(pre_key + key.lower())
                     export_values.append(value)
         elif isinstance(stats, list):

glances/exports/glances_opentsdb/__init__.py

@@ -57,7 +57,7 @@ class Export(GlancesExport):
         return db
 
     def export(self, name, columns, points):
-        """Export the stats to the Statsd server."""
+        """Export the stats to the OpenTSDB server."""
         for i in range(len(columns)):
             if not isinstance(points[i], Number):
                 continue

glances/exports/glances_timescaledb/__init__.py

@@ -0,0 +1,217 @@
+#
+# This file is part of Glances.
+#
+# SPDX-FileCopyrightText: 2025 Nicolas Hennion <nicolas@nicolargo.com>
+#
+# SPDX-License-Identifier: LGPL-3.0-only
+#
+
+"""TimescaleDB interface class."""
+
+import sys
+import time
+from platform import node
+
+import psycopg
+
+from glances.exports.export import GlancesExport
+from glances.logger import logger
+
+# Define the type conversions for TimescaleDB
+# https://www.postgresql.org/docs/current/datatype.html
+convert_types = {
+    'bool': 'BOOLEAN',
+    'int': 'BIGINT',
+    'float': 'DOUBLE PRECISION',
+    'str': 'TEXT',
+    'tuple': 'TEXT',  # Store tuples as TEXT (comma-separated)
+    'list': 'TEXT',  # Store lists as TEXT (comma-separated)
+    'NoneType': 'DOUBLE PRECISION',  # Use DOUBLE PRECISION for NoneType to avoid issues with NULL
+}
+
+
+class Export(GlancesExport):
+    """This class manages the TimescaleDB export module."""
+
+    def __init__(self, config=None, args=None):
+        """Init the TimescaleDB export IF."""
+        super().__init__(config=config, args=args)
+
+        # Mandatory configuration keys (additional to host and port)
+        self.db = None
+
+        # Optional configuration keys
+        self.user = None
+        self.password = None
+        self.hostname = None
+
+        # Load the configuration file
+        self.export_enable = self.load_conf(
+            'timescaledb', mandatories=['host', 'port', 'db'], options=['user', 'password', 'hostname']
+        )
+        if not self.export_enable:
+            exit('Missing TimescaleDB config')
+
+        # The hostname is always add as an identifier in the TimescaleDB table
+        # so we can filter the stats by hostname
+        self.hostname = self.hostname or node().split(".")[0]
+
+        # Init the TimescaleDB client
+        self.client = self.init()
+
+    def init(self):
+        """Init the connection to the TimescaleDB server."""
+        if not self.export_enable:
+            return None
+
+        try:
+            # See https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING
+            conn_str = f"host={self.host} port={self.port} dbname={self.db} user={self.user} password={self.password}"
+            db = psycopg.connect(conn_str)
+        except Exception as e:
+            logger.critical(f"Cannot connect to TimescaleDB server {self.host}:{self.port} ({e})")
+            sys.exit(2)
+        else:
+            logger.info(f"Stats will be exported to TimescaleDB server: {self.host}:{self.port}")
+
+        return db
+
+    def normalize(self, value):
+        """Normalize the value to be exportable to TimescaleDB."""
+        if value is None:
+            return 'NULL'
+        if isinstance(value, bool):
+            return str(value).upper()
+        if isinstance(value, (list, tuple)):
+            return ', '.join([f"'{v}'" for v in value])
+        if isinstance(value, str):
+            return f"'{value}'"
+
+        return f"{value}"
+
+    def update(self, stats):
+        """Update the TimescaleDB export module."""
+        if not self.export_enable:
+            return False
+
+        # Get all the stats & limits
+        # Current limitation with sensors and fs plugins because fields list is not the same
+        self._last_exported_list = [p for p in self.plugins_to_export(stats) if p not in ['sensors', 'fs']]
+        all_stats = stats.getAllExportsAsDict(plugin_list=self.last_exported_list())
+        all_limits = stats.getAllLimitsAsDict(plugin_list=self.last_exported_list())
+
+        # Loop over plugins to export
+        for plugin in self.last_exported_list():
+            if isinstance(all_stats[plugin], dict):
+                all_stats[plugin].update(all_limits[plugin])
+                # Remove the <plugin>_disable field
+                all_stats[plugin].pop(f"{plugin}_disable", None)
+                # user is a special field that should not be exported
+                # rename it to user_<plugin>
+                if 'user' in all_stats[plugin]:
+                    all_stats[plugin][f'user_{plugin}'] = all_stats[plugin].pop('user')
+            elif isinstance(all_stats[plugin], list):
+                for i in all_stats[plugin]:
+                    i.update(all_limits[plugin])
+                    # Remove the <plugin>_disable field
+                    i.pop(f"{plugin}_disable", None)
+                    # user is a special field that should not be exported
+                    # rename it to user_<plugin>
+                    if 'user' in i:
+                        i[f'user_{plugin}'] = i.pop('user')
+            else:
+                continue
+
+            plugin_stats = all_stats[plugin]
+            creation_list = []  # List used to create the TimescaleDB table
+            segmented_by = []  # List of columns used to segment the data
+            values_list = []  # List of values to insert (list of lists, one list per row)
+            if isinstance(plugin_stats, dict):
+                # Stats is a dict
+                # Create the list used to create the TimescaleDB table
+                creation_list.append('time TIMESTAMPTZ NOT NULL')
+                creation_list.append('hostname_id TEXT NOT NULL')
+                segmented_by.extend(['hostname_id'])  # Segment by hostname
+                for key, value in plugin_stats.items():
+                    creation_list.append(f"{key} {convert_types[type(value).__name__]} NULL")
+                values_list.append('NOW()')  # Add the current time (insertion time)
+                values_list.append(f"'{self.hostname}'")  # Add the hostname
+                values_list.extend([self.normalize(value) for value in plugin_stats.values()])
+                values_list = [values_list]
+            elif isinstance(plugin_stats, list) and len(plugin_stats) > 0 and 'key' in plugin_stats[0]:
+                # Stats is a list
+                # Create the list used to create the TimescaleDB table
+                creation_list.append('time TIMESTAMPTZ NOT NULL')
+                creation_list.append('hostname_id TEXT NOT NULL')
+                creation_list.append('key_id TEXT NOT NULL')
+                segmented_by.extend(['hostname_id', 'key_id'])  # Segment by hostname and key
+                for key, value in plugin_stats[0].items():
+                    creation_list.append(f"{key} {convert_types[type(value).__name__]} NULL")
+                # Create the values list (it is a list of list to have a single datamodel for all the plugins)
+                for plugin_item in plugin_stats:
+                    item_list = []
+                    item_list.append('NOW()')  # Add the current time (insertion time)
+                    item_list.append(f"'{self.hostname}'")  # Add the hostname
+                    item_list.append(f"'{plugin_item.get('key')}'")
+                    item_list.extend([self.normalize(value) for value in plugin_item.values()])
+                    values_list.append(item_list[:-1])
+            else:
+                continue
+
+            # Export stats to TimescaleDB
+            self.export(plugin, creation_list, segmented_by, values_list)
+
+        return True
+
+    def export(self, plugin, creation_list, segmented_by, values_list):
+        """Export the stats to the TimescaleDB server."""
+        logger.debug(f"Export {plugin} stats to TimescaleDB")
+
+        with self.client.cursor() as cur:
+            # Is the table exists?
+            cur.execute(f"select exists(select * from information_schema.tables where table_name='{plugin}')")
+            if not cur.fetchone()[0]:
+                # Create the table if it does not exist
+                # https://github.com/timescale/timescaledb/blob/main/README.md#create-a-hypertable
+                # Execute the create table query
+                create_query = f"""
+CREATE TABLE {plugin} (
+    {', '.join(creation_list)}
+)
+WITH (
+    timescaledb.hypertable,
+    timescaledb.partition_column='time',
+    timescaledb.segmentby = '{", ".join(segmented_by)}'
+);"""
+                logger.debug(f"Create table: {create_query}")
+                try:
+                    cur.execute(create_query)
+                except Exception as e:
+                    logger.error(f"Cannot create table {plugin}: {e}")
+                    return
+
+            # Insert the data
+            # https://github.com/timescale/timescaledb/blob/main/README.md#insert-and-query-data
+            insert_list = [f"({','.join(i)})" for i in values_list]
+            insert_query = f"INSERT INTO {plugin} VALUES {','.join(insert_list)};"
+            logger.debug(f"Insert data into table: {insert_query}")
+            try:
+                cur.execute(insert_query)
+            except Exception as e:
+                logger.error(f"Cannot insert data into table {plugin}: {e}")
+                return
+
+        # Commit the changes (for every plugin or to be done at the end ?)
+        self.client.commit()
+
+    def exit(self):
+        """Close the TimescaleDB export module."""
+        # Force last write
+        self.client.commit()
+
+        # Close the TimescaleDB client
+        time.sleep(3)  # Wait a bit to ensure all data is written
+        self.client.close()
+
+        # Call the father method
+        super().exit()

optional-requirements.txt

@@ -16,6 +16,7 @@ influxdb3-python # For InfluxDB 3.x
 jinja2
 kafka-python
 netifaces2
+numpy>=1.22.2 # not directly required, pinned by Snyk to avoid a vulnerability
 nvidia-ml-py
 orjson
 paho-mqtt
@@ -23,6 +24,8 @@ pika
 podman
 potsdb
 prometheus_client
+psycopg[binary]
+pyarrow>=14.0.1 # not directly required, pinned by Snyk to avoid a vulnerability
 pycouchdb
 pydantic
 pygal
@@ -33,12 +36,10 @@ pysnmp-lextudio<6.3.1 # Pinned witing implementation of #2874
 python-dateutil
 pyzmq
 requests
+setuptools>=78.1.1 # not directly required, pinned by Snyk to avoid a vulnerability
 six
 sparklines
 statsd
+urllib3>=2.2.2 # not directly required, pinned by Snyk to avoid a vulnerability
 uvicorn
 zeroconf
-numpy>=1.22.2 # not directly required, pinned by Snyk to avoid a vulnerability
-pyarrow>=14.0.1 # not directly required, pinned by Snyk to avoid a vulnerability
-setuptools>=78.1.1 # not directly required, pinned by Snyk to avoid a vulnerability
-urllib3>=2.2.2 # not directly required, pinned by Snyk to avoid a vulnerability

pyproject.toml

@@ -81,6 +81,7 @@ export = [
   "pika",
   "potsdb",
   "prometheus_client",
+  "psycopg[binary]",
   "pymongo",
   "pyzmq",
   "statsd",

tests/test_export_timescaledb.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+# Pre-requisites:
+# - docker
+# - jq
+
+# Exit on error
+set -e
+
+echo "Clean previous test data..."
+rm -f /tmp/timescaledb-for-glances_cpu.csv
+
+echo "Stop previous TimeScaleDB container..."
+docker stop timescaledb-for-glances || true
+docker rm timescaledb-for-glances || true
+
+echo "Starting TimeScaleDB container..."
+docker run -d \
+    --name timescaledb-for-glances \
+    -p 5432:5432 \
+    -e POSTGRES_PASSWORD=password \
+    timescale/timescaledb-ha:pg17
+
+# Wait for InfluxDB to be ready (15 seconds)
+echo "Waiting for TimeScaleDB to start (~ 15 seconds)..."
+sleep 15
+
+# Create the glances database
+echo "Creating 'glances' database..."
+docker exec timescaledb-for-glances psql -d "postgres://postgres:password@localhost/postgres" -c "CREATE DATABASE glances;"
+
+# Run glances with export to TimescaleDB, stopping after 10 writes
+# This will run synchronously now since we're using --stop-after
+echo "Glances to export system stats to TimescaleDB (duration: ~ 20 seconds)"
+./venv/bin/python -m glances --config ./conf/glances.conf --export timescaledb --stop-after 10 --quiet
+
+
+docker exec timescaledb-for-glances psql -d "postgres://postgres:password@localhost/glances" -c "SELECT * from cpu;" --csv > /tmp/timescaledb-for-glances_cpu.csv
+./venv/bin/python ./tests-data/tools/csvcheck.py -i /tmp/timescaledb-for-glances_cpu.csv -l 9
+
+# Stop and remove the TimescaleDB container
+echo "Stopping and removing TimescaleDB container..."
+# docker stop timescaledb-for-glances && docker rm timescaledb-for-glances
+
+echo "Script completed successfully!"