From 1365d600a3c92483efa42ad67aad8b9aa2769635 Mon Sep 17 00:00:00 2001 From: nicolargo Date: Sun, 22 Jun 2025 18:26:00 +0200 Subject: [PATCH] (postgre)SQL export support / TimeScaleDB #2814 --- Makefile | 5 +- conf/glances.conf | 12 + docker-compose/glances.conf | 35 +- docs/api.rst | 947 ++++++++---------- docs/gw/index.rst | 1 + docs/gw/timescaledb.rst | 48 + docs/man/glances.1 | 2 +- glances/__init__.py | 2 +- glances/exports/export.py | 11 +- glances/exports/glances_opentsdb/__init__.py | 2 +- .../exports/glances_timescaledb/__init__.py | 217 ++++ optional-requirements.txt | 9 +- pyproject.toml | 1 + tests/test_export_timescaledb.sh | 44 + 14 files changed, 793 insertions(+), 543 deletions(-) create mode 100644 docs/gw/timescaledb.rst create mode 100644 glances/exports/glances_timescaledb/__init__.py create mode 100755 tests/test_export_timescaledb.sh diff --git a/Makefile b/Makefile index 8d010e5d..64791c07 100644 --- a/Makefile +++ b/Makefile @@ -126,7 +126,10 @@ test-export-influxdb-v1: ## Run interface tests with InfluxDB version 1 (Legacy) test-export-influxdb-v3: ## Run interface tests with InfluxDB version 3 (Core) /bin/bash ./tests/test_export_influxdb_v3.sh -test-export: test-export-csv test-export-json test-export-influxdb-v1 test-export-influxdb-v3 ## Tests all exports +test-export-timescaledb: ## Run interface tests with TimescaleDB + /bin/bash ./tests/test_export_timescaledb.sh + +test-export: test-export-csv test-export-json test-export-influxdb-v1 test-export-influxdb-v3 test-export-timescaledb## Tests all exports # =================================================================== # Linters, profilers and cyber security diff --git a/conf/glances.conf b/conf/glances.conf index 616a413d..643552d3 100644 --- a/conf/glances.conf +++ b/conf/glances.conf @@ -839,6 +839,18 @@ prefix=glances # By default, system_name = FQDN #system_name=mycomputer +[timescaledb] +# Configuration for the --export timescaledb option +# https://www.timescale.com/ +host=localhost +port=5432 +db=glances +user=postgres +password=password +# Overwrite device name (default is the FQDN) +# Most of the time, you should not overwrite this value +#hostname=mycomputer + ############################################################################## # AMPS # * enable: Enable (true) or disable (false) the AMP diff --git a/docker-compose/glances.conf b/docker-compose/glances.conf index 6f904737..b25aa9c5 100755 --- a/docker-compose/glances.conf +++ b/docker-compose/glances.conf @@ -651,7 +651,28 @@ port=8086 protocol=http org=nicolargo bucket=glances -token=EjFUTWe8U-MIseEAkaVIgVnej_TrnbdvEcRkaB1imstW7gapSqy6_6-8XD-yd51V0zUUpDy-kAdVD1purDLuxA== +token=PUT_YOUR_INFLUXDB2_TOKEN_HERE +# Set the interval between two exports (in seconds) +# If the interval is set to 0, the Glances refresh time is used (default behavor) +#interval=0 +# Prefix will be added for all measurement name +# Ex: prefix=foo +# => foo.cpu +# => foo.mem +# You can also use dynamic values +#prefix=foo +# Following tags will be added for all measurements +# You can also use dynamic values. +# Note: hostname and name (for process) are always added as a tag +#tags=foo:bar,spam:eggs,domain:`domainname` + +[influxdb3] +# Configuration for the --export influxdb3 option +# https://influxdb.com/ +host=http://localhost:8181 +org=nicolargo +database=glances +token=PUT_YOUR_INFLUXDB3_TOKEN_HERE # Set the interval between two exports (in seconds) # If the interval is set to 0, the Glances refresh time is used (default behavor) #interval=0 @@ -817,6 +838,18 @@ prefix=glances # By default, system_name = FQDN #system_name=mycomputer +[timescaledb] +# Configuration for the --export timescaledb option +# https://www.timescale.com/ +host=localhost +port=5432 +db=glances +user=postgres +password=password +# Overwrite device name (default is the FQDN) +# Most of the time, you should not overwrite this value +#hostname=mycomputer + ############################################################################## # AMPS # * enable: Enable (true) or disable (false) the AMP diff --git a/docs/api.rst b/docs/api.rst index 7231544f..d67cb0df 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -160,7 +160,7 @@ Get plugin stats:: "refresh": 3.0, "regex": True, "result": None, - "timer": 0.47667741775512695}, + "timer": 0.3796122074127197}, {"count": 0, "countmax": 20.0, "countmin": None, @@ -169,7 +169,7 @@ Get plugin stats:: "refresh": 3.0, "regex": True, "result": None, - "timer": 0.4765753746032715}] + "timer": 0.37954068183898926}] Fields descriptions: @@ -197,7 +197,7 @@ Get a specific item when field matches the given value:: "refresh": 3.0, "regex": True, "result": None, - "timer": 0.47667741775512695}]} + "timer": 0.3796122074127197}]} GET cloud --------- @@ -238,28 +238,28 @@ GET containers Get plugin stats:: # curl http://localhost:61208/api/4/containers - [{"command": "bash start.sh", + [{"command": "/docker-entrypoint.sh postgres", "cpu": {"total": 0.0}, "cpu_percent": 0.0, - "created": "2025-03-02T18:56:49.050583851Z", + "created": "2025-06-22T15:43:18.364042094Z", "engine": "docker", - "id": "37b79baf008ea602934be8d16d76064f79a032c11e4f6fe6e811b3c46ac20fd0", - "image": ["ghcr.io/open-webui/open-webui:ollama"], - "io": {"cumulative_ior": 618508288, "cumulative_iow": 0}, + "id": "454a8c7f059271f0e7fbb757375014115db3b6e0df2c039e66bb2647f717f67e", + "image": ["timescale/timescaledb-ha:pg17"], + "io": {"cumulative_ior": 24920064, "cumulative_iow": 0}, "io_rx": None, "io_wx": None, "key": "name", - "memory": {"inactive_file": 20922368, - "limit": 16421883904, - "usage": 113471488}, + "memory": {"inactive_file": 7553024, + "limit": 16421875712, + "usage": 170262528}, "memory_percent": None, - "memory_usage": 113471488, - "name": "open-webui", - "network": {"cumulative_rx": 189915, "cumulative_tx": 2159}, + "memory_usage": 170262528, + "name": "timescaledb-for-glances", + "network": {"cumulative_rx": 172655, "cumulative_tx": 77037}, "network_rx": None, "network_tx": None, "status": "running", - "uptime": "yesterday"}] + "uptime": "39 mins"}] Fields descriptions: @@ -283,33 +283,35 @@ Fields descriptions: Get a specific field:: # curl http://localhost:61208/api/4/containers/name - {"name": ["open-webui"]} + {"name": ["timescaledb-for-glances"]} Get a specific item when field matches the given value:: - # curl http://localhost:61208/api/4/containers/name/value/open-webui - {"open-webui": [{"command": "bash start.sh", - "cpu": {"total": 0.0}, - "cpu_percent": 0.0, - "created": "2025-03-02T18:56:49.050583851Z", - "engine": "docker", - "id": "37b79baf008ea602934be8d16d76064f79a032c11e4f6fe6e811b3c46ac20fd0", - "image": ["ghcr.io/open-webui/open-webui:ollama"], - "io": {"cumulative_ior": 618508288, "cumulative_iow": 0}, - "io_rx": None, - "io_wx": None, - "key": "name", - "memory": {"inactive_file": 20922368, - "limit": 16421883904, - "usage": 113471488}, - "memory_percent": None, - "memory_usage": 113471488, - "name": "open-webui", - "network": {"cumulative_rx": 189915, "cumulative_tx": 2159}, - "network_rx": None, - "network_tx": None, - "status": "running", - "uptime": "yesterday"}]} + # curl http://localhost:61208/api/4/containers/name/value/timescaledb-for-glances + {"timescaledb-for-glances": [{"command": "/docker-entrypoint.sh postgres", + "cpu": {"total": 0.0}, + "cpu_percent": 0.0, + "created": "2025-06-22T15:43:18.364042094Z", + "engine": "docker", + "id": "454a8c7f059271f0e7fbb757375014115db3b6e0df2c039e66bb2647f717f67e", + "image": ["timescale/timescaledb-ha:pg17"], + "io": {"cumulative_ior": 24920064, + "cumulative_iow": 0}, + "io_rx": None, + "io_wx": None, + "key": "name", + "memory": {"inactive_file": 7553024, + "limit": 16421875712, + "usage": 170262528}, + "memory_percent": None, + "memory_usage": 170262528, + "name": "timescaledb-for-glances", + "network": {"cumulative_rx": 172655, + "cumulative_tx": 77037}, + "network_rx": None, + "network_tx": None, + "status": "running", + "uptime": "39 mins"}]} GET core -------- @@ -336,19 +338,19 @@ Get plugin stats:: # curl http://localhost:61208/api/4/cpu {"cpucore": 16, - "ctx_switches": 110124988, + "ctx_switches": 769574166, "guest": 0.0, - "idle": 93.2, - "interrupts": 91446752, - "iowait": 0.1, + "idle": 92.4, + "interrupts": 308408102, + "iowait": 1.3, "irq": 0.0, "nice": 0.0, - "soft_interrupts": 37996687, + "soft_interrupts": 61314500, "steal": 0.0, "syscalls": 0, - "system": 2.7, - "total": 5.9, - "user": 3.9} + "system": 2.9, + "total": 7.0, + "user": 3.4} Fields descriptions: @@ -381,7 +383,7 @@ Fields descriptions: Get a specific field:: # curl http://localhost:61208/api/4/cpu/total - {"total": 5.9} + {"total": 7.0} GET diskio ---------- @@ -391,16 +393,16 @@ Get plugin stats:: # curl http://localhost:61208/api/4/diskio [{"disk_name": "nvme0n1", "key": "disk_name", - "read_bytes": 7939079680, - "read_count": 316989, - "write_bytes": 13072667648, - "write_count": 1097171}, + "read_bytes": 11332324864, + "read_count": 603864, + "write_bytes": 19931874304, + "write_count": 2092501}, {"disk_name": "nvme0n1p1", "key": "disk_name", - "read_bytes": 8415232, - "read_count": 953, - "write_bytes": 5120, - "write_count": 3}] + "read_bytes": 12949504, + "read_count": 735, + "write_bytes": 1024, + "write_count": 2}] Fields descriptions: @@ -434,10 +436,10 @@ Get a specific item when field matches the given value:: # curl http://localhost:61208/api/4/diskio/disk_name/value/nvme0n1 {"nvme0n1": [{"disk_name": "nvme0n1", "key": "disk_name", - "read_bytes": 7939079680, - "read_count": 316989, - "write_bytes": 13072667648, - "write_count": 1097171}]} + "read_bytes": 11332324864, + "read_count": 603864, + "write_bytes": 19931874304, + "write_count": 2092501}]} GET folders ----------- @@ -464,14 +466,14 @@ Get plugin stats:: # curl http://localhost:61208/api/4/fs [{"device_name": "/dev/mapper/ubuntu--vg-ubuntu--lv", - "free": 795803656192, + "free": 761794158592, "fs_type": "ext4", "key": "mnt_point", "mnt_point": "/", "options": "rw,relatime", - "percent": 16.5, + "percent": 20.0, "size": 1003736440832, - "used": 156870279168}, + "used": 190879776768}, {"device_name": "zsfpool", "free": 41680896, "fs_type": "zfs", @@ -502,14 +504,14 @@ Get a specific item when field matches the given value:: # curl http://localhost:61208/api/4/fs/mnt_point/value// {"/": [{"device_name": "/dev/mapper/ubuntu--vg-ubuntu--lv", - "free": 795803656192, + "free": 761794158592, "fs_type": "ext4", "key": "mnt_point", "mnt_point": "/", "options": "rw,relatime", - "percent": 16.5, + "percent": 20.0, "size": 1003736440832, - "used": 156870279168}]} + "used": 190879776768}]} GET gpu ------- @@ -542,7 +544,7 @@ GET ip Get plugin stats:: # curl http://localhost:61208/api/4/ip - {"address": "192.168.1.26", + {"address": "192.168.0.28", "mask": "255.255.255.0", "mask_cidr": 24, "public_address": "", @@ -560,7 +562,7 @@ Fields descriptions: Get a specific field:: # curl http://localhost:61208/api/4/ip/address - {"address": "192.168.1.26"} + {"address": "192.168.0.28"} GET irq ------- @@ -582,9 +584,9 @@ Get plugin stats:: # curl http://localhost:61208/api/4/load {"cpucore": 16, - "min1": 2.14111328125, - "min15": 1.15234375, - "min5": 1.4111328125} + "min1": 1.33642578125, + "min15": 0.787109375, + "min5": 1.06787109375} Fields descriptions: @@ -596,7 +598,7 @@ Fields descriptions: Get a specific field:: # curl http://localhost:61208/api/4/load/min1 - {"min1": 2.14111328125} + {"min1": 1.33642578125} GET mem ------- @@ -604,16 +606,16 @@ GET mem Get plugin stats:: # curl http://localhost:61208/api/4/mem - {"active": 8626487296, - "available": 5448634368, - "buffers": 372654080, - "cached": 5088337920, - "free": 5448634368, - "inactive": 4633042944, - "percent": 66.8, - "shared": 859938816, - "total": 16421883904, - "used": 10973249536} + {"active": 9752326144, + "available": 1487818752, + "buffers": 31051776, + "cached": 2230222848, + "free": 1487818752, + "inactive": 4722712576, + "percent": 90.9, + "shared": 769691648, + "total": 16421875712, + "used": 14934056960} Fields descriptions: @@ -632,7 +634,7 @@ Fields descriptions: Get a specific field:: # curl http://localhost:61208/api/4/mem/total - {"total": 16421883904} + {"total": 16421875712} GET memswap ----------- @@ -640,13 +642,13 @@ GET memswap Get plugin stats:: # curl http://localhost:61208/api/4/memswap - {"free": 1380184064, - "percent": 67.9, - "sin": 177889280, - "sout": 3054333952, + {"free": 229376, + "percent": 100.0, + "sin": 1334226944, + "sout": 5940338688, "time_since_update": 1, "total": 4294963200, - "used": 2914779136} + "used": 4294733824} Fields descriptions: @@ -671,32 +673,32 @@ Get plugin stats:: # curl http://localhost:61208/api/4/network [{"alias": None, "bytes_all": None, - "bytes_all_gauge": 1113983505, + "bytes_all_gauge": 555231074, "bytes_all_rate_per_sec": None, "bytes_recv": None, - "bytes_recv_gauge": 881611898, + "bytes_recv_gauge": 341594852, "bytes_recv_rate_per_sec": None, "bytes_sent": None, - "bytes_sent_gauge": 232371607, + "bytes_sent_gauge": 213636222, "bytes_sent_rate_per_sec": None, "interface_name": "wlp0s20f3", "key": "interface_name", "speed": 0, - "time_since_update": 0.4796278476715088}, + "time_since_update": 0.383251428604126}, {"alias": None, "bytes_all": None, - "bytes_all_gauge": 192074, + "bytes_all_gauge": 30225, "bytes_all_rate_per_sec": None, "bytes_recv": None, - "bytes_recv_gauge": 2159, + "bytes_recv_gauge": 0, "bytes_recv_rate_per_sec": None, "bytes_sent": None, - "bytes_sent_gauge": 189915, + "bytes_sent_gauge": 30225, "bytes_sent_rate_per_sec": None, - "interface_name": "veth274f692", + "interface_name": "ipv6leakintrf0", "key": "interface_name", - "speed": 10485760000, - "time_since_update": 0.4796278476715088}] + "speed": 0, + "time_since_update": 0.383251428604126}] Fields descriptions: @@ -718,25 +720,25 @@ Fields descriptions: Get a specific field:: # curl http://localhost:61208/api/4/network/interface_name - {"interface_name": ["wlp0s20f3", "veth274f692"]} + {"interface_name": ["wlp0s20f3", "ipv6leakintrf0", "proton0", "veth620fda1"]} Get a specific item when field matches the given value:: # curl http://localhost:61208/api/4/network/interface_name/value/wlp0s20f3 {"wlp0s20f3": [{"alias": None, "bytes_all": None, - "bytes_all_gauge": 1113983505, + "bytes_all_gauge": 555231074, "bytes_all_rate_per_sec": None, "bytes_recv": None, - "bytes_recv_gauge": 881611898, + "bytes_recv_gauge": 341594852, "bytes_recv_rate_per_sec": None, "bytes_sent": None, - "bytes_sent_gauge": 232371607, + "bytes_sent_gauge": 213636222, "bytes_sent_rate_per_sec": None, "interface_name": "wlp0s20f3", "key": "interface_name", "speed": 0, - "time_since_update": 0.4796278476715088}]} + "time_since_update": 0.383251428604126}]} GET now ------- @@ -744,7 +746,7 @@ GET now Get plugin stats:: # curl http://localhost:61208/api/4/now - {"custom": "2025-05-27 10:44:34 CEST", "iso": "2025-05-27T10:44:34+02:00"} + {"custom": "2025-06-22 18:22:53 CEST", "iso": "2025-06-22T18:22:53+02:00"} Fields descriptions: @@ -754,7 +756,7 @@ Fields descriptions: Get a specific field:: # curl http://localhost:61208/api/4/now/iso - {"iso": "2025-05-27T10:44:34+02:00"} + {"iso": "2025-06-22T18:22:53+02:00"} GET percpu ---------- @@ -766,7 +768,7 @@ Get plugin stats:: "dpc": None, "guest": 0.0, "guest_nice": 0.0, - "idle": 37.0, + "idle": 25.0, "interrupt": None, "iowait": 0.0, "irq": 0.0, @@ -774,14 +776,14 @@ Get plugin stats:: "nice": 0.0, "softirq": 0.0, "steal": 0.0, - "system": 8.0, - "total": 63.0, + "system": 10.0, + "total": 75.0, "user": 0.0}, {"cpu_number": 1, "dpc": None, "guest": 0.0, "guest_nice": 0.0, - "idle": 46.0, + "idle": 32.0, "interrupt": None, "iowait": 0.0, "irq": 0.0, @@ -789,8 +791,8 @@ Get plugin stats:: "nice": 0.0, "softirq": 0.0, "steal": 0.0, - "system": 0.0, - "total": 54.0, + "system": 1.0, + "total": 68.0, "user": 0.0}] Fields descriptions: @@ -822,12 +824,12 @@ Get plugin stats:: # curl http://localhost:61208/api/4/ports [{"description": "DefaultGateway", - "host": "192.168.1.1", + "host": "192.168.0.254", "indice": "port_0", "port": 0, "refresh": 30, "rtt_warning": None, - "status": 0.016056, + "status": 0.016936, "timeout": 3}] Fields descriptions: @@ -844,19 +846,19 @@ Fields descriptions: Get a specific field:: # curl http://localhost:61208/api/4/ports/host - {"host": ["192.168.1.1"]} + {"host": ["192.168.0.254"]} Get a specific item when field matches the given value:: - # curl http://localhost:61208/api/4/ports/host/value/192.168.1.1 - {"192.168.1.1": [{"description": "DefaultGateway", - "host": "192.168.1.1", - "indice": "port_0", - "port": 0, - "refresh": 30, - "rtt_warning": None, - "status": 0.016056, - "timeout": 3}]} + # curl http://localhost:61208/api/4/ports/host/value/192.168.0.254 + {"192.168.0.254": [{"description": "DefaultGateway", + "host": "192.168.0.254", + "indice": "port_0", + "port": 0, + "refresh": 30, + "rtt_warning": None, + "status": 0.016936, + "timeout": 3}]} GET processcount ---------------- @@ -864,7 +866,7 @@ GET processcount Get plugin stats:: # curl http://localhost:61208/api/4/processcount - {"pid_max": 0, "running": 1, "sleeping": 435, "thread": 2312, "total": 566} + {"pid_max": 0, "running": 2, "sleeping": 424, "thread": 2209, "total": 575} Fields descriptions: @@ -877,7 +879,7 @@ Fields descriptions: Get a specific field:: # curl http://localhost:61208/api/4/processcount/total - {"total": 566} + {"total": 575} GET processlist --------------- @@ -885,15 +887,16 @@ GET processlist Get plugin stats:: # curl http://localhost:61208/api/4/processlist - [{"cmdline": ["/snap/code/193/usr/share/code/code", + [{"cmdline": ["/proc/self/exe", "--type=utility", "--utility-sub-type=node.mojom.NodeService", "--lang=en-US", "--service-sandbox-type=none", "--no-sandbox", "--dns-result-order=ipv4first", + "--experimental-network-inspection", "--inspect-port=0", - "--crashpad-handler-pid=12776", + "--crashpad-handler-pid=15887", "--enable-crash-reporter=864d4bb7-dd20-4851-830f-29e81dd93517,no_channel", "--user-data-dir=/home/nicolargo/.config/Code", "--standard-schemes=vscode-webview,vscode-file", @@ -903,182 +906,128 @@ Get plugin stats:: "--service-worker-schemes=vscode-webview", "--code-cache-schemes=vscode-webview,vscode-file", "--shared-files=v8_context_snapshot_data:100", - "--field-trial-handle=3,i,8675254667491921424,564610109630475014,262144", + "--field-trial-handle=3,i,7331693094454094392,6095270110338113916,262144", "--enable-features=DocumentPolicyIncludeJSCallStacksInCrashReports,EarlyEstablishGpuChannel,EstablishGpuChannelAsync", "--disable-features=CalculateNativeWinOcclusion,SpareRendererForSitePerProcess", "--variations-seed-version"], "cpu_percent": 0.0, - "cpu_times": {"children_system": 471.74, - "children_user": 147.05, + "cpu_times": {"children_system": 0.27, + "children_user": 0.08, "iowait": 0.0, - "system": 526.93, - "user": 2401.08}, + "system": 3045.45, + "user": 1877.77}, "gids": {"effective": 1000, "real": 1000, "saved": 1000}, - "io_counters": [437005312, - 607666176, + "io_counters": [102701056, + 32768, 0, 0, 0, - 322137088, - 135368704, + 1365398528, + 760889344, 0, 0, 0, - 76328960, - 491520, + 99730432, + 643072, 0, 0, 0, - 10312704, - 159744, + 116122624, + 344064, 0, 0, 0, - 163840, + 10363904, 0, 0, 0, 0, - 72953856, + 43321344, + 81920, 0, 0, 0, + 147589120, + 132931584, 0, - 13445120, - 126976, 0, 0, + 8920064, 0, - 15028224, 0, 0, 0, + 73924608, 0, - 73690112, - 195317760, 0, 0, 0, - 844800, + 38301696, 0, 0, 0, 0, - 663552, + 1230848, + 8192, 0, 0, 0, + 1818624, 0, - 455680, - 44584960, 0, 0, 0, - 40048640, - 808263680, + 7995392, 0, 0, 0, - 2324480, 0, + 851968, 0, 0, 0, - 6102016, 0, + 7394304, + 6295552, 0, 0, 0, - 36453376, + 552960, 0, 0, 0, 0, - 56055808, - 10477568, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 265216, - 0, - 0, - 0, - 0, - 4233216, - 0, - 0, - 0, - 0, - 803840, - 0, - 0, - 0, - 0, - 3489792, - 0, - 0, - 0, - 0, - 495616, - 0, - 0, - 0, - 0, - 178176, - 0, - 0, - 0, - 0, - 5767168, - 0, - 0, - 0, - 0, - 43987968, - 23666688, - 0, - 0, - 0, - 3426304, - 962560, - 0, - 0, - 0, - 972800, + 1149952, 0, 0, 0, 0], "key": "pid", - "memory_info": {"data": 3651760128, + "memory_info": {"data": 7859228672, "dirty": 0, "lib": 0, - "rss": 1959587840, - "shared": 129040384, - "text": 139026432, - "vms": 1280588754944}, - "memory_percent": 11.932783421533559, + "rss": 6381924352, + "shared": 28991488, + "text": 142934016, + "vms": 1496234172416}, + "memory_percent": 38.862334997070526, "name": "code", "nice": 0, - "num_threads": 61, - "pid": 13009, + "num_threads": 18, + "pid": 16064, "status": "S", "time_since_update": 1, "username": "nicolargo"}, - {"cmdline": ["/snap/code/193/usr/share/code/code", + {"cmdline": ["/proc/self/exe", "--type=utility", "--utility-sub-type=node.mojom.NodeService", "--lang=en-US", "--service-sandbox-type=none", "--no-sandbox", "--dns-result-order=ipv4first", + "--experimental-network-inspection", "--inspect-port=0", - "--crashpad-handler-pid=12776", + "--crashpad-handler-pid=15887", "--enable-crash-reporter=864d4bb7-dd20-4851-830f-29e81dd93517,no_channel", "--user-data-dir=/home/nicolargo/.config/Code", "--standard-schemes=vscode-webview,vscode-file", @@ -1088,31 +1037,31 @@ Get plugin stats:: "--service-worker-schemes=vscode-webview", "--code-cache-schemes=vscode-webview,vscode-file", "--shared-files=v8_context_snapshot_data:100", - "--field-trial-handle=3,i,8675254667491921424,564610109630475014,262144", + "--field-trial-handle=3,i,7331693094454094392,6095270110338113916,262144", "--enable-features=DocumentPolicyIncludeJSCallStacksInCrashReports,EarlyEstablishGpuChannel,EstablishGpuChannelAsync", "--disable-features=CalculateNativeWinOcclusion,SpareRendererForSitePerProcess", "--variations-seed-version"], "cpu_percent": 0.0, - "cpu_times": {"children_system": 245.05, - "children_user": 91.45, + "cpu_times": {"children_system": 760.22, + "children_user": 170.94, "iowait": 0.0, - "system": 175.06, - "user": 181.79}, + "system": 735.2, + "user": 2372.27}, "gids": {"effective": 1000, "real": 1000, "saved": 1000}, - "io_counters": [322137088, 135368704, 0, 0, 0], + "io_counters": [1365398528, 760889344, 0, 0, 0], "key": "pid", - "memory_info": {"data": 2051534848, + "memory_info": {"data": 3502342144, "dirty": 0, "lib": 0, - "rss": 779325440, - "shared": 93945856, - "text": 139026432, - "vms": 1250669309952}, - "memory_percent": 4.745651866471751, + "rss": 1511571456, + "shared": 38739968, + "text": 142934016, + "vms": 1524465074176}, + "memory_percent": 9.204621216901828, "name": "code", "nice": 0, - "num_threads": 18, - "pid": 12982, + "num_threads": 83, + "pid": 16602, "status": "S", "time_since_update": 1, "username": "nicolargo"}] @@ -1139,215 +1088,149 @@ GET programlist Get plugin stats:: # curl http://localhost:61208/api/4/programlist - [{"childrens": [13009, - 12982, - 14962, - 12880, - 14859, - 14754, - 12874, - 15346, - 12755, - 21785, - 21786, - 14763, - 12998, - 12930, - 21733, - 13000, - 21803, - 14993, - 37630, - 21732, - 13677, - 12999, - 13798, - 13711, - 14088, - 12847, - 12758, - 12757], + [{"childrens": [16064, + 16602, + 17005, + 15978, + 16662, + 16074, + 15865, + 16018, + 17530, + 16604, + 16152, + 16938, + 17345, + 16506, + 15941, + 15868, + 15867], "cmdline": ["code"], "cpu_percent": 0, - "cpu_times": {"children_system": 720.6599999999999, - "children_user": 257.92999999999995, - "system": 1076.7100000000007, - "user": 5801.529999999999}, - "io_counters": [437005312, - 607666176, + "cpu_times": {"children_system": 761.63, + "children_user": 175.16, + "system": 4194.55, + "user": 8850.69}, + "io_counters": [102701056, + 32768, 0, 0, 0, - 322137088, - 135368704, + 1365398528, + 760889344, 0, 0, 0, - 76328960, - 491520, + 99730432, + 643072, 0, 0, 0, - 10312704, - 159744, + 116122624, + 344064, 0, 0, 0, - 163840, + 10363904, 0, 0, 0, 0, - 72953856, + 43321344, + 81920, 0, 0, 0, + 147589120, + 132931584, 0, - 13445120, - 126976, 0, 0, + 8920064, 0, - 15028224, 0, 0, 0, + 73924608, 0, - 73690112, - 195317760, 0, 0, 0, - 844800, + 38301696, 0, 0, 0, 0, - 663552, + 1230848, + 8192, 0, 0, 0, + 1818624, 0, - 455680, - 44584960, 0, 0, 0, - 40048640, - 808263680, + 7995392, 0, 0, 0, - 2324480, 0, + 851968, 0, 0, 0, - 6102016, 0, + 7394304, + 6295552, 0, 0, 0, - 36453376, + 552960, 0, 0, 0, 0, - 56055808, - 10477568, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 265216, - 0, - 0, - 0, - 0, - 4233216, - 0, - 0, - 0, - 0, - 803840, - 0, - 0, - 0, - 0, - 3489792, - 0, - 0, - 0, - 0, - 495616, - 0, - 0, - 0, - 0, - 178176, - 0, - 0, - 0, - 0, - 5767168, - 0, - 0, - 0, - 0, - 43987968, - 23666688, - 0, - 0, - 0, - 3426304, - 962560, - 0, - 0, - 0, - 972800, + 1149952, 0, 0, 0, 0], - "memory_info": {"data": 25453400064, - "rss": 7429939200, - "shared": 2272882688, - "text": 3892740096, - "vms": 30116498821120}, - "memory_percent": 45.244134250579094, + "memory_info": {"data": 22324699136, + "rss": 9708691456, + "shared": 525979648, + "text": 2429878272, + "vms": 19632912211968}, + "memory_percent": 59.12047823444154, "name": "code", "nice": 0, - "nprocs": 28, - "num_threads": 462, + "nprocs": 17, + "num_threads": 326, "pid": "_", - "status": "S", + "status": "_", "time_since_update": 1, "username": "nicolargo"}, - {"childrens": [6484], - "cmdline": ["WebExtensions"], + {"childrens": [17473], + "cmdline": ["cloudcode_cli"], "cpu_percent": 0, "cpu_times": {"children_system": 0.0, "children_user": 0.0, "iowait": 0.0, - "system": 60.16, - "user": 378.83}, - "io_counters": [17175552, 0, 0, 0, 0], - "memory_info": {"data": 1012625408, + "system": 24.2, + "user": 186.71}, + "io_counters": [374910976, 0, 0, 0, 0], + "memory_info": {"data": 1479667712, "dirty": 0, "lib": 0, - "rss": 566517760, - "shared": 114372608, - "text": 860160, - "vms": 25258516480}, - "memory_percent": 3.4497732617754595, - "name": "WebExtensions", + "rss": 661827584, + "shared": 12406784, + "text": 33955840, + "vms": 2726150144}, + "memory_percent": 4.030158281592529, + "name": "cloudcode_cli", "nice": 0, "nprocs": 1, - "num_threads": 27, + "num_threads": 20, "pid": "_", "status": "S", "time_since_update": 1, @@ -1383,124 +1266,19 @@ GET quicklook Get plugin stats:: # curl http://localhost:61208/api/4/quicklook - {"cpu": 5.9, + {"cpu": 7.0, "cpu_hz": 4475000000.0, - "cpu_hz_current": 813255500.0, + "cpu_hz_current": 1011126874.9999999, "cpu_log_core": 16, "cpu_name": "13th Gen Intel(R) Core(TM) i7-13620H", "cpu_phys_core": 10, - "load": 7.2, - "mem": 66.8, + "load": 4.9, + "mem": 90.9, "percpu": [{"cpu_number": 0, "dpc": None, "guest": 0.0, "guest_nice": 0.0, - "idle": 37.0, - "interrupt": None, - "iowait": 0.0, - "irq": 0.0, - "key": "cpu_number", - "nice": 0.0, - "softirq": 0.0, - "steal": 0.0, - "system": 8.0, - "total": 63.0, - "user": 0.0}, - {"cpu_number": 1, - "dpc": None, - "guest": 0.0, - "guest_nice": 0.0, - "idle": 46.0, - "interrupt": None, - "iowait": 0.0, - "irq": 0.0, - "key": "cpu_number", - "nice": 0.0, - "softirq": 0.0, - "steal": 0.0, - "system": 0.0, - "total": 54.0, - "user": 0.0}, - {"cpu_number": 2, - "dpc": None, - "guest": 0.0, - "guest_nice": 0.0, - "idle": 46.0, - "interrupt": None, - "iowait": 0.0, - "irq": 0.0, - "key": "cpu_number", - "nice": 0.0, - "softirq": 0.0, - "steal": 0.0, - "system": 0.0, - "total": 54.0, - "user": 0.0}, - {"cpu_number": 3, - "dpc": None, - "guest": 0.0, - "guest_nice": 0.0, - "idle": 47.0, - "interrupt": None, - "iowait": 0.0, - "irq": 0.0, - "key": "cpu_number", - "nice": 0.0, - "softirq": 0.0, - "steal": 0.0, - "system": 0.0, - "total": 53.0, - "user": 0.0}, - {"cpu_number": 4, - "dpc": None, - "guest": 0.0, - "guest_nice": 0.0, - "idle": 39.0, - "interrupt": None, - "iowait": 1.0, - "irq": 0.0, - "key": "cpu_number", - "nice": 0.0, - "softirq": 0.0, - "steal": 0.0, - "system": 5.0, - "total": 61.0, - "user": 1.0}, - {"cpu_number": 5, - "dpc": None, - "guest": 0.0, - "guest_nice": 0.0, - "idle": 47.0, - "interrupt": None, - "iowait": 0.0, - "irq": 0.0, - "key": "cpu_number", - "nice": 0.0, - "softirq": 0.0, - "steal": 0.0, - "system": 0.0, - "total": 53.0, - "user": 0.0}, - {"cpu_number": 6, - "dpc": None, - "guest": 0.0, - "guest_nice": 0.0, - "idle": 42.0, - "interrupt": None, - "iowait": 0.0, - "irq": 0.0, - "key": "cpu_number", - "nice": 0.0, - "softirq": 0.0, - "steal": 0.0, - "system": 4.0, - "total": 58.0, - "user": 0.0}, - {"cpu_number": 7, - "dpc": None, - "guest": 0.0, - "guest_nice": 0.0, - "idle": 24.0, + "idle": 25.0, "interrupt": None, "iowait": 0.0, "irq": 0.0, @@ -1509,28 +1287,133 @@ Get plugin stats:: "softirq": 0.0, "steal": 0.0, "system": 10.0, - "total": 76.0, - "user": 11.0}, + "total": 75.0, + "user": 0.0}, + {"cpu_number": 1, + "dpc": None, + "guest": 0.0, + "guest_nice": 0.0, + "idle": 32.0, + "interrupt": None, + "iowait": 0.0, + "irq": 0.0, + "key": "cpu_number", + "nice": 0.0, + "softirq": 0.0, + "steal": 0.0, + "system": 1.0, + "total": 68.0, + "user": 0.0}, + {"cpu_number": 2, + "dpc": None, + "guest": 0.0, + "guest_nice": 0.0, + "idle": 33.0, + "interrupt": None, + "iowait": 0.0, + "irq": 0.0, + "key": "cpu_number", + "nice": 0.0, + "softirq": 0.0, + "steal": 0.0, + "system": 1.0, + "total": 67.0, + "user": 0.0}, + {"cpu_number": 3, + "dpc": None, + "guest": 0.0, + "guest_nice": 0.0, + "idle": 34.0, + "interrupt": None, + "iowait": 0.0, + "irq": 0.0, + "key": "cpu_number", + "nice": 0.0, + "softirq": 0.0, + "steal": 0.0, + "system": 0.0, + "total": 66.0, + "user": 0.0}, + {"cpu_number": 4, + "dpc": None, + "guest": 0.0, + "guest_nice": 0.0, + "idle": 30.0, + "interrupt": None, + "iowait": 0.0, + "irq": 0.0, + "key": "cpu_number", + "nice": 0.0, + "softirq": 0.0, + "steal": 0.0, + "system": 3.0, + "total": 70.0, + "user": 1.0}, + {"cpu_number": 5, + "dpc": None, + "guest": 0.0, + "guest_nice": 0.0, + "idle": 26.0, + "interrupt": None, + "iowait": 0.0, + "irq": 0.0, + "key": "cpu_number", + "nice": 0.0, + "softirq": 0.0, + "steal": 0.0, + "system": 4.0, + "total": 74.0, + "user": 2.0}, + {"cpu_number": 6, + "dpc": None, + "guest": 0.0, + "guest_nice": 0.0, + "idle": 31.0, + "interrupt": None, + "iowait": 2.0, + "irq": 0.0, + "key": "cpu_number", + "nice": 0.0, + "softirq": 0.0, + "steal": 0.0, + "system": 0.0, + "total": 69.0, + "user": 1.0}, + {"cpu_number": 7, + "dpc": None, + "guest": 0.0, + "guest_nice": 0.0, + "idle": 22.0, + "interrupt": None, + "iowait": 0.0, + "irq": 0.0, + "key": "cpu_number", + "nice": 0.0, + "softirq": 0.0, + "steal": 0.0, + "system": 2.0, + "total": 78.0, + "user": 10.0}, {"cpu_number": 8, "dpc": None, "guest": 0.0, "guest_nice": 0.0, - "idle": 46.0, + "idle": 33.0, "interrupt": None, - "iowait": 0.0, + "iowait": 1.0, "irq": 0.0, "key": "cpu_number", "nice": 0.0, "softirq": 0.0, "steal": 0.0, "system": 0.0, - "total": 54.0, - "user": 1.0}, + "total": 67.0, + "user": 0.0}, {"cpu_number": 9, "dpc": None, "guest": 0.0, "guest_nice": 0.0, - "idle": 46.0, + "idle": 34.0, "interrupt": None, "iowait": 0.0, "irq": 0.0, @@ -1539,13 +1422,13 @@ Get plugin stats:: "softirq": 0.0, "steal": 0.0, "system": 0.0, - "total": 54.0, + "total": 66.0, "user": 0.0}, {"cpu_number": 10, "dpc": None, "guest": 0.0, "guest_nice": 0.0, - "idle": 46.0, + "idle": 32.0, "interrupt": None, "iowait": 0.0, "irq": 0.0, @@ -1554,13 +1437,13 @@ Get plugin stats:: "softirq": 0.0, "steal": 0.0, "system": 1.0, - "total": 54.0, - "user": 0.0}, + "total": 68.0, + "user": 1.0}, {"cpu_number": 11, "dpc": None, "guest": 0.0, "guest_nice": 0.0, - "idle": 47.0, + "idle": 33.0, "interrupt": None, "iowait": 0.0, "irq": 0.0, @@ -1569,13 +1452,13 @@ Get plugin stats:: "softirq": 0.0, "steal": 0.0, "system": 0.0, - "total": 53.0, + "total": 67.0, "user": 0.0}, {"cpu_number": 12, "dpc": None, "guest": 0.0, "guest_nice": 0.0, - "idle": 46.0, + "idle": 33.0, "interrupt": None, "iowait": 0.0, "irq": 0.0, @@ -1584,13 +1467,13 @@ Get plugin stats:: "softirq": 0.0, "steal": 0.0, "system": 1.0, - "total": 54.0, - "user": 1.0}, + "total": 67.0, + "user": 0.0}, {"cpu_number": 13, "dpc": None, "guest": 0.0, "guest_nice": 0.0, - "idle": 46.0, + "idle": 33.0, "interrupt": None, "iowait": 0.0, "irq": 0.0, @@ -1598,14 +1481,14 @@ Get plugin stats:: "nice": 0.0, "softirq": 0.0, "steal": 0.0, - "system": 1.0, - "total": 54.0, + "system": 0.0, + "total": 67.0, "user": 0.0}, {"cpu_number": 14, "dpc": None, "guest": 0.0, "guest_nice": 0.0, - "idle": 46.0, + "idle": 33.0, "interrupt": None, "iowait": 0.0, "irq": 0.0, @@ -1614,13 +1497,13 @@ Get plugin stats:: "softirq": 0.0, "steal": 0.0, "system": 0.0, - "total": 54.0, + "total": 67.0, "user": 0.0}, {"cpu_number": 15, "dpc": None, "guest": 0.0, "guest_nice": 0.0, - "idle": 46.0, + "idle": 32.0, "interrupt": None, "iowait": 0.0, "irq": 0.0, @@ -1628,10 +1511,10 @@ Get plugin stats:: "nice": 0.0, "softirq": 0.0, "steal": 0.0, - "system": 0.0, - "total": 54.0, + "system": 1.0, + "total": 68.0, "user": 0.0}], - "swap": 67.9} + "swap": 100.0} Fields descriptions: @@ -1669,14 +1552,14 @@ Get plugin stats:: "label": "Ambient", "type": "temperature_core", "unit": "C", - "value": 37, + "value": 44, "warning": 0}, {"critical": None, "key": "label", "label": "Ambient 3", "type": "temperature_core", "unit": "C", - "value": 30, + "value": 36, "warning": 0}] Fields descriptions: @@ -1738,7 +1621,7 @@ Get a specific item when field matches the given value:: "label": "Ambient", "type": "temperature_core", "unit": "C", - "value": 37, + "value": 44, "warning": 0}]} GET smart @@ -1756,10 +1639,10 @@ Get plugin stats:: # curl http://localhost:61208/api/4/system {"hostname": "nicolargo-xps15", - "hr_name": "Ubuntu 24.04 64bit / Linux 6.11.0-24-generic", + "hr_name": "Ubuntu 24.04 64bit / Linux 6.11.0-26-generic", "linux_distro": "Ubuntu 24.04", "os_name": "Linux", - "os_version": "6.11.0-24-generic", + "os_version": "6.11.0-26-generic", "platform": "64bit"} Fields descriptions: @@ -1782,7 +1665,7 @@ GET uptime Get plugin stats:: # curl http://localhost:61208/api/4/uptime - "1 day, 17:51:58" + "1 day, 4:58:34" GET version ----------- @@ -1790,7 +1673,7 @@ GET version Get plugin stats:: # curl http://localhost:61208/api/4/version - "4.3.2_dev01" + "4.3.2_dev05" GET vms ------- @@ -1827,8 +1710,8 @@ Get plugin stats:: # curl http://localhost:61208/api/4/wifi [{"key": "ssid", - "quality_level": -59.0, - "quality_link": 51.0, + "quality_level": -66.0, + "quality_link": 44.0, "ssid": "wlp0s20f3"}] Get a specific field:: @@ -1840,8 +1723,8 @@ Get a specific item when field matches the given value:: # curl http://localhost:61208/api/4/wifi/ssid/value/wlp0s20f3 {"wlp0s20f3": [{"key": "ssid", - "quality_level": -59.0, - "quality_link": 51.0, + "quality_level": -66.0, + "quality_link": 44.0, "ssid": "wlp0s20f3"}]} GET all stats @@ -1905,34 +1788,34 @@ GET stats history History of a plugin:: # curl http://localhost:61208/api/4/cpu/history - {"system": [["2025-05-27T10:44:35.577810", 2.7], - ["2025-05-27T10:44:36.687892", 0.9], - ["2025-05-27T10:44:37.721905", 0.9]], - "user": [["2025-05-27T10:44:35.577806", 3.9], - ["2025-05-27T10:44:36.687889", 1.0], - ["2025-05-27T10:44:37.721902", 1.0]]} + {"system": [["2025-06-22T18:22:54.444782", 2.9], + ["2025-06-22T18:22:55.515406", 1.4], + ["2025-06-22T18:22:56.534675", 1.4]], + "user": [["2025-06-22T18:22:54.444780", 3.4], + ["2025-06-22T18:22:55.515405", 1.5], + ["2025-06-22T18:22:56.534674", 1.5]]} Limit history to last 2 values:: # curl http://localhost:61208/api/4/cpu/history/2 - {"system": [["2025-05-27T10:44:36.687892", 0.9], - ["2025-05-27T10:44:37.721905", 0.9]], - "user": [["2025-05-27T10:44:36.687889", 1.0], - ["2025-05-27T10:44:37.721902", 1.0]]} + {"system": [["2025-06-22T18:22:55.515406", 1.4], + ["2025-06-22T18:22:56.534675", 1.4]], + "user": [["2025-06-22T18:22:55.515405", 1.5], + ["2025-06-22T18:22:56.534674", 1.5]]} History for a specific field:: # curl http://localhost:61208/api/4/cpu/system/history - {"system": [["2025-05-27T10:44:34.399773", 2.7], - ["2025-05-27T10:44:35.577810", 2.7], - ["2025-05-27T10:44:36.687892", 0.9], - ["2025-05-27T10:44:37.721905", 0.9]]} + {"system": [["2025-06-22T18:22:53.291867", 2.9], + ["2025-06-22T18:22:54.444782", 2.9], + ["2025-06-22T18:22:55.515406", 1.4], + ["2025-06-22T18:22:56.534675", 1.4]]} Limit history for a specific field to last 2 values:: # curl http://localhost:61208/api/4/cpu/system/history - {"system": [["2025-05-27T10:44:36.687892", 0.9], - ["2025-05-27T10:44:37.721905", 0.9]]} + {"system": [["2025-06-22T18:22:55.515406", 1.4], + ["2025-06-22T18:22:56.534675", 1.4]]} GET limits (used for thresholds) -------------------------------- diff --git a/docs/gw/index.rst b/docs/gw/index.rst index e0b635f2..187f159f 100644 --- a/docs/gw/index.rst +++ b/docs/gw/index.rst @@ -25,4 +25,5 @@ to providing stats to multiple services (see list below). restful riemann statsd + timescaledb zeromq diff --git a/docs/gw/timescaledb.rst b/docs/gw/timescaledb.rst new file mode 100644 index 00000000..4fdaff7a --- /dev/null +++ b/docs/gw/timescaledb.rst @@ -0,0 +1,48 @@ +.. _timescale: + +TimeScaleDB +=========== + +TimescaleDB is a time-series database built on top of PostgreSQL. + +You can export statistics to a ``TimescaleDB`` server. + +The connection should be defined in the Glances configuration file as +following: + +.. code-block:: ini + + [timescaledb] + host=localhost + port=5432 + db=glances + user=postgres + password=password + +and run Glances with: + +.. code-block:: console + + $ glances --export timescaledb + +Data model +----------- + +Each plugin will create an `hypertable`_ in the TimescaleDB database. + +Tables are partitionned by time (using the ``time`` column). + +Tables are segmented by hostname (in order to have multiple host stored in the Glances database). + +For plugin with a key (example network where the key is the interface name), the key will +be added as a column in the table (named key_id) and added to the timescaledb.segmentby option. + +Current limitations +------------------- + +Sensors and Fs plugins are not supported by the TimescaleDB exporter. + +In the cpu plugin, the user field is exported as user_cpu (user_percpu in the percpu plugin) +because user is a reserved keyword in PostgreSQL. + +.. _hypertable: https://docs.tigerdata.com/use-timescale/latest/hypertables/ diff --git a/docs/man/glances.1 b/docs/man/glances.1 index 894e71ab..1396551e 100644 --- a/docs/man/glances.1 +++ b/docs/man/glances.1 @@ -28,7 +28,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "GLANCES" "1" "May 27, 2025" "4.3.2_dev01" "Glances" +.TH "GLANCES" "1" "Jun 22, 2025" "4.3.2_dev05" "Glances" .SH NAME glances \- An eye on your system .SH SYNOPSIS diff --git a/glances/__init__.py b/glances/__init__.py index ade59892..bad2ff85 100644 --- a/glances/__init__.py +++ b/glances/__init__.py @@ -19,7 +19,7 @@ import tracemalloc # Global name # Version should start and end with a numerical char # See https://packaging.python.org/specifications/core-metadata/#version -__version__ = "4.3.2_dev04" +__version__ = "4.3.2_dev05" __apiversion__ = '4' __author__ = 'Nicolas Hennion ' __license__ = 'LGPLv3' diff --git a/glances/exports/export.py b/glances/exports/export.py index 1b648bb9..85ce63c6 100644 --- a/glances/exports/export.py +++ b/glances/exports/export.py @@ -225,7 +225,7 @@ class GlancesExport: The method builds two lists: names and values and calls the export method to export the stats. - Note: this class can be overwritten (for example in CSV and Graph). + Note: if needed this class can be overwritten. """ if not self.export_enable: return False @@ -245,6 +245,8 @@ class GlancesExport: # TypeError: string indices must be integers (Network plugin) #1054 for i in all_stats[plugin]: i.update(all_limits[plugin]) + # Remove the _disable field + i.pop(f"{plugin}_disable", None) else: continue export_names, export_values = self.build_export(all_stats[plugin]) @@ -253,7 +255,11 @@ class GlancesExport: return True def build_export(self, stats): - """Build the export lists.""" + """Build the export lists. + This method builds two lists: names and values. + """ + + # Initialize export lists export_names = [] export_values = [] @@ -278,6 +284,7 @@ class GlancesExport: export_names += item_names export_values += item_values else: + # We are on a simple value export_names.append(pre_key + key.lower()) export_values.append(value) elif isinstance(stats, list): diff --git a/glances/exports/glances_opentsdb/__init__.py b/glances/exports/glances_opentsdb/__init__.py index 2bfeca71..6173f9c7 100644 --- a/glances/exports/glances_opentsdb/__init__.py +++ b/glances/exports/glances_opentsdb/__init__.py @@ -57,7 +57,7 @@ class Export(GlancesExport): return db def export(self, name, columns, points): - """Export the stats to the Statsd server.""" + """Export the stats to the OpenTSDB server.""" for i in range(len(columns)): if not isinstance(points[i], Number): continue diff --git a/glances/exports/glances_timescaledb/__init__.py b/glances/exports/glances_timescaledb/__init__.py new file mode 100644 index 00000000..c2852f31 --- /dev/null +++ b/glances/exports/glances_timescaledb/__init__.py @@ -0,0 +1,217 @@ +# +# This file is part of Glances. +# +# SPDX-FileCopyrightText: 2025 Nicolas Hennion +# +# SPDX-License-Identifier: LGPL-3.0-only +# + +"""TimescaleDB interface class.""" + +import sys +import time +from platform import node + +import psycopg + +from glances.exports.export import GlancesExport +from glances.logger import logger + +# Define the type conversions for TimescaleDB +# https://www.postgresql.org/docs/current/datatype.html +convert_types = { + 'bool': 'BOOLEAN', + 'int': 'BIGINT', + 'float': 'DOUBLE PRECISION', + 'str': 'TEXT', + 'tuple': 'TEXT', # Store tuples as TEXT (comma-separated) + 'list': 'TEXT', # Store lists as TEXT (comma-separated) + 'NoneType': 'DOUBLE PRECISION', # Use DOUBLE PRECISION for NoneType to avoid issues with NULL +} + + +class Export(GlancesExport): + """This class manages the TimescaleDB export module.""" + + def __init__(self, config=None, args=None): + """Init the TimescaleDB export IF.""" + super().__init__(config=config, args=args) + + # Mandatory configuration keys (additional to host and port) + self.db = None + + # Optional configuration keys + self.user = None + self.password = None + self.hostname = None + + # Load the configuration file + self.export_enable = self.load_conf( + 'timescaledb', mandatories=['host', 'port', 'db'], options=['user', 'password', 'hostname'] + ) + if not self.export_enable: + exit('Missing TimescaleDB config') + + # The hostname is always add as an identifier in the TimescaleDB table + # so we can filter the stats by hostname + self.hostname = self.hostname or node().split(".")[0] + + # Init the TimescaleDB client + self.client = self.init() + + def init(self): + """Init the connection to the TimescaleDB server.""" + if not self.export_enable: + return None + + try: + # See https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING + conn_str = f"host={self.host} port={self.port} dbname={self.db} user={self.user} password={self.password}" + db = psycopg.connect(conn_str) + except Exception as e: + logger.critical(f"Cannot connect to TimescaleDB server {self.host}:{self.port} ({e})") + sys.exit(2) + else: + logger.info(f"Stats will be exported to TimescaleDB server: {self.host}:{self.port}") + + return db + + def normalize(self, value): + """Normalize the value to be exportable to TimescaleDB.""" + if value is None: + return 'NULL' + if isinstance(value, bool): + return str(value).upper() + if isinstance(value, (list, tuple)): + return ', '.join([f"'{v}'" for v in value]) + if isinstance(value, str): + return f"'{value}'" + + return f"{value}" + + def update(self, stats): + """Update the TimescaleDB export module.""" + if not self.export_enable: + return False + + # Get all the stats & limits + # Current limitation with sensors and fs plugins because fields list is not the same + self._last_exported_list = [p for p in self.plugins_to_export(stats) if p not in ['sensors', 'fs']] + all_stats = stats.getAllExportsAsDict(plugin_list=self.last_exported_list()) + all_limits = stats.getAllLimitsAsDict(plugin_list=self.last_exported_list()) + + # Loop over plugins to export + for plugin in self.last_exported_list(): + if isinstance(all_stats[plugin], dict): + all_stats[plugin].update(all_limits[plugin]) + # Remove the _disable field + all_stats[plugin].pop(f"{plugin}_disable", None) + # user is a special field that should not be exported + # rename it to user_ + if 'user' in all_stats[plugin]: + all_stats[plugin][f'user_{plugin}'] = all_stats[plugin].pop('user') + elif isinstance(all_stats[plugin], list): + for i in all_stats[plugin]: + i.update(all_limits[plugin]) + # Remove the _disable field + i.pop(f"{plugin}_disable", None) + # user is a special field that should not be exported + # rename it to user_ + if 'user' in i: + i[f'user_{plugin}'] = i.pop('user') + else: + continue + + plugin_stats = all_stats[plugin] + creation_list = [] # List used to create the TimescaleDB table + segmented_by = [] # List of columns used to segment the data + values_list = [] # List of values to insert (list of lists, one list per row) + if isinstance(plugin_stats, dict): + # Stats is a dict + # Create the list used to create the TimescaleDB table + creation_list.append('time TIMESTAMPTZ NOT NULL') + creation_list.append('hostname_id TEXT NOT NULL') + segmented_by.extend(['hostname_id']) # Segment by hostname + for key, value in plugin_stats.items(): + creation_list.append(f"{key} {convert_types[type(value).__name__]} NULL") + values_list.append('NOW()') # Add the current time (insertion time) + values_list.append(f"'{self.hostname}'") # Add the hostname + values_list.extend([self.normalize(value) for value in plugin_stats.values()]) + values_list = [values_list] + elif isinstance(plugin_stats, list) and len(plugin_stats) > 0 and 'key' in plugin_stats[0]: + # Stats is a list + # Create the list used to create the TimescaleDB table + creation_list.append('time TIMESTAMPTZ NOT NULL') + creation_list.append('hostname_id TEXT NOT NULL') + creation_list.append('key_id TEXT NOT NULL') + segmented_by.extend(['hostname_id', 'key_id']) # Segment by hostname and key + for key, value in plugin_stats[0].items(): + creation_list.append(f"{key} {convert_types[type(value).__name__]} NULL") + # Create the values list (it is a list of list to have a single datamodel for all the plugins) + for plugin_item in plugin_stats: + item_list = [] + item_list.append('NOW()') # Add the current time (insertion time) + item_list.append(f"'{self.hostname}'") # Add the hostname + item_list.append(f"'{plugin_item.get('key')}'") + item_list.extend([self.normalize(value) for value in plugin_item.values()]) + values_list.append(item_list[:-1]) + else: + continue + + # Export stats to TimescaleDB + self.export(plugin, creation_list, segmented_by, values_list) + + return True + + def export(self, plugin, creation_list, segmented_by, values_list): + """Export the stats to the TimescaleDB server.""" + logger.debug(f"Export {plugin} stats to TimescaleDB") + + with self.client.cursor() as cur: + # Is the table exists? + cur.execute(f"select exists(select * from information_schema.tables where table_name='{plugin}')") + if not cur.fetchone()[0]: + # Create the table if it does not exist + # https://github.com/timescale/timescaledb/blob/main/README.md#create-a-hypertable + # Execute the create table query + create_query = f""" +CREATE TABLE {plugin} ( + {', '.join(creation_list)} +) +WITH ( + timescaledb.hypertable, + timescaledb.partition_column='time', + timescaledb.segmentby = '{", ".join(segmented_by)}' +);""" + logger.debug(f"Create table: {create_query}") + try: + cur.execute(create_query) + except Exception as e: + logger.error(f"Cannot create table {plugin}: {e}") + return + + # Insert the data + # https://github.com/timescale/timescaledb/blob/main/README.md#insert-and-query-data + insert_list = [f"({','.join(i)})" for i in values_list] + insert_query = f"INSERT INTO {plugin} VALUES {','.join(insert_list)};" + logger.debug(f"Insert data into table: {insert_query}") + try: + cur.execute(insert_query) + except Exception as e: + logger.error(f"Cannot insert data into table {plugin}: {e}") + return + + # Commit the changes (for every plugin or to be done at the end ?) + self.client.commit() + + def exit(self): + """Close the TimescaleDB export module.""" + # Force last write + self.client.commit() + + # Close the TimescaleDB client + time.sleep(3) # Wait a bit to ensure all data is written + self.client.close() + + # Call the father method + super().exit() diff --git a/optional-requirements.txt b/optional-requirements.txt index 19774149..796fd62b 100644 --- a/optional-requirements.txt +++ b/optional-requirements.txt @@ -16,6 +16,7 @@ influxdb3-python # For InfluxDB 3.x jinja2 kafka-python netifaces2 +numpy>=1.22.2 # not directly required, pinned by Snyk to avoid a vulnerability nvidia-ml-py orjson paho-mqtt @@ -23,6 +24,8 @@ pika podman potsdb prometheus_client +psycopg[binary] +pyarrow>=14.0.1 # not directly required, pinned by Snyk to avoid a vulnerability pycouchdb pydantic pygal @@ -33,12 +36,10 @@ pysnmp-lextudio<6.3.1 # Pinned witing implementation of #2874 python-dateutil pyzmq requests +setuptools>=78.1.1 # not directly required, pinned by Snyk to avoid a vulnerability six sparklines statsd +urllib3>=2.2.2 # not directly required, pinned by Snyk to avoid a vulnerability uvicorn zeroconf -numpy>=1.22.2 # not directly required, pinned by Snyk to avoid a vulnerability -pyarrow>=14.0.1 # not directly required, pinned by Snyk to avoid a vulnerability -setuptools>=78.1.1 # not directly required, pinned by Snyk to avoid a vulnerability -urllib3>=2.2.2 # not directly required, pinned by Snyk to avoid a vulnerability diff --git a/pyproject.toml b/pyproject.toml index 688968d4..4d7c7f16 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,6 +81,7 @@ export = [ "pika", "potsdb", "prometheus_client", + "psycopg[binary]", "pymongo", "pyzmq", "statsd", diff --git a/tests/test_export_timescaledb.sh b/tests/test_export_timescaledb.sh new file mode 100755 index 00000000..7bfe8538 --- /dev/null +++ b/tests/test_export_timescaledb.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# Pre-requisites: +# - docker +# - jq + +# Exit on error +set -e + +echo "Clean previous test data..." +rm -f /tmp/timescaledb-for-glances_cpu.csv + +echo "Stop previous TimeScaleDB container..." +docker stop timescaledb-for-glances || true +docker rm timescaledb-for-glances || true + +echo "Starting TimeScaleDB container..." +docker run -d \ + --name timescaledb-for-glances \ + -p 5432:5432 \ + -e POSTGRES_PASSWORD=password \ + timescale/timescaledb-ha:pg17 + +# Wait for InfluxDB to be ready (15 seconds) +echo "Waiting for TimeScaleDB to start (~ 15 seconds)..." +sleep 15 + +# Create the glances database +echo "Creating 'glances' database..." +docker exec timescaledb-for-glances psql -d "postgres://postgres:password@localhost/postgres" -c "CREATE DATABASE glances;" + +# Run glances with export to TimescaleDB, stopping after 10 writes +# This will run synchronously now since we're using --stop-after +echo "Glances to export system stats to TimescaleDB (duration: ~ 20 seconds)" +./venv/bin/python -m glances --config ./conf/glances.conf --export timescaledb --stop-after 10 --quiet + + +docker exec timescaledb-for-glances psql -d "postgres://postgres:password@localhost/glances" -c "SELECT * from cpu;" --csv > /tmp/timescaledb-for-glances_cpu.csv +./venv/bin/python ./tests-data/tools/csvcheck.py -i /tmp/timescaledb-for-glances_cpu.csv -l 9 + +# Stop and remove the TimescaleDB container +echo "Stopping and removing TimescaleDB container..." +# docker stop timescaledb-for-glances && docker rm timescaledb-for-glances + +echo "Script completed successfully!" \ No newline at end of file