diff --git a/NEWS b/NEWS index 831e9b3a..224e397e 100644 --- a/NEWS +++ b/NEWS @@ -7,6 +7,7 @@ Version 2.6 Enhancements and new features: + * Add a connector to ElasticSearch (welcome to Kibana dashboard) (issue #311) * New folders' monitoring plugins (issue #721) * Add an option to disable top menu (issue #766) * Add IOps in the DiskIO plugin (issue #763) diff --git a/conf/glances.conf b/conf/glances.conf index 16bd9a22..9fa9c5e9 100644 --- a/conf/glances.conf +++ b/conf/glances.conf @@ -223,6 +223,14 @@ host=localhost port=8125 #prefix=glances +[elasticsearch] +# Configuration file for the --export-elasticsearch option +# Data are available via the ES Restful API. ex: URL//cpu/system +# https://www.elastic.co +host=localhost +port=9200 +index=glances + [rabbitmq] host=localhost port=5672 diff --git a/docs/glances-doc.rst b/docs/glances-doc.rst index 9a699b0f..37f1a1eb 100644 --- a/docs/glances-doc.rst +++ b/docs/glances-doc.rst @@ -180,10 +180,12 @@ Command-Line Options set the export path for graph history --export-csv EXPORT_CSV export stats to a CSV file - --export-influxdb export stats to an InfluxDB server (influxdb needed) - --export-opentsdb export stats to an OpenTSDB server (potsdb needed) - --export-statsd export stats to a StatsD server (statsd needed) - --export-rabbitmq export stats to a RabbitMQ server (pika needed) + --export-influxdb export stats to an InfluxDB server (influxdb lib needed) + --export-opentsdb export stats to an OpenTSDB server (potsdb lib needed) + --export-statsd export stats to a StatsD server (statsd lib needed) + --export-elasticsearch + export stats to an Elasticsearch server (elasticsearch lib needed) + --export-rabbitmq export stats to a RabbitMQ server (pika lib needed) -c CLIENT, --client CLIENT connect to a Glances server by IPv4/IPv6 address or hostname @@ -953,6 +955,33 @@ Glances will generate stats as: 'glances.load.min1': 0.19, ... +ElasticSearch +------------- + +You can export statistics to an ``ElasticSearch`` server. +The connection should be defined in the Glances configuration file as +following: + +.. code-block:: ini + + [elasticsearch] + host=localhost + port=9200 + index=glances + +and run Glances with: + +.. code-block:: console + + $ glances --export-elasticsearch + +The stats are available through the ElasticSearch API: URL/glances/cpu/system (example for the system CPU). + +.. code-block:: console + + $ curl http://172.17.0.2:9200/glances/cpu/system + {"_index":"glances","_type":"cpu","_id":"system","_version":28,"found":true,"_source":{"timestamp": "2016-02-04T14:11:02.362232", "value": "2.2"}} + RabbitMQ -------- diff --git a/glances/exports/glances_elasticsearch.py b/glances/exports/glances_elasticsearch.py new file mode 100644 index 00000000..86f3a566 --- /dev/null +++ b/glances/exports/glances_elasticsearch.py @@ -0,0 +1,117 @@ +# -*- coding: utf-8 -*- +# +# This file is part of Glances. +# +# Copyright (C) 2015 Nicolargo +# +# Glances is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Glances is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +"""ElasticSearch interface class.""" + +import sys +from datetime import datetime + +from glances.compat import NoOptionError, NoSectionError +from glances.logger import logger +from glances.exports.glances_export import GlancesExport + +from elasticsearch import Elasticsearch, helpers + + +class Export(GlancesExport): + + """This class manages the ElasticSearch (ES) export module.""" + + def __init__(self, config=None, args=None): + """Init the ES export IF.""" + super(Export, self).__init__(config=config, args=args) + + # Load the ES configuration file + self.host = None + self.port = None + self.index = None + self.export_enable = self.load_conf() + if not self.export_enable: + sys.exit(2) + + # Init the ES client + self.client = self.init() + + def load_conf(self, section="elasticsearch"): + """Load the ES configuration in the Glances configuration file.""" + if self.config is None: + return False + try: + self.host = self.config.get_value(section, 'host') + self.port = self.config.get_value(section, 'port') + self.index = self.config.get_value(section, 'index') + except NoSectionError: + logger.critical("No ElasticSearch configuration found") + return False + except NoOptionError as e: + logger.critical("Error in the ElasticSearch configuration (%s)" % e) + return False + else: + logger.debug("Load ElasticSearch from the Glances configuration file") + + return True + + def init(self): + """Init the connection to the ES server.""" + if not self.export_enable: + return None + + try: + es = Elasticsearch(hosts=['{0}:{1}'.format(self.host, self.port)]) + except Exception as e: + logger.critical("Cannot connect to ElasticSearch server %s:%s (%s)" % (self.host, self.port, e)) + sys.exit(2) + else: + logger.info("Connected to the ElasticSearch server %s:%s" % (self.host, self.port)) + + try: + index_count = es.count(index=self.index)['count'] + except Exception as e: + # Index did not exist, it will be created at the first write + # Create it... + es.indices.create(self.index) + else: + logger.info("There is already %s entries in the ElasticSearch %s index" % (index_count, self.index)) + + return es + + def export(self, name, columns, points): + """Write the points to the ES server.""" + logger.debug("Export {0} stats to ElasticSearch".format(name)) + + # Create DB input + # http://elasticsearch-py.readthedocs.org/en/master/helpers.html + actions = [] + for c, p in zip(columns, points): + action = { + "_index": self.index, + "_type": name, + "_id": c, + "_source": { + "value": str(p), + "timestamp": datetime.now() + } + } + actions.append(action) + + # Write input to the ES index + try: + helpers.bulk(self.client, actions) + except Exception as e: + logger.error("Cannot export {0} stats to ElasticSearch ({1})".format(name, e)) diff --git a/glances/logger.py b/glances/logger.py index 549cf0e7..f7936444 100644 --- a/glances/logger.py +++ b/glances/logger.py @@ -70,7 +70,15 @@ LOGGING_CFG = { 'standard': { 'handlers': ['file'], 'level': 'INFO' - } + }, + 'elasticsearch': { + 'handlers': ['file', 'console'], + 'level': 'ERROR', + }, + 'elasticsearch.trace': { + 'handlers': ['file', 'console'], + 'level': 'ERROR', + }, } } diff --git a/glances/main.py b/glances/main.py index fe96871b..cb7ea5d1 100644 --- a/glances/main.py +++ b/glances/main.py @@ -152,13 +152,15 @@ Start the client browser (browser mode):\n\ parser.add_argument('--export-csv', default=None, dest='export_csv', help='export stats to a CSV file') parser.add_argument('--export-influxdb', action='store_true', default=False, - dest='export_influxdb', help='export stats to an InfluxDB server (influxdb needed)') + dest='export_influxdb', help='export stats to an InfluxDB server (influxdb lib needed)') parser.add_argument('--export-opentsdb', action='store_true', default=False, - dest='export_opentsdb', help='export stats to an OpenTSDB server (potsdb needed)') + dest='export_opentsdb', help='export stats to an OpenTSDB server (potsdb lib needed)') parser.add_argument('--export-statsd', action='store_true', default=False, - dest='export_statsd', help='export stats to a StatsD server (statsd needed)') + dest='export_statsd', help='export stats to a StatsD server (statsd lib needed)') + parser.add_argument('--export-elasticsearch', action='store_true', default=False, + dest='export_elasticsearch', help='export stats to an ElasticSearch server (elasticsearch lib needed)') parser.add_argument('--export-rabbitmq', action='store_true', default=False, - dest='export_rabbitmq', help='export stats to rabbitmq broker (pika needed)') + dest='export_rabbitmq', help='export stats to rabbitmq broker (pika lib needed)') # Client/Server option parser.add_argument('-c', '--client', dest='client', help='connect to a Glances server by IPv4/IPv6 address or hostname') @@ -299,7 +301,7 @@ Start the client browser (browser mode):\n\ self.args = args # Export is only available in standalone or client mode (issue #614) - export_tag = args.export_csv or args.export_statsd or args.export_influxdb or args.export_opentsdb or args.export_rabbitmq + export_tag = args.export_csv or args.export_elasticsearch or args.export_statsd or args.export_influxdb or args.export_opentsdb or args.export_rabbitmq if not (self.is_standalone() or self.is_client()) and export_tag: logger.critical("Export is only available in standalone or client mode") sys.exit(2) diff --git a/man/glances.1 b/man/glances.1 index df638621..6917dc2e 100644 --- a/man/glances.1 +++ b/man/glances.1 @@ -109,13 +109,16 @@ set the export path for graph history export stats to a CSV file .TP .B \-\-export-influxdb -export stats to an InfluxDB server (influxdb needed) +export stats to an InfluxDB server (influxdb lib needed) .TP .B \-\-export-opentsdb -export stats to an OpenTSDB server (potsdb needed) +export stats to an OpenTSDB server (potsdb lib needed) .TP .B \-\-export-statsd -export stats to a StatsD server (statsd needed) +export stats to a StatsD server (statsd lib needed) +.TP +.B \-\-export-elasticsearch +export stats to an ElasticSearch server (elasticsearch lib needed) .TP .B \-\-export-rabbitmq export stats to a RabbitMQ server (pika needed)