23#include "mongorrd_thread.h"
27#include <utils/time/wait.h>
30#include <bsoncxx/builder/basic/document.hpp>
31#include <bsoncxx/json.hpp>
32#include <mongocxx/client.hpp>
33#include <mongocxx/exception/exception.hpp>
35using namespace mongocxx;
38#define DB_CONF_PREFIX "/plugins/mongorrd/databases/"
67 opcounters_graph_ = NULL;
69 indexes_graph_ = NULL;
71 std::vector<RRDDataSource> rrds;
72 rrds.push_back(
RRDDataSource(
"insert", RRDDataSource::COUNTER));
73 rrds.push_back(
RRDDataSource(
"query", RRDDataSource::COUNTER));
74 rrds.push_back(
RRDDataSource(
"update", RRDDataSource::COUNTER));
75 rrds.push_back(
RRDDataSource(
"delete", RRDDataSource::COUNTER));
76 rrds.push_back(
RRDDataSource(
"getmore", RRDDataSource::COUNTER));
77 rrds.push_back(
RRDDataSource(
"command", RRDDataSource::COUNTER));
81 rrds.push_back(
RRDDataSource(
"resident", RRDDataSource::GAUGE));
82 rrds.push_back(
RRDDataSource(
"virtual", RRDDataSource::GAUGE));
83 rrds.push_back(
RRDDataSource(
"mapped", RRDDataSource::GAUGE));
87 rrds.push_back(
RRDDataSource(
"accesses", RRDDataSource::COUNTER));
88 rrds.push_back(
RRDDataSource(
"hits", RRDDataSource::COUNTER));
89 rrds.push_back(
RRDDataSource(
"misses", RRDDataSource::COUNTER));
90 rrds.push_back(
RRDDataSource(
"resets", RRDDataSource::COUNTER));
94 rrds.push_back(
RRDDataSource(
"locktime", RRDDataSource::COUNTER));
107 std::vector<RRDGraphDataDefinition> defs;
108 std::vector<RRDGraphElement *> els;
117 els.push_back(
new RRDGraphLine(
"insert", 1,
"FF7200",
"Inserts"));
118 els.push_back(
new RRDGraphGPrint(
"insert", RRDArchive::LAST,
" Current\\:%8.2lf %s"));
119 els.push_back(
new RRDGraphGPrint(
"insert", RRDArchive::AVERAGE,
"Average\\:%8.2lf %s"));
120 els.push_back(
new RRDGraphGPrint(
"insert", RRDArchive::MAX,
"Maximum\\:%8.2lf %s\\n"));
122 els.push_back(
new RRDGraphLine(
"query", 1,
"503001",
"Queries"));
123 els.push_back(
new RRDGraphGPrint(
"query", RRDArchive::LAST,
" Current\\:%8.2lf %s"));
124 els.push_back(
new RRDGraphGPrint(
"query", RRDArchive::AVERAGE,
"Average\\:%8.2lf %s"));
125 els.push_back(
new RRDGraphGPrint(
"query", RRDArchive::MAX,
"Maximum\\:%8.2lf %s\\n"));
127 els.push_back(
new RRDGraphLine(
"update", 1,
"EDAC00",
"Updates"));
128 els.push_back(
new RRDGraphGPrint(
"update", RRDArchive::LAST,
" Current\\:%8.2lf %s"));
129 els.push_back(
new RRDGraphGPrint(
"update", RRDArchive::AVERAGE,
"Average\\:%8.2lf %s"));
130 els.push_back(
new RRDGraphGPrint(
"update", RRDArchive::MAX,
"Maximum\\:%8.2lf %s\\n"));
132 els.push_back(
new RRDGraphLine(
"delete", 1,
"506101",
"Deletes"));
133 els.push_back(
new RRDGraphGPrint(
"delete", RRDArchive::LAST,
" Current\\:%8.2lf %s"));
134 els.push_back(
new RRDGraphGPrint(
"delete", RRDArchive::AVERAGE,
"Average\\:%8.2lf %s"));
135 els.push_back(
new RRDGraphGPrint(
"delete", RRDArchive::MAX,
"Maximum\\:%8.2lf %s\\n"));
137 els.push_back(
new RRDGraphLine(
"getmore", 1,
"0CCCCC",
"Getmores"));
138 els.push_back(
new RRDGraphGPrint(
"getmore", RRDArchive::LAST,
"Current\\:%8.2lf %s"));
139 els.push_back(
new RRDGraphGPrint(
"getmore", RRDArchive::AVERAGE,
"Average\\:%8.2lf %s"));
140 els.push_back(
new RRDGraphGPrint(
"getmore", RRDArchive::MAX,
"Maximum\\:%8.2lf %s\\n"));
142 els.push_back(
new RRDGraphLine(
"command", 1,
"53CA05",
"Commands"));
143 els.push_back(
new RRDGraphGPrint(
"command", RRDArchive::LAST,
"Current\\:%8.2lf %s"));
144 els.push_back(
new RRDGraphGPrint(
"command", RRDArchive::AVERAGE,
"Average\\:%8.2lf %s"));
145 els.push_back(
new RRDGraphGPrint(
"command", RRDArchive::MAX,
"Maximum\\:%8.2lf %s\\n"));
148 "opcounters", opcounters_rrd_,
"MongoDB Op Counters",
"Ops/sec", defs, els);
160 els.push_back(
new RRDGraphArea(
"virtual",
"3B7AD9",
"Virtual"));
161 els.push_back(
new RRDGraphGPrint(
"virtual", RRDArchive::LAST,
" Current\\:%8.2lf %s"));
162 els.push_back(
new RRDGraphGPrint(
"virtual", RRDArchive::AVERAGE,
"Average\\:%8.2lf %s"));
163 els.push_back(
new RRDGraphGPrint(
"virtual", RRDArchive::MAX,
"Maximum\\:%8.2lf %s\\n"));
165 els.push_back(
new RRDGraphArea(
"mapped",
"6FD1BF",
"Mapped"));
166 els.push_back(
new RRDGraphGPrint(
"mapped", RRDArchive::LAST,
" Current\\:%8.2lf %s"));
167 els.push_back(
new RRDGraphGPrint(
"mapped", RRDArchive::AVERAGE,
"Average\\:%8.2lf %s"));
168 els.push_back(
new RRDGraphGPrint(
"mapped", RRDArchive::MAX,
"Maximum\\:%8.2lf %s\\n"));
170 els.push_back(
new RRDGraphArea(
"resident",
"0E6E5C",
"Resident"));
171 els.push_back(
new RRDGraphGPrint(
"resident", RRDArchive::LAST,
"Current\\:%8.2lf %s"));
172 els.push_back(
new RRDGraphGPrint(
"resident", RRDArchive::AVERAGE,
"Average\\:%8.2lf %s"));
173 els.push_back(
new RRDGraphGPrint(
"resident", RRDArchive::MAX,
"Maximum\\:%8.2lf %s\\n"));
176 new RRDGraphDefinition(
"memory", memory_rrd_,
"MongoDB Memory Usage",
"MB", defs, els);
185 els.push_back(
new RRDGraphLine(
"accesses", 1,
"FF7200",
"Accesses"));
186 els.push_back(
new RRDGraphGPrint(
"accesses", RRDArchive::LAST,
"Current\\:%8.2lf %s"));
187 els.push_back(
new RRDGraphGPrint(
"accesses", RRDArchive::AVERAGE,
"Average\\:%8.2lf %s"));
188 els.push_back(
new RRDGraphGPrint(
"accesses", RRDArchive::MAX,
"Maximum\\:%8.2lf %s\\n"));
190 els.push_back(
new RRDGraphLine(
"hits", 1,
"503001",
"Hits"));
191 els.push_back(
new RRDGraphGPrint(
"hits", RRDArchive::LAST,
" Current\\:%8.2lf %s"));
192 els.push_back(
new RRDGraphGPrint(
"hits", RRDArchive::AVERAGE,
"Average\\:%8.2lf %s"));
193 els.push_back(
new RRDGraphGPrint(
"hits", RRDArchive::MAX,
"Maximum\\:%8.2lf %s\\n"));
195 els.push_back(
new RRDGraphLine(
"misses", 1,
"EDAC00",
"Misses"));
196 els.push_back(
new RRDGraphGPrint(
"misses", RRDArchive::LAST,
" Current\\:%8.2lf %s"));
197 els.push_back(
new RRDGraphGPrint(
"misses", RRDArchive::AVERAGE,
"Average\\:%8.2lf %s"));
198 els.push_back(
new RRDGraphGPrint(
"misses", RRDArchive::MAX,
"Maximum\\:%8.2lf %s\\n"));
200 els.push_back(
new RRDGraphLine(
"resets", 1,
"506101",
"Resets"));
201 els.push_back(
new RRDGraphGPrint(
"resets", RRDArchive::LAST,
" Current\\:%8.2lf %s"));
202 els.push_back(
new RRDGraphGPrint(
"resets", RRDArchive::AVERAGE,
"Average\\:%8.2lf %s"));
203 els.push_back(
new RRDGraphGPrint(
"resets", RRDArchive::MAX,
"Maximum\\:%8.2lf %s\\n"));
218 std::string dbprefix = DB_CONF_PREFIX;
224 "Entry %s is not a string, but of type %s, "
232 if (dbname.find(
".") != std::string::npos) {
238 add_dbstats(i->
path(), dbname);
259 for (DbStatsMap::iterator i = dbstats_.begin(); i != dbstats_.end(); ++i) {
260 DbStatsInfo &info = i->second;
269 delete opcounters_graph_;
270 delete memory_graph_;
271 delete indexes_graph_;
273 delete opcounters_rrd_;
280MongoRRDThread::add_dbstats(
const char *path, std::string dbname)
282 if (dbstats_.find(path) != dbstats_.end()) {
283 throw Exception(
"Database stats for config %s already monitored", path);
288 std::vector<RRDDataSource> rrds;
289 rrds.push_back(
RRDDataSource(
"collections", RRDDataSource::GAUGE));
290 rrds.push_back(
RRDDataSource(
"objects", RRDDataSource::GAUGE));
291 rrds.push_back(
RRDDataSource(
"avgObjSize", RRDDataSource::GAUGE));
292 rrds.push_back(
RRDDataSource(
"dataSize", RRDDataSource::GAUGE));
293 rrds.push_back(
RRDDataSource(
"storageSize", RRDDataSource::GAUGE));
294 rrds.push_back(
RRDDataSource(
"numExtents", RRDDataSource::GAUGE));
295 rrds.push_back(
RRDDataSource(
"indexes", RRDDataSource::GAUGE));
296 rrds.push_back(
RRDDataSource(
"indexSize", RRDDataSource::GAUGE));
297 rrds.push_back(
RRDDataSource(
"fileSize", RRDDataSource::GAUGE));
299 info.db_name = dbname;
300 info.rrd_name = std::string(
"dbstats_") + dbname;
303 std::vector<RRDGraphDataDefinition> defs;
304 std::vector<RRDGraphElement *> els;
310 els.push_back(
new RRDGraphLine(
"collections", 1,
"FF7200",
"Collections"));
311 els.push_back(
new RRDGraphGPrint(
"collections", RRDArchive::LAST,
"Current\\:%8.2lf %s"));
312 els.push_back(
new RRDGraphGPrint(
"collections", RRDArchive::AVERAGE,
"Average\\:%8.2lf %s"));
313 els.push_back(
new RRDGraphGPrint(
"collections", RRDArchive::MAX,
"Maximum\\:%8.2lf %s\\n"));
315 els.push_back(
new RRDGraphLine(
"indexes", 1,
"EDAC00",
"Indexes"));
316 els.push_back(
new RRDGraphGPrint(
"indexes", RRDArchive::LAST,
" Current\\:%8.2lf %s"));
317 els.push_back(
new RRDGraphGPrint(
"indexes", RRDArchive::AVERAGE,
"Average\\:%8.2lf %s"));
318 els.push_back(
new RRDGraphGPrint(
"indexes", RRDArchive::MAX,
"Maximum\\:%8.2lf %s\\n"));
320 els.push_back(
new RRDGraphLine(
"numExtents", 1,
"506101",
"Extents"));
321 els.push_back(
new RRDGraphGPrint(
"numExtents", RRDArchive::LAST,
" Current\\:%8.2lf %s"));
322 els.push_back(
new RRDGraphGPrint(
"numExtents", RRDArchive::AVERAGE,
"Average\\:%8.2lf %s"));
323 els.push_back(
new RRDGraphGPrint(
"numExtents", RRDArchive::MAX,
"Maximum\\:%8.2lf %s\\n"));
325 std::string g1name = info.rrd_name +
"_collindext";
326 std::string g1title = std::string(
"MongoDB Collections, Indexes, Extents for ") + dbname;
327 info.graph1 =
new RRDGraphDefinition(g1name.c_str(), info.rrd, g1title.c_str(),
"", defs, els);
333 els.push_back(
new RRDGraphLine(
"objects", 1,
"FF7200",
"Objects"));
334 els.push_back(
new RRDGraphGPrint(
"objects", RRDArchive::LAST,
" Current\\:%8.2lf %s"));
335 els.push_back(
new RRDGraphGPrint(
"objects", RRDArchive::AVERAGE,
"Average\\:%8.2lf %s"));
336 els.push_back(
new RRDGraphGPrint(
"objects", RRDArchive::MAX,
"Maximum\\:%8.2lf %s\\n"));
338 std::string g2name = info.rrd_name +
"_objects";
339 std::string g2title = std::string(
"MongoDB Objects for ") + dbname;
340 info.graph2 =
new RRDGraphDefinition(g2name.c_str(), info.rrd, g2title.c_str(),
"", defs, els);
350 els.push_back(
new RRDGraphLine(
"avgObjSize", 1,
"FF7200",
"Avg Obj Sz"));
351 els.push_back(
new RRDGraphGPrint(
"avgObjSize", RRDArchive::LAST,
"Current\\:%8.2lf %s"));
352 els.push_back(
new RRDGraphGPrint(
"avgObjSize", RRDArchive::AVERAGE,
"Average\\:%8.2lf %s"));
353 els.push_back(
new RRDGraphGPrint(
"avgObjSize", RRDArchive::MAX,
"Maximum\\:%8.2lf %s\\n"));
355 els.push_back(
new RRDGraphLine(
"dataSize", 1,
"503001",
"Data"));
356 els.push_back(
new RRDGraphGPrint(
"dataSize", RRDArchive::LAST,
" Current\\:%8.2lf %s"));
357 els.push_back(
new RRDGraphGPrint(
"dataSize", RRDArchive::AVERAGE,
"Average\\:%8.2lf %s"));
358 els.push_back(
new RRDGraphGPrint(
"dataSize", RRDArchive::MAX,
"Maximum\\:%8.2lf %s\\n"));
360 els.push_back(
new RRDGraphLine(
"storageSize", 1,
"EDAC00",
"Storage"));
361 els.push_back(
new RRDGraphGPrint(
"storageSize", RRDArchive::LAST,
" Current\\:%8.2lf %s"));
362 els.push_back(
new RRDGraphGPrint(
"storageSize", RRDArchive::AVERAGE,
"Average\\:%8.2lf %s"));
363 els.push_back(
new RRDGraphGPrint(
"storageSize", RRDArchive::MAX,
"Maximum\\:%8.2lf %s\\n"));
365 els.push_back(
new RRDGraphLine(
"indexSize", 1,
"506101",
"Index"));
366 els.push_back(
new RRDGraphGPrint(
"indexSize", RRDArchive::LAST,
" Current\\:%8.2lf %s"));
367 els.push_back(
new RRDGraphGPrint(
"indexSize", RRDArchive::AVERAGE,
"Average\\:%8.2lf %s"));
368 els.push_back(
new RRDGraphGPrint(
"indexSize", RRDArchive::MAX,
"Maximum\\:%8.2lf %s\\n"));
370 els.push_back(
new RRDGraphLine(
"fileSize", 1,
"0CCCCC",
"File"));
371 els.push_back(
new RRDGraphGPrint(
"fileSize", RRDArchive::LAST,
" Current\\:%8.2lf %s"));
372 els.push_back(
new RRDGraphGPrint(
"fileSize", RRDArchive::AVERAGE,
"Average\\:%8.2lf %s"));
373 els.push_back(
new RRDGraphGPrint(
"fileSize", RRDArchive::MAX,
"Maximum\\:%8.2lf %s\\n"));
375 std::string g3name = info.rrd_name +
"_sizes";
376 std::string g3title = std::string(
"MongoDB Sizes for ") + dbname;
377 info.graph3 =
new RRDGraphDefinition(g3name.c_str(), info.rrd, g3title.c_str(),
"Mem", defs, els);
385 dbstats_[dbname] = info;
398MongoRRDThread::remove_dbstats(
const char *path)
400 if (dbstats_.find(path) != dbstats_.end()) {
401 DbStatsInfo &info = dbstats_[path];
409 dbstats_.erase(path);
417 using namespace bsoncxx::builder;
421 basic::make_document(basic::kvp(
"serverStatus", 1)));
422 if (check_mongodb_ok(reply.view())) {
423 auto opcounters = reply.view()[
"opcounters"].get_document().view();
424 int64_t insert, query, update, del, getmore, command;
425 insert = opcounters[
"insert"].get_int64();
426 query = opcounters[
"query"].get_int64();
427 update = opcounters[
"update"].get_int64();
428 del = opcounters[
"delete"].get_int64();
429 getmore = opcounters[
"getmore"].get_int64();
430 command = opcounters[
"command"].get_int64();
434 "opcounters",
"N:%i:%i:%i:%i:%i:%i", insert, query, update, del, getmore, command);
437 "Failed to update opcounters RRD, "
438 "exception follows");
442 auto mem = reply.view()[
"mem"].get_document().view();
443 int64_t resident, virtmem, mapped;
444 resident = mem[
"resident"].get_int64();
445 virtmem = mem[
"virtual"].get_int64();
446 mapped = mem[
"mapped"].get_int64();
455 auto indexc = reply.view()[
"indexCounters"][
"btree"].get_document().view();
456 int accesses, hits, misses, resets;
457 accesses = indexc[
"accesses"].get_int64();
458 hits = indexc[
"hits"].get_int64();
459 misses = indexc[
"misses"].get_int64();
460 resets = indexc[
"resets"].get_int64();
466 "Failed to update indexes RRD, "
467 "exception follows");
471 for (DbStatsMap::iterator i = dbstats_.begin(); i != dbstats_.end(); ++i) {
473 .run_command(basic::make_document(basic::kvp(
"dbStats", 1)));
474 if (
int(dbstats.view()[
"ok"].get_double()) == 1) {
476 int64_t collections, objects, numExtents, indexes, dataSize, storageSize, indexSize,
480 collections = dbstats.view()[
"collections"].get_int64();
481 objects = dbstats.view()[
"objects"].get_int64();
482 avgObjSize = dbstats.view()[
"avgObjSize"].get_double();
483 dataSize = dbstats.view()[
"dataSize"].get_int64();
484 storageSize = dbstats.view()[
"storageSize"].get_int64();
485 numExtents = dbstats.view()[
"numExtents"].get_int64();
486 indexes = dbstats.view()[
"indexes"].get_int64();
487 indexSize = dbstats.view()[
"indexSize"].get_int64();
488 fileSize = dbstats.view()[
"fileSize"].get_int64();
492 "N:%li:%li:%f:%li:%li:%li:%li:%li:%li",
504 "Failed to update dbstates RRD for "
505 "%s exception follows",
506 i->second.db_name.c_str());
510 }
catch (mongocxx::exception &e) {
512 "Failed to update MongoDB RRD for database "
514 i->second.db_name.c_str(),
519 "Failed to retrieve db stats for %s, reply: %s",
520 i->second.db_name.c_str(),
521 bsoncxx::to_json(dbstats.view()).c_str());
529 "Failed to retrieve server status, reply: %s",
530 bsoncxx::to_json(reply.view()).c_str());
533 }
catch (mongocxx::exception &e) {
541MongoRRDThread::config_tag_changed(
const char *new_tag)
550 remove_dbstats(v->
path());
563MongoRRDThread::config_value_erased(
const char *path)
565 remove_dbstats(path);
virtual ~MongoRRDThread()
Destructor.
MongoRRDThread()
Constructor.
virtual void finalize()
Finalize the thread.
virtual void loop()
Code to execute in the thread.
virtual void init()
Initialize the thread.
Clock * clock
By means of this member access to the clock is given.
Configuration * config
This is the Configuration member used to access the configuration.
Interface for configuration change handling.
Iterator interface to iterate over config values.
virtual const char * path() const =0
Path of value.
virtual bool next()=0
Check if there is another element and advance to this if possible.
virtual bool is_string() const =0
Check if current value is a string.
virtual const char * type() const =0
Type of value.
virtual std::string get_string() const =0
Get string value.
virtual void rem_change_handler(ConfigurationChangeHandler *h)
Remove a configuration change handler.
virtual ValueIterator * search(const char *path)=0
Iterator with search results.
virtual void add_change_handler(ConfigurationChangeHandler *h)
Add a configuration change handler.
Base class for exceptions in Fawkes.
virtual void log_warn(const char *component, const char *format,...)=0
Log warning message.
virtual void log_info(const char *component, const char *format,...)=0
Log informational message.
Logger * logger
This is the Logger member used to access the logger.
Thread aspect to access MongoDB.
mongocxx::client * mongodb_client
MongoDB client to use to interact with the database.
RRDManager * rrd_manager
Manager class to access RRD features.
Class to represent a RRD data source.
Represent data definition in graph arguments.
Class representing a graph definition.
Print string inside graph.
virtual void add_rrd(RRDDefinition *rrd_def)=0
Add RRD.
virtual void add_graph(RRDGraphDefinition *rrd_graph_def)=0
Add graph.
virtual void add_data(const char *rrd_name, const char *format,...)=0
Add data.
virtual void remove_rrd(RRDDefinition *rrd_def)=0
Remove RRD.
Thread class encapsulation of pthreads.
void set_prepfin_conc_loop(bool concurrent=true)
Set concurrent execution of prepare_finalize() and loop().
const char * name() const
Get name of thread.
void mark_start()
Mark start of loop.
void wait_systime()
Wait until minimum loop time has been reached in real time.
Fawkes library namespace.