Line data Source code
1 : // Copyright (c) 2013-2025 Made to Order Software Corp. All Rights Reserved
2 : //
3 : // https://snapwebsites.org/project/snaplogger
4 : // contact@m2osw.com
5 : //
6 : // This program is free software; you can redistribute it and/or modify
7 : // it under the terms of the GNU General Public License as published by
8 : // the Free Software Foundation; either version 2 of the License, or
9 : // (at your option) any later version.
10 : //
11 : // This program is distributed in the hope that it will be useful,
12 : // but WITHOUT ANY WARRANTY; without even the implied warranty of
13 : // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 : // GNU General Public License for more details.
15 : //
16 : // You should have received a copy of the GNU General Public License along
17 : // with this program; if not, write to the Free Software Foundation, Inc.,
18 : // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19 :
20 : /** \file
21 : * \brief Appenders are used to append data to somewhere.
22 : *
23 : * This file implements the base appender class.
24 : */
25 :
26 :
27 : // self
28 : //
29 : #include "snaplogger/appender.h"
30 :
31 : #include "snaplogger/exception.h"
32 : #include "snaplogger/guard.h"
33 : #include "snaplogger/private_logger.h"
34 :
35 :
36 : // snapdev
37 : //
38 : #include <snapdev/empty_set_intersection.h>
39 : #include <snapdev/not_used.h>
40 :
41 :
42 : // C++
43 : //
44 : #include <iostream>
45 :
46 :
47 : // C
48 : //
49 : #include <math.h>
50 :
51 :
52 : // last include
53 : //
54 : #include <snapdev/poison.h>
55 :
56 :
57 :
58 : namespace snaplogger
59 : {
60 :
61 :
62 : namespace
63 : {
64 :
65 :
66 :
67 : // if we want to be able to reference such we need to create it TBD
68 : // (and it should probably be in the null_appender.cpp file instead)
69 : //APPENDER_FACTORY(null);
70 :
71 :
72 : }
73 :
74 :
75 24 : appender::appender(std::string const & name, std::string const & type)
76 24 : : f_type(type)
77 24 : , f_name(name)
78 96 : , f_normal_component(get_component(COMPONENT_NORMAL))
79 : {
80 24 : guard g;
81 :
82 24 : f_format = get_private_logger()->get_default_format();
83 48 : }
84 :
85 :
86 36 : appender::~appender()
87 : {
88 18 : }
89 :
90 :
91 4 : std::string const & appender::get_type() const
92 : {
93 : // we do not need to guard this one because we set it up on creation
94 : // and it can't be modified later
95 : //
96 4 : return f_type;
97 : }
98 :
99 :
100 1 : void appender::set_name(std::string const & name)
101 : {
102 1 : guard g;
103 :
104 1 : if(f_name != "console"
105 1 : && f_name != "syslog")
106 : {
107 : throw invalid_parameter(
108 3 : "the appender set_name() can only be used for the console & syslog appenders to rename them to your own appender name (and done internally only).");
109 : }
110 :
111 0 : f_name = name;
112 1 : }
113 :
114 :
115 2 : std::string const & appender::get_name() const
116 : {
117 2 : guard g;
118 :
119 2 : return f_name;
120 2 : }
121 :
122 :
123 45796 : bool appender::is_enabled() const
124 : {
125 45796 : guard g;
126 :
127 45796 : return f_enabled;
128 45796 : }
129 :
130 :
131 2 : void appender::set_enabled(bool status)
132 : {
133 2 : guard g;
134 :
135 2 : f_enabled = status;
136 4 : }
137 :
138 :
139 24 : bool appender::unique() const
140 : {
141 24 : return false;
142 : }
143 :
144 :
145 314 : severity_t appender::get_severity() const
146 : {
147 314 : guard g;
148 :
149 314 : return f_severity;
150 314 : }
151 :
152 :
153 294 : void appender::set_severity(severity_t severity_level)
154 : {
155 294 : guard g;
156 :
157 294 : f_severity = severity_level;
158 294 : logger::get_instance()->severity_changed(severity_level);
159 588 : }
160 :
161 :
162 4 : void appender::reduce_severity(severity_t severity_level)
163 : {
164 4 : guard g;
165 :
166 4 : if(severity_level < f_severity)
167 : {
168 3 : set_severity(severity_level);
169 : }
170 8 : }
171 :
172 :
173 2 : void appender::increase_severity(severity_t severity_level)
174 : {
175 2 : guard g;
176 :
177 2 : if(severity_level > f_severity)
178 : {
179 1 : set_severity(severity_level);
180 : }
181 4 : }
182 :
183 :
184 1 : bool appender::operator < (appender const & rhs) const
185 : {
186 1 : guard g;
187 :
188 1 : return f_severity < rhs.f_severity;
189 1 : }
190 :
191 :
192 27 : void appender::set_config(advgetopt::getopt const & opts)
193 : {
194 27 : guard g;
195 :
196 : // ENABLE
197 : //
198 : {
199 27 : std::string const specialized_enabled(f_name + "::enabled");
200 27 : if(opts.is_defined(specialized_enabled))
201 : {
202 0 : f_enabled = !advgetopt::is_false(opts.get_string(specialized_enabled));
203 : }
204 81 : else if(opts.is_defined("enabled"))
205 : {
206 0 : f_enabled = !advgetopt::is_false(opts.get_string("enabled"));
207 : }
208 : else
209 : {
210 27 : f_enabled = true;
211 : }
212 27 : }
213 :
214 : // FORMAT
215 : //
216 : {
217 27 : std::string const specialized_format(f_name + "::format");
218 27 : if(opts.is_defined(specialized_format))
219 : {
220 0 : f_format = std::make_shared<format>(opts.get_string(specialized_format));
221 : }
222 81 : else if(opts.is_defined("format"))
223 : {
224 0 : f_format = std::make_shared<format>(opts.get_string("format"));
225 : }
226 27 : }
227 :
228 : // BITRATE
229 : //
230 : {
231 : // the input is considered to be in mbps
232 : //
233 81 : std::string bitrate("0");
234 27 : std::string const specialized_bitrate(f_name + "::bitrate");
235 27 : if(opts.is_defined(specialized_bitrate))
236 : {
237 0 : bitrate = opts.get_string(specialized_bitrate);
238 : }
239 81 : else if(opts.is_defined("bitrate"))
240 : {
241 0 : bitrate = opts.get_string("bitrate");
242 : }
243 27 : char * end(nullptr);
244 27 : errno = 0;
245 27 : double rate(strtod(bitrate.c_str(), &end));
246 27 : if(rate < 0.0
247 27 : || end == nullptr
248 27 : || *end != '\0'
249 27 : || errno == ERANGE)
250 : {
251 0 : rate = 0.0;
252 : }
253 27 : if(rate > 0.0)
254 : {
255 : // transform the rate to bytes per minute
256 : //
257 0 : rate = rate * (60.0 * 1'000'000.0 / 8.0);
258 : }
259 27 : f_bytes_per_minute = static_cast<decltype(f_bytes_per_minute)>(floor(rate));
260 27 : }
261 :
262 : // SEVERITY
263 : //
264 27 : std::string const specialized_severity(f_name + "::severity");
265 27 : if(opts.is_defined(specialized_severity))
266 : {
267 0 : std::string const severity_name(opts.get_string(specialized_severity));
268 0 : severity::pointer_t sev(snaplogger::get_severity(severity_name));
269 0 : if(sev != nullptr)
270 : {
271 0 : set_severity(sev->get_severity());
272 : }
273 : else
274 : {
275 : throw invalid_severity(
276 : "severity level named \""
277 0 : + severity_name
278 0 : + "\" not found.");
279 : }
280 0 : }
281 81 : else if(opts.is_defined("severity"))
282 : {
283 0 : std::string const severity_name(opts.get_string("severity"));
284 0 : severity::pointer_t sev(snaplogger::get_severity(severity_name));
285 0 : if(sev != nullptr)
286 : {
287 0 : set_severity(sev->get_severity());
288 : }
289 : else
290 : {
291 : throw invalid_severity(
292 : "severity level named \""
293 0 : + severity_name
294 0 : + "\" not found.");
295 : }
296 0 : }
297 :
298 : // COMPONENTS
299 : //
300 27 : std::string comp;
301 27 : std::string const components(f_name + "::components");
302 27 : if(opts.is_defined(components))
303 : {
304 0 : comp = opts.get_string(components);
305 : }
306 81 : else if(opts.is_defined("components"))
307 : {
308 0 : comp = opts.get_string("components");
309 : }
310 27 : if(comp.empty())
311 : {
312 27 : add_component(f_normal_component);
313 : }
314 : else
315 : {
316 0 : advgetopt::string_list_t component_names;
317 0 : advgetopt::split_string(comp, component_names, {","});
318 0 : for(auto name : component_names)
319 : {
320 0 : add_component(get_component(name));
321 0 : }
322 0 : }
323 :
324 : // FILTER
325 : //
326 : {
327 27 : std::string filter;
328 27 : std::string const specialized_filter(f_name + "::filter");
329 27 : if(opts.is_defined(specialized_filter))
330 : {
331 0 : filter = opts.get_string(specialized_filter);
332 : }
333 81 : else if(opts.is_defined("filter"))
334 : {
335 0 : filter = opts.get_string("filter");
336 : }
337 27 : if(!filter.empty())
338 : {
339 0 : std::regex_constants::syntax_option_type flags(std::regex::nosubs | std::regex::optimize);
340 0 : std::regex_constants::syntax_option_type type(std::regex::extended);
341 0 : if(filter[0] == '/')
342 : {
343 0 : std::string::size_type pos(filter.rfind('/'));
344 0 : if(pos == 0)
345 : {
346 : throw invalid_variable(
347 : "invalid filter \""
348 0 : + filter
349 0 : + "\"; missing ending '/'.");
350 : }
351 0 : std::string const flag_list(filter.substr(pos + 1));
352 0 : filter = filter.substr(1, pos - 2);
353 0 : if(filter.empty())
354 : {
355 : throw invalid_variable(
356 : "invalid filter \""
357 0 : + filter
358 0 : + "\"; the regular expression is empty.");
359 : }
360 : // TODO: for errors we would need to iterate using the libutf8
361 : // (since we could have a Unicode character after the /)
362 : //
363 : // TODO: if two type flags are found, err too
364 : //
365 0 : int count(0);
366 0 : for(auto f : flag_list)
367 : {
368 0 : switch(f)
369 : {
370 0 : case 'i':
371 0 : flags |= std::regex::icase;
372 0 : break;
373 :
374 0 : case 'c':
375 0 : flags |= std::regex::collate;
376 0 : break;
377 :
378 0 : case 'j':
379 0 : type = std::regex::ECMAScript;
380 0 : ++count;
381 0 : break;
382 :
383 0 : case 'b':
384 0 : type = std::regex::basic;
385 0 : ++count;
386 0 : break;
387 :
388 0 : case 'x':
389 0 : type = std::regex::extended;
390 0 : ++count;
391 0 : break;
392 :
393 0 : case 'a':
394 0 : type = std::regex::awk;
395 0 : ++count;
396 0 : break;
397 :
398 0 : case 'g':
399 0 : type = std::regex::grep;
400 0 : ++count;
401 0 : break;
402 :
403 0 : case 'e':
404 0 : type = std::regex::egrep;
405 0 : ++count;
406 0 : break;
407 :
408 0 : default:
409 : throw invalid_variable(
410 : "in \""
411 0 : + filter
412 0 : + "\", found invalid flag '"
413 0 : + f
414 0 : + "'.");
415 :
416 : }
417 0 : if(count > 1)
418 : {
419 : throw invalid_variable(
420 : "found multiple types in \""
421 0 : + filter
422 0 : + "\".");
423 : }
424 : }
425 0 : }
426 0 : f_filter = std::make_shared<std::regex>(filter, flags | type);
427 : }
428 27 : }
429 :
430 : // NO REPEAT
431 : //
432 : {
433 27 : std::string no_repeat(f_name + "::no-repeat");
434 27 : if(!opts.is_defined(no_repeat))
435 : {
436 81 : if(opts.is_defined("no-repeat"))
437 : {
438 0 : no_repeat = "no-repeat";
439 : }
440 : else
441 : {
442 27 : no_repeat.clear();
443 : }
444 : }
445 27 : if(!no_repeat.empty())
446 : {
447 0 : std::string const value(opts.get_string(no_repeat));
448 0 : if(value != "off")
449 : {
450 0 : if(value == "max"
451 0 : || value == "maximum")
452 : {
453 0 : f_no_repeat_size = NO_REPEAT_MAXIMUM;
454 : }
455 0 : else if(value == "default")
456 : {
457 0 : f_no_repeat_size = NO_REPEAT_DEFAULT;
458 : }
459 : else
460 : {
461 0 : f_no_repeat_size = opts.get_long(no_repeat, 0, 0, NO_REPEAT_MAXIMUM);
462 : }
463 : }
464 0 : }
465 27 : }
466 54 : }
467 :
468 :
469 1 : void appender::reopen()
470 : {
471 1 : }
472 :
473 :
474 29 : void appender::add_component(component::pointer_t comp)
475 : {
476 29 : guard g;
477 :
478 29 : f_components.insert(comp);
479 58 : }
480 :
481 :
482 3 : format::pointer_t appender::get_format() const
483 : {
484 3 : guard g;
485 :
486 6 : return f_format;
487 3 : }
488 :
489 :
490 47 : format::pointer_t appender::set_format(format::pointer_t new_format)
491 : {
492 47 : guard g;
493 :
494 47 : format::pointer_t old(f_format);
495 47 : f_format = new_format;
496 94 : return old;
497 47 : }
498 :
499 :
500 1 : long appender::get_bytes_per_minute() const
501 : {
502 1 : return f_bytes_per_minute;
503 : }
504 :
505 :
506 : /** \brief Return the number of dropped messages due to bitrate restrictions.
507 : *
508 : * It is possible to set the bit rate at which an appender accepts messages.
509 : * Anything beyond that number gets dropped. The bit rate defined in the
510 : * appender configuration gets transformed in a number of bytes per minute.
511 : *
512 : * Each time a message is sent, the number of bytes in that message
513 : * string is added to a counter. If that counter reaches a number of bytes
514 : * in a minute larger than the allowed bytes per minute, then the following
515 : * messages get dropped until that one minute has elapsed.
516 : *
517 : * This function returns the number of messages that were dropped because
518 : * the bytes per minute limit was reached.
519 : *
520 : * \note
521 : * This counter doesn't get reset so reading it always returns a grand
522 : * total of all the messages that were dropped so far. This counter is
523 : * per appender.
524 : *
525 : * \return The number of messages that were dropped because the bitrate was
526 : * reached.
527 : */
528 1 : std::size_t appender::get_bitrate_dropped_messages() const
529 : {
530 1 : return f_bitrate_dropped_messages;
531 : }
532 :
533 :
534 45793 : void appender::send_message(message const & msg)
535 : {
536 45793 : guard g;
537 :
538 45793 : if(!is_enabled()
539 45793 : || msg.get_severity() < f_severity)
540 : {
541 4138 : return;
542 : }
543 :
544 41655 : component::set_t const & components(msg.get_components());
545 41655 : if(components.empty())
546 : {
547 : // user did not supply any component in 'msg', check for
548 : // the normal component
549 : //
550 83290 : if(!f_components.empty()
551 83290 : && f_components.find(f_normal_component) == f_components.end())
552 : {
553 0 : return;
554 : }
555 : }
556 : else
557 : {
558 10 : if(snapdev::empty_set_intersection(f_components, components))
559 : {
560 3 : return;
561 : }
562 : }
563 :
564 41652 : std::string formatted_message(f_format->process_message(msg));
565 41646 : if(formatted_message.empty())
566 : {
567 0 : return;
568 : }
569 :
570 41646 : if(f_filter != nullptr
571 41646 : && !std::regex_match(formatted_message, *f_filter))
572 : {
573 0 : return;
574 : }
575 :
576 41646 : if(formatted_message.back() != '\n'
577 41646 : && formatted_message.back() != '\r')
578 : {
579 : // TODO: add support to define line terminator (cr, nl, cr nl)
580 : //
581 41646 : formatted_message += '\n';
582 : }
583 :
584 : // TBD: should we use the time of the message rather than 'now'?
585 : //
586 41646 : if(f_bytes_per_minute != 0)
587 : {
588 0 : time_t const current_minute(time(0) / 60);
589 0 : if(current_minute != f_bytes_minute)
590 : {
591 0 : f_bytes_minute = current_minute;
592 0 : f_bytes_received = 0;
593 : }
594 0 : else if(f_bytes_received + static_cast<long>(formatted_message.length())
595 0 : >= f_bytes_per_minute)
596 : {
597 : // overflow
598 : //
599 : // IMPORTANT NOTE: this algorithm may kick out a very long log
600 : // message and then accept a smaller one which still fits in the
601 : // `f_bytes_per_minute` bitrate
602 : //
603 0 : ++f_bitrate_dropped_messages;
604 0 : return;
605 : }
606 0 : f_bytes_received += formatted_message.length();
607 : }
608 :
609 41646 : if(f_no_repeat_size > NO_REPEAT_OFF)
610 : {
611 0 : std::string const non_changing_message(f_format->process_message(msg, true));
612 0 : auto it(std::find(f_last_messages.rbegin(), f_last_messages.rend(), non_changing_message));
613 0 : if(it != f_last_messages.rend())
614 : {
615 : // TODO: count the number of times this message occurred and
616 : // at a certain number, show it again -- add an option to
617 : // defined that certain number
618 : //
619 : // check the timestamp and if the message happened more
620 : // than X seconds prior, repeat it anyway (this can be
621 : // a timeout of our cache as far as implementation is
622 : // concerned) -- add option to define time and this check
623 : // could go "the other way around" where we cumulate messages
624 : // and if not repeat within X seconds, then send them out
625 : // so here we would just push messages on a "stack" and
626 : // the thread would take care of send messages and if
627 : // repeated then we can show a "count" in the message
628 : // (i.e. something like "(message repeated N times in
629 : // the last X seconds)") -- I think syslog sends the
630 : // first message and then records the repeats for
631 : // X seconds and finally sends the results to the screen
632 : //
633 : // the current implementation helps, but it's weak in
634 : // its results... (the results look weird)
635 : //
636 0 : return;
637 : }
638 0 : f_last_messages.push_back(non_changing_message);
639 0 : if(f_last_messages.size() > f_no_repeat_size)
640 : {
641 0 : f_last_messages.pop_front();
642 : }
643 0 : }
644 :
645 41646 : process_message(msg, formatted_message);
646 45793 : }
647 :
648 :
649 0 : void appender::process_message(message const & msg, std::string const & formatted_message)
650 : {
651 : // the default is a "null appender" -- do nothing
652 0 : snapdev::NOT_USED(msg, formatted_message);
653 0 : }
654 :
655 :
656 :
657 :
658 :
659 8 : appender_factory::appender_factory(std::string const & type)
660 8 : : f_type(type)
661 : {
662 8 : char const * appender_factory_debug(getenv("APPENDER_FACTORY_DEBUG"));
663 8 : if(appender_factory_debug != nullptr
664 0 : && *appender_factory_debug != '\0')
665 : {
666 0 : std::cerr << "appender_factory:debug: adding appender factory \"" << type << "\".\n";
667 : }
668 8 : }
669 :
670 :
671 4 : appender_factory::~appender_factory()
672 : {
673 4 : }
674 :
675 :
676 16 : std::string const & appender_factory::get_type() const
677 : {
678 16 : return f_type;
679 : }
680 :
681 :
682 :
683 :
684 8 : void register_appender_factory(appender_factory::pointer_t factory)
685 : {
686 8 : get_private_logger()->register_appender_factory(factory);
687 8 : }
688 :
689 :
690 3 : appender::pointer_t create_appender(std::string const & type, std::string const & name)
691 : {
692 6 : return get_private_logger()->create_appender(type, name);
693 : }
694 :
695 :
696 :
697 :
698 :
699 :
700 1 : safe_format::safe_format(appender::pointer_t a, format::pointer_t new_format)
701 1 : : f_appender(a)
702 1 : , f_old_format(a->set_format(new_format))
703 : {
704 1 : }
705 :
706 :
707 1 : safe_format::~safe_format()
708 : {
709 1 : snapdev::NOT_USED(f_appender->set_format(f_old_format));
710 1 : }
711 :
712 :
713 :
714 :
715 :
716 :
717 :
718 : } // snaplogger namespace
719 : // vim: ts=4 sw=4 et
|