Description of problem: Enable sent status data to file in rsyslog configmap, check the impstats.json, get: Wed Jul 17 09:07:45 2019: @cee: { "name": "global", "origin": "dynstats", "values": { } } Wed Jul 17 09:07:45 2019: @cee: { "name": "omelasticsearch", "origin": "omelasticsearch", "submitted": 13453, "failed.http": 8, "failed.httprequests": 1, "failed.checkConn": 5, "failed.es": 157, "response.success": 0, "response.bad": 0, "response.duplicate": 0, "response.badargument": 0, "response.bulkrejection": 0, "response.other": 0, "rebinds": 2827 } Wed Jul 17 09:07:45 2019: @cee: { "name": "prometheus_stats", "origin": "core.action", "processed": 0, "failed": 0, "suspended": 0, "suspended.duration": 0, "resumed": 0 } Wed Jul 17 09:07:45 2019: @cee: { "name": "action-1-omstdout", "origin": "core.action", "processed": 0, "failed": 0, "suspended": 0, "suspended.duration": 0, "resumed": 0 } Wed Jul 17 09:07:45 2019: @cee: { "name": "action-2-builtin:omfile", "origin": "core.action", "processed": 19, "failed": 0, "suspended": 0, "suspended.duration": 0, "resumed": 0 } Wed Jul 17 09:07:45 2019: @cee: { "name": "action-3-builtin:omfile", "origin": "core.action", "processed": 0, "failed": 0, "suspended": 0, "suspended.duration": 0, "resumed": 0 } Wed Jul 17 09:07:45 2019: @cee: { "name": "elasticsearch", "origin": "core.action", "processed": 15514, "failed": 0, "suspended": 1, "suspended.duration": 150, "resumed": 0 } Wed Jul 17 09:07:45 2019: @cee: { "name": "elasticsearch", "origin": "core.action", "processed": 0, "failed": 0, "suspended": 0, "suspended.duration": 0, "resumed": 0 } Wed Jul 17 09:07:45 2019: @cee: { "name": "action-6-mmnormalize", "origin": "core.action", "processed": 0, "failed": 0, "suspended": 0, "suspended.duration": 0, "resumed": 0 } Wed Jul 17 09:07:45 2019: @cee: { "name": "action-7-mmnormalize", "origin": "core.action", "processed": 6742, "failed": 0, "suspended": 0, "suspended.duration": 0, "resumed": 0 } Wed Jul 17 09:07:45 2019: @cee: { "name": "action-8-mmkubernetes", "origin": "core.action", "processed": 6742, "failed": 0, "suspended": 0, "suspended.duration": 0, "resumed": 0 } Wed Jul 17 09:07:45 2019: @cee: { "name": "action-9-mmnormalize", "origin": "core.action", "processed": 0, "failed": 0, "suspended": 0, "suspended.duration": 0, "resumed": 0 } Wed Jul 17 09:07:45 2019: @cee: { "name": "action-10-mmnormalize", "origin": "core.action", "processed": 0, "failed": 0, "suspended": 0, "suspended.duration": 0, "resumed": 0 } Wed Jul 17 09:07:45 2019: @cee: { "name": "action-11-mmexternal", "origin": "core.action", "processed": 15514, "failed": 0, "suspended": 0, "suspended.duration": 0, "resumed": 0 } Wed Jul 17 09:07:45 2019: @cee: { "name": "action-12-mmnormalize", "origin": "core.action", "processed": 0, "failed": 0, "suspended": 0, "suspended.duration": 0, "resumed": 0 } Wed Jul 17 09:07:45 2019: @cee: { "name": "action-13-builtin:omfile", "origin": "core.action", "processed": 0, "failed": 0, "suspended": 0, "suspended.duration": 0, "resumed": 0 } Wed Jul 17 09:07:45 2019: @cee: { "name": "imjournal", "origin": "imjournal", "submitted": 8772, "read": 8772, "discarded": 0, "failed": 0, "poll_failed": 0, "rotations": 0, "recovery_attempts": 0, "ratelimit_discarded_in_interval": 0, "disk_usage_bytes": 218112000 } Wed Jul 17 09:07:45 2019: @cee: { "name": "resource-usage", "origin": "impstats", "utime": 79500902, "stime": 5042774, "maxrss": 247676, "minflt": 62323, "majflt": 0, "inblock": 48, "oublock": 227608, "nvcsw": 123806, "nivcsw": 1358, "openfiles": 2781 } Wed Jul 17 09:07:45 2019: @cee: { "name": "elasticsearch queue", "origin": "core.queue", "size": 1, "enqueued": 15514, "full": 0, "discarded.full": 0, "discarded.nf": 0, "maxqsize": 119 } Wed Jul 17 09:07:45 2019: @cee: { "name": "elasticsearch queue", "origin": "core.queue", "size": 0, "enqueued": 0, "full": 0, "discarded.full": 0, "discarded.nf": 0, "maxqsize": 0 } Wed Jul 17 09:07:45 2019: @cee: { "name": "main Q[DA]", "origin": "core.queue", "size": 0, "enqueued": 0, "full": 0, "discarded.full": 0, "discarded.nf": 0, "maxqsize": 0 } Wed Jul 17 09:07:45 2019: @cee: { "name": "main Q", "origin": "core.queue", "size": 0, "enqueued": 15533, "full": 0, "discarded.full": 0, "discarded.nf": 0, "maxqsize": 364 } Wed Jul 17 09:07:45 2019: @cee: { "name": "mmkubernetes(https:\/\/kubernetes.default.svc.cluster.local:443)", "origin": "mmkubernetes", "recordseen": 6742, "namespacemetadatasuccess": 9, "namespacemetadatanotfound": 0, "namespacemetadatabusy": 0, "namespacemetadataerror": 0, "podmetadatasuccess": 14, "podmetadatanotfound": 0, "podmetadatabusy": 0, "podmetadataerror": 0, "namespacecachenumentries": 9, "podcachenumentries": 14, "namespacecachehits": 5, "podcachehits": 6728, "namespacecachemisses": 9, "podcachemisses": 14 } Check the rsyslog prometheus metrics, the metrics of mmkubernetes and omelasticsearch modules are not found. Besides, there are two elasticsearch modules in rsyslog, in the rsyslog prometheus metrics, only have one module rsyslog_action_processed{action="elasticsearch"} . The rsyslog prometheus metrics: # HELP rsyslog_action_failed messages failed # TYPE rsyslog_action_failed counter rsyslog_action_failed{action="action-1-omstdout"} 0 rsyslog_action_failed{action="action-10-mmnormalize"} 0 rsyslog_action_failed{action="action-11-mmexternal"} 0 rsyslog_action_failed{action="action-12-mmnormalize"} 0 rsyslog_action_failed{action="action-13-builtin:omfile"} 0 rsyslog_action_failed{action="action-2-builtin:omfile"} 0 rsyslog_action_failed{action="action-3-builtin:omfile"} 0 rsyslog_action_failed{action="action-6-mmnormalize"} 0 rsyslog_action_failed{action="action-7-mmnormalize"} 0 rsyslog_action_failed{action="action-8-mmkubernetes"} 0 rsyslog_action_failed{action="action-9-mmnormalize"} 0 rsyslog_action_failed{action="elasticsearch"} 0 rsyslog_action_failed{action="prometheus_stats"} 0 # HELP rsyslog_action_processed messages processed # TYPE rsyslog_action_processed counter rsyslog_action_processed{action="action-1-omstdout"} 0 rsyslog_action_processed{action="action-10-mmnormalize"} 0 rsyslog_action_processed{action="action-11-mmexternal"} 6801 rsyslog_action_processed{action="action-12-mmnormalize"} 0 rsyslog_action_processed{action="action-13-builtin:omfile"} 0 rsyslog_action_processed{action="action-2-builtin:omfile"} 5 rsyslog_action_processed{action="action-3-builtin:omfile"} 0 rsyslog_action_processed{action="action-6-mmnormalize"} 0 rsyslog_action_processed{action="action-7-mmnormalize"} 3620 rsyslog_action_processed{action="action-8-mmkubernetes"} 3620 rsyslog_action_processed{action="action-9-mmnormalize"} 0 rsyslog_action_processed{action="elasticsearch"} 0 rsyslog_action_processed{action="prometheus_stats"} 691 # HELP rsyslog_action_resumed times resumed # TYPE rsyslog_action_resumed counter rsyslog_action_resumed{action="action-1-omstdout"} 0 rsyslog_action_resumed{action="action-10-mmnormalize"} 0 rsyslog_action_resumed{action="action-11-mmexternal"} 0 rsyslog_action_resumed{action="action-12-mmnormalize"} 0 rsyslog_action_resumed{action="action-13-builtin:omfile"} 0 rsyslog_action_resumed{action="action-2-builtin:omfile"} 0 rsyslog_action_resumed{action="action-3-builtin:omfile"} 0 rsyslog_action_resumed{action="action-6-mmnormalize"} 0 rsyslog_action_resumed{action="action-7-mmnormalize"} 0 rsyslog_action_resumed{action="action-8-mmkubernetes"} 0 rsyslog_action_resumed{action="action-9-mmnormalize"} 0 rsyslog_action_resumed{action="elasticsearch"} 0 rsyslog_action_resumed{action="prometheus_stats"} 0 # HELP rsyslog_action_suspended times suspended # TYPE rsyslog_action_suspended counter rsyslog_action_suspended{action="action-1-omstdout"} 0 rsyslog_action_suspended{action="action-10-mmnormalize"} 0 rsyslog_action_suspended{action="action-11-mmexternal"} 0 rsyslog_action_suspended{action="action-12-mmnormalize"} 0 rsyslog_action_suspended{action="action-13-builtin:omfile"} 0 rsyslog_action_suspended{action="action-2-builtin:omfile"} 0 rsyslog_action_suspended{action="action-3-builtin:omfile"} 0 rsyslog_action_suspended{action="action-6-mmnormalize"} 0 rsyslog_action_suspended{action="action-7-mmnormalize"} 0 rsyslog_action_suspended{action="action-8-mmkubernetes"} 0 rsyslog_action_suspended{action="action-9-mmnormalize"} 0 rsyslog_action_suspended{action="elasticsearch"} 0 rsyslog_action_suspended{action="prometheus_stats"} 0 # HELP rsyslog_action_suspended_duration time spent suspended # TYPE rsyslog_action_suspended_duration counter rsyslog_action_suspended_duration{action="action-1-omstdout"} 0 rsyslog_action_suspended_duration{action="action-10-mmnormalize"} 0 rsyslog_action_suspended_duration{action="action-11-mmexternal"} 0 rsyslog_action_suspended_duration{action="action-12-mmnormalize"} 0 rsyslog_action_suspended_duration{action="action-13-builtin:omfile"} 0 rsyslog_action_suspended_duration{action="action-2-builtin:omfile"} 0 rsyslog_action_suspended_duration{action="action-3-builtin:omfile"} 0 rsyslog_action_suspended_duration{action="action-6-mmnormalize"} 0 rsyslog_action_suspended_duration{action="action-7-mmnormalize"} 0 rsyslog_action_suspended_duration{action="action-8-mmkubernetes"} 0 rsyslog_action_suspended_duration{action="action-9-mmnormalize"} 0 rsyslog_action_suspended_duration{action="elasticsearch"} 0 rsyslog_action_suspended_duration{action="prometheus_stats"} 0 # HELP rsyslog_input_submitted messages submitted # TYPE rsyslog_input_submitted counter rsyslog_input_submitted{input="imjournal"} 3181 rsyslog_input_submitted{input="omelasticsearch"} 6801 # HELP rsyslog_queue_discarded_full messages discarded due to queue being full # TYPE rsyslog_queue_discarded_full counter rsyslog_queue_discarded_full{queue="elasticsearch queue"} 0 rsyslog_queue_discarded_full{queue="main Q"} 0 rsyslog_queue_discarded_full{queue="main Q[DA]"} 0 # HELP rsyslog_queue_discarded_not_full messages discarded when queue not full # TYPE rsyslog_queue_discarded_not_full counter rsyslog_queue_discarded_not_full{queue="elasticsearch queue"} 0 rsyslog_queue_discarded_not_full{queue="main Q"} 0 rsyslog_queue_discarded_not_full{queue="main Q[DA]"} 0 # HELP rsyslog_queue_enqueued total messages enqueued # TYPE rsyslog_queue_enqueued counter rsyslog_queue_enqueued{queue="elasticsearch queue"} 0 rsyslog_queue_enqueued{queue="main Q"} 7517 rsyslog_queue_enqueued{queue="main Q[DA]"} 0 # HELP rsyslog_queue_full times queue was full # TYPE rsyslog_queue_full counter rsyslog_queue_full{queue="elasticsearch queue"} 0 rsyslog_queue_full{queue="main Q"} 0 rsyslog_queue_full{queue="main Q[DA]"} 0 # HELP rsyslog_queue_max_size maximum size queue has reached # TYPE rsyslog_queue_max_size gauge rsyslog_queue_max_size{queue="elasticsearch queue"} 0 rsyslog_queue_max_size{queue="main Q"} 630 rsyslog_queue_max_size{queue="main Q[DA]"} 0 # HELP rsyslog_queue_size messages currently in queue # TYPE rsyslog_queue_size gauge rsyslog_queue_size{queue="elasticsearch queue"} 0 rsyslog_queue_size{queue="main Q"} 2 rsyslog_queue_size{queue="main Q[DA]"} 0 # HELP rsyslog_resource_inblock filesystem input operations # TYPE rsyslog_resource_inblock counter rsyslog_resource_inblock{resource="resource-usage"} 48 # HELP rsyslog_resource_majflt total major faults # TYPE rsyslog_resource_majflt counter rsyslog_resource_majflt{resource="resource-usage"} 0 # HELP rsyslog_resource_maxrss maximum resident set size # TYPE rsyslog_resource_maxrss gauge rsyslog_resource_maxrss{resource="resource-usage"} 88800 # HELP rsyslog_resource_minflt total minor faults # TYPE rsyslog_resource_minflt counter rsyslog_resource_minflt{resource="resource-usage"} 25963 # HELP rsyslog_resource_nivcsw involuntary context switches # TYPE rsyslog_resource_nivcsw counter rsyslog_resource_nivcsw{resource="resource-usage"} 10008 # HELP rsyslog_resource_nvcsw voluntary context switches # TYPE rsyslog_resource_nvcsw counter rsyslog_resource_nvcsw{resource="resource-usage"} 36262 # HELP rsyslog_resource_oublock filesystem output operations # TYPE rsyslog_resource_oublock counter rsyslog_resource_oublock{resource="resource-usage"} 94736 # HELP rsyslog_resource_stime system time used in microsends # TYPE rsyslog_resource_stime counter rsyslog_resource_stime{resource="resource-usage"} 1.757466e+06 # HELP rsyslog_resource_utime user time used in microseconds # TYPE rsyslog_resource_utime counter rsyslog_resource_utime{resource="resource-usage"} 1.8830195e+07 Version-Release number of selected component (if applicable): ose-cluster-logging-operator-v4.2.0-201907161019 ose-logging-rsyslog-v4.2.0-201907141446 How reproducible: Always Steps to Reproduce: 1. deploy logging using rsyslog as log collector 2. enable rsyslog prometheus endpoint 3. check the rsyslog metrics and compare it with the stats data created by the impstat module in rsyslog pod Actual results: Expected results: Additional info:
Verified with ose-logging-rsyslog-v4.2.0-201908011419, the metrics are: # HELP rsyslog_action_failed messages failed # TYPE rsyslog_action_failed counter rsyslog_action_failed{action="debug_undefined_field"} 0 rsyslog_action_failed{action="elasticsearch"} 0 rsyslog_action_failed{action="es_error_file"} 0 rsyslog_action_failed{action="impstats_json"} 0 rsyslog_action_failed{action="kubernetes_metadata"} 0 rsyslog_action_failed{action="parse_json_containers"} 0 rsyslog_action_failed{action="parse_json_journald"} 0 rsyslog_action_failed{action="parse_multiline_crio_file"} 0 rsyslog_action_failed{action="parse_multiline_docker_json"} 0 rsyslog_action_failed{action="prometheus_stats"} 0 rsyslog_action_failed{action="rsyslog_logs_to_console"} 0 rsyslog_action_failed{action="rsyslog_logs_to_file"} 0 rsyslog_action_failed{action="skip_empty"} 0 rsyslog_action_failed{action="undefined_field"} 0 # HELP rsyslog_action_processed messages processed # TYPE rsyslog_action_processed counter rsyslog_action_processed{action="debug_undefined_field"} 0 rsyslog_action_processed{action="elasticsearch"} 0 rsyslog_action_processed{action="es_error_file"} 0 rsyslog_action_processed{action="impstats_json"} 0 rsyslog_action_processed{action="kubernetes_metadata"} 14297 rsyslog_action_processed{action="parse_json_containers"} 14297 rsyslog_action_processed{action="parse_json_journald"} 8607 rsyslog_action_processed{action="parse_multiline_crio_file"} 14297 rsyslog_action_processed{action="parse_multiline_docker_json"} 0 rsyslog_action_processed{action="prometheus_stats"} 2575 rsyslog_action_processed{action="rsyslog_logs_to_console"} 0 rsyslog_action_processed{action="rsyslog_logs_to_file"} 2 rsyslog_action_processed{action="skip_empty"} 0 rsyslog_action_processed{action="undefined_field"} 22904 # HELP rsyslog_action_resumed times resumed # TYPE rsyslog_action_resumed counter rsyslog_action_resumed{action="debug_undefined_field"} 0 rsyslog_action_resumed{action="elasticsearch"} 0 rsyslog_action_resumed{action="es_error_file"} 0 rsyslog_action_resumed{action="impstats_json"} 0 rsyslog_action_resumed{action="kubernetes_metadata"} 0 rsyslog_action_resumed{action="parse_json_containers"} 0 rsyslog_action_resumed{action="parse_json_journald"} 0 rsyslog_action_resumed{action="parse_multiline_crio_file"} 0 rsyslog_action_resumed{action="parse_multiline_docker_json"} 0 rsyslog_action_resumed{action="prometheus_stats"} 0 rsyslog_action_resumed{action="rsyslog_logs_to_console"} 0 rsyslog_action_resumed{action="rsyslog_logs_to_file"} 0 rsyslog_action_resumed{action="skip_empty"} 0 rsyslog_action_resumed{action="undefined_field"} 0 # HELP rsyslog_action_suspended times suspended # TYPE rsyslog_action_suspended counter rsyslog_action_suspended{action="debug_undefined_field"} 0 rsyslog_action_suspended{action="elasticsearch"} 0 rsyslog_action_suspended{action="es_error_file"} 0 rsyslog_action_suspended{action="impstats_json"} 0 rsyslog_action_suspended{action="kubernetes_metadata"} 0 rsyslog_action_suspended{action="parse_json_containers"} 0 rsyslog_action_suspended{action="parse_json_journald"} 0 rsyslog_action_suspended{action="parse_multiline_crio_file"} 0 rsyslog_action_suspended{action="parse_multiline_docker_json"} 0 rsyslog_action_suspended{action="prometheus_stats"} 0 rsyslog_action_suspended{action="rsyslog_logs_to_console"} 0 rsyslog_action_suspended{action="rsyslog_logs_to_file"} 0 rsyslog_action_suspended{action="skip_empty"} 0 rsyslog_action_suspended{action="undefined_field"} 0 # HELP rsyslog_action_suspended_duration time spent suspended # TYPE rsyslog_action_suspended_duration counter rsyslog_action_suspended_duration{action="debug_undefined_field"} 0 rsyslog_action_suspended_duration{action="elasticsearch"} 0 rsyslog_action_suspended_duration{action="es_error_file"} 0 rsyslog_action_suspended_duration{action="impstats_json"} 0 rsyslog_action_suspended_duration{action="kubernetes_metadata"} 0 rsyslog_action_suspended_duration{action="parse_json_containers"} 0 rsyslog_action_suspended_duration{action="parse_json_journald"} 0 rsyslog_action_suspended_duration{action="parse_multiline_crio_file"} 0 rsyslog_action_suspended_duration{action="parse_multiline_docker_json"} 0 rsyslog_action_suspended_duration{action="prometheus_stats"} 0 rsyslog_action_suspended_duration{action="rsyslog_logs_to_console"} 0 rsyslog_action_suspended_duration{action="rsyslog_logs_to_file"} 0 rsyslog_action_suspended_duration{action="skip_empty"} 0 rsyslog_action_suspended_duration{action="undefined_field"} 0 # HELP rsyslog_imjournal_discarded messages that were read but discarded # TYPE rsyslog_imjournal_discarded counter rsyslog_imjournal_discarded{imjournal="imjournal"} 0 # HELP rsyslog_imjournal_disk_usage_bytes disk usage of journal in bytes # TYPE rsyslog_imjournal_disk_usage_bytes gauge rsyslog_imjournal_disk_usage_bytes{imjournal="imjournal"} 1.84557568e+08 # HELP rsyslog_imjournal_failed messages that could not be read due to failure # TYPE rsyslog_imjournal_failed counter rsyslog_imjournal_failed{imjournal="imjournal"} 0 # HELP rsyslog_imjournal_read messages read from journal # TYPE rsyslog_imjournal_read counter 100 23165 0 23165 0 0 181k 0 --:--:-- --:--:-- --:--:-- 182k ournal_read{imjournal="imjournal"} 8607 # HELP rsyslog_imjournal_recovery_attempts number of recovery attempts # TYPE rsyslog_imjournal_recovery_attempts counter rsyslog_imjournal_recovery_attempts{imjournal="imjournal"} 0 # HELP rsyslog_imjournal_rotations number of journal rotations detected # TYPE rsyslog_imjournal_rotations counter rsyslog_imjournal_rotations{imjournal="imjournal"} 0 # HELP rsyslog_imjournal_submitted messages submitted # TYPE rsyslog_imjournal_submitted counter rsyslog_imjournal_submitted{imjournal="imjournal"} 8607 # HELP rsyslog_mmkubernetes_namespacecachehits number of namespace metadata cache hits # TYPE rsyslog_mmkubernetes_namespacecachehits counter rsyslog_mmkubernetes_namespacecachehits{mmkubernetes="mmkubernetes(https://kubernetes.default.svc.cluster.local:443)"} 3 # HELP rsyslog_mmkubernetes_namespacecachemisses number of namespace metadata cache misses # TYPE rsyslog_mmkubernetes_namespacecachemisses counter rsyslog_mmkubernetes_namespacecachemisses{mmkubernetes="mmkubernetes(https://kubernetes.default.svc.cluster.local:443)"} 15 # HELP rsyslog_mmkubernetes_namespacecachenumentries number of entries in namespace metadata cache # TYPE rsyslog_mmkubernetes_namespacecachenumentries gauge rsyslog_mmkubernetes_namespacecachenumentries{mmkubernetes="mmkubernetes(https://kubernetes.default.svc.cluster.local:443)"} 15 # HELP rsyslog_mmkubernetes_namespacemetadatabusy number of unsuccessful queries for namespace metadata due to busy (HTTP 429) response # TYPE rsyslog_mmkubernetes_namespacemetadatabusy counter rsyslog_mmkubernetes_namespacemetadatabusy{mmkubernetes="mmkubernetes(https://kubernetes.default.svc.cluster.local:443)"} 0 # HELP rsyslog_mmkubernetes_namespacemetadataerror number of unsuccessful queries for namespace metadata due to unknown response # TYPE rsyslog_mmkubernetes_namespacemetadataerror counter rsyslog_mmkubernetes_namespacemetadataerror{mmkubernetes="mmkubernetes(https://kubernetes.default.svc.cluster.local:443)"} 0 # HELP rsyslog_mmkubernetes_namespacemetadatanotfound number of unsuccessful queries for namespace metadata due to missing namespace (HTTP 404) # TYPE rsyslog_mmkubernetes_namespacemetadatanotfound counter rsyslog_mmkubernetes_namespacemetadatanotfound{mmkubernetes="mmkubernetes(https://kubernetes.default.svc.cluster.local:443)"} 0 # HELP rsyslog_mmkubernetes_namespacemetadatasuccess number of successful queries for namespace metadata # TYPE rsyslog_mmkubernetes_namespacemetadatasuccess counter rsyslog_mmkubernetes_namespacemetadatasuccess{mmkubernetes="mmkubernetes(https://kubernetes.default.svc.cluster.local:443)"} 15 # HELP rsyslog_mmkubernetes_podcachehits number of pod metadata cache hits # TYPE rsyslog_mmkubernetes_podcachehits counter rsyslog_mmkubernetes_podcachehits{mmkubernetes="mmkubernetes(https://kubernetes.default.svc.cluster.local:443)"} 14279 # HELP rsyslog_mmkubernetes_podcachemisses number of pod metadata cache misses # TYPE rsyslog_mmkubernetes_podcachemisses counter rsyslog_mmkubernetes_podcachemisses{mmkubernetes="mmkubernetes(https://kubernetes.default.svc.cluster.local:443)"} 18 # HELP rsyslog_mmkubernetes_podcachenumentries number of entries in pod metadata cache # TYPE rsyslog_mmkubernetes_podcachenumentries gauge rsyslog_mmkubernetes_podcachenumentries{mmkubernetes="mmkubernetes(https://kubernetes.default.svc.cluster.local:443)"} 18 # HELP rsyslog_mmkubernetes_podmetadatabusy number of unsuccessful queries for pod metadata due to busy (HTTP 429) response # TYPE rsyslog_mmkubernetes_podmetadatabusy counter rsyslog_mmkubernetes_podmetadatabusy{mmkubernetes="mmkubernetes(https://kubernetes.default.svc.cluster.local:443)"} 0 # HELP rsyslog_mmkubernetes_podmetadataerror number of unsuccessful queries for pod metadata due to unknown response # TYPE rsyslog_mmkubernetes_podmetadataerror counter rsyslog_mmkubernetes_podmetadataerror{mmkubernetes="mmkubernetes(https://kubernetes.default.svc.cluster.local:443)"} 0 # HELP rsyslog_mmkubernetes_podmetadatanotfound number of unsuccessful queries for pod metadata due to missing pod (HTTP 404) # TYPE rsyslog_mmkubernetes_podmetadatanotfound counter rsyslog_mmkubernetes_podmetadatanotfound{mmkubernetes="mmkubernetes(https://kubernetes.default.svc.cluster.local:443)"} 0 # HELP rsyslog_mmkubernetes_podmetadatasuccess number of successful queries for pod metadata # TYPE rsyslog_mmkubernetes_podmetadatasuccess counter rsyslog_mmkubernetes_podmetadatasuccess{mmkubernetes="mmkubernetes(https://kubernetes.default.svc.cluster.local:443)"} 18 # HELP rsyslog_mmkubernetes_recordseen number of messages processed from Kubernetes container logs # TYPE rsyslog_mmkubernetes_recordseen counter rsyslog_mmkubernetes_recordseen{mmkubernetes="mmkubernetes(https://kubernetes.default.svc.cluster.local:443)"} 14297 # HELP rsyslog_omelasticsearch_failedcheckconn number of times we failed to get a connection to Elasticsearch # TYPE rsyslog_omelasticsearch_failedcheckconn counter rsyslog_omelasticsearch_failedcheckconn{omelasticsearch="omelasticsearch"} 0 # HELP rsyslog_omelasticsearch_failedes number of times Elasticsearch response contained an error - detailed in responses # TYPE rsyslog_omelasticsearch_failedes counter rsyslog_omelasticsearch_failedes{omelasticsearch="omelasticsearch"} 0 # HELP rsyslog_omelasticsearch_failedhttp number of messages rejected due to error returned from HTTP # TYPE rsyslog_omelasticsearch_failedhttp counter rsyslog_omelasticsearch_failedhttp{omelasticsearch="omelasticsearch"} 0 # HELP rsyslog_omelasticsearch_failedhttprequests number of requests rejected due to error returned from the HTTP request # TYPE rsyslog_omelasticsearch_failedhttprequests counter rsyslog_omelasticsearch_failedhttprequests{omelasticsearch="omelasticsearch"} 0 # HELP rsyslog_omelasticsearch_rebinds number of times rsyslog reconnected to Elasticsearch # TYPE rsyslog_omelasticsearch_rebinds counter rsyslog_omelasticsearch_rebinds{omelasticsearch="omelasticsearch"} 42 # HELP rsyslog_omelasticsearch_responsebad number of times the Elasticsearch reponse could not be parsed, or was malformed # TYPE rsyslog_omelasticsearch_responsebad counter rsyslog_omelasticsearch_responsebad{omelasticsearch="omelasticsearch"} 0 # HELP rsyslog_omelasticsearch_responsebadargument number of records rejected due to syntax errors, formatting, unknown arguments (e.g. HTTP 400) # TYPE rsyslog_omelasticsearch_responsebadargument counter rsyslog_omelasticsearch_responsebadargument{omelasticsearch="omelasticsearch"} 0 # HELP rsyslog_omelasticsearch_responsebulkrejection number of records rejected due to Bulk Index Rejection (HTTP 429) # TYPE rsyslog_omelasticsearch_responsebulkrejection counter rsyslog_omelasticsearch_responsebulkrejection{omelasticsearch="omelasticsearch"} 0 # HELP rsyslog_omelasticsearch_responseduplicate number of duplicate records created e.g. by bulk index retry (HTTP 409) # TYPE rsyslog_omelasticsearch_responseduplicate counter rsyslog_omelasticsearch_responseduplicate{omelasticsearch="omelasticsearch"} 0 # HELP rsyslog_omelasticsearch_responseother number of records rejected due to unknown reasons # TYPE rsyslog_omelasticsearch_responseother counter rsyslog_omelasticsearch_responseother{omelasticsearch="omelasticsearch"} 0 # HELP rsyslog_omelasticsearch_responsesuccess number of times Elasticsearch response was successful (HTTP 200) # TYPE rsyslog_omelasticsearch_responsesuccess counter rsyslog_omelasticsearch_responsesuccess{omelasticsearch="omelasticsearch"} 0 # HELP rsyslog_omelasticsearch_submitted number of messages submitted for output # TYPE rsyslog_omelasticsearch_submitted counter rsyslog_omelasticsearch_submitted{omelasticsearch="omelasticsearch"} 22904 # HELP rsyslog_queue_discarded_full messages discarded due to queue being full # TYPE rsyslog_queue_discarded_full counter rsyslog_queue_discarded_full{queue="elasticsearch queue"} 0 rsyslog_queue_discarded_full{queue="main Q"} 0 rsyslog_queue_discarded_full{queue="main Q[DA]"} 0 rsyslog_queue_discarded_full{queue="prometheus_stats"} 0 # HELP rsyslog_queue_discarded_not_full messages discarded when queue not full # TYPE rsyslog_queue_discarded_not_full counter rsyslog_queue_discarded_not_full{queue="elasticsearch queue"} 0 rsyslog_queue_discarded_not_full{queue="main Q"} 0 rsyslog_queue_discarded_not_full{queue="main Q[DA]"} 0 rsyslog_queue_discarded_not_full{queue="prometheus_stats"} 0 # HELP rsyslog_queue_enqueued total messages enqueued # TYPE rsyslog_queue_enqueued counter rsyslog_queue_enqueued{queue="elasticsearch queue"} 0 rsyslog_queue_enqueued{queue="main Q"} 22906 rsyslog_queue_enqueued{queue="main Q[DA]"} 0 rsyslog_queue_enqueued{queue="prometheus_stats"} 2596 # HELP rsyslog_queue_full times queue was full # TYPE rsyslog_queue_full counter rsyslog_queue_full{queue="elasticsearch queue"} 0 rsyslog_queue_full{queue="main Q"} 0 rsyslog_queue_full{queue="main Q[DA]"} 0 rsyslog_queue_full{queue="prometheus_stats"} 0 # HELP rsyslog_queue_max_size maximum size queue has reached # TYPE rsyslog_queue_max_size gauge rsyslog_queue_max_size{queue="elasticsearch queue"} 0 rsyslog_queue_max_size{queue="main Q"} 523 rsyslog_queue_max_size{queue="main Q[DA]"} 0 rsyslog_queue_max_size{queue="prometheus_stats"} 25 # HELP rsyslog_queue_size messages currently in queue # TYPE rsyslog_queue_size gauge rsyslog_queue_size{queue="elasticsearch queue"} 0 rsyslog_queue_size{queue="main Q"} 0 rsyslog_queue_size{queue="main Q[DA]"} 0 rsyslog_queue_size{queue="prometheus_stats"} 21 # HELP rsyslog_resource_inblock filesystem input operations # TYPE rsyslog_resource_inblock counter rsyslog_resource_inblock{resource="resource-usage"} 48 # HELP rsyslog_resource_majflt total major faults # TYPE rsyslog_resource_majflt counter rsyslog_resource_majflt{resource="resource-usage"} 0 # HELP rsyslog_resource_maxrss maximum resident set size # TYPE rsyslog_resource_maxrss gauge rsyslog_resource_maxrss{resource="resource-usage"} 33024 # HELP rsyslog_resource_minflt total minor faults # TYPE rsyslog_resource_minflt counter rsyslog_resource_minflt{resource="resource-usage"} 9058 # HELP rsyslog_resource_nivcsw involuntary context switches # TYPE rsyslog_resource_nivcsw counter rsyslog_resource_nivcsw{resource="resource-usage"} 19281 # HELP rsyslog_resource_nvcsw voluntary context switches # TYPE rsyslog_resource_nvcsw counter rsyslog_resource_nvcsw{resource="resource-usage"} 120738 # HELP rsyslog_resource_oublock filesystem output operations # TYPE rsyslog_resource_oublock counter rsyslog_resource_oublock{resource="resource-usage"} 309488 # HELP rsyslog_resource_stime system time used in microsends # TYPE rsyslog_resource_stime counter rsyslog_resource_stime{resource="resource-usage"} 2.946874e+06 # HELP rsyslog_resource_utime user time used in microseconds # TYPE rsyslog_resource_utime counter rsyslog_resource_utime{resource="resource-usage"} 8.859452e+06
Since the problem described in this bug report should be resolved in a recent advisory, it has been closed with a resolution of ERRATA. For information on the advisory, and where to find the updated files, follow the link below. If the solution does not work for you, open a new bug report. https://access.redhat.com/errata/RHBA-2019:2922