Attachment 1469923 Details for Bug 1183244 – potential journalctl sizelimit patch for 7.5

[patch] potential journalctl sizelimit patch for 7.5

1120-sizelimit.patch (text/plain), 7.83 KB, created by Pavel Moravec on 2018-07-23 11:31:42 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Pavel Moravec

Created: 2018-07-23 11:31:42 UTC

Size: 7.83 KB

patch

obsolete

>From f2e7e8e11fd5bfdc3e3d591d97996af69aacf8da Mon Sep 17 00:00:00 2001
>From: Jake Hunsaker <jhunsake@redhat.com>
>Date: Fri, 6 Oct 2017 15:52:06 -0400
>Subject: [PATCH] [utilities] Allow size limits for command output
>
>Previously, any command output generated from plugins from
>add_cmd_output and add_journal would be collected in full in memory. For
>example, if a journal was 4GB in size, then 4GB would be read into
>memory and subsequently written to the final sos archive. This lead to
>not only potentially large archives, but in some cases failure to
>collect data or produce an archive due to memory constraints on the
>system.
>
>This patch adds the ability to use a sizelimit option in both
>add_cmd_output and add_journal. This will limit the collected output
>from commands or journals to the given limit, both what is read into
>memory and what is written to the final archive. If not given,
>sizelimit will default to --log-size. For journal collection, if no
>sizelimit is given then the larger of either --log-size or 100mb is
>used.
>
>Signed-off-by: Jake Hunsaker <jhunsake@redhat.com>
>---
> sos/plugins/__init__.py | 45 ++++++++++++++++++++++++------------
> sos/utilities.py        | 61 ++++++++++++++++++++++++++++++++++++++++++++++---
> 2 files changed, 89 insertions(+), 17 deletions(-)
>
>diff --git a/sos/plugins/__init__.py b/sos/plugins/__init__.py
>index 78de61f6a..cca862eba 100644
>--- a/sos/plugins/__init__.py
>+++ b/sos/plugins/__init__.py
>@@ -606,7 +606,7 @@ def getmtime(path):
> 
>     def get_command_output(self, prog, timeout=300, stderr=True,
>                            chroot=True, runat=None, env=None,
>-                           binary=False):
>+                           binary=False, sizelimit=None):
>         if chroot or self.commons['cmdlineopts'].chroot == 'always':
>             root = self.sysroot
>         else:
>@@ -614,7 +614,8 @@ def get_command_output(self, prog, timeout=300, stderr=True,
> 
>         result = sos_get_command_output(prog, timeout=timeout, stderr=stderr,
>                                         chroot=root, chdir=runat,
>-                                        env=env, binary=binary)
>+                                        env=env, binary=binary,
>+                                        sizelimit=sizelimit)
> 
>         if result['status'] == 124:
>             self._log_warn("command '%s' timed out after %ds"
>@@ -652,14 +653,16 @@ def check_ext_prog(self, prog):
> 
>     def _add_cmd_output(self, cmd, suggest_filename=None,
>                         root_symlink=None, timeout=300, stderr=True,
>-                        chroot=True, runat=None, env=None, binary=False):
>+                        chroot=True, runat=None, env=None, binary=False,
>+                        sizelimit=None):
>         """Internal helper to add a single command to the collection list."""
>         cmdt = (
>             cmd, suggest_filename,
>             root_symlink, timeout, stderr,
>-            chroot, runat, env, binary
>+            chroot, runat, env, binary, sizelimit
>         )
>-        _tuplefmt = "('%s', '%s', '%s', %s, '%s', '%s', '%s', '%s', '%s')"
>+        _tuplefmt = ("('%s', '%s', '%s', %s, '%s', '%s', '%s', '%s', '%s', "
>+                     "'%s')")
>         _logstr = "packed command tuple: " + _tuplefmt
>         self._log_debug(_logstr % cmdt)
>         self.collect_cmds.append(cmdt)
>@@ -667,8 +670,12 @@ def _add_cmd_output(self, cmd, suggest_filename=None,
> 
>     def add_cmd_output(self, cmds, suggest_filename=None,
>                        root_symlink=None, timeout=300, stderr=True,
>-                       chroot=True, runat=None, env=None, binary=False):
>+                       chroot=True, runat=None, env=None, binary=False,
>+                       sizelimit=None):
>         """Run a program or a list of programs and collect the output"""
>+        if sizelimit is None:
>+            sizelimit = int(self.get_option('log_size'))
>+
>         if isinstance(cmds, six.string_types):
>             cmds = [cmds]
>         if len(cmds) > 1 and (suggest_filename or root_symlink):
>@@ -676,7 +683,7 @@ def add_cmd_output(self, cmds, suggest_filename=None,
>         for cmd in cmds:
>             self._add_cmd_output(cmd, suggest_filename,
>                                  root_symlink, timeout, stderr,
>-                                 chroot, runat, env, binary)
>+                                 chroot, runat, env, binary, sizelimit)
> 
>     def get_cmd_output_path(self, name=None, make=True):
>         """Return a path into which this module should store collected
>@@ -733,14 +740,15 @@ def add_string_as_file(self, content, filename):
>     def get_cmd_output_now(self, exe, suggest_filename=None,
>                            root_symlink=False, timeout=300, stderr=True,
>                            chroot=True, runat=None, env=None,
>-                           binary=False):
>+                           binary=False, sizelimit=None):
>         """Execute a command and save the output to a file for inclusion in the
>         report.
>         """
>         start = time()
>         result = self.get_command_output(exe, timeout=timeout, stderr=stderr,
>                                          chroot=chroot, runat=runat,
>-                                         env=env, binary=binary)
>+                                         env=env, binary=binary,
>+                                         sizelimit=sizelimit)
>         self._log_debug("collected output of '%s' in %s"
>                         % (exe.split()[0], time() - start))
> 
>@@ -788,7 +788,8 @@ class Plugin(object):
>         self.custom_text += text
> 
>     def add_journal(self, units=None, boot=None, since=None, until=None,
>-                    lines=None, allfields=False, output=None, timeout=None):
>+                    lines=None, allfields=False, output=None, timeout=None,
>+                    sizelimit=None):
>         """ Collect journald logs from one of more units.
> 
>         Keyword arguments:
>
>@@ -799,6 +799,8 @@ class Plugin(object):
>         output    -- A journalctl output control string, for example
>                      "verbose".
>         timeout   -- An optional timeout in seconds.
>+        sizelimit -- Limit to the size of output returned in MB. Defaults
>+                     to --log-size
>         """
>         journal_cmd = "journalctl --no-pager "
>         unit_opt = " --unit %s"
>@@ -857,8 +867,13 @@ def add_journal(self, units=None, boot=None, since=None, until=None,
>         if output:
>             journal_cmd += output_opt % output
> 
>+        if sizelimit is None:
>+            sizelimit = max(100, int(self.get_option('log_size')))
>+
>         self._log_debug("collecting journal: %s" % journal_cmd)
>-        self._add_cmd_output(journal_cmd, None, None, timeout)
>+        self._add_cmd_output(journal_cmd, None, None, timeout,
>+                             sizelimit=sizelimit
>+                             )
> 
>     def _expand_copy_spec(self, copyspec):
>         return glob.glob(copyspec)
>@@ -876,16 +891,18 @@ def _collect_cmd_output(self):
>                 timeout,
>                 stderr,
>                 chroot, runat,
>-                env, binary
>+                env, binary,
>+                sizelimit
>             ) = progs[0]
>             self._log_debug(("unpacked command tuple: " +
>                              "('%s', '%s', '%s', %s, '%s', '%s', '%s', '%s'," +
>-                             "'%s')") % progs[0])
>+                             "'%s %s')") % progs[0])
>             self._log_info("collecting output of '%s'" % prog)
>             self.get_cmd_output_now(prog, suggest_filename=suggest_filename,
>                                     root_symlink=root_symlink, timeout=timeout,
>                                     stderr=stderr, chroot=chroot, runat=runat,
>-                                    env=env, binary=binary)
>+                                    env=env, binary=binary,
>+                                    sizelimit=sizelimit)
> 
>     def _collect_strings(self):
>         for string, file_name in self.copy_strings:

Actions: View | Diff

Attachments on bug 1183244: 1469923 | 1477901