Attachment 881516 Details for Bug 1083287 – Use a lockfile around things that modify the cachedir

[patch] Use a lockfile around things that modify the cachedir

0002-Use-a-lockfile-around-things-that-modify-the-cachedi.patch (text/plain), 8.10 KB, created by Ralph Bean on 2014-04-01 20:05:08 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Ralph Bean

Created: 2014-04-01 20:05:08 UTC

Size: 8.10 KB

patch

obsolete

>From 345ef7607c9c8cc5cc84f7a535ca22b320ff5cfb Mon Sep 17 00:00:00 2001
>From: Ralph Bean <rbean@redhat.com>
>Date: Tue, 1 Apr 2014 15:56:29 -0400
>Subject: [PATCH 2/2] Use a lockfile around things that modify the cachedir.
>
>---
> src/bin/pungi.py        | 77 +++++++++++++++++++++++++------------------------
> src/pypungi/__init__.py | 46 ++++++++++++++++++++++++++++-
> 2 files changed, 85 insertions(+), 38 deletions(-)
>
>diff --git a/src/bin/pungi.py b/src/bin/pungi.py
>index 261939a..9d0e706 100755
>--- a/src/bin/pungi.py
>+++ b/src/bin/pungi.py
>@@ -103,49 +103,52 @@ def main():
>     # Actually do work.
>     mypungi = pypungi.Pungi(config, ksparser)
> 
>-    if not opts.sourceisos:
>-        if opts.do_all or opts.do_gather or opts.do_buildinstall:
>-            mypungi._inityum() # initialize the yum object for things that need it
>-        if opts.do_all or opts.do_gather:
>-            mypungi.gather()
>-            if opts.nodownload:
>-                for line in mypungi.list_packages():
>-                    flags_str = ",".join(line["flags"])
>-                    if flags_str:
>-                        flags_str = "(%s)" % flags_str
>-                    sys.stdout.write("RPM%s: %s\n" % (flags_str, line["path"]))
>-                sys.stdout.flush()
>-            else:
>-                mypungi.downloadPackages()
>-            mypungi.makeCompsFile()
>-            if not opts.nodebuginfo:
>-                mypungi.getDebuginfoList()
>-                if opts.nodownload:
>-                    for line in mypungi.list_debuginfo():
>-                        flags_str = ",".join(line["flags"])
>-                        if flags_str:
>-                            flags_str = "(%s)" % flags_str
>-                        sys.stdout.write("DEBUGINFO%s: %s\n" % (flags_str, line["path"]))
>-                    sys.stdout.flush()
>-                else:
>-                    mypungi.downloadDebuginfo()
>-            if not opts.nosource:
>+    with mypungi.yumlock:
>+        if not opts.sourceisos:
>+            if opts.do_all or opts.do_gather or opts.do_buildinstall:
>+                mypungi._inityum() # initialize the yum object for things that need it
>+            if opts.do_all or opts.do_gather:
>+                mypungi.gather()
>                 if opts.nodownload:
>-                    for line in mypungi.list_srpms():
>+                    for line in mypungi.list_packages():
>                         flags_str = ",".join(line["flags"])
>                         if flags_str:
>                             flags_str = "(%s)" % flags_str
>-                        sys.stdout.write("SRPM%s: %s\n" % (flags_str, line["path"]))
>+                        sys.stdout.write("RPM%s: %s\n" % (flags_str, line["path"]))
>                     sys.stdout.flush()
>                 else:
>-                    mypungi.downloadSRPMs()
>-
>-            print "RPM size:       %s MiB" % (mypungi.size_packages() / 1024 ** 2)
>-            if not opts.nodebuginfo:
>-                print "DEBUGINFO size: %s MiB" % (mypungi.size_debuginfo() / 1024 ** 2)
>-            if not opts.nosource:
>-                print "SRPM size:      %s MiB" % (mypungi.size_srpms() / 1024 ** 2)
>-
>+                    mypungi.downloadPackages()
>+                mypungi.makeCompsFile()
>+                if not opts.nodebuginfo:
>+                    mypungi.getDebuginfoList()
>+                    if opts.nodownload:
>+                        for line in mypungi.list_debuginfo():
>+                            flags_str = ",".join(line["flags"])
>+                            if flags_str:
>+                                flags_str = "(%s)" % flags_str
>+                            sys.stdout.write("DEBUGINFO%s: %s\n" % (flags_str, line["path"]))
>+                        sys.stdout.flush()
>+                    else:
>+                        mypungi.downloadDebuginfo()
>+                if not opts.nosource:
>+                    if opts.nodownload:
>+                        for line in mypungi.list_srpms():
>+                            flags_str = ",".join(line["flags"])
>+                            if flags_str:
>+                                flags_str = "(%s)" % flags_str
>+                            sys.stdout.write("SRPM%s: %s\n" % (flags_str, line["path"]))
>+                        sys.stdout.flush()
>+                    else:
>+                        mypungi.downloadSRPMs()
>+
>+                print "RPM size:       %s MiB" % (mypungi.size_packages() / 1024 ** 2)
>+                if not opts.nodebuginfo:
>+                    print "DEBUGINFO size: %s MiB" % (mypungi.size_debuginfo() / 1024 ** 2)
>+                if not opts.nosource:
>+                    print "SRPM size:      %s MiB" % (mypungi.size_srpms() / 1024 ** 2)
>+
>+    # Furthermore (but without the yumlock...)
>+    if not opts.sourceisos:
>         if opts.do_all or opts.do_createrepo:
>             mypungi.doCreaterepo()
> 
>diff --git a/src/pypungi/__init__.py b/src/pypungi/__init__.py
>index b28a41e..9e80287 100644
>--- a/src/pypungi/__init__.py
>+++ b/src/pypungi/__init__.py
>@@ -23,6 +23,7 @@ import sys
> import gzip
> import pypungi.util
> import pprint
>+import lockfile
> import logging
> import urlgrabber.progress
> import subprocess
>@@ -35,6 +36,40 @@ import arch as arch_module
> import multilib
> 
> 
>+class ReentrantYumLock(object):
>+    """ A lock that can be acquired multiple times by the same process. """
>+
>+    def __init__(self, lock, log):
>+        self.lock = lock
>+        self.log = log
>+        self.count = 0
>+
>+    def __enter__(self):
>+        if not self.count:
>+            self.log.info("Waiting on %r" % self.lock.lock_file)
>+            self.lock.acquire()
>+            self.log.info("Got %r" % self.lock.lock_file)
>+        self.count = self.count + 1
>+        self.log.info("Lock count upped to %i" % self.count)
>+
>+    def __exit__(self, type, value, tb):
>+        self.count = self.count - 1
>+        self.log.info("Lock count downed to %i" % self.count)
>+        self.log.info("%r %r %r" % (type, value, tb))
>+        if not self.count:
>+            self.lock.release()
>+            self.log.info("Released %r" % self.lock.lock_file)
>+
>+
>+def yumlocked(method):
>+    """ A locking decorator. """
>+    def wrapper(self, *args, **kwargs):
>+        with self.yumlock:
>+            return method(self, *args, **kwargs)
>+    # TODO - replace argspec, signature, etc..
>+    return wrapper
>+
>+
> def is_debug(po):
>     if "debuginfo" in po.name:
>         return True
>@@ -163,10 +198,15 @@ class PungiYum(yum.YumBase):
> class Pungi(pypungi.PungiBase):
>     def __init__(self, config, ksparser):
>         pypungi.PungiBase.__init__(self, config)
>- 
>+
>         # Set our own logging name space
>         self.logger = logging.getLogger('Pungi')
> 
>+        # Create a lock object for later use.
>+        filename = self.config.get('pungi', 'cachedir') + "/yumlock"
>+        lock = lockfile.LockFile(filename)
>+        self.yumlock = ReentrantYumLock(lock, self.logger)
>+
>         # Create the stdout/err streams and only send INFO+ stuff there
>         formatter = logging.Formatter('%(name)s:%(levelname)s: %(message)s')
>         console = logging.StreamHandler()
>@@ -294,6 +334,7 @@ class Pungi(pypungi.PungiBase):
>         if os.path.exists(os.path.join(thisrepo.cachedir, 'repomd.xml')):
>             os.remove(os.path.join(thisrepo.cachedir, 'repomd.xml'))
> 
>+    @yumlocked
>     def _inityum(self):
>         """Initialize the yum object.  Only needed for certain actions."""
> 
>@@ -1066,6 +1107,7 @@ class Pungi(pypungi.PungiBase):
> 
>         self.logger.info('Finished downloading packages.')
> 
>+    @yumlocked
>     def downloadPackages(self):
>         """Download the package objects obtained in getPackageObjects()."""
> 
>@@ -1114,6 +1156,7 @@ class Pungi(pypungi.PungiBase):
> 
>         #pypungi.util._doRunCommand(compsfilter, self.logger)
> 
>+    @yumlocked
>     def downloadSRPMs(self):
>         """Cycle through the list of srpms and
>            find the package objects for them, Then download them."""
>@@ -1121,6 +1164,7 @@ class Pungi(pypungi.PungiBase):
>         # do the downloads
>         self._downloadPackageList(self.srpm_po_list, os.path.join('source', 'SRPMS'))
> 
>+    @yumlocked
>     def downloadDebuginfo(self):
>         """Cycle through the list of debuginfo rpms and
>            download them."""
>-- 
>1.9.0
>

Actions: View | Diff

Attachments on bug 1083287: 881515 | 881516