Attachment 1258173 Details for Bug 1404345 – Patch to compress RPM metadata, compatible with pulp_rpm 2.8.7

[patch] Patch to compress RPM metadata, compatible with pulp_rpm 2.8.7

compressed_metadata_pulp_rpm_2.8.7.patch (text/plain), 26.51 KB, created by Tanya Tereshchenko on 2017-02-27 17:52:58 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Tanya Tereshchenko

Created: 2017-02-27 17:52:58 UTC

Size: 26.51 KB

patch

obsolete

>From 753970e884b81b45176726430f10066842c1d4ca Mon Sep 17 00:00:00 2001
>From: Tatiana Tereshchenko <ttereshc@redhat.com>
>Date: Sat, 28 Jan 2017 01:32:59 +0100
>Subject: [PATCH] Compressed metadata patch for pulp_rpm 2.8.7
>
>It will decrease a probability of hitting maximum document size in MongoDB
>for RPM/SRPM content with huge metadata.
>
>Changes are similar to PR for https://pulp.plan.io/issues/723
>---
> common/pulp_rpm/common/constants.py                |  2 +
> plugins/pulp_rpm/plugins/db/models.py              | 27 +++++++
> .../plugins/distributors/yum/metadata/filelists.py |  2 +-
> .../plugins/distributors/yum/metadata/metadata.py  | 51 -------------
> .../plugins/distributors/yum/metadata/other.py     |  2 +-
> .../plugins/distributors/yum/metadata/primary.py   |  2 +-
> .../plugins/importers/yum/repomd/metadata.py       |  6 +-
> plugins/pulp_rpm/plugins/importers/yum/upload.py   | 24 +++---
> .../plugins/migrations/0039_gzip_repodata.py       | 85 ++++++++++++++++++++++
> .../distributors/yum/metadata/test_filelists.py    | 17 +----
> .../distributors/yum/metadata/test_metadata.py     | 56 +-------------
> .../distributors/yum/metadata/test_other.py        | 15 +---
> .../distributors/yum/metadata/test_primary.py      | 15 +---
> .../test/unit/plugins/importers/yum/test_upload.py | 11 +--
> .../plugins/migrations/test_0039_gzip_repodata.py  | 54 ++++++++++++++
> 15 files changed, 208 insertions(+), 161 deletions(-)
> create mode 100644 plugins/pulp_rpm/plugins/migrations/0039_gzip_repodata.py
> create mode 100644 plugins/test/unit/plugins/migrations/test_0039_gzip_repodata.py
>
>diff --git a/common/pulp_rpm/common/constants.py b/common/pulp_rpm/common/constants.py
>index 75e0ec5..7f54ca1 100644
>--- a/common/pulp_rpm/common/constants.py
>+++ b/common/pulp_rpm/common/constants.py
>@@ -180,3 +180,5 @@
> # used in the scratchpad
> REPOMD_REVISION_KEY = 'repomd_revision'
> PREVIOUS_SKIP_LIST = 'previous_skip_list'
>+
>+MANDATORY_METADATA_TYPES = ('primary', 'filelists', 'other')
>diff --git a/plugins/pulp_rpm/plugins/db/models.py b/plugins/pulp_rpm/plugins/db/models.py
>index 6d2fe90..793fc86 100644
>--- a/plugins/pulp_rpm/plugins/db/models.py
>+++ b/plugins/pulp_rpm/plugins/db/models.py
>@@ -1,11 +1,13 @@
> from collections import namedtuple
> import csv
>+import gzip
> import logging
> import os
> from gettext import gettext as _
> from operator import itemgetter
> from urlparse import urljoin
> 
>+import bson
> import mongoengine
> from pulp.plugins.util import verification
> from pulp.server.db.model import ContentUnit, FileContentUnit
>@@ -567,6 +569,31 @@ def download_path(self):
>         """
>         return self.relativepath
> 
>+    def set_repodata(self, metadata_type, xml_snippet):
>+        """
>+        Compress metadata and put it into `repodata` attribute which will be saved to the db later.
>+
>+        :param metadata_type: key for the `repodata` dictionary which indicates type of metadata
>+        :type  metadata_type: str
>+        :param xml_snippet: utf-8 string which will be compressed and put into `repodata` dictionary
>+        :type  xml_snippet: str
>+        """
>+        if isinstance(xml_snippet, unicode):
>+            xml_snippet = xml_snippet.encode('utf-8')
>+        self.repodata[metadata_type] = bson.binary.Binary(gzip.zlib.compress(xml_snippet))
>+
>+    def get_repodata(self, metadata_type):
>+        """
>+        Get metadata from db and decompress it.
>+
>+        :param metadata_type: key for the `repodata` dictionary which indicates type of metadata
>+        :type  metadata_type: str
>+
>+        :return: requested xml snippet
>+        :rtype:  unicode
>+        """
>+        return gzip.zlib.decompress(self.repodata[metadata_type]).decode('utf-8')
>+
> 
> class RPM(RpmBase):
>     # TODO add docstring to this class
>diff --git a/plugins/pulp_rpm/plugins/distributors/yum/metadata/filelists.py b/plugins/pulp_rpm/plugins/distributors/yum/metadata/filelists.py
>index 03e8200..43e667d 100644
>--- a/plugins/pulp_rpm/plugins/distributors/yum/metadata/filelists.py
>+++ b/plugins/pulp_rpm/plugins/distributors/yum/metadata/filelists.py
>@@ -39,7 +39,7 @@ def add_unit_metadata(self, unit):
>         :param unit: unit whose metadata is to be written
>         :type  unit: pulp_rpm.plugins.db.models.RpmBase
>         """
>-        metadata = unit.repodata['filelists']
>+        metadata = unit.get_repodata('filelists')
>         if isinstance(metadata, unicode):
>             metadata = metadata.encode('utf-8')
>         self.metadata_file_handle.write(metadata)
>diff --git a/plugins/pulp_rpm/plugins/distributors/yum/metadata/metadata.py b/plugins/pulp_rpm/plugins/distributors/yum/metadata/metadata.py
>index 80e55c3..86fe181 100644
>--- a/plugins/pulp_rpm/plugins/distributors/yum/metadata/metadata.py
>+++ b/plugins/pulp_rpm/plugins/distributors/yum/metadata/metadata.py
>@@ -201,54 +201,3 @@ def _is_closed(file_object):
>                 return file_object.myfileobj is None or file_object.myfileobj.closed
>             else:
>                 raise
>-
>-
>-# -- pre-generated metadata context --------------------------------------------
>-
>-class PreGeneratedMetadataContext(MetadataFileContext):
>-    """
>-    Intermediate context manager for metadata files that have had their content
>-    pre-generated and stored on the unit model.
>-    """
>-
>-    def _add_unit_pre_generated_metadata(self, metadata_category, unit):
>-        """
>-        Write a unit's pre-generated metadata, from the given metadata category,
>-        into the metadata file.
>-
>-        :param metadata_category: metadata category to get pre-generated metadata for
>-        :type  metadata_category: str
>-        :param unit: unit whose metadata is being written
>-        :type  unit: pulp.plugins.model.Unit
>-        """
>-        _LOG.debug('Writing pre-generated %s metadata for unit: %s' %
>-                   (metadata_category, unit.unit_key.get('name', 'unknown')))
>-
>-        if 'repodata' not in unit.metadata or metadata_category not in unit.metadata['repodata']:
>-            msg = _('No pre-generated metadata found for unit [%(u)s], [%(c)s]')
>-            _LOG.error(msg % {'u': str(unit.unit_key), 'c': metadata_category})
>-
>-            return
>-
>-        metadata = unit.metadata['repodata'][metadata_category]
>-
>-        if not isinstance(metadata, basestring):
>-            msg = _('%(c)s metadata for [%(u)s] must be a string, but is a %(t)s')
>-            _LOG.error(
>-                msg % {'c': metadata_category.title(), 'u': unit.id, 't': str(type(metadata))})
>-
>-            return
>-
>-        # this should already be unicode if it came from the db
>-        # but, you know, testing...
>-        metadata = unicode(metadata)
>-        self.metadata_file_handle.write(metadata.encode('utf-8'))
>-
>-    def add_unit_metadata(self, unit):
>-        """
>-        Write the metadata for a given unit to the file handle.
>-
>-        :param unit: unit whose metadata is being written
>-        :type  unit: pulp.plugins.model.Unit
>-        """
>-        raise NotImplementedError()
>diff --git a/plugins/pulp_rpm/plugins/distributors/yum/metadata/other.py b/plugins/pulp_rpm/plugins/distributors/yum/metadata/other.py
>index 3652041..36bee45 100644
>--- a/plugins/pulp_rpm/plugins/distributors/yum/metadata/other.py
>+++ b/plugins/pulp_rpm/plugins/distributors/yum/metadata/other.py
>@@ -39,7 +39,7 @@ def add_unit_metadata(self, unit):
>         :param unit: unit whose metadata is to be written
>         :type  unit: pulp_rpm.plugins.db.models.RpmBase
>         """
>-        metadata = unit.repodata['other']
>+        metadata = unit.get_repodata('other')
>         if isinstance(metadata, unicode):
>             metadata = metadata.encode('utf-8')
>         self.metadata_file_handle.write(metadata)
>diff --git a/plugins/pulp_rpm/plugins/distributors/yum/metadata/primary.py b/plugins/pulp_rpm/plugins/distributors/yum/metadata/primary.py
>index 94343c4..b2b448c 100644
>--- a/plugins/pulp_rpm/plugins/distributors/yum/metadata/primary.py
>+++ b/plugins/pulp_rpm/plugins/distributors/yum/metadata/primary.py
>@@ -51,7 +51,7 @@ def add_unit_metadata(self, unit):
>         :param unit: unit whose metadata is to be written
>         :type  unit: pulp_rpm.plugins.db.models.RpmBase
>         """
>-        metadata = unit.repodata['primary']
>+        metadata = unit.get_repodata('primary')
>         if isinstance(metadata, unicode):
>             metadata = metadata.encode('utf-8')
>         self.metadata_file_handle.write(metadata)
>diff --git a/plugins/pulp_rpm/plugins/importers/yum/repomd/metadata.py b/plugins/pulp_rpm/plugins/importers/yum/repomd/metadata.py
>index 22e8c97..4bc462a 100644
>--- a/plugins/pulp_rpm/plugins/importers/yum/repomd/metadata.py
>+++ b/plugins/pulp_rpm/plugins/importers/yum/repomd/metadata.py
>@@ -338,13 +338,13 @@ def add_repodata(self, model):
>                 raw_xml = db_file[db_key]
>             finally:
>                 db_file.close()
>-            model.repodata[filename] = raw_xml
>+            model.set_repodata(filename, raw_xml)
>             element = ElementTree.fromstring(raw_xml)
>             unit_key, items = process_func(element)
>             setattr(model, metadata_key, items)
> 
>-        raw_xml = model.raw_xml
>-        model.repodata['primary'] = change_location_tag(raw_xml, model.filename)
>+        primary_xml_snippet = change_location_tag(model.raw_xml, model.filename)
>+        model.set_repodata('primary', primary_xml_snippet)
> 
> 
> def process_repomd_data_element(data_element):
>diff --git a/plugins/pulp_rpm/plugins/importers/yum/upload.py b/plugins/pulp_rpm/plugins/importers/yum/upload.py
>index b609ce4..c81e423 100644
>--- a/plugins/pulp_rpm/plugins/importers/yum/upload.py
>+++ b/plugins/pulp_rpm/plugins/importers/yum/upload.py
>@@ -403,9 +403,11 @@ def _handle_package(repo, type_id, unit_key, metadata, file_path, conduit, confi
>         raise ModelInstantiationError()
> 
>     # Extract/adjust the repodata snippets
>-    unit.repodata = rpm_parse.get_package_xml(file_path, sumtype=unit.checksumtype)
>-    _update_provides_requires(unit)
>-    _update_location(unit)
>+    repodata = rpm_parse.get_package_xml(file_path, sumtype=unit.checksumtype)
>+    _update_provides_requires(unit, repodata)
>+    _update_location(unit, repodata)
>+    for metadata_key in repodata:
>+        unit.set_repodata(metadata_key, repodata[metadata_key])
> 
>     # check if the unit has duplicate nevra
>     purge.remove_unit_duplicate_nevra(unit, repo)
>@@ -442,7 +444,7 @@ def _fake_xml_element(repodata_snippet):
>     return ET.fromstring(fake_xml.encode(codec))
> 
> 
>-def _update_provides_requires(unit):
>+def _update_provides_requires(unit, repodata):
>     """
>     Determines the provides and requires fields based on the RPM's XML snippet and updates
>     the model instance.
>@@ -450,8 +452,10 @@ def _update_provides_requires(unit):
>     :param unit: the unit being added to Pulp; the metadata attribute must already have
>                  a key called 'repodata'
>     :type  unit: subclass of pulp.server.db.model.ContentUnit
>+    :param repodata: xml snippets to analyze
>+    :type  repodata: dict
>     """
>-    fake_element = _fake_xml_element(unit.repodata['primary'])
>+    fake_element = _fake_xml_element(repodata['primary'])
>     utils.strip_ns(fake_element)
>     primary_element = fake_element.find('package')
>     format_element = primary_element.find('format')
>@@ -463,7 +467,7 @@ def _update_provides_requires(unit):
>                         requires_element.findall('entry')) if requires_element else []
> 
> 
>-def _update_location(unit):
>+def _update_location(unit, repodata):
>     """
>     Fix a unit's repodata primary xml
> 
>@@ -476,20 +480,22 @@ def _update_location(unit):
>     :param unit: the unit being added to Pulp; the metadata attribute must already have
>                  a key called 'repodata'
>     :type  unit: subclass of pulp.server.db.model.ContentUnit
>+    :param repodata: xml snippets to analyze
>+    :type  repodata: dict
>     """
>-    fake_element = _fake_xml_element(unit.repodata['primary'])
>+    fake_element = _fake_xml_element(repodata['primary'])
>     primary_element = fake_element.find('package')
>     location_element = primary_element.find('location')
>     location_element.set('href', unit.filename)
>     new_location = ET.tostring(location_element).strip()
>-    lines = list(unit.repodata['primary'].splitlines())
>+    lines = list(repodata['primary'].splitlines())
>     # rather than deal with the xml namespaces, just
>     # replace the old location with the new one in-place
>     for i, line in enumerate(lines):
>         index = line.find('<location')
>         if index != -1:
>             lines[i] = line[:index] + new_location
>-    unit.repodata['primary'] = '\n'.join(lines)
>+    repodata['primary'] = '\n'.join(lines)
> 
> 
> def _extract_rpm_data(type_id, rpm_filename):
>diff --git a/plugins/pulp_rpm/plugins/migrations/0039_gzip_repodata.py b/plugins/pulp_rpm/plugins/migrations/0039_gzip_repodata.py
>new file mode 100644
>index 0000000..7d51c0a
>--- /dev/null
>+++ b/plugins/pulp_rpm/plugins/migrations/0039_gzip_repodata.py
>@@ -0,0 +1,85 @@
>+import gzip
>+import logging
>+
>+import bson
>+
>+from pulp.server.db import connection
>+
>+
>+_logger = logging.getLogger(__name__)
>+
>+
>+def migrate(*args, **kwargs):
>+    """
>+    Compress the content of repodata field for RPM and SRPM units.
>+    Log the progress after every 10% of the each collection is migrated.
>+
>+    :param args:   unused
>+    :type  args:   list
>+    :param kwargs: unused
>+    :type  kwargs: dict
>+    """
>+    db = connection.get_database()
>+    rpm_collection = db['units_rpm']
>+    srpm_collection = db['units_srpm']
>+
>+    total_rpm_units = rpm_collection.count()
>+    total_srpm_units = srpm_collection.count()
>+
>+    msg = '* NOTE: This migration may take some time depending on the size of your Pulp content. *'
>+    stars = '*' * len(msg)
>+    progress_msg = '* Migrated units: %s of %s'
>+
>+    _logger.info(stars)
>+    _logger.info(msg)
>+    _logger.info(stars)
>+
>+    if total_rpm_units:
>+        _logger.info('* Migrating RPM content...')
>+
>+    migrated_units = 0
>+    for rpm in rpm_collection.find({}, ['repodata']).batch_size(100):
>+        migrate_rpm_base(rpm_collection, rpm)
>+
>+        migrated_units += 1
>+        another_ten_percent_completed = total_rpm_units >= 10 and \
>+            not migrated_units % (total_rpm_units // 10)
>+        all_units_migrated = migrated_units == total_rpm_units
>+        if another_ten_percent_completed or all_units_migrated:
>+            _logger.info(progress_msg % (migrated_units, total_rpm_units))
>+
>+    if total_srpm_units:
>+        _logger.info('* Migrating SRPM content...')
>+
>+    migrated_units = 0
>+    for srpm in srpm_collection.find({}, ['repodata']).batch_size(100):
>+        migrate_rpm_base(srpm_collection, srpm)
>+
>+        migrated_units += 1
>+        another_ten_percent_completed = total_srpm_units >= 10 and \
>+            not migrated_units % (total_srpm_units // 10)
>+        all_units_migrated = migrated_units == total_srpm_units
>+        if another_ten_percent_completed or all_units_migrated:
>+            _logger.info(progress_msg % (migrated_units, total_srpm_units))
>+
>+    _logger.info(stars)
>+
>+
>+def migrate_rpm_base(collection, unit):
>+    """
>+    Compress 'primary', 'other' and 'filelists' metadata for a given unit.
>+
>+    :param collection:  collection of RPM units
>+    :type  collection:  pymongo.collection.Collection
>+    :param unit:        the RPM unit being migrated
>+    :type  unit:        dict
>+    """
>+
>+    delta = {'repodata': {}}
>+    for metadata_type in ['primary', 'other', 'filelists']:
>+        metadata = unit['repodata'].get(metadata_type)
>+        if metadata and not isinstance(metadata, bson.binary.Binary):
>+            compressed_data = gzip.zlib.compress(metadata.encode('utf-8'))
>+            delta['repodata'][metadata_type] = bson.binary.Binary(compressed_data)
>+    if delta['repodata']:
>+        collection.update_one({'_id': unit['_id']}, {'$set': delta})
>diff --git a/plugins/test/unit/plugins/distributors/yum/metadata/test_filelists.py b/plugins/test/unit/plugins/distributors/yum/metadata/test_filelists.py
>index 760d846..ca05327 100644
>--- a/plugins/test/unit/plugins/distributors/yum/metadata/test_filelists.py
>+++ b/plugins/test/unit/plugins/distributors/yum/metadata/test_filelists.py
>@@ -4,6 +4,7 @@
> 
> import mock
> 
>+from pulp_rpm.plugins.db import models
> from pulp_rpm.plugins.distributors.yum.metadata.filelists import FilelistsXMLFileContext
> 
> 
>@@ -22,19 +23,9 @@ def test_init(self):
>     def test_add_unit_metadata(self):
>         context = FilelistsXMLFileContext(self.working_dir, 3)
>         context.metadata_file_handle = mock.Mock()
>+        unit = models.RPM()
>+        unit.set_repodata('filelists', 'bar')
> 
>-        context.add_unit_metadata(mock.Mock(repodata={'filelists': 'bar'}))
>+        context.add_unit_metadata(unit)
> 
>         context.metadata_file_handle.write.assert_called_once_with('bar')
>-
>-    def test_add_unit_metadata_unicode(self):
>-        """
>-        Test that the filelists repodata is passed as a str even if it's a unicode object.
>-        """
>-        context = FilelistsXMLFileContext(self.working_dir, 3)
>-        context.metadata_file_handle = mock.Mock()
>-        expected_call = 'some unicode'
>-        repodata = {'filelists': unicode(expected_call)}
>-
>-        context.add_unit_metadata(mock.Mock(repodata=repodata))
>-        context.metadata_file_handle.write.assert_called_once_with(expected_call)
>diff --git a/plugins/test/unit/plugins/distributors/yum/metadata/test_metadata.py b/plugins/test/unit/plugins/distributors/yum/metadata/test_metadata.py
>index 0d9d27b..f5bdd72 100644
>--- a/plugins/test/unit/plugins/distributors/yum/metadata/test_metadata.py
>+++ b/plugins/test/unit/plugins/distributors/yum/metadata/test_metadata.py
>@@ -12,7 +12,7 @@
> from pulp_rpm.common.ids import TYPE_ID_RPM
> from pulp_rpm.devel.skip import skip_broken
> from pulp_rpm.plugins.distributors.yum.metadata.metadata import (
>-    MetadataFileContext, PreGeneratedMetadataContext, REPO_DATA_DIR_NAME)
>+    MetadataFileContext, REPO_DATA_DIR_NAME)
> from pulp_rpm.plugins.distributors.yum.metadata.prestodelta import (
>     PrestodeltaXMLFileContext, PRESTO_DELTA_FILE_NAME)
> from pulp_rpm.plugins.distributors.yum.metadata.repomd import (
>@@ -261,60 +261,6 @@ def test_finalize_for_repomd_file_with_valid_checksum_type(self):
> 
>         self.assertEqual(context.metadata_file_path, path)
> 
>-    # -- pre-generated metadata context tests ----------------------------------
>-
>-    def test_pre_generated_metadata(self):
>-
>-        path = os.path.join(self.metadata_file_dir, 'pre-gen.xml')
>-        context = PreGeneratedMetadataContext(path)
>-        unit = self._generate_rpm('test_rpm')
>-
>-        context._open_metadata_file_handle()
>-        context._add_unit_pre_generated_metadata('primary', unit)
>-        context._close_metadata_file_handle()
>-
>-        self.assertEqual(os.path.getsize(path), len('PRIMARY'))
>-
>-    def test_pre_generated_metadata_no_repodata(self):
>-
>-        path = os.path.join(self.metadata_file_dir, 'no-repodata.xml')
>-        context = PreGeneratedMetadataContext(path)
>-        unit = self._generate_rpm('no_repodata')
>-
>-        unit.metadata.pop('repodata')
>-
>-        context._open_metadata_file_handle()
>-        context._add_unit_pre_generated_metadata('primary', unit)
>-        context._close_metadata_file_handle()
>-
>-        self.assertEqual(os.path.getsize(path), 0)
>-
>-    def test_pre_generated_metadata_wrong_category(self):
>-
>-        path = os.path.join(self.metadata_file_dir, 'wrong-category.xml')
>-        context = PreGeneratedMetadataContext(path)
>-        unit = self._generate_rpm('wrong_category')
>-
>-        context._open_metadata_file_handle()
>-        context._add_unit_pre_generated_metadata('not_found', unit)
>-        context._close_metadata_file_handle()
>-
>-        self.assertEqual(os.path.getsize(path), 0)
>-
>-    def test_pre_generated_metadata_not_string(self):
>-
>-        path = os.path.join(self.metadata_file_dir, 'not-string.xml')
>-        context = PreGeneratedMetadataContext(path)
>-        unit = self._generate_rpm('not_string')
>-
>-        unit.metadata['repodata']['whatisthis'] = 1
>-
>-        context._open_metadata_file_handle()
>-        context._add_unit_pre_generated_metadata('whatisthis', unit)
>-        context._close_metadata_file_handle()
>-
>-        self.assertEqual(os.path.getsize(path), 0)
>-
>     # -- updateinfo.xml testing ------------------------------------------------
> 
>     def test_updateinfo_file_creation(self):
>diff --git a/plugins/test/unit/plugins/distributors/yum/metadata/test_other.py b/plugins/test/unit/plugins/distributors/yum/metadata/test_other.py
>index ca5dad2..8a9c33b 100644
>--- a/plugins/test/unit/plugins/distributors/yum/metadata/test_other.py
>+++ b/plugins/test/unit/plugins/distributors/yum/metadata/test_other.py
>@@ -4,6 +4,7 @@
> 
> import mock
> 
>+from pulp_rpm.plugins.db import models
> from pulp_rpm.plugins.distributors.yum.metadata.other import OtherXMLFileContext
> 
> 
>@@ -20,16 +21,8 @@ def test_init(self):
>         self.assertEquals(self.context.num_packages, 3)
> 
>     def test_add_unit_metadata(self):
>+        unit = models.RPM()
>+        unit.set_repodata('other', 'bar')
>         self.context.metadata_file_handle = mock.Mock()
>-        self.context.add_unit_metadata(mock.Mock(repodata={'other': 'bar'}))
>+        self.context.add_unit_metadata(unit)
>         self.context.metadata_file_handle.write.assert_called_once_with('bar')
>-
>-    def test_add_unit_metadata_unicode(self):
>-        """
>-        Test that the other repodata is passed as a str even if it's a unicode object.
>-        """
>-        self.context.metadata_file_handle = mock.Mock()
>-        expected_call = 'some unicode'
>-        repodata = {'other': unicode(expected_call)}
>-        self.context.add_unit_metadata(mock.Mock(repodata=repodata))
>-        self.context.metadata_file_handle.write.assert_called_once_with(expected_call)
>diff --git a/plugins/test/unit/plugins/distributors/yum/metadata/test_primary.py b/plugins/test/unit/plugins/distributors/yum/metadata/test_primary.py
>index e4eb27b..83de5d2 100644
>--- a/plugins/test/unit/plugins/distributors/yum/metadata/test_primary.py
>+++ b/plugins/test/unit/plugins/distributors/yum/metadata/test_primary.py
>@@ -4,6 +4,7 @@
> 
> import mock
> 
>+from pulp_rpm.plugins.db import models
> from pulp_rpm.plugins.distributors.yum.metadata.primary import PrimaryXMLFileContext
> 
> 
>@@ -21,15 +22,7 @@ def test_init(self):
> 
>     def test_add_unit_metadata(self):
>         self.context.metadata_file_handle = mock.Mock()
>-        self.context.add_unit_metadata(mock.Mock(repodata={'primary': 'bar'}))
>+        unit = models.RPM()
>+        unit.set_repodata('primary', 'bar')
>+        self.context.add_unit_metadata(unit)
>         self.context.metadata_file_handle.write.assert_called_once_with('bar')
>-
>-    def test_add_unit_metadata_unicode(self):
>-        """
>-        Test that the primary repodata is passed as a str even if it's a unicode object.
>-        """
>-        self.context.metadata_file_handle = mock.Mock()
>-        expected_call = 'some unicode'
>-        repodata = {'primary': unicode(expected_call)}
>-        self.context.add_unit_metadata(mock.Mock(repodata=repodata))
>-        self.context.metadata_file_handle.write.assert_called_once_with(expected_call)
>diff --git a/plugins/test/unit/plugins/importers/yum/test_upload.py b/plugins/test/unit/plugins/importers/yum/test_upload.py
>index 613a562..89da21b 100644
>--- a/plugins/test/unit/plugins/importers/yum/test_upload.py
>+++ b/plugins/test/unit/plugins/importers/yum/test_upload.py
>@@ -671,15 +671,16 @@ class TestMangleRepodataPrimaryXML(unittest.TestCase):
> 
>     def setUp(self):
>         self.unit = models.RPM()
>-        self.unit.repodata['primary'] = self.PRIMARY_XML
>+        self.repodata = {'primary': self.PRIMARY_XML}
>         self.unit.filename = 'fixed-filename.rpm'
> 
>     def test_update_provides_requires(self):
>-        upload._update_provides_requires(self.unit)
>+        upload._update_provides_requires(self.unit, self.repodata)
>         self.assertEqual(len(self.unit.provides), 1)
>         self.assertEqual(len(self.unit.requires), 2)
> 
>     def test_update_location(self):
>-        upload._update_location(self.unit)
>-        self.assertTrue('fixme' not in self.unit.repodata['primary'])
>-        self.assertTrue(self.unit.filename in self.unit.repodata['primary'])
>+        upload._update_location(self.unit, self.repodata)
>+        self.unit.set_repodata('primary', self.repodata['primary'])
>+        self.assertTrue('fixme' not in self.unit.get_repodata('primary'))
>+        self.assertTrue(self.unit.filename in self.unit.get_repodata('primary'))
>diff --git a/plugins/test/unit/plugins/migrations/test_0039_gzip_repodata.py b/plugins/test/unit/plugins/migrations/test_0039_gzip_repodata.py
>new file mode 100644
>index 0000000..7f31bd1
>--- /dev/null
>+++ b/plugins/test/unit/plugins/migrations/test_0039_gzip_repodata.py
>@@ -0,0 +1,54 @@
>+import copy
>+import gzip
>+
>+import bson
>+import mock
>+
>+from pulp.common.compat import unittest
>+from pulp.server.db.migrate.models import _import_all_the_way
>+
>+
>+PATH_TO_MODULE = 'pulp_rpm.plugins.migrations.0039_gzip_repodata'
>+migration = _import_all_the_way(PATH_TO_MODULE)
>+
>+
>+@mock.patch('pulp.server.db.connection.get_database')
>+@mock.patch.object(migration, 'migrate_rpm_base')
>+class TestMigrate(unittest.TestCase):
>+    def test_calls_migrate_rpm_base(self, mock_migrate_rpm_base, mock_get_db):
>+        mock_db = mock_get_db.return_value
>+        mock_unit = mock.MagicMock()
>+        mock_rpm_collection = mock_db['units_rpm']
>+        mock_rpm_collection.find.return_value.batch_size.return_value = [mock_unit]
>+        mock_srpm_collection = mock_db['units_srpm']
>+        mock_srpm_collection.find.return_value.batch_size.return_value = [mock_unit]
>+
>+        migration.migrate()
>+
>+        self.assertEqual(mock_migrate_rpm_base.call_count, 2)
>+        mock_migrate_rpm_base.assert_called_with(mock_rpm_collection, mock_unit)
>+        mock_migrate_rpm_base.assert_called_with(mock_srpm_collection, mock_unit)
>+
>+
>+class TestMigrateRPMBase(unittest.TestCase):
>+    def setUp(self):
>+        super(TestMigrateRPMBase, self).setUp()
>+        self.repodata = {'primary': u'\u041f\u0440\u0438\u0432\u0435\u0442! stuff',
>+                         'filelists': u'\u041f\u0440\u0438\u0432\u0435\u0442! stuff',
>+                         'other': u'\u041f\u0440\u0438\u0432\u0435\u0442! stuff'}
>+        self.rpm = {
>+            '_id': '1234',
>+            'repodata': copy.copy(self.repodata)
>+        }
>+
>+    def test_calls_update(self):
>+        mock_collection = mock.MagicMock()
>+
>+        migration.migrate_rpm_base(mock_collection, self.rpm)
>+
>+        expected_delta = {
>+            'repodata': {mtype: bson.binary.Binary(gzip.zlib.compress(metadata.encode('utf-8')))
>+                         for mtype, metadata in self.rpm['repodata'].items()}
>+        }
>+        mock_collection.update_one.assert_called_once_with({'_id': '1234'},
>+                                                           {'$set': expected_delta})

Actions: View | Diff

Attachments on bug 1404345: 1258173 | 1258189 | 1262699