Description of problem: Using Centos7.1 python-glanceclient-0.17.0-2.el7.noarch.rpm distributed in RDO Kilo both: https://repos.fedorapeople.org/repos/openstack/openstack-kilo/el7/python-glanceclient-0.17.0-2.el7.noarch.rpm https://repos.fedorapeople.org/repos/openstack/openstack-kilo/testing/el7/python-glanceclient-0.17.0-2.el7.noarch.rpm Below is a diff of glanceclient/common/https.py first is the one in Ubuntu14.04 OS Kilo stable release, the last is the one from the el7 repos above notice right away the missing import poolmanager in the beggining I was getting exceptions when instantiating (nova create) interacting with glance client API diff /usr/lib/python2.7/dist-packages/glanceclient/common/https.py https.py 16a17 > import ssl 20a22 > from requests import compat 23d24 < from requests.packages.urllib3 import poolmanager 26d26 < from urllib3 import poolmanager 27a28 > from oslo_utils import encodeutils 29c30,31 < import ssl --- > # NOTE(jokke): simplified transition to py3, behaves like py2 xrange > from six.moves import range 53c55,130 < from glanceclient.openstack.common import strutils --- > > > def verify_callback(host=None): > """ > We use a partial around the 'real' verify_callback function > so that we can stash the host value without holding a > reference on the VerifiedHTTPSConnection. > """ > def wrapper(connection, x509, errnum, > depth, preverify_ok, host=host): > return do_verify_callback(connection, x509, errnum, > depth, preverify_ok, host=host) > return wrapper > > > def do_verify_callback(connection, x509, errnum, > depth, preverify_ok, host=None): > """ > Verify the server's SSL certificate. > > This is a standalone function rather than a method to avoid > issues around closing sockets if a reference is held on > a VerifiedHTTPSConnection by the callback function. > """ > if x509.has_expired(): > msg = "SSL Certificate expired on '%s'" % x509.get_notAfter() > raise exc.SSLCertificateError(msg) > > if depth == 0 and preverify_ok: > # We verify that the host matches against the last > # certificate in the chain > return host_matches_cert(host, x509) > else: > # Pass through OpenSSL's default result > return preverify_ok > > > def host_matches_cert(host, x509): > """ > Verify that the x509 certificate we have received > from 'host' correctly identifies the server we are > connecting to, ie that the certificate's Common Name > or a Subject Alternative Name matches 'host'. > """ > def check_match(name): > # Directly match the name > if name == host: > return True > > # Support single wildcard matching > if name.startswith('*.') and host.find('.') > 0: > if name[2:] == host.split('.', 1)[1]: > return True > > common_name = x509.get_subject().commonName > > # First see if we can match the CN > if check_match(common_name): > return True > # Also try Subject Alternative Names for a match > san_list = None > for i in range(x509.get_extension_count()): > ext = x509.get_extension(i) > if ext.get_short_name() == b'subjectAltName': > san_list = str(ext) > for san in ''.join(san_list.split()).split(','): > if san.startswith('DNS:'): > if check_match(san.split(':', 1)[1]): > return True > > # Server certificate does not match host > msg = ('Host "%s" does not match x509 certificate contents: ' > 'CommonName "%s"' % (host, common_name)) > if san_list is not None: > msg = msg + ', subjectAltName "%s"' % san_list > raise exc.SSLCertificateError(msg) 73,79d149 < def __init__(self, *args, **kwargs): < # NOTE(flaper87): This line forces poolmanager to use < # glanceclient HTTPSConnection < classes_by_scheme = poolmanager.pool_classes_by_scheme < classes_by_scheme["glance+https"] = HTTPSConnectionPool < super(HTTPSAdapter, self).__init__(*args, **kwargs) < 84c154,179 < return strutils.safe_encode(url) --- > return encodeutils.safe_encode(url) > > def _create_glance_httpsconnectionpool(self, url): > kw = self.poolmanager.connection_kw > # Parse the url to get the scheme, host, and port > parsed = compat.urlparse(url) > # If there is no port specified, we should use the standard HTTPS port > port = parsed.port or 443 > pool = HTTPSConnectionPool(parsed.host, port, **kw) > > with self.poolmanager.pools.lock: > self.poolmanager.pools[(parsed.scheme, parsed.host, port)] = pool > > return pool > > def get_connection(self, url, proxies=None): > try: > return super(HTTPSAdapter, self).get_connection(url, proxies) > except KeyError: > # NOTE(sigamvirus24): This works around modifying a module global > # which fixes bug #1396550 > # The scheme is most likely glance+https but check anyway > if not url.startswith('glance+https://'): > raise > > return self._create_glance_httpsconnectionpool(url) 143a239,243 > # Restrict the set of client supported cipher suites > CIPHERS = 'ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:'\ > 'eCDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:'\ > 'RSA+AES:RSA+3DES:!aNULL:!MD5:!DSS' > 178,232d277 < @staticmethod < def host_matches_cert(host, x509): < """ < Verify that the x509 certificate we have received < from 'host' correctly identifies the server we are < connecting to, ie that the certificate's Common Name < or a Subject Alternative Name matches 'host'. < """ < def check_match(name): < # Directly match the name < if name == host: < return True < < # Support single wildcard matching < if name.startswith('*.') and host.find('.') > 0: < if name[2:] == host.split('.', 1)[1]: < return True < < common_name = x509.get_subject().commonName < < # First see if we can match the CN < if check_match(common_name): < return True < # Also try Subject Alternative Names for a match < san_list = None < for i in range(x509.get_extension_count()): < ext = x509.get_extension(i) < if ext.get_short_name() == b'subjectAltName': < san_list = str(ext) < for san in ''.join(san_list.split()).split(','): < if san.startswith('DNS:'): < if check_match(san.split(':', 1)[1]): < return True < < # Server certificate does not match host < msg = ('Host "%s" does not match x509 certificate contents: ' < 'CommonName "%s"' % (host, common_name)) < if san_list is not None: < msg = msg + ', subjectAltName "%s"' % san_list < raise exc.SSLCertificateError(msg) < < def verify_callback(self, connection, x509, errnum, < depth, preverify_ok): < if x509.has_expired(): < msg = "SSL Certificate expired on '%s'" % x509.get_notAfter() < raise exc.SSLCertificateError(msg) < < if depth == 0 and preverify_ok: < # We verify that the host matches against the last < # certificate in the chain < return self.host_matches_cert(self.host, x509) < else: < # Pass through OpenSSL's default result < return preverify_ok < 237a283 > self.context.set_cipher_list(self.CIPHERS) 244c290 < self.verify_callback) --- > verify_callback(host=self.host)) Version-Release number of selected component (if applicable): 0.17.0-2.el7 How reproducible: Always Steps to Reproduce: 1. 2. 3. Actual results: In the nova log 2015-07-14 17:18:52.908 16054 TRACE nova.api.openstack File "/usr/lib/python2.7/site-packages/glanceclient/common/https.py", line 179, in get_connection 2015-07-14 17:18:52.908 16054 TRACE nova.api.openstack return self._create_glance_httpsconnectionpool(url) 2015-07-14 17:18:52.908 16054 TRACE nova.api.openstack File "/usr/lib/python2.7/site-packages/glanceclient/common/https.py", line 157, in _create_glance_httpsconnectionpool 2015-07-14 17:18:52.908 16054 TRACE nova.api.openstack kw = self.poolmanager.connection_kw 2015-07-14 17:18:52.908 16054 TRACE nova.api.openstack AttributeError: 'PoolManager' object has no attribute 'connection_kw' Expected results: Additional info:
> notice right away the missing import poolmanager in the beggining No, it's not missing, looks like some code was not updated accordingly. Urllib3's PoolManager has no attribute connection_kw but connection_pool_kw, so it was a legitimate error. Look at your diff, and you'll see that attribute was never used directly previously. I already submitted a fix for liberty and kilo branches. https://review.openstack.org/#/c/204521/ https://review.openstack.org/#/c/204523/ If you could apply that patch and comment these reviews, I would be grateful :)
hi I will apply the patch and will give a review/comment Mario
hi the patch is not enough to solve the problem After applying the patch 'substitute connection_kw by connection_pool_kw' 2015-07-22 15:21:17.234 6051 TRACE nova.api.openstack File "/usr/lib/python2.7/site-packages/glanceclient/common/https.py", line 162, in _create_glance_httpsconnectionpool 2015-07-22 15:21:17.234 6051 TRACE nova.api.openstack pool = HTTPSConnectionPool(parsed.host, port, **kw) 2015-07-22 15:21:17.234 6051 TRACE nova.api.openstack AttributeError: 'ParseResult' object has no attribute 'host' I got more wrong named attrs, so I did the following, modified _create_glance_httpsconnectionpool # modified mariojmdavid self.host = parsed.netloc.split(':')[0] self.port = parsed.netloc.split(':')[1] or 443 self.scheme = parsed.scheme pool = HTTPSConnectionPool(self.host, self.port, **kw) with self.poolmanager.pools.lock: self.poolmanager.pools[(self.scheme, self.host, self.port)] = pool After I got this exception 2015-07-22 16:35:30.125 6053 TRACE nova.api.openstack File "/usr/lib/python2.7/site-packages/nova/image/glance.py", line 237, in call 2015-07-22 16:35:30.125 6053 TRACE nova.api.openstack host=host, port=port, reason=six.text_type(e)) 2015-07-22 16:35:30.125 6053 TRACE nova.api.openstack GlanceConnectionFailed: Connection to glance host nimbus.ncg.ingrid.pt:9292 failed: Error finding address for glance+https://nimbus.ncg.ingrid.pt:9292/v1/images/CentOS-7-x86_64-GenericCloud: ('Connection aborted.', error('getaddrinfo() argument 2 must be integer or string',)) 2015-07-22 16:35:30.125 6053 TRACE nova.api.openstack The connection is done with the scheme = 'glance+https' and it should be with https I don't have knowledge about how this scheme would work on an actual connection I saw in another bug report that this scheme was added in commit 695dbc910d2068aff89a68d6f5aa088cc835766e Still I modify https.py and http.py and taken out glance+ from the scheme def get_connection(self, url, proxies=None): try: return super(HTTPSAdapter, self).get_connection(url, proxies) except KeyError: # NOTE(sigamvirus24): This works around modifying a module global # which fixes bug #1396550 # The scheme is most likely glance+https but check anyway # modified by mariojmdavid just https if not url.startswith('https://'): class HTTPSConnectionPool(connectionpool.HTTPSConnectionPool): scheme = 'https' #mariojmdavid In http.py class HTTPClient(object): ... if not compression: self.session.mount("https://", https.HTTPSAdapter()) 2015-07-22 17:44:42.378 16711 TRACE nova.api.openstack File "/usr/lib/python2.7/site-packages/requests/sessions.py", line 641, in get_adapter 2015-07-22 17:44:42.378 16711 TRACE nova.api.openstack raise InvalidSchema("No connection adapters were found for '%s'" % url) 2015-07-22 17:44:42.378 16711 TRACE nova.api.openstack InvalidSchema: No connection adapters were found for 'glance+https://nimbus.ncg.ingrid.pt:9292/v1/images/CentOS-7-x86_64-GenericCloud' finally removed from http.py the line self.endpoint = 'glance+' + self.endpoint and got an 2015-07-22 18:25:47.698 20691 TRACE nova.image.glance File "/usr/lib/python2.7/site-packages/glanceclient/common/http.py", line 218, in _request 2015-07-22 18:25:47.698 20691 TRACE nova.image.glance raise exc.CommunicationError(message=message) 2015-07-22 18:25:47.698 20691 TRACE nova.image.glance CommunicationError: Error finding address for https://nimbus.ncg.ingrid.pt:9292/v1/images/CentOS-7-x86_64-GenericCloud: [Errno 1] _ssl.c:504: error:14090086:SSL routines:SSL3_GET_SERVER_CERTIFICATE:certificate verify failed 2015-07-22 18:25:47.698 20691 TRACE nova.image.glance 2015-07-22 18:25:47.699 20691 ERROR nova.api.openstack [req-9dbaf88f-22ac-45b0-ba2a-8aafc219907c 2ed32b53df014cb6a9b444e4f38257a0 21f61c3b2ce04ff3aae52bc5428d0d22 - - -] Caught error: Connection to glance host nimbus.ncg.ingrid.pt:9292 failed: Error finding address for https://nimbus.ncg.ingrid.pt:9292/v1/images/CentOS-7-x86_64-GenericCloud: [Errno 1] _ssl.c:504: error:14090086:SSL routines:SSL3_GET_SERVER_CERTIFICATE:certificate verify failed -------------------------------------- Reaching here, I would have to ask first about ssl_compression or no_ssl_compression I don't know where it is set, if its some conf file, and ssl_compression related to which service I have glance over swift, so the images are in swift the images are qcow2, so I understand that there is no need for ssl_compression in Although I have at the moment in glance-api.conf swift_store_ssl_compression = True is this the what you are referring class HTTPSConnectionPool(connectionpool.HTTPSConnectionPool): """ HTTPSConnectionPool will be instantiated when a new connection is requested to the HTTPSAdapter.This implementation overwrites the _new_conn method and returns an instances of glanceclient's VerifiedHTTPSConnection which handles no compression. ssl_compression is hard-coded to False because this will be used just when the user sets --no-ssl-compression. """ ---------------------------------------- now a final thing about my setup I have a virtual IP, 2 haproxys, 3 controller nodes expose the dashboard and all OS apis under that single VIP that is nimbus.ncg.ingrid.pt all goes to https, and I have a x.509 certificate for nimbus.ncg.ingrid.pt at the haproxies all https to nimbus transforms into http for the underlying control nodes tell me if you need more info best Mario
error/mistake on my part I have reverted 'https' again to 'glance+https' AND had a mistake in https.py self.host = parsed.netloc.split(':')[0] should be in fact self.host = parsed.netloc.rsplit(':', 1)[0] After this change, I now get to the next exception which is the latest reported in my previous comment: ERROR nova.api.openstack [req-46d37997-6cc6-4449-b2bb-438972d0c624 2ed32b53df014cb6a9b444e4f38257a0 21f61c3b2ce04ff3aae52bc5428d0d22 - - -] Caught error: [('SSL routines', 'SSL3_GET_SERVER_CERTIFICATE', 'certificate verify failed')]
the ssl cert error has been solved by including our "private" ca in the trusted bundle I can confirm what I wrote in the review comment to the patches, the following patch is needed /root/https.py is the patched one # diff https.py /root/https.py 157c157 < kw = self.poolmanager.connection_kw --- > kw = self.poolmanager.connection_pool_kw 162c162,163 < pool = HTTPSConnectionPool(parsed.host, port, **kw) --- > self.host = parsed.netloc.rsplit(':', 1)[0] > pool = HTTPSConnectionPool(self.host, port, **kw) 165c166 < self.poolmanager.pools[(parsed.scheme, parsed.host, port)] = pool --- > self.poolmanager.pools[(parsed.scheme, self.host, port)] = pool
Thanks for your feedback, it's appreciated :)
I think this is solved alredy in python-glanceclient 0.17.2. I tested this by simply replace https.py from python-glanceclient 0.17.2.
Hello, I have the same issue in a very similar context (RDO kilo on CentOS with reverse https proxy). I tried python-glanceclient-0.17.2-1.fc23 which did not solve the problem. Upgrading to cbs.centos.org python-glanceclient-1.1.0-1.el7 did. Is there a pending official RDO release to fix this ? All the best.
Issue being clients are consummed by services which relies on very specific versions. https://github.com/openstack/requirements/blob/stable/kilo/global-requirements.txt#L125 As F23 ships Kilo, we can't upgrade this, workaround would be using rawhide version of python-glanceclient. So we'll try to backport the fix. Note that we're removing services from Fedora and so we'll be able to upgrade clients more aggressively starting F24.
Hi again, obviously, using 1.1.0 leads to other issues while addressing the former. I followed your advise to give a try with rawhide 0.17.2 but I still have the problem. Exemple for a nova boot command : nova-controller[19838]: nova.api.openstack Caught error: Expecting value: line 1 column 1 (char 0) 2015-10-21 13:12:55.692 19838 TRACE nova.api.openstack Traceback (most recent call last): 2015-10-21 13:12:55.692 19838 TRACE nova.api.openstack File "/usr/lib/python2.7/site-packages/nova/api/openstack/__init__.py", line 125, in __call__ 2015-10-21 13:12:55.692 19838 TRACE nova.api.openstack return req.get_response(self.application) 2015-10-21 13:12:55.692 19838 TRACE nova.api.openstack File "/usr/lib/python2.7/site-packages/webob/request.py", line 1296, in send 2015-10-21 13:12:55.692 19838 TRACE nova.api.openstack application, catch_exc_info=False) 2015-10-21 13:12:55.692 19838 TRACE nova.api.openstack File "/usr/lib/python2.7/site-packages/webob/request.py", line 1260, in call_application 2015-10-21 13:12:55.692 19838 TRACE nova.api.openstack app_iter = application(self.environ, start_response) 2015-10-21 13:12:55.692 19838 TRACE nova.api.openstack File "/usr/lib/python2.7/site-packages/webob/dec.py", line 144, in __call__ 2015-10-21 13:12:55.692 19838 TRACE nova.api.openstack return resp(environ, start_response) 2015-10-21 13:12:55.692 19838 TRACE nova.api.openstack File "/usr/lib/python2.7/site-packages/keystonemiddleware/auth_token/__init__.py", line 634, in __call__ 2015-10-21 13:12:55.692 19838 TRACE nova.api.openstack return self._call_app(env, start_response) 2015-10-21 13:12:55.692 19838 TRACE nova.api.openstack File "/usr/lib/python2.7/site-packages/keystonemiddleware/auth_token/__init__.py", line 554, in _call_app 2015-10-21 13:12:55.692 19838 TRACE nova.api.openstack return self._app(env, _fake_start_response) 2015-10-21 13:12:55.692 19838 TRACE nova.api.openstack File "/usr/lib/python2.7/site-packages/webob/dec.py", line 144, in __call__ 2015-10-21 13:12:55.692 19838 TRACE nova.api.openstack return resp(environ, start_response) 2015-10-21 13:12:55.692 19838 TRACE nova.api.openstack File "/usr/lib/python2.7/site-packages/webob/dec.py", line 144, in __call__ 2015-10-21 13:12:55.692 19838 TRACE nova.api.openstack return resp(environ, start_response) 2015-10-21 13:12:55.692 19838 TRACE nova.api.openstack File "/usr/lib/python2.7/site-packages/routes/middleware.py", line 131, in __call__ 2015-10-21 13:12:55.692 19838 TRACE nova.api.openstack response = self.app(environ, start_response) 2015-10-21 13:12:55.692 19838 TRACE nova.api.openstack File "/usr/lib/python2.7/site-packages/webob/dec.py", line 144, in __call__ 2015-10-21 13:12:55.692 19838 TRACE nova.api.openstack return resp(environ, start_response) 2015-10-21 13:12:55.692 19838 TRACE nova.api.openstack File "/usr/lib/python2.7/site-packages/webob/dec.py", line 130, in __call__ 2015-10-21 13:12:55.692 19838 TRACE nova.api.openstack resp = self.call_func(req, *args, **self.kwargs) 2015-10-21 13:12:55.692 19838 TRACE nova.api.openstack File "/usr/lib/python2.7/site-packages/webob/dec.py", line 195, in call_func 2015-10-21 13:12:55.692 19838 TRACE nova.api.openstack return self.func(req, *args, **kwargs) 2015-10-21 13:12:55.692 19838 TRACE nova.api.openstack File "/usr/lib/python2.7/site-packages/nova/api/openstack/wsgi.py", line 756, in __call__ 2015-10-21 13:12:55.692 19838 TRACE nova.api.openstack content_type, body, accept) 2015-10-21 13:12:55.692 19838 TRACE nova.api.openstack File "/usr/lib/python2.7/site-packages/nova/api/openstack/wsgi.py", line 821, in _process_stack 2015-10-21 13:12:55.692 19838 TRACE nova.api.openstack action_result = self.dispatch(meth, request, action_args) 2015-10-21 13:12:55.692 19838 TRACE nova.api.openstack File "/usr/lib/python2.7/site-packages/nova/api/openstack/wsgi.py", line 911, in dispatch 2015-10-21 13:12:55.692 19838 TRACE nova.api.openstack return method(req=request, **action_args) 2015-10-21 13:12:55.692 19838 TRACE nova.api.openstack File "/usr/lib/python2.7/site-packages/nova/api/openstack/compute/images.py", line 125, in index 2015-10-21 13:12:55.692 19838 TRACE nova.api.openstack **page_params) 2015-10-21 13:12:55.692 19838 TRACE nova.api.openstack File "/usr/lib/python2.7/site-packages/nova/image/api.py", line 68, in get_all 2015-10-21 13:12:55.692 19838 TRACE nova.api.openstack return session.detail(context, **kwargs) 2015-10-21 13:12:55.692 19838 TRACE nova.api.openstack File "/usr/lib/python2.7/site-packages/nova/image/glance.py", line 274, in detail 2015-10-21 13:12:55.692 19838 TRACE nova.api.openstack for image in images: 2015-10-21 13:12:55.692 19838 TRACE nova.api.openstack File "/usr/lib/python2.7/site-packages/glanceclient/v1/images.py", line 249, in list 2015-10-21 13:12:55.692 19838 TRACE nova.api.openstack for image in paginate(params, return_request_id): 2015-10-21 13:12:55.692 19838 TRACE nova.api.openstack File "/usr/lib/python2.7/site-packages/glanceclient/v1/images.py", line 233, in paginate 2015-10-21 13:12:55.692 19838 TRACE nova.api.openstack images, resp = self._list(url, "images") 2015-10-21 13:12:55.692 19838 TRACE nova.api.openstack File "/usr/lib/python2.7/site-packages/glanceclient/v1/images.py", line 63, in _list 2015-10-21 13:12:55.692 19838 TRACE nova.api.openstack resp, body = self.client.get(url) 2015-10-21 13:12:55.692 19838 TRACE nova.api.openstack File "/usr/lib/python2.7/site-packages/glanceclient/common/http.py", line 262, in get 2015-10-21 13:12:55.692 19838 TRACE nova.api.openstack return self._request('GET', url, **kwargs) 2015-10-21 13:12:55.692 19838 TRACE nova.api.openstack File "/usr/lib/python2.7/site-packages/glanceclient/common/http.py", line 249, in _request 2015-10-21 13:12:55.692 19838 TRACE nova.api.openstack body_iter = resp.json() 2015-10-21 13:12:55.692 19838 TRACE nova.api.openstack File "/usr/lib/python2.7/site-packages/requests/models.py", line 819, in json 2015-10-21 13:12:55.692 19838 TRACE nova.api.openstack return json.loads(self.text, **kwargs) 2015-10-21 13:12:55.692 19838 TRACE nova.api.openstack File "/usr/lib64/python2.7/site-packages/simplejson/__init__.py", line 488, in loads 2015-10-21 13:12:55.692 19838 TRACE nova.api.openstack return _default_decoder.decode(s) 2015-10-21 13:12:55.692 19838 TRACE nova.api.openstack File "/usr/lib64/python2.7/site-packages/simplejson/decoder.py", line 370, in decode 2015-10-21 13:12:55.692 19838 TRACE nova.api.openstack obj, end = self.raw_decode(s) 2015-10-21 13:12:55.692 19838 TRACE nova.api.openstack File "/usr/lib64/python2.7/site-packages/simplejson/decoder.py", line 389, in raw_decode 2015-10-21 13:12:55.692 19838 TRACE nova.api.openstack return self.scan_once(s, idx=_w(s, idx).end()) 2015-10-21 13:12:55.692 19838 TRACE nova.api.openstack JSONDecodeError: Expecting value: line 1 column 1 (char 0) 2015-10-21 13:12:55.692 19838 TRACE nova.api.openstack Am I missing something ? Can I safely bypass the kilo requirement for glanceclient<0.18 and which version should I use ? Do you know which specific bugfixes and patchsets for 0.17.2 circumvent this particular issue ? Best regards,
hi, i notice that the python glance client from http://rdo.fedorapeople.org/openstack-kilo/rdo-release-kilo.rpm is still version 0.17.0.2.el7; which still has this bug. mariojmdavid's patch fixes the bug in my testing. When will it be merged?
This bug is against a Version which has reached End of Life. If it's still present in supported release (http://releases.openstack.org), please update Version and reopen.