Bug 1393469

$ cat enable-tls.yaml 
# Use this environment to pass in certificates for SSL deployments.
# For these values to take effect, one of the tls-endpoints-*.yaml environments
# must also be used.
parameter_defaults:
  SSLCertificate: |
    -----BEGIN CERTIFICATE-----
    MIIDmDCCAoACAQEwDQYJKoZIhvcNAQEFBQAwgZExCzAJBgNVBAYTAlVTMQ4wDAYD
    VQQIDAVUZXhhczEPMA0GA1UEBwwGQXVzdGluMRAwDgYDVQQKDAdSZWQgSGF0MREw
    DwYDVQQLDAhGaWVsZCBQTTEYMBYGA1UEAwwPMTkyLjE2OC4xMjIuMTUwMSIwIAYJ
    KoZIhvcNAQkBFhNqbGliZXJtYUByZWRoYXQuY29tMB4XDTE2MTEwODIyMzExNloX
    DTE3MTEwODIyMzExNlowgZExCzAJBgNVBAYTAlVTMQ4wDAYDVQQIDAVUZXhhczEP
    MA0GA1UEBwwGQXVzdGluMRAwDgYDVQQKDAdSZWQgSGF0MREwDwYDVQQLDAhGaWVs
    ZCBQTTEYMBYGA1UEAwwPMTkyLjE2OC4xMjIuMTUwMSIwIAYJKoZIhvcNAQkBFhNq
    bGliZXJtYUByZWRoYXQuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC
    AQEArsgLMqS/98JB1BeP+or5GpDSjT1jvqUavSXFOqI3XawqjLI8I1uKpHqDOo2G
    I341wS6M35SRv6CARMjrMiLR9I2+DEpp+uYJsdHf8cX9bJyMr+NNo746mYnf3nGd
    SY0DMHRu3Y/FqGqZud86GHqhQjf9l/bJ3RvBBKYxc/fwbJYi4L52IBzcogwUi2Pa
    2syXv89V9NM9/jigCuojRXJ4nBWbrSHeBwk+TGOWJrs64PhHaJqLRSMUqFgEEi3S
    KwYexjDaXQunu7882CigZ8uZcYdXkKEnQWurWJ3mwZM43xfGsiSfscFOd1DEIw7n
    4eAtnGjV+hyujqJpfMmT+ggG2wIDAQABMA0GCSqGSIb3DQEBBQUAA4IBAQABBQq+
    HM49tKGUvjA96gH8wnW2LhWGIcHYW/kYH+s9LBSNzuMbWVga9fNI0klprGoutBO9
    oIiuNP1D/bjH1emr6mIKSbhleTnK3e9vM2+RAOEgrGtzTDawI/bCc9gyCouDH9bz
    V5dHnAUPM9to9mGaFYzEqnELc6ylATug/Y79+2NUoLJ8vEUDab9gLAFTQnR4fGKi
    9dzCi77rE2Rj+f5Ec3Z6D8eGzD9i1kfkiQ8RSAoBimtUTDvPRHKuxRWgOwROjc4w
    miYhRNoIXRM1MGLMq/oMSfPHdBzR1AaTKwQQcHkoVvWyiVT5vJzOItOlcOOkrdt1
    3yWx11QDjC1wkB64
    -----END CERTIFICATE-----
  SSLIntermediateCertificate: ''
  SSLKey: |
    -----BEGIN RSA PRIVATE KEY-----
    MIIEpAIBAAKCAQEArsgLMqS/98JB1BeP+or5GpDSjT1jvqUavSXFOqI3XawqjLI8
    I1uKpHqDOo2GI341wS6M35SRv6CARMjrMiLR9I2+DEpp+uYJsdHf8cX9bJyMr+NN
    o746mYnf3nGdSY0DMHRu3Y/FqGqZud86GHqhQjf9l/bJ3RvBBKYxc/fwbJYi4L52
    IBzcogwUi2Pa2syXv89V9NM9/jigCuojRXJ4nBWbrSHeBwk+TGOWJrs64PhHaJqL
    RSMUqFgEEi3SKwYexjDaXQunu7882CigZ8uZcYdXkKEnQWurWJ3mwZM43xfGsiSf
    scFOd1DEIw7n4eAtnGjV+hyujqJpfMmT+ggG2wIDAQABAoIBAFZ/phsQMfk56D5A
    0v6ZnKKrHajMwvps14jYkn1sHp57QCuVYfHVsg2onB10QPh707qhgRJ9gowlsJVa
    IhwT43h2VaHbigujoRSh24TaCYuhBnddtOTelj26YFLEQ6VM5lEqrlt0dcvdxeqZ
    MTVAK2KMNzLn7gtBWwsj/MF56UCzUukbFSN0WCsEH8P/w4f8xymMsOea3rOHICat
    DZ2qysh1ParFrpZcFdCSWe+r1tCW+aXSGz3p5OBvaYI8dFsyabYm7RtYTSD6mZV3
    9lZZ9gG7eldBC3kqbI7ocBP2WI0X3ZobEdH4ir7tFemS0/X88G8koygqDfUbGpSp
    DZbFtHECgYEA48ywqTPZJqqrrkYp+99Hw5LHNlcpOfbxFkdUYx5OXiYQBqNM3Ila
    vLlLWgTF5UxS78S+M8tnHU88MJ+n3uaKbYM9r4gsMLjEXKbVyVGgbOMoNyO9+pcu
    zjPoh663orNTdSXgaIf7y2UuvxFVyJY6u8fiJscm0OchPt4d6RXpVB8CgYEAxGsh
    BpmxSKNgoU8IZtaDps38aCJMj46Q5dQ2u6Hnn79QyQVy3rLPzBg7TqIwYrICYfEm
    guL4Sy8166ZhVow9ZOKGSDDw44eVblwngmiVoaB3X4wh7vjWKbXWEm4Qu9MkOmv6
    n0deA5KBVBaYWrkVNJyeqSmbCGS0vdNBnan0VcUCgYEAr5vqTGYV7wL/GngogWzd
    S91pCXEj7PV7YWtXmJmSXG4HSLa22ARjGL3XYuvvCxdNkF0hK5iQQz4D7pAFv4YG
    DOpxsVHOjzjA15QdlvcALzDmnatGF1pY0MmfZonAMwL/QX4Tg0HhUCkOYmkgsmNt
    n7k1lCNOvxiOvoJImJk1qI0CgYEAwHluQoaHSHP48/l7dTLOjb2akvzRY3fEG8OJ
    4vI0BMG4S1SQjRRSNmCkdVjP61ceqJVkNKxvvVVFFGfVSLdiTiMAjWuQEpbBYwTh
    HpSX1GtnrvSmKAQl8XejkCnVMgkkni9Dx6NqyDtfimQd5gEqe4TioUgJCP+Ocdm7
    meF6cjUCgYAY4sb5pvYIckADxRChAz1GcO4J/56rjHaIdXEaB/TSj4g/gG/kzQuw
    EpiCoYO/9uNh/KyJ/7Jzb1eiB5Ts/uUCNybNF4miJU8vLykq2C88SUhZzdZfu+ag
    8oNBU1Ol9w1KVwvUsJRGTi5KkRv1jKQ9fVFM3bxP5C8/CcyoxEUdGA==
    -----END RSA PRIVATE KEY-----

resource_registry:
  OS::TripleO::NodeTLSData: /usr/share/openstack-tripleo-heat-templates/puppet/extraconfig/tls/tls-cert-inject.yaml
[stack@undercloud environments]$ cat inject-trust-anchor.yaml 
parameter_defaults:
  SSLRootCertificate: |
    -----BEGIN CERTIFICATE-----
    MIID9zCCAt+gAwIBAgIJAPgD8fwqZ00hMA0GCSqGSIb3DQEBCwUAMIGRMQswCQYD
    VQQGEwJVUzEOMAwGA1UECAwFVGV4YXMxDzANBgNVBAcMBkF1c3RpbjEQMA4GA1UE
    CgwHUmVkIEhhdDERMA8GA1UECwwIRmllbGQgUE0xGDAWBgNVBAMMDzE5Mi4xNjgu
    MTIyLjE1MDEiMCAGCSqGSIb3DQEJARYTamxpYmVybWFAcmVkaGF0LmNvbTAeFw0x
    NjExMDgyMjI4NTVaFw0xNzExMDgyMjI4NTVaMIGRMQswCQYDVQQGEwJVUzEOMAwG
    A1UECAwFVGV4YXMxDzANBgNVBAcMBkF1c3RpbjEQMA4GA1UECgwHUmVkIEhhdDER
    MA8GA1UECwwIRmllbGQgUE0xGDAWBgNVBAMMDzE5Mi4xNjguMTIyLjE1MDEiMCAG
    CSqGSIb3DQEJARYTamxpYmVybWFAcmVkaGF0LmNvbTCCASIwDQYJKoZIhvcNAQEB
    BQADggEPADCCAQoCggEBAPM12G+3xZHlsR8Uiw9RwIQVjXr8k6KTl5x7ke73Zgcp
    /ybaEt3+s1OQz4q31eYRUvnUDFPSC99Ddi7sJFI2unlsjP2F9IaPE5j9TMB97vjy
    VTiBQ4jmTNy7+QCLhL157ZbbA0JTNsX5ZBpHzw+Kawwf2s/kOvHcMXO3Xao/pfc3
    u/KGT1SkDz5RlqQs+udBJ4g62E2t/YEVaco6L2+8Tjq5K4WP6JLGz37eepyeHd1Y
    Aa1I1VBMqzDs1TGAtEHR8dqYce7me2YLMwMe2N6lm6QWdZsDUvHh5sXI97UEbdBE
    l29Nvpnzt9FOgoNULcVz3Wpa7CcVGB9I0JRPf/v6rpMCAwEAAaNQME4wHQYDVR0O
    BBYEFOn1GtVes1fitdkNVwU2633bZsmYMB8GA1UdIwQYMBaAFOn1GtVes1fitdkN
    VwU2633bZsmYMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAOzvLWub
    mqkaEt+tydwFikPcLctpDyTeGViscHMpSOx1gTObiQiYIa3tBJudIIc7zXIUeCJF
    my9JdIO7Ni6utNJ8M2/FUrynfvnnr1bX57luQGAuCxIZ//lJTFTOEi7UWFecsRiy
    lF09Z6DFBydw3zv/TZWCDYzX+DpbC93fvYGE5ZA8BHFNMOKOW+LEGcaQ1p9MGjVL
    MrTSLxV3zuyWuqRgXWfNXx314gIMttWOlUElFDjYe/ShrbOdAzzA6SAuqMqBPBcU
    swyGGr6RQbk6GkwHvoZKnDxE3uOYdH4zp1adDgfHDGXB4U32vTrlyBfqhc6TMUrU
    ndyy8E5cJp0W0+E=
    -----END CERTIFICATE-----

resource_registry:
  OS::TripleO::NodeTLSCAData: /usr/share/openstack-tripleo-heat-templates/puppet/extraconfig/tls/ca-inject.yaml

$ cat tls-endpoints-public-ip.yaml 
# Use this environment when deploying an SSL-enabled overcloud where the public
# endpoint is an IP address.
parameter_defaults:
  EndpointMap:
    AodhAdmin: {protocol: 'http', port: '8042', host: 'IP_ADDRESS'}
    AodhInternal: {protocol: 'http', port: '8042', host: 'IP_ADDRESS'}
    AodhPublic: {protocol: 'https', port: '13042', host: 'IP_ADDRESS'}
    CeilometerAdmin: {protocol: 'http', port: '8777', host: 'IP_ADDRESS'}
    CeilometerInternal: {protocol: 'http', port: '8777', host: 'IP_ADDRESS'}
    CeilometerPublic: {protocol: 'https', port: '13777', host: 'IP_ADDRESS'}
    CephRgwAdmin: {protocol: 'http', port: '8080', host: 'IP_ADDRESS'}
    CephRgwInternal: {protocol: 'http', port: '8080', host: 'IP_ADDRESS'}
    CephRgwPublic: {protocol: 'https', port: '13808', host: 'IP_ADDRESS'}
    CinderAdmin: {protocol: 'http', port: '8776', host: 'IP_ADDRESS'}
    CinderInternal: {protocol: 'http', port: '8776', host: 'IP_ADDRESS'}
    CinderPublic: {protocol: 'https', port: '13776', host: 'IP_ADDRESS'}
    GlanceAdmin: {protocol: 'http', port: '9292', host: 'IP_ADDRESS'}
    GlanceInternal: {protocol: 'http', port: '9292', host: 'IP_ADDRESS'}
    GlancePublic: {protocol: 'https', port: '13292', host: 'IP_ADDRESS'}
    GlanceRegistryInternal: {protocol: 'http', port: '9191', host: 'IP_ADDRESS'}
    GnocchiAdmin: {protocol: 'http', port: '8041', host: 'IP_ADDRESS'}
    GnocchiInternal: {protocol: 'http', port: '8041', host: 'IP_ADDRESS'}
    GnocchiPublic: {protocol: 'https', port: '13041', host: 'IP_ADDRESS'}
    HeatAdmin: {protocol: 'http', port: '8004', host: 'IP_ADDRESS'}
    HeatInternal: {protocol: 'http', port: '8004', host: 'IP_ADDRESS'}
    HeatPublic: {protocol: 'https', port: '13004', host: 'IP_ADDRESS'}
    HeatCfnAdmin: {protocol: 'http', port: '8000', host: 'IP_ADDRESS'}
    HeatCfnInternal: {protocol: 'http', port: '8000', host: 'IP_ADDRESS'}
    HeatCfnPublic: {protocol: 'https', port: '13005', host: 'IP_ADDRESS'}
    HorizonPublic: {protocol: 'https', port: '443', host: 'IP_ADDRESS'}
    IronicAdmin: {protocol: 'http', port: '6385', host: 'IP_ADDRESS'}
    IronicInternal: {protocol: 'http', port: '6385', host: 'IP_ADDRESS'}
    IronicPublic: {protocol: 'https', port: '13385', host: 'IP_ADDRESS'}
    KeystoneAdmin: {protocol: 'http', port: '35357', host: 'IP_ADDRESS'}
    KeystoneInternal: {protocol: 'http', port: '5000', host: 'IP_ADDRESS'}
    KeystonePublic: {protocol: 'https', port: '13000', host: 'IP_ADDRESS'}
    ManilaAdmin: {protocol: 'http', port: '8786', host: 'IP_ADDRESS'}
    ManilaInternal: {protocol: 'http', port: '8786', host: 'IP_ADDRESS'}
    ManilaPublic: {protocol: 'https', port: '13786', host: 'IP_ADDRESS'}
    MysqlInternal: {protocol: 'mysql+pymysql', port: '3306', host: 'IP_ADDRESS'}
    NeutronAdmin: {protocol: 'http', port: '9696', host: 'IP_ADDRESS'}
    NeutronInternal: {protocol: 'http', port: '9696', host: 'IP_ADDRESS'}
    NeutronPublic: {protocol: 'https', port: '13696', host: 'IP_ADDRESS'}
    NovaAdmin: {protocol: 'http', port: '8774', host: 'IP_ADDRESS'}
    NovaInternal: {protocol: 'http', port: '8774', host: 'IP_ADDRESS'}
    NovaPublic: {protocol: 'https', port: '13774', host: 'IP_ADDRESS'}
    NovaVNCProxyAdmin: {protocol: 'http', port: '6080', host: 'IP_ADDRESS'}
    NovaVNCProxyInternal: {protocol: 'http', port: '6080', host: 'IP_ADDRESS'}
    NovaVNCProxyPublic: {protocol: 'https', port: '13080', host: 'IP_ADDRESS'}
    SaharaAdmin: {protocol: 'http', port: '8386', host: 'IP_ADDRESS'}
    SaharaInternal: {protocol: 'http', port: '8386', host: 'IP_ADDRESS'}
    SaharaPublic: {protocol: 'https', port: '13386', host: 'IP_ADDRESS'}
    SwiftAdmin: {protocol: 'http', port: '8080', host: 'IP_ADDRESS'}
    SwiftInternal: {protocol: 'http', port: '8080', host: 'IP_ADDRESS'}
    SwiftPublic: {protocol: 'https', port: '13808', host: 'IP_ADDRESS'}

I am double checking to make sure this was not a user error, but wanted to get the bug filed ASAP just in case.

FWIW we're keeping track of converting non-ssl to ssl in bz#1320379 and the last issue that I've seen when testing it was that haproxy config didn't load the config containing the certificates. Could you please check the haproxy config and see if it got updated?

Marius, this is a different issue. heat is not taking into account the resource NodeTLSData at all, so it doesn't set the service_certificate parameter that will end up enabling TLS in haproxy. So, as a result, the haproxy configuration doesn't have any trace of SSL at all (the ports don't match, and the certificate is not being served). So this seems to be an issue with how resource definitions are being passed to heat.

Having reproduced this, it seems that user-environment is being set correctly by the pythonclient, however, heat still doesn't get this resource definition:

[stack@undercloud overcloud]$ grep TLS user-environment.yaml 
  OS::TripleO::NodeTLSCAData: puppet/extraconfig/tls/ca-inject.yaml
  OS::TripleO::NodeTLSData: puppet/extraconfig/tls/tls-cert-inject.yaml
[stack@undercloud overcloud]$ openstack stack environment show overcloud | grep TLS
  OS::TripleO::NodeTLSCAData: OS::Heat::None
  OS::TripleO::NodeTLSData: OS::Heat::None
  OS::TripleO::Services::ApacheTLS: OS::Heat::None
  OS::TripleO::Services::HAProxyInternalTLS: OS::Heat::None
  OS::TripleO::Services::HAProxyPublicTLS: OS::Heat::None

Note that I downloaded the user-environment.yaml from the swift files for the overcloud.

I had a successful run of converting non-ssl to ssl with yesterday's build. Maybe it's also worth trying the latest puddle build? 

Deployed non-ssl overcloud with:

export THT=/usr/share/openstack-tripleo-heat-templates
openstack overcloud deploy --templates \
-e $THT/environments/network-isolation.yaml \
-e $THT/environments/network-management.yaml \
-e ~/templates/network-environment.yaml \
-e $THT/environments/storage-environment.yaml \
-e ~/templates/disk-layout.yaml \
--control-scale 3 \
--control-flavor controller-d75f3dec-c770-5f88-9d4c-3fea1bf9c484 \
--compute-scale 1 \
--compute-flavor compute-b634c10a-570f-59ba-bdbf-0c313d745a10 \
--ceph-storage-scale 1 \
--ceph-storage-flavor ceph-cf1f074b-dadb-5eb8-9eb0-55828273fab7 \
--ntp-server clock.redhat.com \
--log-file overcloud_deployment.log &> overcloud_install.log

Then ran the deploy command for enabling SSL. Update completed ok and was able to reach the public VIP via SSL:

export THT=/usr/share/openstack-tripleo-heat-templates
openstack overcloud deploy --templates \
-e $THT/environments/network-isolation.yaml \
-e $THT/environments/network-management.yaml \
-e ~/templates/network-environment.yaml \
-e $THT/environments/storage-environment.yaml \
-e ~/templates/disk-layout.yaml \
-e ~/templates/enable-tls.yaml \
-e ~/templates/inject-trust-anchor.yaml \
-e ~/templates/tls-endpoints-public-ip.yaml \
--control-scale 3 \
--control-flavor controller-d75f3dec-c770-5f88-9d4c-3fea1bf9c484 \
--compute-scale 1 \
--compute-flavor compute-b634c10a-570f-59ba-bdbf-0c313d745a10 \
--ceph-storage-scale 1 \
--ceph-storage-flavor ceph-cf1f074b-dadb-5eb8-9eb0-55828273fab7 \
--ntp-server clock.redhat.com \
--log-file overcloud_deployment.log &> overcloud_install.log

I will redeploy with the latest puddle today and try again.

Thanks, Jacob

Actually now that I checked my haproxy config, it is correctly set up... so this is pretty strange... So actually I couldn't reproduce it in master. The heat environment seems screwed up, but the change got reflected. Can someone else verify that this is an issue in the latest OSP10?

Looking back through my deploy script, it looks like I might have left off a "\" which would have included the environment files.

exec openstack overcloud deploy \
        --templates /usr/share/openstack-tripleo-heat-templates \
        --ntp-server 10.16.255.1 \
        --control-flavor control --control-scale 3 \
        --compute-flavor compute --compute-scale 1 \
        --ceph-storage-flavor ceph-storage --ceph-storage-scale 3 \
        --neutron-tunnel-types vxlan --neutron-network-type vxlan \
        -e /usr/share/openstack-tripleo-heat-templates/environments/network-isolation.yaml \
        -e /home/stack/templates/network-environment.yaml \
        -e /home/stack/templates/HostnameMap.yaml \
        -e /home/stack/templates/ips-from-pool-all.yaml \
        -e /home/stack/templates/storage-environment.yaml
^^^^
        -e /home/stack/templates/limits.yaml \
	-e /home/stack/templates/environments/enable-tls.yaml \
	-e /home/stack/templates/environments/tls-endpoints-public-ip.yaml \
        -e /home/stack/templates/environments/inject-trust-anchor.yaml

However, the keys were injected in the plan and also in the environment file, so I'm not sure if this was the case or not.

My plan is:
1. try with frsh install
2. try with update on current puddle
3. update puddle and repeat 1&2

After a clean deploy the haproxy.cfg includes the ssl information but I get connection refused errors when connecting to overcloud endpoints.

    $ openstack stack list
    +--------------------------------------+------------+-----------------+----------------------+--------------+
    | ID                                   | Stack Name | Stack Status    | Creation Time        | Updated Time |
    +--------------------------------------+------------+-----------------+----------------------+--------------+
    | b22e1f7c-21c8-4cd9-b437-eb35587f3e9c | overcloud  | CREATE_COMPLETE | 2016-11-09T17:14:44Z | None         |
    +--------------------------------------+------------+-----------------+----------------------+--------------+

    $ openstack stack environment show overcloud | grep TLS
      OS::TripleO::NodeTLSCAData: http://172.16.0.1:8080/v1/AUTH_3f474f7ad3c7405b8782a7df90c648bd/overcloud/user-files/usr/share/openstack-tripleo-heat-templates/puppet/extraconfig/tls/ca-inject.yaml
      OS::TripleO::NodeTLSData: http://172.16.0.1:8080/v1/AUTH_3f474f7ad3c7405b8782a7df90c648bd/overcloud/user-files/usr/share/openstack-tripleo-heat-templates/puppet/extraconfig/tls/tls-cert-inject.yaml

    $ source overcloudrc

    $ openstack endpoint list
    Discovering versions from the identity service failed when creating the password plugin. Attempting to determine version from URL.
    Unable to establish connection to http://192.168.122.150:5000/v2.0/tokens: HTTPConnectionPool(host='192.168.122.150', port=5000): Max retries exceeded with url: /v2.0/tokens (Caused by NewConnectionError('<requests.packages.urllib3.connection.HTTPConnection object at 0x3724210>: Failed to establish a new connection: [Errno 111] Connection refused',))

    $ ssh -l heat-admin 172.16.0.31
    Last login: Wed Nov  9 21:24:12 2016 from 172.16.0.1

    $ sudo -i

    # grep ssl /etc/haproxy/haproxy.cfg | head
      ssl-default-bind-ciphers  !SSLv2:kEECDH:kRSA:kEDH:kPSK:+3DES:!aNULL:!eNULL:!MD5:!EXP:!RC4:!SEED:!IDEA:!DES
      ssl-default-bind-options  no-sslv3
      bind 192.168.122.150:13042 transparent ssl crt /etc/pki/tls/private/overcloud_endpoint.pem
      bind 192.168.122.150:13777 transparent ssl crt /etc/pki/tls/private/overcloud_endpoint.pem
      bind 192.168.122.150:13776 transparent ssl crt /etc/pki/tls/private/overcloud_endpoint.pem
      http-request set-header X-Forwarded-Proto https if { ssl_fc }
      http-request set-header X-Forwarded-Proto http if !{ ssl_fc }
      bind 192.168.122.150:13292 transparent ssl crt /etc/pki/tls/private/overcloud_endpoint.pem
      http-request set-header X-Forwarded-Proto https if { ssl_fc }
      http-request set-header X-Forwarded-Proto http if !{ ssl_fc }


Any ideas for troubleshooting the connection errors I am seeing?

I ran an update deployment without changing anything after the deployment above.

The update completed successfully.

Now I get a new error:

$ openstack stack list
+--------------------------------------+------------+-----------------+----------------------+----------------------+
| ID                                   | Stack Name | Stack Status    | Creation Time        | Updated Time         |
+--------------------------------------+------------+-----------------+----------------------+----------------------+
| b22e1f7c-21c8-4cd9-b437-eb35587f3e9c | overcloud  | UPDATE_COMPLETE | 2016-11-09T17:14:44Z | 2016-11-09T21:50:49Z |
+--------------------------------------+------------+-----------------+----------------------+----------------------+

$ source overcloudrc

$ nova list
No handlers could be found for logger "keystoneauth.identity.generic.base"
ERROR (SSLError): SSL exception connecting to https://192.168.122.150:13000/v2.0/tokens: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed (_ssl.c:579)

jcoufal, is converting from non-ssl to ssl a blocker?

The issue with the clean deploy sounds like the overcloudrc is wrong. It was trying to connect to the wrong port for some reason. After the update, the port in the overcloudrc was updated (as you can see it's now using 13000) however, the certificate verification failed, which is an expected issue if the CN or the SubjectAltName is wrong. Does the certificate match the VIP that was deployed?

I did a fresh install of the overcloud and the haproxy.cfg was populated correctly. When I tried to connect I recived the following error:

SSLError: SSL exception connecting to https://192.168.122.150:13000/v2.0/tokens: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed (_ssl.c:579)
ERROR (SSLError): SSL exception connecting to https://192.168.122.150:13000/v2.0/tokens: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed (_ssl.c:579)

I checked the key and cert in /etc/pki/tls/private/overcloud_endpoint.pem on the controller nodes and they are correct.

I also checked the SSLKey, SSLCertificate, and SSLRootCertificate in the mistral environment and they look correct. 

I am redeploying and will try with the latest build if I can get it through rhos-release.

The initial problem with the update may have been a misconfiguration.

The new recommended process for using a self-signed CA (intead of just a self-signed cert) should be documented.

(In reply to jliberma from comment #16)
> I did a fresh install of the overcloud and the haproxy.cfg was populated
> correctly. When I tried to connect I recived the following error:
> 
> SSLError: SSL exception connecting to
> https://192.168.122.150:13000/v2.0/tokens: [SSL: CERTIFICATE_VERIFY_FAILED]
> certificate verify failed (_ssl.c:579)
> ERROR (SSLError): SSL exception connecting to
> https://192.168.122.150:13000/v2.0/tokens: [SSL: CERTIFICATE_VERIFY_FAILED]
> certificate verify failed (_ssl.c:579)
> 
> I checked the key and cert in /etc/pki/tls/private/overcloud_endpoint.pem on
> the controller nodes and they are correct.
> 
> I also checked the SSLKey, SSLCertificate, and SSLRootCertificate in the
> mistral environment and they look correct. 
> 
> I am redeploying and will try with the latest build if I can get it through
> rhos-release.
> 
> The initial problem with the update may have been a misconfiguration.
> 
> The new recommended process for using a self-signed CA (intead of just a
> self-signed cert) should be documented.

Maybe the issue now is with the client. IIUC this is a self signed certificate. Have you updated the undercloud trusted certificate store to contain it?

sudo cp overcloud-cert.pem /etc/pki/ca-trust/source/anchors/
sudo update-ca-trust extract 

Also pasting the link to the docs which contain the self signed cert generation steps:
http://docs.openstack.org/developer/tripleo-docs/advanced_deployment/ssl.html#certificate-details

(In reply to Juan Antonio Osorio from comment #13)
> The issue with the clean deploy sounds like the overcloudrc is wrong. It was
> trying to connect to the wrong port for some reason. After the update, the
> port in the overcloudrc was updated (as you can see it's now using 13000)
> however, the certificate verification failed, which is an expected issue if
> the CN or the SubjectAltName is wrong. Does the certificate match the VIP
> that was deployed?

Os,

How do I verify I used the correct address?

I used the ExternalNetworkVip from ips-from-pool-all.yaml (predictable addresses feature) when I generated the certificates.

undercloud$ grep ExternalNetworkVip templates/ips-from-pool-all.yaml
  ExternalNetworkVip: 192.168.122.150

I also use IP_ADDRESS in the endpoint map (not DNS):

EndpointMap:
    CeilometerAdmin: {protocol: 'http', port: '8777', host: 'IP_ADDRESS'}
    CeilometerInternal: {protocol: 'http', port: '8777', host: 'IP_ADDRESS'}
    CeilometerPublic: {protocol: 'https', port: '13777', host: 'IP_ADDRESS'}
    CinderAdmin: {protocol: 'http', port: '8776', host: 'IP_ADDRESS'}
    CinderInternal: {protocol: 'http', port: '8776', host: 'IP_ADDRESS'}
    CinderPublic: {protocol: 'https', port: '13776', host: 'IP_ADDRESS'}
    GlanceAdmin: {protocol: 'http', port: '9292', host: 'IP_ADDRESS'}
    GlanceInternal: {protocol: 'http', port: '9292', host: 'IP_ADDRESS'}
    GlancePublic: {protocol: 'https', port: '13292', host: 'IP_ADDRESS'}
    GlanceRegistryAdmin: {protocol: 'http', port: '9191', host: 'IP_ADDRESS'}

(In reply to Marius Cornea from comment #18)
> (In reply to jliberma from comment #16)
> > I did a fresh install of the overcloud and the haproxy.cfg was populated
> > correctly. When I tried to connect I recived the following error:
> > 
> > SSLError: SSL exception connecting to
> > https://192.168.122.150:13000/v2.0/tokens: [SSL: CERTIFICATE_VERIFY_FAILED]
> > certificate verify failed (_ssl.c:579)
> > ERROR (SSLError): SSL exception connecting to
> > https://192.168.122.150:13000/v2.0/tokens: [SSL: CERTIFICATE_VERIFY_FAILED]
> > certificate verify failed (_ssl.c:579)
> > 
> > I checked the key and cert in /etc/pki/tls/private/overcloud_endpoint.pem on
> > the controller nodes and they are correct.
> > 
> > I also checked the SSLKey, SSLCertificate, and SSLRootCertificate in the
> > mistral environment and they look correct. 
> > 
> > I am redeploying and will try with the latest build if I can get it through
> > rhos-release.
> > 
> > The initial problem with the update may have been a misconfiguration.
> > 
> > The new recommended process for using a self-signed CA (intead of just a
> > self-signed cert) should be documented.
> 
> Maybe the issue now is with the client. IIUC this is a self signed
> certificate. Have you updated the undercloud trusted certificate store to
> contain it?
> 
> sudo cp overcloud-cert.pem /etc/pki/ca-trust/source/anchors/
> sudo update-ca-trust extract 
> 
> Also pasting the link to the docs which contain the self signed cert
> generation steps:
> http://docs.openstack.org/developer/tripleo-docs/advanced_deployment/ssl.
> html#certificate-details


Yes, that is the doc I followed.

I used 

- overcloud-cert.pem for the self-signed CA cert (SSLRootCertificate in inject-trust-anchor.yaml)
- server-cert.pem for SSLCertificate (in enable-tls.yaml)
- server-key.pem for SSLKey (in enable-tls.yaml)
- no intermediate SSL cert

Is this correct ^^?

Do I need to include an intermediate SSL cert if I am using a self-signed CA?

I did see an upstream bug in previous versions where haproxy was not updated with the new cert because it is restarted before the new cert is added.

(In reply to jliberma from comment #20)
> (In reply to Marius Cornea from comment #18)
> > (In reply to jliberma from comment #16)
> 
> Yes, that is the doc I followed.
> 
> I used 
> 
> - overcloud-cert.pem for the self-signed CA cert (SSLRootCertificate in
> inject-trust-anchor.yaml)
> - server-cert.pem for SSLCertificate (in enable-tls.yaml)
> - server-key.pem for SSLKey (in enable-tls.yaml)
> - no intermediate SSL cert
> 
> Is this correct ^^?
> 
> Do I need to include an intermediate SSL cert if I am using a self-signed CA?
> 
> I did see an upstream bug in previous versions where haproxy was not updated
> with the new cert because it is restarted before the new cert is added.

It looks that the self signed CA instructions in the docs don't work as expected. I was able to reproduce the certificate validation issue on my system. 

As a workaround(only for test purposes) you could add the leaf certificate itself to the undercloud trusted store: 

cat server-cert.pem server-key.pem > server.pem
sudo cp server.pem /etc/pki/ca-trust/source/anchors/
sudo update-ca-trust extract

(In reply to Marius Cornea from comment #21)
> (In reply to jliberma from comment #20)
> > (In reply to Marius Cornea from comment #18)
> > > (In reply to jliberma from comment #16)
> > 
> > Yes, that is the doc I followed.
> > 
> > I used 
> > 
> > - overcloud-cert.pem for the self-signed CA cert (SSLRootCertificate in
> > inject-trust-anchor.yaml)
> > - server-cert.pem for SSLCertificate (in enable-tls.yaml)
> > - server-key.pem for SSLKey (in enable-tls.yaml)
> > - no intermediate SSL cert
> > 
> > Is this correct ^^?
> > 
> > Do I need to include an intermediate SSL cert if I am using a self-signed CA?
> > 
> > I did see an upstream bug in previous versions where haproxy was not updated
> > with the new cert because it is restarted before the new cert is added.
> 
> It looks that the self signed CA instructions in the docs don't work as
> expected. I was able to reproduce the certificate validation issue on my
> system. 
> 
> As a workaround(only for test purposes) you could add the leaf certificate
> itself to the undercloud trusted store: 
> 
> cat server-cert.pem server-key.pem > server.pem
> sudo cp server.pem /etc/pki/ca-trust/source/anchors/
> sudo update-ca-trust extract

Hi Marius,

I ran these commands ^^ and then redeployed. 

I still get this error:

$ nova list
No handlers could be found for logger "keystoneauth.identity.generic.base"
ERROR (SSLError): SSL exception connecting to https://192.168.122.150:13000/v2.0/tokens: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed (_ssl.c:579)

How can I tell whether the certificates match the IP address that I installed with?

I am including the tls-endpoints-public-ip.yaml and I used the ExternalNetworkVip during certificate generation.

(In reply to jliberma from comment #22)
> (In reply to Marius Cornea from comment #21)
> > (In reply to jliberma from comment #20)
> > > (In reply to Marius Cornea from comment #18)
> > > > (In reply to jliberma from comment #16)
> How can I tell whether the certificates match the IP address that I
> installed with?
>

You could try this:

echo | openssl s_client -showcerts -servername 192.168.122.150 -connect 192.168.122.150:13000 2>/dev/null | openssl x509 -inform pem -noout -text

and see whether the CN matches the vip - 192.168.122.150

(In reply to Marius Cornea from comment #23)
> (In reply to jliberma from comment #22)
> > (In reply to Marius Cornea from comment #21)
> > > (In reply to jliberma from comment #20)
> > > > (In reply to Marius Cornea from comment #18)
> > > > > (In reply to jliberma from comment #16)
> > How can I tell whether the certificates match the IP address that I
> > installed with?
> >
> 
> You could try this:
> 
> echo | openssl s_client -showcerts -servername 192.168.122.150 -connect
> 192.168.122.150:13000 2>/dev/null | openssl x509 -inform pem -noout -text
> 
> and see whether the CN matches the vip - 192.168.122.150

$ echo | openssl s_client -showcerts -servername 192.168.122.150 -connect \
> 192.168.122.150:13000 2>/dev/null | openssl x509 -inform pem -noout -text
Certificate:
    Data:
        Version: 1 (0x0)
        Serial Number: 1 (0x1)
    Signature Algorithm: sha1WithRSAEncryption
        Issuer: C=US, ST=Texas, L=Austin, O=Red Hat, OU=Field PM, CN=192.168.122.150/emailAddress=jliberma
        Validity
            Not Before: Nov  8 22:31:16 2016 GMT
            Not After : Nov  8 22:31:16 2017 GMT
        Subject: C=US, ST=Texas, L=Austin, O=Red Hat, OU=Field PM, CN=192.168.122.150/emailAddress=jliberma
        Subject Public Key Info:
            Public Key Algorithm: rsaEncryption
                Public-Key: (2048 bit)
                Modulus:
                    00:ae:c8:0b:32:a4:bf:f7:c2:41:d4:17:8f:fa:8a:
                    f9:1a:90:d2:8d:3d:63:be:a5:1a:bd:25:c5:3a:a2:
                    37:5d:ac:2a:8c:b2:3c:23:5b:8a:a4:7a:83:3a:8d:
                    86:23:7e:35:c1:2e:8c:df:94:91:bf:a0:80:44:c8:
                    eb:32:22:d1:f4:8d:be:0c:4a:69:fa:e6:09:b1:d1:
                    df:f1:c5:fd:6c:9c:8c:af:e3:4d:a3:be:3a:99:89:
                    df:de:71:9d:49:8d:03:30:74:6e:dd:8f:c5:a8:6a:
                    99:b9:df:3a:18:7a:a1:42:37:fd:97:f6:c9:dd:1b:
                    c1:04:a6:31:73:f7:f0:6c:96:22:e0:be:76:20:1c:
                    dc:a2:0c:14:8b:63:da:da:cc:97:bf:cf:55:f4:d3:
                    3d:fe:38:a0:0a:ea:23:45:72:78:9c:15:9b:ad:21:
                    de:07:09:3e:4c:63:96:26:bb:3a:e0:f8:47:68:9a:
                    8b:45:23:14:a8:58:04:12:2d:d2:2b:06:1e:c6:30:
                    da:5d:0b:a7:bb:bf:3c:d8:28:a0:67:cb:99:71:87:
                    57:90:a1:27:41:6b:ab:58:9d:e6:c1:93:38:df:17:
                    c6:b2:24:9f:b1:c1:4e:77:50:c4:23:0e:e7:e1:e0:
                    2d:9c:68:d5:fa:1c:ae:8e:a2:69:7c:c9:93:fa:08:
                    06:db
                Exponent: 65537 (0x10001)
    Signature Algorithm: sha1WithRSAEncryption
         01:05:0a:be:1c:ce:3d:b4:a1:94:be:30:3d:ea:01:fc:c2:75:
         b6:2e:15:86:21:c1:d8:5b:f9:18:1f:eb:3d:2c:14:8d:ce:e3:
         1b:59:58:1a:f5:f3:48:d2:49:69:ac:6a:2e:b4:13:bd:a0:88:
         ae:34:fd:43:fd:b8:c7:d5:e9:ab:ea:62:0a:49:b8:65:79:39:
         ca:dd:ef:6f:33:6f:91:00:e1:20:ac:6b:73:4c:36:b0:23:f6:
         c2:73:d8:32:0a:8b:83:1f:d6:f3:57:97:47:9c:05:0f:33:db:
         68:f6:61:9a:15:8c:c4:aa:71:0b:73:ac:a5:01:3b:a0:fd:8e:
         fd:fb:63:54:a0:b2:7c:bc:45:03:69:bf:60:2c:01:53:42:74:
         78:7c:62:a2:f5:dc:c2:8b:be:eb:13:64:63:f9:fe:44:73:76:
         7a:0f:c7:86:cc:3f:62:d6:47:e4:89:0f:11:48:0a:01:8a:6b:
         54:4c:3b:cf:44:72:ae:c5:15:a0:3b:04:4e:8d:ce:30:9a:26:
         21:44:da:08:5d:13:35:30:62:cc:ab:fa:0c:49:f3:c7:74:1c:
         d1:d4:06:93:2b:04:10:70:79:28:56:f5:b2:89:54:f9:bc:9c:
         ce:22:d3:a5:70:e3:a4:ad:db:75:df:25:b1:d7:54:03:8c:2d:
         70:90:1e:b8

By the way, does the undercloud trust the overcloud's certificate? You'll also get that error if the cert is not trusted.

(In reply to Juan Antonio Osorio from comment #25)
> By the way, does the undercloud trust the overcloud's certificate? You'll
> also get that error if the cert is not trusted.

How do I test that?

I'm a bit confused by these certificates in general, where did they come from?

The CA and the server cert both have the same subject and it isn't clear to me at all that the server cert was issued by the CA, it looks self-signed to me. That would cause these verification issues.

http://docs.openstack.org/developer/tripleo-docs/advanced_deployment/ssl.html#overcloud-ssl

Hi Rob, I followed the instructions here ^^ (upstream tripleo TLS docs for Newton) to create a self-signed CA, which is new feature supported in Newton.

I performed the "Self-signed IP-based certificate" steps to create the certs.

This is not a cert signed by an external CA.

If you have alternate steps to generate a cert, please let me know.

I am current reinstalling my undercloud with the 11.11.1 puddle to test again.

Thanks, Jacob

openssl genrsa -out overcloud-ca-privkey.pem 2048
openssl req -new -x509 -key overcloud-ca-privkey.pem \
     -out overcloud-cacert.pem -days 365
sudo cp overcloud-cacert.pem /etc/pki/ca-trust/source/anchors/
sudo update-ca-trust extract
openssl req -newkey rsa:2048 -days 365 \
     -nodes -keyout server-key.pem -out server-req.pem
openssl rsa -in server-key.pem -out server-key.pem
openssl x509 -req -in server-req.pem -days 365 \
      -CA overcloud-cacert.pem -CAkey overcloud-ca-privkey.pem \
      -set_serial 01 -out server-cert.pem

I also appended these commands per mcornea's suggestion:

(adding leaf certificate to undercloud's trusted store)

cat server-cert.pem server-key.pem > server.pem
sudo cp server.pem /etc/pki/ca-trust/source/anchors/
sudo update-ca-trust extract

Deploy with: 
-e ~/ssl-heat-templates/environments/enable-tls.yaml -e ~/ssl-heat-templates/environments/tls-endpoints-public-ip.yaml -e ~/ssl-heat-templates/environments/inject-trust-anchor.yaml

What is confusing is the CA has the same subject as the server certificate. Can you retest and use a unique name for the CA? The CA doesn't need to have the IP embedded in the subject, for example.

Closed this as QE and Engineering can not reproduce this issue.  We can reopen if needed.

Marius, when you reproduced it did your suggested workaround fix it for you?

(In reply to Rob Crittenden from comment #33)
> Marius, when you reproduced it did your suggested workaround fix it for you?

Yes, I was able to workaround by storing the server certificate into the undercloud trusted store. I think the issue's we've been tracking in this BZ are related to the certificate generation/validation which is outside the scope of the overcloud SSL functionality. I'm dropping the blocker flag for this bug and keep it open until we find the correct instructions to generate a self signed CA.

Note that our automated tests which use a self signed certificate are passing. These are the steps for the certificate generation:
https://github.com/rhosqeauto/InfraRed/blob/master/roles/installer/ospd/overcloud/ssl/tasks/ssl.yml#L13-L28

I retried by using a different CN for the self signed CA and it worked ok this time. Juan also adjusted the docs[1] so we now explicitly mention that the CA CN should be different than the server CN.

These are the steps that I did:

Generate self signed CA:
openssl genrsa -out overcloud-ca-privkey.pem 2048
openssl req -new -x509 -key overcloud-ca-privkey.pem -out overcloud-cacert.pem -days 365 -subj '/C=US/ST=CA/L=LA/O=Cloudy/OU=cloud/CN=cloudy.net'

Update undercloud trusted store with the CA certificate:
sudo cp overcloud-cacert.pem /etc/pki/ca-trust/source/anchors/
sudo update-ca-trust extract

Generate certificate signed by the CA created in the previous step:
openssl req -newkey rsa:2048 -days 365 -nodes -keyout server-key.pem -out server-req.pem -subj '/C=US/ST=CA/L=LA/O=Cloudy/OU=cloud/CN=172.16.18.27'
openssl rsa -in server-key.pem -out server-key.pem
openssl x509 -req -in server-req.pem -days 365 -CA overcloud-cacert.pem -CAkey overcloud-ca-privkey.pem -set_serial 01 -out server-cert.pem

Verify the certificate agains the undercloud trusted store:
openssl verify -verbose -CAfile /etc/ssl/certs/ca-bundle.crt  server-cert.pem
it should return => server-cert.pem: OK

Add the servert certificate and key to enable-tls.yaml and the CA certificate to inject-trust-anchor.yaml.
overcloud-cacert.pem => environments/inject-trust-anchor.yaml
server-cert.pem =>  environments/enable-tls.yaml
server-key.pem => environments/enable-tls.yaml

To make sure that the public VIP matches the one passed during certificate creation add this in an environment file(in my case I used network-environment.yaml):
parameter_defaults:
  PublicVirtualFixedIPs: [{'ip_address':'172.16.18.27'}]

Deploy command:

source ~/stackrc
#export THT=/usr/share/openstack-tripleo-heat-templates/
export THT=/home/stack/openstack_deployment/templates

openstack overcloud deploy --templates $THT \
-r ~/openstack_deployment/roles/roles_data.yaml \
-e $THT/environments/network-isolation.yaml \
-e $THT/environments/network-management.yaml \
-e $THT/environments/storage-environment.yaml \
-e $THT/environments/tls-endpoints-public-ip.yaml \
-e ~/openstack_deployment/environments/nodes.yaml \
-e ~/openstack_deployment/environments/network-environment.yaml \
-e ~/openstack_deployment/environments/disk-layout.yaml \
-e ~/openstack_deployment/environments/neutron-settings.yaml \
-e ~/openstack_deployment/environments/enable-tls.yaml \
-e ~/openstack_deployment/environments/inject-trust-anchor.yaml \
--log-file overcloud_deployment.log &> overcloud_install.log
  
Note: if you're converting from non-ssl to ssl please apply the fix for BZ#1395124. It's a regression that has been introduced in the latest build and it's affecting non-ssl to ssl conversion as well.

[1] http://docs.openstack.org/developer/tripleo-docs/advanced_deployment/ssl.html#certificate-details

It seems like this is resolved, any opposition to closing it?