Bug 1743402

Description of problem: While trying to replace controller nodes, after redeploying overcloud, deployment failed on "Failed to start memcached-w4dspvrv container." error


Steps to Reproduce:
1. Reproduction steps attached (controller-replacement.log)
2. Deploy OSP 15 with 3 controllers, 2 compute
3. [root@controller-0 ~]# ls -la /var/log/containers/nova/ |grep manage
-rw-------.  1 42436 42436    91905 Aug 16 23:43 nova-manage.log
[root@controller-0 ~]# rm /var/log/containers/nova/nova-manage.log
rm: remove regular file '/var/log/containers/nova/nova-manage.log'? yes
[root@controller-0 ~]# ls -la /var/log/containers/nova/ |grep manage
[root@controller-0 ~]# docker exec -it -u root nova_api nova-manage db version
Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg.
391
[root@controller-0 ~]# ls -la /var/log/containers/nova/ |grep manage
-rw-r--r--.  1 root  root         0 Aug 17 20:33 nova-manage.log

4. Follow guide (Chapter 14. Replacing Controller Nodes)14.3 specifically [1] 
5. Redeploy overcloud 

[1] https://access.redhat.com/documentation/en-us/red_hat_openstack_platform/15-beta/html-single/director_installation_and_usage/index#preparing-the-cluster-for-controller-replacement

Actual results:
TASK [Debug output for task: Start containers for step 1] **********************                                                                                                                                                    [189/1905]
Saturday 17 August 2019  17:29:24 -0400 (0:00:15.795)       0:15:21.052 *******
fatal: [controller-1]: FAILED! => {
    "failed_when_result": true,
    "outputs.stdout_lines | default([]) | union(outputs.stderr_lines | default([]))": [                                                                                                                                                      
        "stdout: aad88dc4b754806593a8e4722028ea60287aa320e25bb7e9dfd8ca460d9616d9",
        "",
        "stderr: Trying to pull 192.168.24.1:8787/rhosp15/openstack-mariadb:20190815.1...Getting image source signatures",
        "Copying blob sha256:e61d8721e62e50814b162c8341bb235d3453b9c95bd26439bf9100fcf88338c7",
        "Copying blob sha256:c585fd5093c62ee42a56af6c09813ac8384d8145c4285bc62c357b41224b1970",
        "Copying blob sha256:305e8a83063c1cc33517b21472472f9e213387f47be0996ce4f2aeeb1f004486",
        "Copying blob sha256:f31644cdb030c658b0d6603affab726881a90a2bc0dff8729f2e85321eb0899a",
        "Copying config sha256:aad88dc4b754806593a8e4722028ea60287aa320e25bb7e9dfd8ca460d9616d9",
        "Writing manifest to image destination",
        "Storing signatures",                                                                                                                                                                                                                
        "stdout: 721514fcd22b506794578a5fe494c0937fc4aabc1f6a618e5b10f49eae348490",                                                                                                                                                          
        "stderr: Trying to pull 192.168.24.1:8787/rhosp15/openstack-memcached:20190815.1...Getting image source signatures",                                                                                                                 
        "Copying blob sha256:5f7e9eb9bee5b3dd0b676a79077c102fc564d2ea98c092d3ffc344d899067307",
        "Copying config sha256:721514fcd22b506794578a5fe494c0937fc4aabc1f6a618e5b10f49eae348490",                                                                                                                                            
        "stdout: 214660d3aad60dac1132c01c78211392e1f6670948c87592607a457b704de807",                                                                                                                                                          
        "stderr: Trying to pull 192.168.24.1:8787/rhosp15/openstack-rabbitmq:20190815.1...Getting image source signatures",                                                                                                                  
        "Copying blob sha256:46430179a92aa221117e5f2af5738c2c357c4fb911b3d068df4135d47d1efecf",                                                                                                                                              
        "Copying config sha256:214660d3aad60dac1132c01c78211392e1f6670948c87592607a457b704de807",
        "No systemd unit file was found for tripleo_mysql_bootstrap.service",                                                                                                                                                                
        "No systemd unit file was found for tripleo_mysql_bootstrap_healthcheck.service",                                                                                                                                                    
        "No systemd unit file was found for tripleo_mysql_bootstrap_healthcheck.timer",                                                                                                                                                      
        "Removed /etc/systemd/system/multi-user.target.wants/tripleo_memcached.service.",
        "Removed /etc/systemd/system/timers.target.wants/tripleo_memcached_healthcheck.timer.",
        "Removed /etc/systemd/system/tripleo_memcached.service.requires/tripleo_memcached_healthcheck.timer.",
        "stdout: c68424af4f8840ec126d46d434a3372b5da3d01333435992370b5ac8ddc11ab1",
        "stderr: ",                                                                                                                                                                                                                          
        "Created symlink /etc/systemd/system/multi-user.target.wants/tripleo_memcached-w4dspvrv.service → /etc/systemd/system/tripleo_memcached-w4dspvrv.service.",                                                                          
        "Job for tripleo_memcached-w4dspvrv.service failed because the service did not take the steps required by its unit configuration.",                                                                                                  
        "See \"systemctl status tripleo_memcached-w4dspvrv.service\" and \"journalctl -xe\" for details.",                                                                                                                                   
        "systemctl failed",                                                                                                                                                                                                                  
        "Traceback (most recent call last):",                                                                                                                                                                                                
        "  File \"/usr/lib/python3.6/site-packages/paunch/utils/systemctl.py\", line 31, in systemctl",                                                                                                                                      
        "    subprocess.check_call(cmd)",
        "  File \"/usr/lib64/python3.6/subprocess.py\", line 311, in check_call",
        "    raise CalledProcessError(retcode, cmd)",
        "subprocess.CalledProcessError: Command '['systemctl', 'enable', '--now', 'tripleo_memcached-w4dspvrv']' returned non-zero exit status 1.",                                                                                          
        "During handling of the above exception, another exception occurred:",
        "  File \"/usr/lib/python3.6/site-packages/paunch/utils/systemd.py\", line 111, in service_create",
        "    systemctl.enable(service, now=True)",
        "  File \"/usr/lib/python3.6/site-packages/paunch/utils/systemctl.py\", line 53, in enable",
        "    systemctl(cmd, log)",
        "  File \"/usr/lib/python3.6/site-packages/paunch/utils/systemctl.py\", line 33, in systemctl",
        "    raise SystemctlException(str(err))",
        "paunch.utils.systemctl.SystemctlException: Command '['systemctl', 'enable', '--now', 'tripleo_memcached-w4dspvrv']' returned non-zero exit status 1.",                                                                              
        "Command '['systemctl', 'enable', '--now', 'tripleo_memcached-w4dspvrv']' returned non-zero exit status 1."


Expected results:
Overcloud deployment passed

Additional info: