Description of problem:
configuration drive isn't returning the same information as the metadata services and nova-join fails to enroll new nodes because of the following code logic:

     if ! get_metadata_config_drive; then
        if ! get_metadata_network; then
            echo \"FATAL: No metadata available\"
            exit 1
        fi
     fi

as you can see with the following outputs:



{
  "static": {
    "cloud-init": "#cloud-config\npackages:\n - python-simplejson\n - ipa-client\n - ipa-admintools\n - openldap-clients\n - hostname\nwrite_files:\n - content: |\n     #!/bin/sh\n     \n     function get_metadata_config_drive {\n         if [ -f /run/cloud-init/status.json ]; then\n             # Get metadata from config drive\n             data=`cat /run/cloud-init/status.json`\n             config_drive=`echo $data | python -c 'import json,re,sys;obj=json.load(sys.stdin);ds=obj.get(\"v1\", {}).get(\"datasource\"); print(re.findall(r\"source=(.*)]\", ds)[0])'`\n             if [[ -b $config_drive ]]; then\n                 temp_dir=`mktemp -d`\n                 mount $config_drive $temp_dir\n                 if [ -f $temp_dir/openstack/latest/vendor_data2.json ]; then\n                     data=`cat $temp_dir/openstack/latest/vendor_data2.json`\n                     umount $config_drive\n                     rmdir $temp_dir\n                 else\n                     umount $config_drive\n                     rmdir $temp_dir\n                 fi\n             else \n                 echo \"Unable to retrieve metadata from config drive.\"\n                 return 1\n             fi\n         else\n             echo \"Unable to retrieve metadata from config drive.\"\n             return 1\n         fi\n     \n         return 0\n     }\n     \n     function get_metadata_network {\n         # Get metadata over the network\n         data=$(timeout 300 /bin/bash -c 'data=\"\"; while [ -z \"$data\" ]; do sleep $[ ( $RANDOM % 10 )  + 1 ]s; data=`curl -s http://169.254.169.254/openstack/2016-10-06/vendor_data2.json 2>/dev/null`; done; echo $data')\n     \n         if [[ $? != 0 ]] ; then\n             echo \"Unable to retrieve metadata from metadata service.\"\n             return 1\n         fi\n     }\n     \n     \n     if ! get_metadata_config_drive; then\n        if ! get_metadata_network; then\n            echo \"FATAL: No metadata available\"\n            exit 1\n        fi\n     fi\n     \n     # Get the instance hostname out of the metadata\n     fqdn=`echo $data | python -c 'import json,sys;obj=json.load(sys.stdin);print(obj.get(\"join\", {}).get(\"hostname\", \"\"))'`\n      \n     if [ -z \"$fqdn\" ]; then\n         echo \"Unable to determine hostname\"\n         exit 1\n     fi\n      \n     realm=`echo $data | python -c 'import json,sys;obj=json.load(sys.stdin);print(obj.get(\"join\", {}).get(\"krb_realm\", \"\"))'`\n     otp=`echo $data | python -c 'import json,sys;obj=json.load(sys.stdin);print(obj.get(\"join\", {}).get(\"ipaotp\", \"\"))'`\n     \n     hostname=`/bin/hostname -f`\n      \n     # run ipa-client-install\n     OPTS=\"-U -w $otp\"\n     if [ $hostname != $fqdn ]; then\n         OPTS=\"$OPTS --hostname $fqdn\"\n     fi\n     if [ -n \"$realm\" ]; then\n         OPTS=\"$OPTS --realm=$realm\"\n     fi\n     ipa-client-install $OPTS\n   path: /root/setup-ipa-client.sh\n   permissions: '0700'\n   owner: root:root\nruncmd:\n- sh -x /root/setup-ipa-client.sh > /var/log/setup-ipa-client.log 2>&1"
  },
  "join": {
    "krb_realm": "IDM.localdomain",
    "hostname": "overcloud-controller-1.idm.localdomain"
  }
}

versus:


[root@overcloud-controller-1 ~]# curl -s http://169.254.169.254/openstack/2016-10-06/vendor_data2.json | jq .
{
  "static": {
    "cloud-init": "#cloud-config\npackages:\n - python-simplejson\n - ipa-client\n - ipa-admintools\n - openldap-clients\n - hostname\nwrite_files:\n - content: |\n     #!/bin/sh\n     \n     function get_metadata_config_drive {\n         if [ -f /run/cloud-init/status.json ]; then\n             # Get metadata from config drive\n             data=`cat /run/cloud-init/status.json`\n             config_drive=`echo $data | python -c 'import json,re,sys;obj=json.load(sys.stdin);ds=obj.get(\"v1\", {}).get(\"datasource\"); print(re.findall(r\"source=(.*)]\", ds)[0])'`\n             if [[ -b $config_drive ]]; then\n                 temp_dir=`mktemp -d`\n                 mount $config_drive $temp_dir\n                 if [ -f $temp_dir/openstack/latest/vendor_data2.json ]; then\n                     data=`cat $temp_dir/openstack/latest/vendor_data2.json`\n                     umount $config_drive\n                     rmdir $temp_dir\n                 else\n                     umount $config_drive\n                     rmdir $temp_dir\n                 fi\n             else \n                 echo \"Unable to retrieve metadata from config drive.\"\n                 return 1\n             fi\n         else\n             echo \"Unable to retrieve metadata from config drive.\"\n             return 1\n         fi\n     \n         return 0\n     }\n     \n     function get_metadata_network {\n         # Get metadata over the network\n         data=$(timeout 300 /bin/bash -c 'data=\"\"; while [ -z \"$data\" ]; do sleep $[ ( $RANDOM % 10 )  + 1 ]s; data=`curl -s http://169.254.169.254/openstack/2016-10-06/vendor_data2.json 2>/dev/null`; done; echo $data')\n     \n         if [[ $? != 0 ]] ; then\n             echo \"Unable to retrieve metadata from metadata service.\"\n             return 1\n         fi\n     }\n     \n     \n     if ! get_metadata_config_drive; then\n        if ! get_metadata_network; then\n            echo \"FATAL: No metadata available\"\n            exit 1\n        fi\n     fi\n     \n     # Get the instance hostname out of the metadata\n     fqdn=`echo $data | python -c 'import json,sys;obj=json.load(sys.stdin);print(obj.get(\"join\", {}).get(\"hostname\", \"\"))'`\n      \n     if [ -z \"$fqdn\" ]; then\n         echo \"Unable to determine hostname\"\n         exit 1\n     fi\n      \n     realm=`echo $data | python -c 'import json,sys;obj=json.load(sys.stdin);print(obj.get(\"join\", {}).get(\"krb_realm\", \"\"))'`\n     otp=`echo $data | python -c 'import json,sys;obj=json.load(sys.stdin);print(obj.get(\"join\", {}).get(\"ipaotp\", \"\"))'`\n     \n     hostname=`/bin/hostname -f`\n      \n     # run ipa-client-install\n     OPTS=\"-U -w $otp\"\n     if [ $hostname != $fqdn ]; then\n         OPTS=\"$OPTS --hostname $fqdn\"\n     fi\n     if [ -n \"$realm\" ]; then\n         OPTS=\"$OPTS --realm=$realm\"\n     fi\n     ipa-client-install $OPTS\n   path: /root/setup-ipa-client.sh\n   permissions: '0700'\n   owner: root:root\nruncmd:\n- sh -x /root/setup-ipa-client.sh > /var/log/setup-ipa-client.log 2>&1"
  },
  "join": {
    "krb_realm": "IDM.localdomain",
    "ipaotp": "41cca7447d054e5eaa06b100dac38629", 
    "hostname": "overcloud-controller-1.idm.localdomain"
  }
}


the ipaotp value is missing in the later one.





Version-Release number of selected component (if applicable):
Latest

How reproducible:
All nodes

Steps to Reproduce:
1. Deploy this overcloud in this environment.
2.
3.

Actual results:
Nodes fail to enroll with IPA

Expected results:
No issues

Additional info: