Bug 1883018

Description of problem:
Manila csi storage can not be used on the SSC OSP16 cluster

Version-Release number of selected component (if applicable):
4.6.0-0.nightly-2020-09-26-202331

How reproducible:
Always

Steps to Reproduce:
1. Install an OCP cluster on a ssc OSP16 cluster
2. Create PVC with manila csi sc
3. Create a Pod using this PVC

Actual results:
Pod can not be run successfully

  Warning  FailedMount  <invalid> (x2 over 110s)    kubelet            Unable to attach or mount volumes: unmounted volumes=[local], unattached volumes=[local default-token-86269]: timed out waiting for the condition
  Warning  FailedMount  <invalid> (x11 over 3m53s)  kubelet            MountVolume.SetUp failed for volume "pvc-5fd86a54-74d9-4195-b7ee-1a64a2a7144d" : rpc error: code = Unauthenticated desc = failed to create Manila v2 client: failed to authenticate: failed to read and parse /etc/kubernetes/static-pod-resources/configmaps/cloud-config/ca-bundle.pem certificate: open /etc/kubernetes/static-pod-resources/configmaps/cloud-config/ca-bundle.pem: no such file or directory

$ oc -n openshift-manila-csi-driver exec openstack-manila-csi-nodeplugin-mf7ss -c csi-driver -- ls /etc/kubernetes/
ls: cannot access '/etc/kubernetes/': No such file or directory
command terminated with exit code 2


Expected results:
Pod can be run successfully.

Master Log:

Node Log (of failed PODs):

PV Dump:

PVC Dump:

StorageClass Dump (if StorageClass used by PV/PVC):

Additional info:
Looks we difined cert volume, but didn't use it.
$ oc -n openshift-manila-csi-driver get pod openstack-manila-csi-nodeplugin-mf7ss -o json|jq .spec
{
  "affinity": {
    "nodeAffinity": {
      "requiredDuringSchedulingIgnoredDuringExecution": {
        "nodeSelectorTerms": [
          {
            "matchFields": [
              {
                "key": "metadata.name",
                "operator": "In",
                "values": [
                  "piqin-0928-xwgzw-worker-0-zvk4d"
                ]
              }
            ]
          }
        ]
      }
    }
  },
  "containers": [
    {
      "args": [
        "--v=2",
        "--nodeid=$(NODE_ID)",
        "--endpoint=$(CSI_ENDPOINT)",
        "--drivername=$(DRIVER_NAME)",
        "--share-protocol-selector=$(MANILA_SHARE_PROTO)",
        "--fwdendpoint=$(FWD_CSI_ENDPOINT)"
      ],
      "env": [
        {
          "name": "DRIVER_NAME",
          "value": "manila.csi.openstack.org"
        },
        {
          "name": "NODE_ID",
          "valueFrom": {
            "fieldRef": {
              "apiVersion": "v1",
              "fieldPath": "spec.nodeName"
            }
          }
        },
        {
          "name": "CSI_ENDPOINT",
          "value": "unix:///var/lib/kubelet/plugins/manila.csi.openstack.org/csi.sock"
        },
        {
          "name": "FWD_CSI_ENDPOINT",
          "value": "unix:///var/lib/kubelet/plugins/csi-nfsplugin/csi.sock"
        },
        {
          "name": "MANILA_SHARE_PROTO",
          "value": "NFS"
        }
      ],
      "image": "quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ec8d3a17d052484ac20a1cf9d145ebcfd4fbbc6edaaff92f8d064191f462f926",
      "imagePullPolicy": "IfNotPresent",
      "name": "csi-driver",
      "resources": {
        "requests": {
          "cpu": "10m",
          "memory": "50Mi"
        }
      },
      "securityContext": {
        "privileged": true
      },
      "terminationMessagePath": "/dev/termination-log",
      "terminationMessagePolicy": "File",
      "volumeMounts": [
        {
          "mountPath": "/var/lib/kubelet/plugins/manila.csi.openstack.org",
          "name": "plugin-dir"
        },
        {
          "mountPath": "/var/lib/kubelet/plugins/csi-nfsplugin",
          "name": "fwd-plugin-dir"
        },
        {
          "mountPath": "/usr/share/pki/ca-trust-source",
          "name": "cacert"
        },
        {
          "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount",
          "name": "manila-csi-driver-node-sa-token-fvrzq",
          "readOnly": true
        }
      ]
    },
    {
      "args": [
        "--v=2",
        "--csi-address=/csi/csi.sock",
        "--kubelet-registration-path=/var/lib/kubelet/plugins/manila.csi.openstack.org/csi.sock"
      ],
      "env": [
        {
          "name": "KUBE_NODE_NAME",
          "valueFrom": {
            "fieldRef": {
              "apiVersion": "v1",
              "fieldPath": "spec.nodeName"
            }
          }
        }
      ],
      "image": "quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5e8681dcb767b9d52776ae2aba625aba535056b422d4494ff76e2f56e0a21be0",
      "imagePullPolicy": "IfNotPresent",
      "lifecycle": {
        "preStop": {
          "exec": {
            "command": [
              "/bin/sh",
              "-c",
              "rm -rf /var/lib/kubelet/plugins/manila.csi.openstack.org/csi.sock"
            ]
          }
        }
      },
      "name": "csi-node-driver-registrar",
      "resources": {
        "requests": {
          "cpu": "5m",
          "memory": "20Mi"
        }
      },
      "securityContext": {
        "privileged": true
      },
      "terminationMessagePath": "/dev/termination-log",
      "terminationMessagePolicy": "File",
      "volumeMounts": [
        {
          "mountPath": "/csi",
          "name": "plugin-dir"
        },
        {
          "mountPath": "/registration",
          "name": "registration-dir"
        },
        {
          "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount",
          "name": "manila-csi-driver-node-sa-token-fvrzq",
          "readOnly": true
        }
      ]
    }
  ],
  "dnsPolicy": "ClusterFirstWithHostNet",
  "enableServiceLinks": true,
  "hostNetwork": true,
  "imagePullSecrets": [
    {
      "name": "manila-csi-driver-node-sa-dockercfg-r78gd"
    }
  ],
  "nodeName": "piqin-0928-xwgzw-worker-0-zvk4d",
  "preemptionPolicy": "PreemptLowerPriority",
  "priority": 2000001000,
  "priorityClassName": "system-node-critical",
  "restartPolicy": "Always",
  "schedulerName": "default-scheduler",
  "securityContext": {},
  "serviceAccount": "manila-csi-driver-node-sa",
  "serviceAccountName": "manila-csi-driver-node-sa",
  "terminationGracePeriodSeconds": 30,
  "tolerations": [
    {
      "operator": "Exists"
    }
  ],
  "volumes": [
    {
      "hostPath": {
        "path": "/var/lib/kubelet/plugins_registry/",
        "type": "Directory"
      },
      "name": "registration-dir"
    },
    {
      "hostPath": {
        "path": "/var/lib/kubelet/plugins/manila.csi.openstack.org",
        "type": "DirectoryOrCreate"
      },
      "name": "plugin-dir"
    },
    {
      "hostPath": {
        "path": "/var/lib/kubelet/plugins/csi-nfsplugin",
        "type": "DirectoryOrCreate"
      },
      "name": "fwd-plugin-dir"
    },
    {
      "configMap": {
        "defaultMode": 420,
        "items": [
          {
            "key": "ca-bundle.pem",
            "path": "ca-bundle.pem"
          }
        ],
        "name": "cloud-provider-config",
        "optional": true
      },
      "name": "cacert"
    },
    {
      "name": "manila-csi-driver-node-sa-token-fvrzq",
      "secret": {
        "defaultMode": 420,
        "secretName": "manila-csi-driver-node-sa-token-fvrzq"
      }
    }
  ]
}