Description of problem: sriov-dp-admission-controller fails due to empty pod.ObjectMeta.Namespace in body when pod is created without explicit namespace Here's the full reproducer: ## Reproducer: ### With oc The oc client will inject the namespace no matter what - in order to reproduce this, first create a pod with oc and with --loglevel=10 and look at the curl POST and its data: ~~~ [root@openshift-jumpserver-0 ~]# cat sriovpodc.yaml apiVersion: v1 kind: Pod metadata: name: sriovpodc annotations: k8s.v1.cni.cncf.io/networks: '[ { "name": "sriov-net-ens6f1-netdev", "ips": ["192.168.10.12/24", "2001::12/64"] } ]' spec: containers: - name: sample-container image: centos:8 imagePullPolicy: IfNotPresent command: ["sleep", "infinity"] ~~~ ~~~ [root@openshift-jumpserver-0 ~]# oc create --loglevel=10 -f sriovpodc.yaml I0511 16:17:59.036802 2323309 loader.go:375] Config loaded from file: /root/openshift-install/auth/kubeconfig I0511 16:17:59.039486 2323309 cached_discovery.go:114] returning cached discovery info from /root/.kube/cache/discovery/api.ipi_cluster.example.com_6443/servergroups.json I(...) I0511 16:17:59.041254 2323309 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/api.ipi_cluster.example.com_6443/migration.k8s.io/v1alpha1/serverresources.json I0511 16:17:59.043190 2323309 request.go:1097] Request Body: {"apiVersion":"v1","kind":"Pod","metadata":{"annotations":{"k8s.v1.cni.cncf.io/networks":"[ { \"name\": \"sriov-net-ens6f1-netdev\", \"ips\": [\"192.168.10.12/24\", \"2001::12/64\"] } ]"},"name":"sriovpodc","namespace":"sriov-testing"},"spec":{"containers":[{"command":["sleep","infinity"],"image":"centos:8","imagePullPolicy":"IfNotPresent","name":"sample-container"}]}} I0511 16:17:59.043277 2323309 round_trippers.go:423] curl -k -v -XPOST -H "Accept: application/json" -H "Content-Type: application/json" -H "User-Agent: oc/4.6.0 (linux/amd64) kubernetes/c3dc785" 'https://api.ipi-cluster.example.com:6443/api/v1/namespaces/sriov-testing/pods?fieldManager=kubectl-create' I0511 16:17:59.132660 2323309 round_trippers.go:443] POST https://api.ipi-cluster.example.com:6443/api/v1/namespaces/sriov-testing/pods?fieldManager=kubectl-create 201 Created in 89 milliseconds I0511 16:17:59.132696 2323309 round_trippers.go:449] Response Headers: I0511 16:17:59.132702 2323309 round_trippers.go:452] Audit-Id: b52f7f43-98cc-485c-a872-d77930ede248 I0511 16:17:59.132715 2323309 round_trippers.go:452] Cache-Control: no-cache, private I0511 16:17:59.132725 2323309 round_trippers.go:452] Content-Type: application/json I0511 16:17:59.132730 2323309 round_trippers.go:452] X-Kubernetes-Pf-Flowschema-Uid: 63e45eb4-07cf-45c8-9614-6c6a54b79cb8 I0511 16:17:59.132739 2323309 round_trippers.go:452] X-Kubernetes-Pf-Prioritylevel-Uid: 2eb5c603-28bf-44fd-b5d0-8e94d5f7de34 I0511 16:17:59.132746 2323309 round_trippers.go:452] Content-Length: 2577 I0511 16:17:59.132752 2323309 round_trippers.go:452] Date: Tue, 11 May 2021 16:24:35 GMT I0511 16:17:59.132816 2323309 request.go:1097] Response Body: {"kind":"Pod","apiVersion":"v1","metadata":{"name":"sriovpodc","namespace":"sriov-testing","selfLink":"/api/v1/namespaces/sriov-testing/pods/sriovpodc","uid":"bf422017-a2ec-4764-b366-f95ad8c23fad","resourceVersion":"16267559","creationTimestamp":"2021-05-11T16:24:35Z","annotations":{"k8s.v1.cni.cncf.io/networks":"[ { \"name\": \"sriov-net-ens6f1-netdev\", \"ips\": [\"192.168.10.12/24\", \"2001::12/64\"] } ]","openshift.io/scc":"anyuid"},"managedFields":[{"manager":"kubectl-create","operation":"Update","apiVersion":"v1","time":"2021-05-11T16:24:35Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:k8s.v1.cni.cncf.io/networks":{}}},"f:spec":{"f:containers":{"k:{\"name\":\"sample-container\"}":{".":{},"f:command":{},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:resources":{},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:terminationGracePeriodSeconds":{}}}}]},"spec":{"volumes":[{"name":"default-token-n2f4s","secret":{"secretName":"default-token-n2f4s","defaultMode":420}},{"name":"podnetinfo","downwardAPI":{"items":[{"path":"labels","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.labels"}},{"path":"annotations","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.annotations"}}],"defaultMode":420}}],"containers":[{"name":"sample-container","image":"centos:8","command":["sleep","infinity"],"resources":{"limits":{"openshift.io/ens6f1Netdev":"1"},"requests":{"openshift.io/ens6f1Netdev":"1"}},"volumeMounts":[{"name":"default-token-n2f4s","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"},{"name":"podnetinfo","mountPath":"/etc/podnetinfo"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"IfNotPresent","securityContext":{"capabilities":{"drop":["MKNOD"]}}}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","serviceAccountName":"default","serviceAccount":"default","securityContext":{"seLinuxOptions":{"level":"s0:c27,c9"}},"imagePullSecrets":[{"name":"default-dockercfg-2dhdn"}],"schedulerName":"default-scheduler","tolerations":[{"key":"node.kubernetes.io/not-ready","operator":"Exists","effect":"NoExecute","tolerationSeconds":300},{"key":"node.kubernetes.io/unreachable","operator":"Exists","effect":"NoExecute","tolerationSeconds":300}],"priority":0,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Pending","qosClass":"BestEffort"}} pod/sriovpodc created ~~~ See above: ~~~ I0511 16:17:59.043190 2323309 request.go:1097] Request Body: {"apiVersion":"v1","kind":"Pod","metadata":{"annotations":{"k8s.v1.cni.cncf.io/networks":"[ { \"name\": \"sriov-net-ens6f1-netdev\", \"ips\": [\"192.168.10.12/24\", \"2001::12/64\"] } ]"},"name":"sriovpodc","namespace":"sriov-testing"},"spec":{"containers":[{"command":["sleep","infinity"],"image":"centos:8","imagePullPolicy":"IfNotPresent","name":"sample-container"}]}} ~~~ ### Without oc, with curl, with namespace Now, delete the pod. Then, reproduce the exact same test again, this time with `oc proxy` and manually with curl. ~~~ [root@openshift-jumpserver-0 ~]# oc proxy & [1] 2324267 [root@openshift-jumpserver-0 ~]# Starting to serve on 127.0.0.1:8001 ~~~ Now, run: ~~~ cat <<'EOF' > post-data {"apiVersion":"v1","kind":"Pod","metadata":{"annotations":{"k8s.v1.cni.cncf.io/networks":"[ { \"name\": \"sriov-net-ens6f1-netdev\", \"ips\": [\"192.168.10.12/24\", \"2001::12/64\"] } ]"},"name":"sriovpodc","namespace":"sriov-testing"},"spec":{"containers":[{"command":["sleep","infinity"],"image":"centos:8","imagePullPolicy":"IfNotPresent","name":"sample-container"}]}} EOF curl -k -v -XPOST -H "Accept: application/json" -H "Content-Type: application/json" -H "User-Agent: oc/4.6.0 (linux/amd64) kubernetes/c3dc785" -d @post-data '127.0.0.1:8001/api/v1/namespaces/sriov-testing/pods?fieldManager=kubectl-create' ~~~ This will succeed: ~~~ [root@openshift-jumpserver-0 ~]# cat <<'EOF' > post-data > {"apiVersion":"v1","kind":"Pod","metadata":{"annotations":{"k8s.v1.cni.cncf.io/networks":"[ { \"name\": \"sriov-net-ens6f1-netdev\", \"ips\": [\"192.168.10.12/24\", \"2001::12/64\"] } ]"},"name":"sriovpodc","namespace":"sriov-testing"},"spec":{"containers":[{"command":["sleep","infinity"],"image":"centos:8","imagePullPolicy":"IfNotPresent","name":"sample-container"}]}} > EOF [root@openshift-jumpserver-0 ~]# curl -k -v -XPOST -H "Accept: application/json" -H "Content-Type: application/json" -H "User-Agent: oc/4.6.0 (linux/amd64) kubernetes/c3dc785" -d @post-data '127.0.0.1:8001/api/v1/namespaces/sriov-testing/pods?fieldManager=kubectl-create' Note: Unnecessary use of -X or --request, POST is already inferred. * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8001 (#0) > POST /api/v1/namespaces/sriov-testing/pods?fieldManager=kubectl-create HTTP/1.1 > Host: 127.0.0.1:8001 > Accept: application/json > Content-Type: application/json > User-Agent: oc/4.6.0 (linux/amd64) kubernetes/c3dc785 > Content-Length: 371 > * upload completely sent off: 371 out of 371 bytes < HTTP/1.1 201 Created < Audit-Id: 41b1070a-886d-4227-81a9-3429c680413e < Cache-Control: no-cache, private < Content-Length: 2577 < Content-Type: application/json < Date: Tue, 11 May 2021 17:04:02 GMT < X-Kubernetes-Pf-Flowschema-Uid: 63e45eb4-07cf-45c8-9614-6c6a54b79cb8 < X-Kubernetes-Pf-Prioritylevel-Uid: 2eb5c603-28bf-44fd-b5d0-8e94d5f7de34 < {"kind":"Pod","apiVersion":"v1","metadata":{"name":"sriovpodc","namespace":"sriov-testing","selfLink":"/api/v1/namespaces/sriov-testing/pods/sriovpodc","uid":"3fc52367-f44b-4e47-8379-7bd1fd34d96e","resourceVersion":"16281051","creationTimestamp":"2021-05-11T17:04:02Z","annotations":{"k8s.v1.cni.cncf.io/networks":"[ { \"name\": \"sriov-net-ens6f1-netdev\", \"ips\": [\"192.168.10.12/24\", \"2001::12/64\"] } ]","openshift.io/scc":"anyuid"},"managedFields":[{"manager":"kubectl-create","operation":"Update","apiVersion":"v1","time":"2021-05-11T17:04:02Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:k8s.v1.cni.cncf.io/networks":{}}},"f:spec":{"f:containers":{"k:{\"name\":\"sample-container\"}":{".":{},"f:command":{},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:resources":{},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:terminationGracePeriodSeconds":{}}}}]},"spec":{"volumes":[{"name":"default-token-n2f4s","secret":{"secretName":"default-token-n2f4s","defaultMode":420}},{"name":"podnetinfo","downwardAPI":{"items":[{"path":"labels","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.labels"}},{"path":"annotations","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.annotations"}}],"defaultMode":420}}],"containers":[{"name":"sample-container","image":"centos:8","command":["sleep","infinity"],"resources":{"limits":{"openshift.io/ens6f1Netdev":"1"},"requests":{"openshift.io/ens6f1Netdev":"1"}},"volumeMounts":[{"name":"default-token-n2f4s","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"},{"name":"podnetinfo","mountPath":"/etc/podnetinfo"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"IfNotPresent","securityContext":{"capabilities":{"drop":["MKNOD"]}}}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","serviceAccountName":"default","serviceAccount":"default","securityContext":{"seLinuxOptions":{"level":"s0:c27,c9"}},"imagePullSecrets":[{"name":"default-dockercfg-2dhdn"}],"schedulerName":"default-scheduler","tolerations":[{"key":"node.kubernetes.io/not-ready","operator":"Exists","effect":"NoExecute","tolerationSeconds":300},{"key":"node.kubernetes.io/unreachable","operator":"Exists","effect":"NoExecute","tolerationSeconds":300}],"priority":0,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Pending","qosClass":"BestEffort"}} * Connection #0 to host 127.0.0.1 left intact [root@openshift-jumpserver-0 ~]# ~~~ ~~~ [root@openshift-jumpserver-0 ~]# oc get pods NAME READY STATUS RESTARTS AGE sriovpodc 1/1 Running 0 18s ~~~ ### Without oc, with curl, without namespace Now, delete the pod. This time, remove the namespace from the post-data: ~~~ cat <<'EOF' > post-data {"apiVersion":"v1","kind":"Pod","metadata":{"annotations":{"k8s.v1.cni.cncf.io/networks":"[ { \"name\": \"sriov-net-ens6f1-netdev\", \"ips\": [\"192.168.10.12/24\", \"2001::12/64\"] } ]"},"name":"sriovpodc"},"spec":{"containers":[{"command":["sleep","infinity"],"image":"centos:8","imagePullPolicy":"IfNotPresent","name":"sample-container"}]}} EOF curl -k -v -XPOST -H "Accept: application/json" -H "Content-Type: application/json" -H "User-Agent: oc/4.6.0 (linux/amd64) kubernetes/c3dc785" -d @post-data '127.0.0.1:8001/api/v1/namespaces/sriov-testing/pods?fieldManager=kubectl-create' ~~~ And this fails: ~~~ [root@openshift-jumpserver-0 ~]# oc get pods No resources found in sriov-testing namespace. [root@openshift-jumpserver-0 ~]# cat <<'EOF' > post-data > {"apiVersion":"v1","kind":"Pod","metadata":{"annotations":{"k8s.v1.cni.cncf.io/networks":"[ { \"name\": \"sriov-net-ens6f1-netdev\", \"ips\": [\"192.168.10.12/24\", \"2001::12/64\"] } ]"},"name":"sriovpodc"},"spec":{"containers":[{"command":["sleep","infinity"],"image":"centos:8","imagePullPolicy":"IfNotPresent","name":"sample-container"}]}} > EOF [root@openshift-jumpserver-0 ~]# curl -k -v -XPOST -H "Accept: application/json" -H "Content-Type: application/json" -H "User-Agent: oc/4.6.0 (linux/amd64) kubernetes/c3dc785" -d @post-data '127.0.0.1:8001/api/v1/namespaces/sriov-testing/pods?fieldManager=kubectl-create' Note: Unnecessary use of -X or --request, POST is already inferred. * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8001 (#0) > POST /api/v1/namespaces/sriov-testing/pods?fieldManager=kubectl-create HTTP/1.1 > Host: 127.0.0.1:8001 > Accept: application/json > Content-Type: application/json > User-Agent: oc/4.6.0 (linux/amd64) kubernetes/c3dc785 > Content-Length: 343 > * upload completely sent off: 343 out of 343 bytes < HTTP/1.1 400 Bad Request < Audit-Id: e211d659-2d33-44d1-ab62-ade7fec86a0e < Cache-Control: no-cache, private < Content-Length: 326 < Content-Type: application/json < Date: Tue, 11 May 2021 17:06:01 GMT < X-Kubernetes-Pf-Flowschema-Uid: 63e45eb4-07cf-45c8-9614-6c6a54b79cb8 < X-Kubernetes-Pf-Prioritylevel-Uid: 2eb5c603-28bf-44fd-b5d0-8e94d5f7de34 < {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"admission webhook \"network-resources-injector-config.k8s.io\" denied the request: could not find network attachment definition '/sriov-net-ens6f1-netdev': could not get Network Attachment Definition /sriov-net-ens6f1-netdev: unknown","code":400} * Connection #0 to host 127.0.0.1 left intact [root@openshift-jumpserver-0 ~]# oc get pods No resources found in sriov-testing namespace. ~~~ The resources-network-injector pods will complain with: ~~~ [root@openshift-jumpserver-0 ~]# for p in $(oc get pods -n openshift-sriov-network-operator -o name | grep network-resources-injector) ; do oc logs -n openshift-sriov-network-operator $p --tail=0 -f & done [1] 2324567 [2] 2324568 [3] 2324569 [root@openshift-jumpserver-0 ~]# I0511 17:06:02.362079 1 webhook.go:411] Received mutation request E0511 17:06:02.374924 1 webhook.go:315] could not get Network Attachment Definition /sriov-net-ens6f1-netdev: unknown I0511 17:06:02.376827 1 webhook.go:440] could not find network attachment definition '/sriov-net-ens6f1-netdev': could not get Network Attachment Definition /sriov-net-ens6f1-netdev: unknown I0511 17:06:02.376858 1 webhook.go:336] sending response to the Kubernetes API server [root@openshift-jumpserver-0 ~]# fg oc logs -n openshift-sriov-network-operator $p --tail=0 -f ^C [root@openshift-jumpserver-0 ~]# ^C [root@openshift-jumpserver-0 ~]# fg oc logs -n openshift-sriov-network-operator $p --tail=0 -f ^C [root@openshift-jumpserver-0 ~]# fg oc logs -n openshift-sriov-network-operator $p --tail=0 -f ^C ~~~ Version-Release number of selected component (if applicable): How reproducible: Steps to Reproduce: 1. 2. 3. Actual results: Expected results: Additional info:
I think that the SRIOV admission controller is making an invalid assumption here: ~~~ 409 // MutateHandler handles AdmissionReview requests and sends responses back to the K8s API server 410 func MutateHandler(w http.ResponseWriter, req *http.Request) { 411 glog.Infof("Received mutation request") 412 413 /* read AdmissionReview from the HTTP request */ 414 ar, httpStatus, err := readAdmissionReview(req) 415 if err != nil { 416 http.Error(w, err.Error(), httpStatus) 417 return 418 } 419 420 /* read pod annotations */ 421 /* if networks missing skip everything */ 422 pod, err := deserializePod(ar) 423 if err != nil { 424 handleValidationError(w, ar, err) 425 return 426 } 427 if netSelections, exists := pod.ObjectMeta.Annotations[networksAnnotationKey]; exists && netSelections != "" { 428 /* map of resources request needed by a pod and a number of them */ 429 resourceRequests := make(map[string]int64) 430 431 /* unmarshal list of network selection objects */ 432 networks, _ := parsePodNetworkSelections(netSelections, pod.ObjectMeta.Namespace) ~~~ We only extract the HTTP body: ~~~ ar, httpStatus, err := readAdmissionReview(req) ~~~ Here: ~~~ 73 func readAdmissionReview(req *http.Request) (*v1beta1.AdmissionReview, int, error) { 74 var body []byte 75 76 if req.Body != nil { 77 if data, err := ioutil.ReadAll(req.Body); err == nil { 78 body = data 79 } 80 } 81 82 if len(body) == 0 { 83 err := errors.New("Error reading HTTP request: empty body") 84 glog.Errorf("%s", err) 85 return nil, http.StatusBadRequest, err 86 } 87 88 /* validate HTTP request headers */ 89 contentType := req.Header.Get("Content-Type") 90 if contentType != "application/json" { 91 err := errors.Errorf("Invalid Content-Type='%s', expected 'application/json'", contentType) 92 glog.Errorf("%v", err) 93 return nil, http.StatusUnsupportedMediaType, err 94 } 95 96 /* read AdmissionReview from the request body */ 97 ar, err := deserializeAdmissionReview(body) 98 if err != nil { 99 err := errors.Wrap(err, "error deserializing AdmissionReview") 100 glog.Errorf("%v", err) 101 return nil, http.StatusBadRequest, err 102 } 103 104 return ar, http.StatusOK, nil 105 } ~~~ However, kubernetes can implicitly determine a pod's namespace from the HTTP POST (from /api/v1/namespaces/sriov-testing/pods, we know that this is namespace "sriov-testing"): ~~~ [root@openshift-jumpserver-0 ~]# curl -k -v -XPOST -H "Accept: application/json" -H "Content-Type: application/json" -H "User-Agent: oc/4.6.0 (linux/amd64) kubernetes/c3dc785" -d @post-data '127.0.0.1:8001/api/v1/namespaces/sriov-testing/pods?fieldManager=kubectl-create' Note: Unnecessary use of -X or --request, POST is already inferred. * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8001 (#0) > POST /api/v1/namespaces/sriov-testing/pods?fieldManager=kubectl-create HTTP/1.1 > Host: 127.0.0.1:8001 > Accept: application/json > Content-Type: application/json > User-Agent: oc/4.6.0 (linux/amd64) kubernetes/c3dc785 > Content-Length: 343 > ~~~ The question is: does this work *without* the SRIOV admission webhook in the way? ~~~ cat <<'EOF' > post-data {"apiVersion":"v1","kind":"Pod","metadata":{"name":"normalpod"},"spec":{"containers":[{"command":["sleep","infinity"],"image":"centos:8","imagePullPolicy":"IfNotPresent","name":"sample-container"}]}} EOF curl -k -v -XPOST -H "Accept: application/json" -H "Content-Type: application/json" -H "User-Agent: oc/4.6.0 (linux/amd64) kubernetes/c3dc785" -d @post-data '127.0.0.1:8001/api/v1/namespaces/normalpod/pods?fieldManager=kubectl-create' ~~~ And indeed, it does: ~~~ [root@openshift-jumpserver-0 ~]# oc get pods No resources found in normalpod namespace. [root@openshift-jumpserver-0 ~]# cat <<'EOF' > post-data > {"apiVersion":"v1","kind":"Pod","metadata":{"name":"normalpod"},"spec":{"containers":[{"command":["sleep","infinity"],"image":"centos:8","imagePullPolicy":"IfNotPresent","name":"sample-container"}]}} > EOF [root@openshift-jumpserver-0 ~]# curl -k -v -XPOST -H "Accept: application/json" -H "Content-Type: application/json" -H "User-Agent: oc/4.6.0 (linux/amd64) kubernetes/c3dc785" -d @post-data '127.0.0.1:8001/api/v1/namespaces/normalpod/pods?fieldManager=kubectl-create' Note: Unnecessary use of -X or --request, POST is already inferred. * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8001 (#0) > POST /api/v1/namespaces/normalpod/pods?fieldManager=kubectl-create HTTP/1.1 > Host: 127.0.0.1:8001 > Accept: application/json > Content-Type: application/json > User-Agent: oc/4.6.0 (linux/amd64) kubernetes/c3dc785 > Content-Length: 199 > * upload completely sent off: 199 out of 199 bytes < HTTP/1.1 201 Created < Audit-Id: d8f90c06-76ea-487e-a9b9-77960c8d7466 < Cache-Control: no-cache, private < Content-Length: 1994 < Content-Type: application/json < Date: Tue, 11 May 2021 17:24:04 GMT < X-Kubernetes-Pf-Flowschema-Uid: 63e45eb4-07cf-45c8-9614-6c6a54b79cb8 < X-Kubernetes-Pf-Prioritylevel-Uid: 2eb5c603-28bf-44fd-b5d0-8e94d5f7de34 < {"kind":"Pod","apiVersion":"v1","metadata":{"name":"normalpod","namespace":"normalpod","selfLink":"/api/v1/namespaces/normalpod/pods/normalpod","uid":"5160a336-ac64-49c9-9e4f-559de735845e","resourceVersion":"16287824","creationTimestamp":"2021-05-11T17:24:04Z","annotations":{"openshift.io/scc":"anyuid"},"managedFields":[{"manager":"kubectl-create","operation":"Update","apiVersion":"v1","time":"2021-05-11T17:24:04Z","fieldsType":"FieldsV1","fieldsV1":{"f:spec":{"f:containers":{"k:{\"name\":\"sample-container\"}":{".":{},"f:command":{},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:resources":{},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:terminationGracePeriodSeconds":{}}}}]},"spec":{"volumes":[{"name":"default-token-jd48h","secret":{"secretName":"default-token-jd48h","defaultMode":420}}],"containers":[{"name":"sample-container","image":"centos:8","command":["sleep","infinity"],"resources":{},"volumeMounts":[{"name":"default-token-jd48h","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"IfNotPresent","securityContext":{"capabilities":{"drop":["MKNOD"]}}}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","serviceAccountName":"default","serviceAccount":"default","securityContext":{"seLinuxOptions":{"level":"s0:c27,c14"}},"imagePullSecrets":[{"name":"default-dockercfg-98vbq"}],"schedulerName":"default-scheduler","tolerations":[{"key":"node.kubernetes.io/not-ready","operator":"Exists","effect":"NoExecute","tolerationSeconds":300},{"key":"node.kubernetes.io/unreachable","operator":"Exists","effect":"NoExecute","tolerationSeconds":300}],"priority":0,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Pending","qosClass":"BestEffort"}} * Connection #0 to host 127.0.0.1 left intact [root@openshift-jumpserver-0 ~]# oc get pods NAME READY STATUS RESTARTS AGE normalpod 1/1 Running 0 3s ~~~ So, the request is totally legit, kubernetes can implicitly get the namespace from the Request-URI
Also, just out of curiosity, I checked this here: In case the 2 namespaces (URL and body) do not match, kubernetes will not admit this: ~~~ [root@openshift-jumpserver-0 ~]# cat <<'EOF' > post-data > {"apiVersion":"v1","kind":"Pod","metadata":{"name":"normalpod", "namespace": "test"},"spec":{"containers":[{"command":["sleep","infinity"],"image":"centos:8","imagePullPolicy":"IfNotPresent","name":"sample-container"}]}} > EOF [root@openshift-jumpserver-0 ~]# curl -k -v -XPOST -H "Accept: application/json" -H "Content-Type: application/json" -H "User-Agent: oc/4.6.0 (linux/amd64) kubernetes/c3dc785" -d @post-data '127.0.0.1:8001/api/v1/namespaces/normalpod/pods?fieldManager=kubectl-create' Note: Unnecessary use of -X or --request, POST is already inferred. * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8001 (#0) > POST /api/v1/namespaces/normalpod/pods?fieldManager=kubectl-create HTTP/1.1 > Host: 127.0.0.1:8001 > Accept: application/json > Content-Type: application/json > User-Agent: oc/4.6.0 (linux/amd64) kubernetes/c3dc785 > Content-Length: 220 > * upload completely sent off: 220 out of 220 bytes < HTTP/1.1 400 Bad Request < Audit-Id: a5987fbe-5964-4941-bfb1-f33f90ebe599 < Cache-Control: no-cache, private < Content-Length: 200 < Content-Type: application/json < Date: Tue, 11 May 2021 17:42:12 GMT < X-Kubernetes-Pf-Flowschema-Uid: 63e45eb4-07cf-45c8-9614-6c6a54b79cb8 < X-Kubernetes-Pf-Prioritylevel-Uid: 2eb5c603-28bf-44fd-b5d0-8e94d5f7de34 < {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"the namespace of the provided object does not match the namespace sent on the request","reason":"BadRequest","code":400} * Connection #0 to host 127.0.0.1 left intact ~~~
I have reproduced the issue following the steps in description. Pod could be created successfully when "With oc" and "Without oc, with curl, with namespace". But when testing creating pod Without oc, with curl, without namespace, the results are different when the post-data with ipv6 ip and without ipv6 ip. ### without ipv6 ip, pod could be created successfully. [root@f21-h01-000-r640 ying]# cat <<'EOF' > post-data {"apiVersion":"v1","kind":"Pod","metadata":{"annotations":{"k8s.v1.cni.cncf.io/networks":"[ { \"name\": \"sriovnet\", \"ips\": \"192.168.10.12/24\" } ]"},"name":"sriovpodc"},"spec":{"containers":[{"command":["sleep","infinity"],"image":"centos:8","imagePullPolicy":"IfNotPresent","name":"sample-container"}]}} EOF [root@f21-h01-000-r640 ying]# curl -k -v -XPOST -H "Accept: application/json" -H "Content-Type: application/json" -H "User-Agent: oc/4.6.0 (linux/amd64) kubernetes/c3dc785" -d @post-data '127.0.0.1:8001/api/v1/namespaces/sriov-testing/pods?fieldManager=kubectl-create' Note: Unnecessary use of -X or --request, POST is already inferred. * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8001 (#0) > POST /api/v1/namespaces/sriov-testing/pods?fieldManager=kubectl-create HTTP/1.1 > Host: 127.0.0.1:8001 > Accept: application/json > Content-Type: application/json > User-Agent: oc/4.6.0 (linux/amd64) kubernetes/c3dc785 > Content-Length: 309 > * upload completely sent off: 309 out of 309 bytes < HTTP/1.1 201 Created < Audit-Id: 6c6b3a6a-77ad-4543-b25f-0cc7080f0ccd < Cache-Control: no-cache, private < Content-Length: 2108 < Content-Type: application/json < Date: Fri, 14 May 2021 09:45:02 GMT < X-Kubernetes-Pf-Flowschema-Uid: 7a11cb04-d19f-4625-8c76-2ea0198f9709 < X-Kubernetes-Pf-Prioritylevel-Uid: 33385cde-f1e8-4c04-bedd-b0551ed6af78 < {"kind":"Pod","apiVersion":"v1","metadata":{"name":"sriovpodc","namespace":"sriov-testing","uid":"2d4ce43c-9941-4d4b-8206-56040abd3716","resourceVersion":"8853559","creationTimestamp":"2021-05-14T09:45:02Z","annotations":{"k8s.v1.cni.cncf.io/networks":"[ { \"name\": \"sriovnet\", \"ips\": \"192.168.10.12/24\" } ]","openshift.io/scc":"anyuid"},"managedFields":[{"manager":"kubectl-create","operation":"Update","apiVersion":"v1","time":"2021-05-14T09:45:02Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:k8s.v1.cni.cncf.io/networks":{}}},"f:spec":{"f:containers":{"k:{\"name\":\"sample-container\"}":{".":{},"f:command":{},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:resources":{},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:terminationGracePeriodSeconds":{}}}}]},"spec":{"volumes":[{"name":"default-token-rs785","secret":{"secretName":"default-token-rs785","defaultMode":420}}],"containers":[{"name":"sample-container","image":"centos:8","command":["sleep","infinity"],"resources":{},"volumeMounts":[{"name":"default-token-rs785","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"IfNotPresent","securityContext":{"capabilities":{"drop":["MKNOD"]}}}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","serviceAccountName":"default","serviceAccount":"default","securityContext":{"seLinuxOptions":{"level":"s0:c26,c5"}},"imagePullSecrets":[{"name":"default-dockercfg-pjgzb"}],"schedulerName":"default-scheduler","tolerations":[{"key":"node.kubernetes.io/not-ready","operator":"Exists","effect":"NoExecute","tolerationSeconds":300},{"key":"node.kubernetes.io/unreachable","operator":"Exists","effect":"NoExecute","tolerationSeconds":300}],"priority":0,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Pending","qosClass":"BestEffort"}} ### with ipv6 ip, pod creation was failed [root@f21-h01-000-r640 ying]# cat <<'EOF' > post-data {"apiVersion":"v1","kind":"Pod","metadata":{"annotations":{"k8s.v1.cni.cncf.io/networks":"[ { \"name\": \"sriovnet\", \"ips\": [\"192.168.10.12/24\", \"2001::12/64\"] } ]"},"name":"sriovpodc"},"spec":{"containers":[{"command":["sleep","infinity"],"image":"centos:8","imagePullPolicy":"IfNotPresent","name":"sample-container"}]}} EOF [root@f21-h01-000-r640 ying]# oc delete pod/sriovpodc pod "sriovpodc" deleted [root@f21-h01-000-r640 ying]# curl -k -v -XPOST -H "Accept: application/json" -H "Content-Type: application/json" -H "User-Agent: oc/4.6.0 (linux/amd64) kubernetes/c3dc785" -d @post-data '127.0.0.1:8001/api/v1/namespaces/sriov-testing/pods?fieldManager=kubectl-create' Note: Unnecessary use of -X or --request, POST is already inferred. * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8001 (#0) > POST /api/v1/namespaces/sriov-testing/pods?fieldManager=kubectl-create HTTP/1.1 > Host: 127.0.0.1:8001 > Accept: application/json > Content-Type: application/json > User-Agent: oc/4.6.0 (linux/amd64) kubernetes/c3dc785 > Content-Length: 328 > * upload completely sent off: 328 out of 328 bytes < HTTP/1.1 400 Bad Request < Audit-Id: e5817a78-a9c7-42da-88f9-b3d44b3fa28d < Cache-Control: no-cache, private < Content-Length: 296 < Content-Type: application/json < Date: Fri, 14 May 2021 10:07:43 GMT < X-Kubernetes-Pf-Flowschema-Uid: 7a11cb04-d19f-4625-8c76-2ea0198f9709 < X-Kubernetes-Pf-Prioritylevel-Uid: 33385cde-f1e8-4c04-bedd-b0551ed6af78 < {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"admission webhook \"network-resources-injector-config.k8s.io\" denied the request: could not find network attachment definition '/sriovnet': could not get Network Attachment Definition /sriovnet: unknown","code":400} * Connection #0 to host 127.0.0.1 left intact SRIOV network is configured as below: [root@f21-h01-000-r640 ying]# cat sriovnet.yaml apiVersion: sriovnetwork.openshift.io/v1 kind: SriovNetwork metadata: name: sriovnet namespace: openshift-sriov-network-operator spec: resourceName: ens3f0vf1115 networkNamespace: sriov-testing ipam: |- { "type": "host-local", "subnet": "192.168.10.0/24", "rangeStart": "192.168.10.2", "rangeEnd": "192.168.10.10", "gateway": "192.168.10.1" } [root@f21-h01-000-r640 ying]#
additional info of comment#4 [root@f21-h01-000-r640 ~]# oc version Client Version: 4.8.0-0.nightly-2021-04-23-131610 Server Version: 4.8.0-0.nightly-2021-05-10-002052 Kubernetes Version: v1.21.0-rc.0+86f0080 [root@f21-h01-000-r640 ~]#
upstream PR: https://github.com/k8snetworkplumbingwg/network-resources-injector/pull/100
I verified on sriov-network-operator.4.8.0-202106030339, creating pod Without oc, with curl, without namespace still failed with error "could not get Network Attachment Definition /sriovnet1" [root@dell-per740-36 ~]# oc get csv -n openshift-sriov-network-operator NAME DISPLAY VERSION REPLACES PHASE sriov-network-operator.4.8.0-202106030339 SR-IOV Network Operator 4.8.0-202106030339 sriov-network-operator.4.8.0-202106021449 Succeeded [root@dell-per740-36 ~]# [root@dell-per740-36 ~]# oc version Client Version: 4.6.0-202012172338.p0-02c1100 Server Version: 4.8.0-0.nightly-2021-06-01-231204 Kubernetes Version: v1.21.0-rc.0+4b2b6ff
(In reply to Ying Wang from comment #8) > I verified on sriov-network-operator.4.8.0-202106030339, creating pod > Without oc, with curl, without namespace still failed with error "could not > get Network Attachment Definition /sriovnet1" > Could you attach the log from network-resources-injector pods (three pods on the master nodes)?
(In reply to zenghui.shi from comment #9) > (In reply to Ying Wang from comment #8) > > I verified on sriov-network-operator.4.8.0-202106030339, creating pod > > Without oc, with curl, without namespace still failed with error "could not > > get Network Attachment Definition /sriovnet1" > > > > Could you attach the log from network-resources-injector pods (three pods on > the master nodes)? Hi Zenghui, I retried on build below, it succeeded this time. Do you still need the logs? [root@dell-per740-36 yingwang]# oc version Client Version: 4.8.0-0.nightly-2021-04-23-131610 Server Version: 4.8.0-0.nightly-2021-06-13-101614 Kubernetes Version: v1.21.0-rc.0+120883f [root@dell-per740-36 yingwang]# [root@dell-per740-36 yingwang]# [root@dell-per740-36 yingwang]# [root@dell-per740-36 yingwang]# oc get csv -n openshift-sriov-network-operator NAME DISPLAY VERSION REPLACES PHASE sriov-network-operator.4.8.0-202106152230 SR-IOV Network Operator 4.8.0-202106152230 sriov-network-operator.4.8.0-202106102328 Succeeded Without oc, with curl, without namespace, creating pod succeeded. [root@dell-per740-36 yingwang]# cat <<'EOF' > post-data > {"apiVersion":"v1","kind":"Pod","metadata":{"annotations":{"k8s.v1.cni.cncf.io/networks":"[ { \"name\": \"sriovnet2\", \"ips\": [\"192.168.10.12/24\", \"2001::12/64\"] } ]"},"name":"sriovpodc"},"spec":{"containers":[{"command":["sleep","infinity"],"image":"centos","imagePullPolicy":"IfNotPresent","name":"sample-container"}]}} > EOF [root@dell-per740-36 yingwang]# [root@dell-per740-36 yingwang]# [root@dell-per740-36 yingwang]# [root@dell-per740-36 yingwang]# curl -k -v -XPOST -H "Accept: application/json" -H "Content-Type: application/json" -H "User-Agent: oc/4.8.0 (linux/amd64) kubernetes/a5f16de" -d @post-data '127.0.0.1:8001/api/v1/namespaces/sriov-testing/pods?fieldManager=kubectl-create' * About to connect() to 127.0.0.1 port 8001 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8001 (#0) > POST /api/v1/namespaces/sriov-testing/pods?fieldManager=kubectl-create HTTP/1.1 > Host: 127.0.0.1:8001 > Accept: application/json > Content-Type: application/json > User-Agent: oc/4.8.0 (linux/amd64) kubernetes/a5f16de > Content-Length: 327 > * upload completely sent off: 327 out of 327 bytes < HTTP/1.1 201 Created < Audit-Id: 761d4a56-a24e-46ac-94b3-2408d6fd5421 < Cache-Control: no-cache, private < Content-Length: 2784 < Content-Type: application/json < Date: Thu, 17 Jun 2021 09:06:50 GMT < X-Kubernetes-Pf-Flowschema-Uid: 64222950-9dd7-41f5-b025-d5132db8bc7f < X-Kubernetes-Pf-Prioritylevel-Uid: d80debc0-a182-4d4b-84ad-51a5667e28fe < {"kind":"Pod","apiVersion":"v1","metadata":{"name":"sriovpodc","namespace":"sriov-testing","uid":"fb116c23-0c26-470a-b7d4-eb391920bdbf","resourceVersion":"860450","creationTimestamp":"2021-06-17T09:06:50Z","annotations":{"k8s.v1.cni.cncf.io/networks":"[ { \"name\": \"sriovnet2\", \"ips\": [\"192.168.10.12/24\", \"2001::12/64\"] } ]","openshift.io/scc":"anyuid"},"managedFields":[{"manager":"kubectl-create","operation":"Update","apiVersion":"v1","time":"2021-06-17T09:06:50Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:k8s.v1.cni.cncf.io/networks":{}}},"f:spec":{"f:containers":{"k:{\"name\":\"sample-container\"}":{".":{},"f:command":{},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:resources":{},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:terminationGracePeriodSeconds":{}}}}]},"spec":{"volumes":[{"name":"kube-api-access-xq7cw","projected":{"sources":[{"serviceAccountToken":{"expirationSeconds":3607,"path":"token"}},{"configMap":{"name":"kube-root-ca.crt","items":[{"key":"ca.crt","path":"ca.crt"}]}},{"downwardAPI":{"items":[{"path":"namespace","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}]}},{"configMap":{"name":"openshift-service-ca.crt","items":[{"key":"service-ca.crt","path":"service-ca.crt"}]}}],"defaultMode":420}},{"name":"podnetinfo","downwardAPI":{"items":[{"path":"annotations","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.annotations"}}],"defaultMode":420}}],"containers":[{"name":"sample-container","image":"centos","command":["sleep","infinity"],"resources":{"limits":{"openshift.io/ens1f0vf":"1"},"requests":{"openshift.io/ens1f0vf":"1"}},"volumeMounts":[{"name":"kube-api-access-xq7cw","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"},{"name":"podnetinfo","readOnly":true,"mountPath":"/etc/podnetinfo"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"IfNotPresent","securityContext":{"capabilities":{"drop":["MKNOD"]}}}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","serviceAccountName":"default","serviceAccount":"default","securityContext":{"seLinuxOptions":{"level":"s0:c27,c9"}},"imagePullSecrets":[{"name":"default-dockercfg-nrtv9"}],"schedulerName":"default-scheduler","tolerations":[{"key":"node.kubernetes.io/not-ready","operator":"Exists","effect":"NoExecute","tolerationSeconds":300},{"key":"node.kubernetes.io/unreachable","operator":"Exists","effect":"NoExecute","tolerationSeconds":300}],"priority":0,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Pending","qosClass":"BestEffort"}} * Connection #0 to host 127.0.0.1 left intact [root@dell-per740-36 yingwang]# oc get pods NAME READY STATUS RESTARTS AGE sriovpodc 1/1 Running 0 7s