Bug 1959520 - sriov-dp-admission-controller fails due to empty pod.ObjectMeta.Namespace in body when pod is created without explicit namespace
Summary: sriov-dp-admission-controller fails due to empty pod.ObjectMeta.Namespace in ...
Keywords:
Status: CLOSED CURRENTRELEASE
Alias: None
Product: OpenShift Container Platform
Classification: Red Hat
Component: Networking
Version: 4.6
Hardware: Unspecified
OS: Unspecified
high
high
Target Milestone: ---
: 4.8.0
Assignee: zenghui.shi
QA Contact: Ying Wang
URL:
Whiteboard: Telco
Depends On:
Blocks:
TreeView+ depends on / blocked
 
Reported: 2021-05-11 17:39 UTC by Andreas Karis
Modified: 2024-12-20 20:03 UTC (History)
5 users (show)

Fixed In Version:
Doc Type: Bug Fix
Doc Text:
Clone Of:
Environment:
Last Closed: 2021-08-23 09:42:02 UTC
Target Upstream Version:
Embargoed:


Attachments (Terms of Use)


Links
System ID Private Priority Status Summary Last Updated
Github openshift sriov-dp-admission-controller pull 37 0 None open Bug 1959520: Ignore AdmissionReview request with empty namespace 2021-06-01 03:24:43 UTC

Description Andreas Karis 2021-05-11 17:39:14 UTC
Description of problem:

sriov-dp-admission-controller fails due to empty pod.ObjectMeta.Namespace in body when pod is created without explicit namespace

Here's the full reproducer:

## Reproducer:

### With oc

The oc client will inject the namespace no matter what - in order to reproduce this, first create a pod with oc and with --loglevel=10 and look at the curl POST and its data:
~~~
[root@openshift-jumpserver-0 ~]# cat sriovpodc.yaml 
apiVersion: v1
kind: Pod
metadata:
  name: sriovpodc
  annotations:
    k8s.v1.cni.cncf.io/networks: '[
        {
                "name": "sriov-net-ens6f1-netdev",
                "ips": ["192.168.10.12/24", "2001::12/64"]
        }
]'
spec:
  containers:
  - name: sample-container
    image: centos:8
    imagePullPolicy: IfNotPresent
    command: ["sleep", "infinity"]
~~~

~~~
[root@openshift-jumpserver-0 ~]# oc create --loglevel=10 -f sriovpodc.yaml 
I0511 16:17:59.036802 2323309 loader.go:375] Config loaded from file:  /root/openshift-install/auth/kubeconfig
I0511 16:17:59.039486 2323309 cached_discovery.go:114] returning cached discovery info from /root/.kube/cache/discovery/api.ipi_cluster.example.com_6443/servergroups.json
I(...)
I0511 16:17:59.041254 2323309 cached_discovery.go:71] returning cached discovery info from /root/.kube/cache/discovery/api.ipi_cluster.example.com_6443/migration.k8s.io/v1alpha1/serverresources.json
I0511 16:17:59.043190 2323309 request.go:1097] Request Body: {"apiVersion":"v1","kind":"Pod","metadata":{"annotations":{"k8s.v1.cni.cncf.io/networks":"[ { \"name\": \"sriov-net-ens6f1-netdev\", \"ips\": [\"192.168.10.12/24\", \"2001::12/64\"] } ]"},"name":"sriovpodc","namespace":"sriov-testing"},"spec":{"containers":[{"command":["sleep","infinity"],"image":"centos:8","imagePullPolicy":"IfNotPresent","name":"sample-container"}]}}
I0511 16:17:59.043277 2323309 round_trippers.go:423] curl -k -v -XPOST  -H "Accept: application/json" -H "Content-Type: application/json" -H "User-Agent: oc/4.6.0 (linux/amd64) kubernetes/c3dc785" 'https://api.ipi-cluster.example.com:6443/api/v1/namespaces/sriov-testing/pods?fieldManager=kubectl-create'
I0511 16:17:59.132660 2323309 round_trippers.go:443] POST https://api.ipi-cluster.example.com:6443/api/v1/namespaces/sriov-testing/pods?fieldManager=kubectl-create 201 Created in 89 milliseconds
I0511 16:17:59.132696 2323309 round_trippers.go:449] Response Headers:
I0511 16:17:59.132702 2323309 round_trippers.go:452]     Audit-Id: b52f7f43-98cc-485c-a872-d77930ede248
I0511 16:17:59.132715 2323309 round_trippers.go:452]     Cache-Control: no-cache, private
I0511 16:17:59.132725 2323309 round_trippers.go:452]     Content-Type: application/json
I0511 16:17:59.132730 2323309 round_trippers.go:452]     X-Kubernetes-Pf-Flowschema-Uid: 63e45eb4-07cf-45c8-9614-6c6a54b79cb8
I0511 16:17:59.132739 2323309 round_trippers.go:452]     X-Kubernetes-Pf-Prioritylevel-Uid: 2eb5c603-28bf-44fd-b5d0-8e94d5f7de34
I0511 16:17:59.132746 2323309 round_trippers.go:452]     Content-Length: 2577
I0511 16:17:59.132752 2323309 round_trippers.go:452]     Date: Tue, 11 May 2021 16:24:35 GMT
I0511 16:17:59.132816 2323309 request.go:1097] Response Body: {"kind":"Pod","apiVersion":"v1","metadata":{"name":"sriovpodc","namespace":"sriov-testing","selfLink":"/api/v1/namespaces/sriov-testing/pods/sriovpodc","uid":"bf422017-a2ec-4764-b366-f95ad8c23fad","resourceVersion":"16267559","creationTimestamp":"2021-05-11T16:24:35Z","annotations":{"k8s.v1.cni.cncf.io/networks":"[ { \"name\": \"sriov-net-ens6f1-netdev\", \"ips\": [\"192.168.10.12/24\", \"2001::12/64\"] } ]","openshift.io/scc":"anyuid"},"managedFields":[{"manager":"kubectl-create","operation":"Update","apiVersion":"v1","time":"2021-05-11T16:24:35Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:k8s.v1.cni.cncf.io/networks":{}}},"f:spec":{"f:containers":{"k:{\"name\":\"sample-container\"}":{".":{},"f:command":{},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:resources":{},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:terminationGracePeriodSeconds":{}}}}]},"spec":{"volumes":[{"name":"default-token-n2f4s","secret":{"secretName":"default-token-n2f4s","defaultMode":420}},{"name":"podnetinfo","downwardAPI":{"items":[{"path":"labels","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.labels"}},{"path":"annotations","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.annotations"}}],"defaultMode":420}}],"containers":[{"name":"sample-container","image":"centos:8","command":["sleep","infinity"],"resources":{"limits":{"openshift.io/ens6f1Netdev":"1"},"requests":{"openshift.io/ens6f1Netdev":"1"}},"volumeMounts":[{"name":"default-token-n2f4s","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"},{"name":"podnetinfo","mountPath":"/etc/podnetinfo"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"IfNotPresent","securityContext":{"capabilities":{"drop":["MKNOD"]}}}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","serviceAccountName":"default","serviceAccount":"default","securityContext":{"seLinuxOptions":{"level":"s0:c27,c9"}},"imagePullSecrets":[{"name":"default-dockercfg-2dhdn"}],"schedulerName":"default-scheduler","tolerations":[{"key":"node.kubernetes.io/not-ready","operator":"Exists","effect":"NoExecute","tolerationSeconds":300},{"key":"node.kubernetes.io/unreachable","operator":"Exists","effect":"NoExecute","tolerationSeconds":300}],"priority":0,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Pending","qosClass":"BestEffort"}}
pod/sriovpodc created
~~~

See above:
~~~
I0511 16:17:59.043190 2323309 request.go:1097] Request Body: {"apiVersion":"v1","kind":"Pod","metadata":{"annotations":{"k8s.v1.cni.cncf.io/networks":"[ { \"name\": \"sriov-net-ens6f1-netdev\", \"ips\": [\"192.168.10.12/24\", \"2001::12/64\"] } ]"},"name":"sriovpodc","namespace":"sriov-testing"},"spec":{"containers":[{"command":["sleep","infinity"],"image":"centos:8","imagePullPolicy":"IfNotPresent","name":"sample-container"}]}}
~~~

### Without oc, with curl, with namespace

Now, delete the pod.

Then, reproduce the exact same test again, this time with `oc proxy` and manually with curl.

~~~
[root@openshift-jumpserver-0 ~]# oc proxy &
[1] 2324267
[root@openshift-jumpserver-0 ~]# Starting to serve on 127.0.0.1:8001
~~~

Now, run:
~~~
cat <<'EOF' > post-data
{"apiVersion":"v1","kind":"Pod","metadata":{"annotations":{"k8s.v1.cni.cncf.io/networks":"[ { \"name\": \"sriov-net-ens6f1-netdev\", \"ips\": [\"192.168.10.12/24\", \"2001::12/64\"] } ]"},"name":"sriovpodc","namespace":"sriov-testing"},"spec":{"containers":[{"command":["sleep","infinity"],"image":"centos:8","imagePullPolicy":"IfNotPresent","name":"sample-container"}]}}
EOF
curl -k -v -XPOST  -H "Accept: application/json" -H "Content-Type: application/json" -H "User-Agent: oc/4.6.0 (linux/amd64) kubernetes/c3dc785" -d @post-data '127.0.0.1:8001/api/v1/namespaces/sriov-testing/pods?fieldManager=kubectl-create'
~~~

This will succeed:
~~~
[root@openshift-jumpserver-0 ~]# cat <<'EOF' > post-data
> {"apiVersion":"v1","kind":"Pod","metadata":{"annotations":{"k8s.v1.cni.cncf.io/networks":"[ { \"name\": \"sriov-net-ens6f1-netdev\", \"ips\": [\"192.168.10.12/24\", \"2001::12/64\"] } ]"},"name":"sriovpodc","namespace":"sriov-testing"},"spec":{"containers":[{"command":["sleep","infinity"],"image":"centos:8","imagePullPolicy":"IfNotPresent","name":"sample-container"}]}}
> EOF
[root@openshift-jumpserver-0 ~]# curl -k -v -XPOST  -H "Accept: application/json" -H "Content-Type: application/json" -H "User-Agent: oc/4.6.0 (linux/amd64) kubernetes/c3dc785" -d @post-data '127.0.0.1:8001/api/v1/namespaces/sriov-testing/pods?fieldManager=kubectl-create'
Note: Unnecessary use of -X or --request, POST is already inferred.
*   Trying 127.0.0.1...
* TCP_NODELAY set
* Connected to 127.0.0.1 (127.0.0.1) port 8001 (#0)
> POST /api/v1/namespaces/sriov-testing/pods?fieldManager=kubectl-create HTTP/1.1
> Host: 127.0.0.1:8001
> Accept: application/json
> Content-Type: application/json
> User-Agent: oc/4.6.0 (linux/amd64) kubernetes/c3dc785
> Content-Length: 371
> 
* upload completely sent off: 371 out of 371 bytes
< HTTP/1.1 201 Created
< Audit-Id: 41b1070a-886d-4227-81a9-3429c680413e
< Cache-Control: no-cache, private
< Content-Length: 2577
< Content-Type: application/json
< Date: Tue, 11 May 2021 17:04:02 GMT
< X-Kubernetes-Pf-Flowschema-Uid: 63e45eb4-07cf-45c8-9614-6c6a54b79cb8
< X-Kubernetes-Pf-Prioritylevel-Uid: 2eb5c603-28bf-44fd-b5d0-8e94d5f7de34
< 
{"kind":"Pod","apiVersion":"v1","metadata":{"name":"sriovpodc","namespace":"sriov-testing","selfLink":"/api/v1/namespaces/sriov-testing/pods/sriovpodc","uid":"3fc52367-f44b-4e47-8379-7bd1fd34d96e","resourceVersion":"16281051","creationTimestamp":"2021-05-11T17:04:02Z","annotations":{"k8s.v1.cni.cncf.io/networks":"[ { \"name\": \"sriov-net-ens6f1-netdev\", \"ips\": [\"192.168.10.12/24\", \"2001::12/64\"] } ]","openshift.io/scc":"anyuid"},"managedFields":[{"manager":"kubectl-create","operation":"Update","apiVersion":"v1","time":"2021-05-11T17:04:02Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:k8s.v1.cni.cncf.io/networks":{}}},"f:spec":{"f:containers":{"k:{\"name\":\"sample-container\"}":{".":{},"f:command":{},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:resources":{},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:terminationGracePeriodSeconds":{}}}}]},"spec":{"volumes":[{"name":"default-token-n2f4s","secret":{"secretName":"default-token-n2f4s","defaultMode":420}},{"name":"podnetinfo","downwardAPI":{"items":[{"path":"labels","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.labels"}},{"path":"annotations","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.annotations"}}],"defaultMode":420}}],"containers":[{"name":"sample-container","image":"centos:8","command":["sleep","infinity"],"resources":{"limits":{"openshift.io/ens6f1Netdev":"1"},"requests":{"openshift.io/ens6f1Netdev":"1"}},"volumeMounts":[{"name":"default-token-n2f4s","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"},{"name":"podnetinfo","mountPath":"/etc/podnetinfo"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"IfNotPresent","securityContext":{"capabilities":{"drop":["MKNOD"]}}}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","serviceAccountName":"default","serviceAccount":"default","securityContext":{"seLinuxOptions":{"level":"s0:c27,c9"}},"imagePullSecrets":[{"name":"default-dockercfg-2dhdn"}],"schedulerName":"default-scheduler","tolerations":[{"key":"node.kubernetes.io/not-ready","operator":"Exists","effect":"NoExecute","tolerationSeconds":300},{"key":"node.kubernetes.io/unreachable","operator":"Exists","effect":"NoExecute","tolerationSeconds":300}],"priority":0,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Pending","qosClass":"BestEffort"}}
* Connection #0 to host 127.0.0.1 left intact
[root@openshift-jumpserver-0 ~]# 
~~~

~~~
[root@openshift-jumpserver-0 ~]# oc get pods
NAME        READY   STATUS    RESTARTS   AGE
sriovpodc   1/1     Running   0          18s
~~~

### Without oc, with curl, without namespace

Now, delete the pod. This time, remove the namespace from the post-data:
~~~
cat <<'EOF' > post-data
{"apiVersion":"v1","kind":"Pod","metadata":{"annotations":{"k8s.v1.cni.cncf.io/networks":"[ { \"name\": \"sriov-net-ens6f1-netdev\", \"ips\": [\"192.168.10.12/24\", \"2001::12/64\"] } ]"},"name":"sriovpodc"},"spec":{"containers":[{"command":["sleep","infinity"],"image":"centos:8","imagePullPolicy":"IfNotPresent","name":"sample-container"}]}}
EOF
curl -k -v -XPOST  -H "Accept: application/json" -H "Content-Type: application/json" -H "User-Agent: oc/4.6.0 (linux/amd64) kubernetes/c3dc785" -d @post-data '127.0.0.1:8001/api/v1/namespaces/sriov-testing/pods?fieldManager=kubectl-create'
~~~

And this fails:
~~~
[root@openshift-jumpserver-0 ~]# oc get pods
No resources found in sriov-testing namespace.
[root@openshift-jumpserver-0 ~]# cat <<'EOF' > post-data
> {"apiVersion":"v1","kind":"Pod","metadata":{"annotations":{"k8s.v1.cni.cncf.io/networks":"[ { \"name\": \"sriov-net-ens6f1-netdev\", \"ips\": [\"192.168.10.12/24\", \"2001::12/64\"] } ]"},"name":"sriovpodc"},"spec":{"containers":[{"command":["sleep","infinity"],"image":"centos:8","imagePullPolicy":"IfNotPresent","name":"sample-container"}]}}
> EOF
[root@openshift-jumpserver-0 ~]# curl -k -v -XPOST  -H "Accept: application/json" -H "Content-Type: application/json" -H "User-Agent: oc/4.6.0 (linux/amd64) kubernetes/c3dc785" -d @post-data '127.0.0.1:8001/api/v1/namespaces/sriov-testing/pods?fieldManager=kubectl-create'
Note: Unnecessary use of -X or --request, POST is already inferred.
*   Trying 127.0.0.1...
* TCP_NODELAY set
* Connected to 127.0.0.1 (127.0.0.1) port 8001 (#0)
> POST /api/v1/namespaces/sriov-testing/pods?fieldManager=kubectl-create HTTP/1.1
> Host: 127.0.0.1:8001
> Accept: application/json
> Content-Type: application/json
> User-Agent: oc/4.6.0 (linux/amd64) kubernetes/c3dc785
> Content-Length: 343
> 
* upload completely sent off: 343 out of 343 bytes
< HTTP/1.1 400 Bad Request
< Audit-Id: e211d659-2d33-44d1-ab62-ade7fec86a0e
< Cache-Control: no-cache, private
< Content-Length: 326
< Content-Type: application/json
< Date: Tue, 11 May 2021 17:06:01 GMT
< X-Kubernetes-Pf-Flowschema-Uid: 63e45eb4-07cf-45c8-9614-6c6a54b79cb8
< X-Kubernetes-Pf-Prioritylevel-Uid: 2eb5c603-28bf-44fd-b5d0-8e94d5f7de34
< 
{"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"admission webhook \"network-resources-injector-config.k8s.io\" denied the request: could not find network attachment definition '/sriov-net-ens6f1-netdev': could not get Network Attachment Definition /sriov-net-ens6f1-netdev: unknown","code":400}
* Connection #0 to host 127.0.0.1 left intact
[root@openshift-jumpserver-0 ~]# oc get pods
No resources found in sriov-testing namespace.
~~~

The resources-network-injector pods will complain with:
~~~
[root@openshift-jumpserver-0 ~]# for p in $(oc get pods -n openshift-sriov-network-operator -o name | grep  network-resources-injector) ; do oc logs -n openshift-sriov-network-operator $p --tail=0 -f & done
[1] 2324567
[2] 2324568
[3] 2324569
[root@openshift-jumpserver-0 ~]# I0511 17:06:02.362079       1 webhook.go:411] Received mutation request
E0511 17:06:02.374924       1 webhook.go:315] could not get Network Attachment Definition /sriov-net-ens6f1-netdev: unknown
I0511 17:06:02.376827       1 webhook.go:440] could not find network attachment definition '/sriov-net-ens6f1-netdev': could not get Network Attachment Definition /sriov-net-ens6f1-netdev: unknown
I0511 17:06:02.376858       1 webhook.go:336] sending response to the Kubernetes API server

[root@openshift-jumpserver-0 ~]# fg
oc logs -n openshift-sriov-network-operator $p --tail=0 -f
^C
[root@openshift-jumpserver-0 ~]# ^C
[root@openshift-jumpserver-0 ~]# fg
oc logs -n openshift-sriov-network-operator $p --tail=0 -f
^C
[root@openshift-jumpserver-0 ~]# fg
oc logs -n openshift-sriov-network-operator $p --tail=0 -f
^C
~~~


Version-Release number of selected component (if applicable):


How reproducible:


Steps to Reproduce:
1.
2.
3.

Actual results:


Expected results:


Additional info:

Comment 1 Andreas Karis 2021-05-11 17:39:36 UTC
I think that the SRIOV admission controller is making an invalid assumption here:
~~~
409 // MutateHandler handles AdmissionReview requests and sends responses back to the K8s API server
410 func MutateHandler(w http.ResponseWriter, req *http.Request) {
411         glog.Infof("Received mutation request")
412 
413         /* read AdmissionReview from the HTTP request */
414         ar, httpStatus, err := readAdmissionReview(req)
415         if err != nil {
416                 http.Error(w, err.Error(), httpStatus)
417                 return
418         }
419 
420         /* read pod annotations */
421         /* if networks missing skip everything */
422         pod, err := deserializePod(ar)
423         if err != nil {
424                 handleValidationError(w, ar, err)
425                 return
426         }
427         if netSelections, exists := pod.ObjectMeta.Annotations[networksAnnotationKey]; exists && netSelections != "" {
428                 /* map of resources request needed by a pod and a number of them */
429                 resourceRequests := make(map[string]int64)
430 
431                 /* unmarshal list of network selection objects */
432                 networks, _ := parsePodNetworkSelections(netSelections, pod.ObjectMeta.Namespace)
~~~

We only extract the HTTP body:
~~~
ar, httpStatus, err := readAdmissionReview(req)
~~~

Here:
~~~
 73 func readAdmissionReview(req *http.Request) (*v1beta1.AdmissionReview, int, error) {
 74         var body []byte
 75 
 76         if req.Body != nil {
 77                 if data, err := ioutil.ReadAll(req.Body); err == nil {
 78                         body = data
 79                 }
 80         }
 81 
 82         if len(body) == 0 {
 83                 err := errors.New("Error reading HTTP request: empty body")
 84                 glog.Errorf("%s", err)
 85                 return nil, http.StatusBadRequest, err
 86         }
 87 
 88         /* validate HTTP request headers */
 89         contentType := req.Header.Get("Content-Type")
 90         if contentType != "application/json" {
 91                 err := errors.Errorf("Invalid Content-Type='%s', expected 'application/json'", contentType)
 92                 glog.Errorf("%v", err)
 93                 return nil, http.StatusUnsupportedMediaType, err
 94         }
 95 
 96         /* read AdmissionReview from the request body */
 97         ar, err := deserializeAdmissionReview(body)
 98         if err != nil {
 99                 err := errors.Wrap(err, "error deserializing AdmissionReview")
100                 glog.Errorf("%v", err)
101                 return nil, http.StatusBadRequest, err
102         }
103 
104         return ar, http.StatusOK, nil
105 }
~~~

However, kubernetes can implicitly determine a pod's namespace from the HTTP POST (from /api/v1/namespaces/sriov-testing/pods, we know that this is namespace "sriov-testing"):
~~~
[root@openshift-jumpserver-0 ~]# curl -k -v -XPOST  -H "Accept: application/json" -H "Content-Type: application/json" -H "User-Agent: oc/4.6.0 (linux/amd64) kubernetes/c3dc785" -d @post-data '127.0.0.1:8001/api/v1/namespaces/sriov-testing/pods?fieldManager=kubectl-create'
Note: Unnecessary use of -X or --request, POST is already inferred.
*   Trying 127.0.0.1...
* TCP_NODELAY set
* Connected to 127.0.0.1 (127.0.0.1) port 8001 (#0)
> POST /api/v1/namespaces/sriov-testing/pods?fieldManager=kubectl-create HTTP/1.1
> Host: 127.0.0.1:8001
> Accept: application/json
> Content-Type: application/json
> User-Agent: oc/4.6.0 (linux/amd64) kubernetes/c3dc785
> Content-Length: 343
> 
~~~

The question is: does this work *without* the SRIOV admission webhook in the way?

~~~
cat <<'EOF' > post-data
{"apiVersion":"v1","kind":"Pod","metadata":{"name":"normalpod"},"spec":{"containers":[{"command":["sleep","infinity"],"image":"centos:8","imagePullPolicy":"IfNotPresent","name":"sample-container"}]}}
EOF
curl -k -v -XPOST  -H "Accept: application/json" -H "Content-Type: application/json" -H "User-Agent: oc/4.6.0 (linux/amd64) kubernetes/c3dc785" -d @post-data '127.0.0.1:8001/api/v1/namespaces/normalpod/pods?fieldManager=kubectl-create'
~~~

And indeed, it does:
~~~
[root@openshift-jumpserver-0 ~]# oc get pods
No resources found in normalpod namespace.
[root@openshift-jumpserver-0 ~]# cat <<'EOF' > post-data
> {"apiVersion":"v1","kind":"Pod","metadata":{"name":"normalpod"},"spec":{"containers":[{"command":["sleep","infinity"],"image":"centos:8","imagePullPolicy":"IfNotPresent","name":"sample-container"}]}}
> EOF
[root@openshift-jumpserver-0 ~]# curl -k -v -XPOST  -H "Accept: application/json" -H "Content-Type: application/json" -H "User-Agent: oc/4.6.0 (linux/amd64) kubernetes/c3dc785" -d @post-data '127.0.0.1:8001/api/v1/namespaces/normalpod/pods?fieldManager=kubectl-create'
Note: Unnecessary use of -X or --request, POST is already inferred.
*   Trying 127.0.0.1...
* TCP_NODELAY set
* Connected to 127.0.0.1 (127.0.0.1) port 8001 (#0)
> POST /api/v1/namespaces/normalpod/pods?fieldManager=kubectl-create HTTP/1.1
> Host: 127.0.0.1:8001
> Accept: application/json
> Content-Type: application/json
> User-Agent: oc/4.6.0 (linux/amd64) kubernetes/c3dc785
> Content-Length: 199
> 
* upload completely sent off: 199 out of 199 bytes
< HTTP/1.1 201 Created
< Audit-Id: d8f90c06-76ea-487e-a9b9-77960c8d7466
< Cache-Control: no-cache, private
< Content-Length: 1994
< Content-Type: application/json
< Date: Tue, 11 May 2021 17:24:04 GMT
< X-Kubernetes-Pf-Flowschema-Uid: 63e45eb4-07cf-45c8-9614-6c6a54b79cb8
< X-Kubernetes-Pf-Prioritylevel-Uid: 2eb5c603-28bf-44fd-b5d0-8e94d5f7de34
< 
{"kind":"Pod","apiVersion":"v1","metadata":{"name":"normalpod","namespace":"normalpod","selfLink":"/api/v1/namespaces/normalpod/pods/normalpod","uid":"5160a336-ac64-49c9-9e4f-559de735845e","resourceVersion":"16287824","creationTimestamp":"2021-05-11T17:24:04Z","annotations":{"openshift.io/scc":"anyuid"},"managedFields":[{"manager":"kubectl-create","operation":"Update","apiVersion":"v1","time":"2021-05-11T17:24:04Z","fieldsType":"FieldsV1","fieldsV1":{"f:spec":{"f:containers":{"k:{\"name\":\"sample-container\"}":{".":{},"f:command":{},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:resources":{},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:terminationGracePeriodSeconds":{}}}}]},"spec":{"volumes":[{"name":"default-token-jd48h","secret":{"secretName":"default-token-jd48h","defaultMode":420}}],"containers":[{"name":"sample-container","image":"centos:8","command":["sleep","infinity"],"resources":{},"volumeMounts":[{"name":"default-token-jd48h","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"IfNotPresent","securityContext":{"capabilities":{"drop":["MKNOD"]}}}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","serviceAccountName":"default","serviceAccount":"default","securityContext":{"seLinuxOptions":{"level":"s0:c27,c14"}},"imagePullSecrets":[{"name":"default-dockercfg-98vbq"}],"schedulerName":"default-scheduler","tolerations":[{"key":"node.kubernetes.io/not-ready","operator":"Exists","effect":"NoExecute","tolerationSeconds":300},{"key":"node.kubernetes.io/unreachable","operator":"Exists","effect":"NoExecute","tolerationSeconds":300}],"priority":0,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Pending","qosClass":"BestEffort"}}
* Connection #0 to host 127.0.0.1 left intact
[root@openshift-jumpserver-0 ~]# oc get pods
NAME        READY   STATUS    RESTARTS   AGE
normalpod   1/1     Running   0          3s
~~~

So, the request is totally legit, kubernetes can implicitly get the namespace from the Request-URI

Comment 2 Andreas Karis 2021-05-11 17:45:38 UTC
Also, just out of curiosity, I checked this here:

In case the 2 namespaces (URL and body) do not match, kubernetes will not admit this:
~~~
[root@openshift-jumpserver-0 ~]# cat <<'EOF' > post-data
> {"apiVersion":"v1","kind":"Pod","metadata":{"name":"normalpod", "namespace": "test"},"spec":{"containers":[{"command":["sleep","infinity"],"image":"centos:8","imagePullPolicy":"IfNotPresent","name":"sample-container"}]}}
> EOF
[root@openshift-jumpserver-0 ~]# curl -k -v -XPOST  -H "Accept: application/json" -H "Content-Type: application/json" -H "User-Agent: oc/4.6.0 (linux/amd64) kubernetes/c3dc785" -d @post-data '127.0.0.1:8001/api/v1/namespaces/normalpod/pods?fieldManager=kubectl-create'
Note: Unnecessary use of -X or --request, POST is already inferred.
*   Trying 127.0.0.1...
* TCP_NODELAY set
* Connected to 127.0.0.1 (127.0.0.1) port 8001 (#0)
> POST /api/v1/namespaces/normalpod/pods?fieldManager=kubectl-create HTTP/1.1
> Host: 127.0.0.1:8001
> Accept: application/json
> Content-Type: application/json
> User-Agent: oc/4.6.0 (linux/amd64) kubernetes/c3dc785
> Content-Length: 220
> 
* upload completely sent off: 220 out of 220 bytes
< HTTP/1.1 400 Bad Request
< Audit-Id: a5987fbe-5964-4941-bfb1-f33f90ebe599
< Cache-Control: no-cache, private
< Content-Length: 200
< Content-Type: application/json
< Date: Tue, 11 May 2021 17:42:12 GMT
< X-Kubernetes-Pf-Flowschema-Uid: 63e45eb4-07cf-45c8-9614-6c6a54b79cb8
< X-Kubernetes-Pf-Prioritylevel-Uid: 2eb5c603-28bf-44fd-b5d0-8e94d5f7de34
< 
{"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"the namespace of the provided object does not match the namespace sent on the request","reason":"BadRequest","code":400}
* Connection #0 to host 127.0.0.1 left intact
~~~

Comment 4 Ying Wang 2021-05-14 10:35:15 UTC
I have reproduced the issue following the steps in description.

Pod could be created successfully when "With oc" and "Without oc, with curl, with namespace".

But when testing creating pod Without oc, with curl, without namespace, the results are different when the post-data with ipv6 ip and without ipv6 ip.

### without ipv6 ip, pod could be created successfully.

[root@f21-h01-000-r640 ying]# cat <<'EOF' > post-data                                                                                                                                     {"apiVersion":"v1","kind":"Pod","metadata":{"annotations":{"k8s.v1.cni.cncf.io/networks":"[ { \"name\": \"sriovnet\", \"ips\": \"192.168.10.12/24\" } ]"},"name":"sriovpodc"},"spec":{"containers":[{"command":["sleep","infinity"],"image":"centos:8","imagePullPolicy":"IfNotPresent","name":"sample-container"}]}}
EOF
[root@f21-h01-000-r640 ying]#  curl -k -v -XPOST  -H "Accept: application/json" -H "Content-Type: application/json" -H "User-Agent: oc/4.6.0 (linux/amd64) kubernetes/c3dc785" -d @post-data '127.0.0.1:8001/api/v1/namespaces/sriov-testing/pods?fieldManager=kubectl-create'
Note: Unnecessary use of -X or --request, POST is already inferred.
*   Trying 127.0.0.1...
* TCP_NODELAY set
* Connected to 127.0.0.1 (127.0.0.1) port 8001 (#0)
> POST /api/v1/namespaces/sriov-testing/pods?fieldManager=kubectl-create HTTP/1.1
> Host: 127.0.0.1:8001
> Accept: application/json
> Content-Type: application/json
> User-Agent: oc/4.6.0 (linux/amd64) kubernetes/c3dc785
> Content-Length: 309
> 
* upload completely sent off: 309 out of 309 bytes
< HTTP/1.1 201 Created
< Audit-Id: 6c6b3a6a-77ad-4543-b25f-0cc7080f0ccd
< Cache-Control: no-cache, private
< Content-Length: 2108
< Content-Type: application/json
< Date: Fri, 14 May 2021 09:45:02 GMT
< X-Kubernetes-Pf-Flowschema-Uid: 7a11cb04-d19f-4625-8c76-2ea0198f9709
< X-Kubernetes-Pf-Prioritylevel-Uid: 33385cde-f1e8-4c04-bedd-b0551ed6af78
< 
{"kind":"Pod","apiVersion":"v1","metadata":{"name":"sriovpodc","namespace":"sriov-testing","uid":"2d4ce43c-9941-4d4b-8206-56040abd3716","resourceVersion":"8853559","creationTimestamp":"2021-05-14T09:45:02Z","annotations":{"k8s.v1.cni.cncf.io/networks":"[ { \"name\": \"sriovnet\", \"ips\": \"192.168.10.12/24\" } ]","openshift.io/scc":"anyuid"},"managedFields":[{"manager":"kubectl-create","operation":"Update","apiVersion":"v1","time":"2021-05-14T09:45:02Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:k8s.v1.cni.cncf.io/networks":{}}},"f:spec":{"f:containers":{"k:{\"name\":\"sample-container\"}":{".":{},"f:command":{},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:resources":{},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:terminationGracePeriodSeconds":{}}}}]},"spec":{"volumes":[{"name":"default-token-rs785","secret":{"secretName":"default-token-rs785","defaultMode":420}}],"containers":[{"name":"sample-container","image":"centos:8","command":["sleep","infinity"],"resources":{},"volumeMounts":[{"name":"default-token-rs785","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"IfNotPresent","securityContext":{"capabilities":{"drop":["MKNOD"]}}}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","serviceAccountName":"default","serviceAccount":"default","securityContext":{"seLinuxOptions":{"level":"s0:c26,c5"}},"imagePullSecrets":[{"name":"default-dockercfg-pjgzb"}],"schedulerName":"default-scheduler","tolerations":[{"key":"node.kubernetes.io/not-ready","operator":"Exists","effect":"NoExecute","tolerationSeconds":300},{"key":"node.kubernetes.io/unreachable","operator":"Exists","effect":"NoExecute","tolerationSeconds":300}],"priority":0,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Pending","qosClass":"BestEffort"}}



### with ipv6 ip, pod creation was failed

[root@f21-h01-000-r640 ying]# cat <<'EOF' > post-data                                                                                                                                     {"apiVersion":"v1","kind":"Pod","metadata":{"annotations":{"k8s.v1.cni.cncf.io/networks":"[ { \"name\": \"sriovnet\", \"ips\": [\"192.168.10.12/24\", \"2001::12/64\"] } ]"},"name":"sriovpodc"},"spec":{"containers":[{"command":["sleep","infinity"],"image":"centos:8","imagePullPolicy":"IfNotPresent","name":"sample-container"}]}}
EOF
[root@f21-h01-000-r640 ying]# oc delete pod/sriovpodc
pod "sriovpodc" deleted
[root@f21-h01-000-r640 ying]#  curl -k -v -XPOST  -H "Accept: application/json" -H "Content-Type: application/json" -H "User-Agent: oc/4.6.0 (linux/amd64) kubernetes/c3dc785" -d @post-data '127.0.0.1:8001/api/v1/namespaces/sriov-testing/pods?fieldManager=kubectl-create'
Note: Unnecessary use of -X or --request, POST is already inferred.
*   Trying 127.0.0.1...
* TCP_NODELAY set
* Connected to 127.0.0.1 (127.0.0.1) port 8001 (#0)
> POST /api/v1/namespaces/sriov-testing/pods?fieldManager=kubectl-create HTTP/1.1
> Host: 127.0.0.1:8001
> Accept: application/json
> Content-Type: application/json
> User-Agent: oc/4.6.0 (linux/amd64) kubernetes/c3dc785
> Content-Length: 328
> 
* upload completely sent off: 328 out of 328 bytes
< HTTP/1.1 400 Bad Request
< Audit-Id: e5817a78-a9c7-42da-88f9-b3d44b3fa28d
< Cache-Control: no-cache, private
< Content-Length: 296
< Content-Type: application/json
< Date: Fri, 14 May 2021 10:07:43 GMT
< X-Kubernetes-Pf-Flowschema-Uid: 7a11cb04-d19f-4625-8c76-2ea0198f9709
< X-Kubernetes-Pf-Prioritylevel-Uid: 33385cde-f1e8-4c04-bedd-b0551ed6af78
< 
{"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"admission webhook \"network-resources-injector-config.k8s.io\" denied the request: could not find network attachment definition '/sriovnet': could not get Network Attachment Definition /sriovnet: unknown","code":400}
* Connection #0 to host 127.0.0.1 left intact


SRIOV network is configured as below:
[root@f21-h01-000-r640 ying]# cat sriovnet.yaml
apiVersion: sriovnetwork.openshift.io/v1
kind: SriovNetwork
metadata:
  name: sriovnet
  namespace: openshift-sriov-network-operator
spec:
  resourceName: ens3f0vf1115
  networkNamespace: sriov-testing
  ipam: |-
    {
      "type": "host-local",
      "subnet": "192.168.10.0/24",
      "rangeStart": "192.168.10.2",
      "rangeEnd": "192.168.10.10",
      "gateway": "192.168.10.1"
    }
[root@f21-h01-000-r640 ying]#

Comment 5 Ying Wang 2021-05-17 03:09:14 UTC
additional info of comment#4

[root@f21-h01-000-r640 ~]# oc version
Client Version: 4.8.0-0.nightly-2021-04-23-131610
Server Version: 4.8.0-0.nightly-2021-05-10-002052
Kubernetes Version: v1.21.0-rc.0+86f0080
[root@f21-h01-000-r640 ~]#

Comment 8 Ying Wang 2021-06-04 01:23:58 UTC
I verified on sriov-network-operator.4.8.0-202106030339, creating pod Without oc, with curl, without namespace still failed with error "could not get Network Attachment Definition /sriovnet1"

[root@dell-per740-36 ~]# oc get csv -n openshift-sriov-network-operator
NAME                                        DISPLAY                   VERSION              REPLACES                                    PHASE
sriov-network-operator.4.8.0-202106030339   SR-IOV Network Operator   4.8.0-202106030339   sriov-network-operator.4.8.0-202106021449   Succeeded
[root@dell-per740-36 ~]# 
[root@dell-per740-36 ~]# oc version
Client Version: 4.6.0-202012172338.p0-02c1100
Server Version: 4.8.0-0.nightly-2021-06-01-231204
Kubernetes Version: v1.21.0-rc.0+4b2b6ff

Comment 9 zenghui.shi 2021-06-15 09:29:08 UTC
(In reply to Ying Wang from comment #8)
> I verified on sriov-network-operator.4.8.0-202106030339, creating pod
> Without oc, with curl, without namespace still failed with error "could not
> get Network Attachment Definition /sriovnet1"
> 

Could you attach the log from network-resources-injector pods (three pods on the master nodes)?

Comment 10 Ying Wang 2021-06-17 09:10:49 UTC
(In reply to zenghui.shi from comment #9)
> (In reply to Ying Wang from comment #8)
> > I verified on sriov-network-operator.4.8.0-202106030339, creating pod
> > Without oc, with curl, without namespace still failed with error "could not
> > get Network Attachment Definition /sriovnet1"
> > 
> 
> Could you attach the log from network-resources-injector pods (three pods on
> the master nodes)?

Hi Zenghui,

I retried on build below, it succeeded this time. Do you still need the logs?

[root@dell-per740-36 yingwang]# oc version
Client Version: 4.8.0-0.nightly-2021-04-23-131610
Server Version: 4.8.0-0.nightly-2021-06-13-101614
Kubernetes Version: v1.21.0-rc.0+120883f
[root@dell-per740-36 yingwang]# 
[root@dell-per740-36 yingwang]# 
[root@dell-per740-36 yingwang]# 
[root@dell-per740-36 yingwang]# oc get csv -n openshift-sriov-network-operator 
NAME                                        DISPLAY                   VERSION              REPLACES                                    PHASE
sriov-network-operator.4.8.0-202106152230   SR-IOV Network Operator   4.8.0-202106152230   sriov-network-operator.4.8.0-202106102328   Succeeded


Without oc, with curl, without namespace, creating pod succeeded. 

[root@dell-per740-36 yingwang]# cat <<'EOF' > post-data
> {"apiVersion":"v1","kind":"Pod","metadata":{"annotations":{"k8s.v1.cni.cncf.io/networks":"[ { \"name\": \"sriovnet2\", \"ips\": [\"192.168.10.12/24\", \"2001::12/64\"] } ]"},"name":"sriovpodc"},"spec":{"containers":[{"command":["sleep","infinity"],"image":"centos","imagePullPolicy":"IfNotPresent","name":"sample-container"}]}}
> EOF
[root@dell-per740-36 yingwang]# 
[root@dell-per740-36 yingwang]# 
[root@dell-per740-36 yingwang]# 
[root@dell-per740-36 yingwang]# curl -k -v -XPOST  -H "Accept: application/json" -H "Content-Type: application/json" -H "User-Agent: oc/4.8.0 (linux/amd64) kubernetes/a5f16de" -d @post-data '127.0.0.1:8001/api/v1/namespaces/sriov-testing/pods?fieldManager=kubectl-create'
* About to connect() to 127.0.0.1 port 8001 (#0)
*   Trying 127.0.0.1...
* Connected to 127.0.0.1 (127.0.0.1) port 8001 (#0)
> POST /api/v1/namespaces/sriov-testing/pods?fieldManager=kubectl-create HTTP/1.1
> Host: 127.0.0.1:8001
> Accept: application/json
> Content-Type: application/json
> User-Agent: oc/4.8.0 (linux/amd64) kubernetes/a5f16de
> Content-Length: 327
> 
* upload completely sent off: 327 out of 327 bytes
< HTTP/1.1 201 Created
< Audit-Id: 761d4a56-a24e-46ac-94b3-2408d6fd5421
< Cache-Control: no-cache, private
< Content-Length: 2784
< Content-Type: application/json
< Date: Thu, 17 Jun 2021 09:06:50 GMT
< X-Kubernetes-Pf-Flowschema-Uid: 64222950-9dd7-41f5-b025-d5132db8bc7f
< X-Kubernetes-Pf-Prioritylevel-Uid: d80debc0-a182-4d4b-84ad-51a5667e28fe
< 
{"kind":"Pod","apiVersion":"v1","metadata":{"name":"sriovpodc","namespace":"sriov-testing","uid":"fb116c23-0c26-470a-b7d4-eb391920bdbf","resourceVersion":"860450","creationTimestamp":"2021-06-17T09:06:50Z","annotations":{"k8s.v1.cni.cncf.io/networks":"[ { \"name\": \"sriovnet2\", \"ips\": [\"192.168.10.12/24\", \"2001::12/64\"] } ]","openshift.io/scc":"anyuid"},"managedFields":[{"manager":"kubectl-create","operation":"Update","apiVersion":"v1","time":"2021-06-17T09:06:50Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:k8s.v1.cni.cncf.io/networks":{}}},"f:spec":{"f:containers":{"k:{\"name\":\"sample-container\"}":{".":{},"f:command":{},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:resources":{},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:terminationGracePeriodSeconds":{}}}}]},"spec":{"volumes":[{"name":"kube-api-access-xq7cw","projected":{"sources":[{"serviceAccountToken":{"expirationSeconds":3607,"path":"token"}},{"configMap":{"name":"kube-root-ca.crt","items":[{"key":"ca.crt","path":"ca.crt"}]}},{"downwardAPI":{"items":[{"path":"namespace","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.namespace"}}]}},{"configMap":{"name":"openshift-service-ca.crt","items":[{"key":"service-ca.crt","path":"service-ca.crt"}]}}],"defaultMode":420}},{"name":"podnetinfo","downwardAPI":{"items":[{"path":"annotations","fieldRef":{"apiVersion":"v1","fieldPath":"metadata.annotations"}}],"defaultMode":420}}],"containers":[{"name":"sample-container","image":"centos","command":["sleep","infinity"],"resources":{"limits":{"openshift.io/ens1f0vf":"1"},"requests":{"openshift.io/ens1f0vf":"1"}},"volumeMounts":[{"name":"kube-api-access-xq7cw","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"},{"name":"podnetinfo","readOnly":true,"mountPath":"/etc/podnetinfo"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"IfNotPresent","securityContext":{"capabilities":{"drop":["MKNOD"]}}}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","serviceAccountName":"default","serviceAccount":"default","securityContext":{"seLinuxOptions":{"level":"s0:c27,c9"}},"imagePullSecrets":[{"name":"default-dockercfg-nrtv9"}],"schedulerName":"default-scheduler","tolerations":[{"key":"node.kubernetes.io/not-ready","operator":"Exists","effect":"NoExecute","tolerationSeconds":300},{"key":"node.kubernetes.io/unreachable","operator":"Exists","effect":"NoExecute","tolerationSeconds":300}],"priority":0,"enableServiceLinks":true,"preemptionPolicy":"PreemptLowerPriority"},"status":{"phase":"Pending","qosClass":"BestEffort"}}
* Connection #0 to host 127.0.0.1 left intact

[root@dell-per740-36 yingwang]# oc get pods
NAME        READY   STATUS    RESTARTS   AGE
sriovpodc   1/1     Running   0          7s


Note You need to log in before you can comment on or make changes to this bug.