Confirmed with latest oc client, the issue has fixed:

[root@dhcp-140-138 ~]# oc version -o yaml 
clientVersion:
  buildDate: "2019-12-13T16:45:38Z"
  compiler: gc
  gitCommit: 60ccef2e295e103fa6bd01b9ae5de9061386b948
  gitTreeState: clean
  gitVersion: v4.4.0
  goVersion: go1.12.12
  major: ""
  minor: ""
  platform: linux/amd64


[root@dhcp-140-138 ~]# oc debug -h
Launch a command shell to debug a running application
......
      --to-namespace='': Override the namespace to create the pod into (instead of using --namespace).

[root@dhcp-140-138 ~]# oc debug po/myapp-pod -c init-mydb --keep-init-containers=false
Starting pod/myapp-pod-debug, command was: sh -c until nslookup mydb; do echo waiting for mydb; sleep 2; done;

Pod IP: 10.128.2.15
If you don't see a command prompt, try pressing enter.

/ # 


[root@dhcp-140-138 ~]# oc get po 
NAME              READY   STATUS     RESTARTS   AGE
myapp-pod         1/1     Running    0          2m5s
myapp-pod-debug   0/1     Init:0/1   0          36s
[root@dhcp-140-138 ~]# oc describe po/myapp-pod-debug
Name:         myapp-pod-debug
Namespace:    zhouyt
Priority:     0
....
Init Containers:
  init-mydb:
    Container ID:  cri-o://4bb5485a944d423bf860037f4b953ca48b2a262dbe911da4486503476e7a2fc1
    Image:         busybox:1.28
    Image ID:      docker.io/library/busybox@sha256:141c253bc4c3fd0a201d32dc1f493bcf3fff003b6df416dea4f41046e0f37d47
    Port:          <none>
    Host Port:     <none>
    Command:
      /bin/sh
    State:          Running
      Started:      Mon, 16 Dec 2019 11:02:08 +0800
    Ready:          False
    Restart Count:  0
    Environment:    <none>
    Mounts:
      /var/run/secrets/kubernetes.io/serviceaccount from default-token-qj89w (ro)
Containers:
  myapp-container:
    Container ID:  
    Image:         busybox:1.28
    Image ID:      
    Port:          <none>
    Host Port:     <none>
    Command:
      sh
      -c
      echo The app is running! && sleep 3600
    State:          Waiting
      Reason:       PodInitializing
    Ready:          False
    Restart Count:  0
    Environment:    <none>
    Mounts:
      /var/run/secrets/kubernetes.io/serviceaccount from default-token-qj89w (ro)
Conditions:
  Type              Status
  Initialized       False 
  Ready             False 
  ContainersReady   False 
  PodScheduled      True

Since the problem described in this bug report should be
resolved in a recent advisory, it has been closed with a
resolution of ERRATA.

For information on the advisory, and where to find the updated
files, follow the link below.

If the solution does not work for you, open a new bug report.

https://access.redhat.com/errata/RHBA-2020:0581