Attachment 1839888 Details for Bug 2019931 – VR log from Primary

VR log from Primary

vr.primary.log (text/plain), 61.92 KB, created by Benamar Mekhissi on 2021-11-04 16:14:07 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Benamar Mekhissi

Created: 2021-11-04 16:14:07 UTC

Size: 61.92 KB

patch

obsolete

>I1029 09:02:51.930404       1 request.go:665] Waited for 1.01722575s due to client-side throttling, not priority and fairness, request: GET:https://172.31.0.1:443/apis/operators.coreos.com/v1?timeout=32s
>I1029 09:02:53.907024       1 common.go:111] Probing CSI driver for readiness
>{"level":"info","timestamp":"2021-10-29T09:02:53.908Z","logger":"setup","caller":"app/main.go:160","msg":"starting manager"}
>I1029 09:02:53.908690       1 leaderelection.go:248] attempting to acquire leader lease openshift-storage/openshift-storage.rbd.csi.ceph.comvolume-replication-openshift-storage...
>I1029 09:02:53.921835       1 leaderelection.go:258] successfully acquired lease openshift-storage/openshift-storage.rbd.csi.ceph.comvolume-replication-openshift-storage
>{"level":"info","timestamp":"2021-10-29T09:02:53.922Z","logger":"controller-runtime.manager.controller.volumereplication","caller":"controller/controller.go:221","msg":"Starting EventSource","reconciler group":"replication.storage.openshift.io","reconciler kind":"VolumeReplication","source":"kind source: /, Kind="}
>{"level":"info","timestamp":"2021-10-29T09:02:53.922Z","logger":"controller-runtime.manager.controller.volumereplication","caller":"controller/controller.go:221","msg":"Starting Controller","reconciler group":"replication.storage.openshift.io","reconciler kind":"VolumeReplication"}
>{"level":"info","timestamp":"2021-10-29T09:02:54.022Z","logger":"controller-runtime.manager.controller.volumereplication","caller":"controller/controller.go:221","msg":"Starting workers","reconciler group":"replication.storage.openshift.io","reconciler kind":"VolumeReplication","worker count":1}
>{"level":"info","timestamp":"2021-11-02T11:39:06.599Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume handle","Request.Name":"busybox-pvc-3","Request.Namespace":"busybox-workloads","VolumeHandleName":"0001-0011-openshift-storage-0000000000000001-f036de0c-389c-11ec-bca9-0a580a800241"}
>{"level":"info","timestamp":"2021-11-02T11:39:06.599Z","logger":"controllers.VolumeReplication","caller":"controllers/volumereplication_controller.go:160","msg":"adding finalizer to volumeReplication object","Request.Name":"busybox-pvc-3","Request.Namespace":"busybox-workloads","Finalizer":"replication.storage.openshift.io"}
>{"level":"info","timestamp":"2021-11-02T11:39:06.611Z","logger":"controllers.VolumeReplication","caller":"controllers/volumereplication_controller.go:164","msg":"adding finalizer to PersistentVolumeClaim object","Request.Name":"busybox-pvc-3","Request.Namespace":"busybox-workloads","Finalizer":"replication.storage.openshift.io/pvc-protection"}
>{"level":"info","timestamp":"2021-11-02T11:39:06.788Z","logger":"controllers.VolumeReplication","caller":"controllers/volumereplication_controller.go:210","msg":"force promoting volume due to known grpc error","Request.Name":"busybox-pvc-3","Request.Namespace":"busybox-workloads","error":"rpc error: code = FailedPrecondition desc = failed to promote image \"ocs-storagecluster-cephblockpool/csi-vol-f036de0c-389c-11ec-bca9-0a580a800241\" with error: rbd: ret=-16, Device or resource busy"}
>{"level":"info","timestamp":"2021-11-02T11:39:10.464Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume is marked primary","Request.Name":"busybox-pvc-3","Request.Namespace":"busybox-workloads"}
>{"level":"info","timestamp":"2021-11-02T11:39:10.464Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume handle","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","VolumeHandleName":"0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241"}
>{"level":"info","timestamp":"2021-11-02T11:39:10.464Z","logger":"controllers.VolumeReplication","caller":"controllers/volumereplication_controller.go:160","msg":"adding finalizer to volumeReplication object","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","Finalizer":"replication.storage.openshift.io"}
>{"level":"info","timestamp":"2021-11-02T11:39:10.473Z","logger":"controllers.VolumeReplication","caller":"controllers/volumereplication_controller.go:164","msg":"adding finalizer to PersistentVolumeClaim object","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","Finalizer":"replication.storage.openshift.io/pvc-protection"}
>{"level":"info","timestamp":"2021-11-02T11:39:11.490Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume is marked primary","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads"}
>{"level":"info","timestamp":"2021-11-02T11:39:11.490Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume handle","Request.Name":"busybox-pvc-4","Request.Namespace":"busybox-workloads","VolumeHandleName":"0001-0011-openshift-storage-0000000000000001-f0cb7c21-389c-11ec-bca9-0a580a800241"}
>{"level":"info","timestamp":"2021-11-02T11:39:11.490Z","logger":"controllers.VolumeReplication","caller":"controllers/volumereplication_controller.go:160","msg":"adding finalizer to volumeReplication object","Request.Name":"busybox-pvc-4","Request.Namespace":"busybox-workloads","Finalizer":"replication.storage.openshift.io"}
>{"level":"info","timestamp":"2021-11-02T11:39:11.505Z","logger":"controllers.VolumeReplication","caller":"controllers/volumereplication_controller.go:164","msg":"adding finalizer to PersistentVolumeClaim object","Request.Name":"busybox-pvc-4","Request.Namespace":"busybox-workloads","Finalizer":"replication.storage.openshift.io/pvc-protection"}
>{"level":"info","timestamp":"2021-11-02T11:39:11.718Z","logger":"controllers.VolumeReplication","caller":"controllers/volumereplication_controller.go:210","msg":"force promoting volume due to known grpc error","Request.Name":"busybox-pvc-4","Request.Namespace":"busybox-workloads","error":"rpc error: code = FailedPrecondition desc = failed to promote image \"ocs-storagecluster-cephblockpool/csi-vol-f0cb7c21-389c-11ec-bca9-0a580a800241\" with error: rbd: ret=-16, Device or resource busy"}
>{"level":"info","timestamp":"2021-11-02T11:39:15.902Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume is marked primary","Request.Name":"busybox-pvc-4","Request.Namespace":"busybox-workloads"}
>{"level":"info","timestamp":"2021-11-02T11:39:15.902Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume handle","Request.Name":"busybox-pvc-5","Request.Namespace":"busybox-workloads","VolumeHandleName":"0001-0011-openshift-storage-0000000000000001-f06f752a-389c-11ec-bca9-0a580a800241"}
>{"level":"info","timestamp":"2021-11-02T11:39:15.902Z","logger":"controllers.VolumeReplication","caller":"controllers/volumereplication_controller.go:160","msg":"adding finalizer to volumeReplication object","Request.Name":"busybox-pvc-5","Request.Namespace":"busybox-workloads","Finalizer":"replication.storage.openshift.io"}
>{"level":"info","timestamp":"2021-11-02T11:39:15.918Z","logger":"controllers.VolumeReplication","caller":"controllers/volumereplication_controller.go:164","msg":"adding finalizer to PersistentVolumeClaim object","Request.Name":"busybox-pvc-5","Request.Namespace":"busybox-workloads","Finalizer":"replication.storage.openshift.io/pvc-protection"}
>{"level":"info","timestamp":"2021-11-02T11:39:16.143Z","logger":"controllers.VolumeReplication","caller":"controllers/volumereplication_controller.go:210","msg":"force promoting volume due to known grpc error","Request.Name":"busybox-pvc-5","Request.Namespace":"busybox-workloads","error":"rpc error: code = FailedPrecondition desc = failed to promote image \"ocs-storagecluster-cephblockpool/csi-vol-f06f752a-389c-11ec-bca9-0a580a800241\" with error: rbd: ret=-16, Device or resource busy"}
>{"level":"info","timestamp":"2021-11-02T11:39:19.962Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume is marked primary","Request.Name":"busybox-pvc-5","Request.Namespace":"busybox-workloads"}
>{"level":"info","timestamp":"2021-11-02T11:39:19.962Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume handle","Request.Name":"busybox-pvc-1","Request.Namespace":"busybox-workloads","VolumeHandleName":"0001-0011-openshift-storage-0000000000000001-f0aca7cb-389c-11ec-bca9-0a580a800241"}
>{"level":"info","timestamp":"2021-11-02T11:39:19.962Z","logger":"controllers.VolumeReplication","caller":"controllers/volumereplication_controller.go:160","msg":"adding finalizer to volumeReplication object","Request.Name":"busybox-pvc-1","Request.Namespace":"busybox-workloads","Finalizer":"replication.storage.openshift.io"}
>{"level":"info","timestamp":"2021-11-02T11:39:19.974Z","logger":"controllers.VolumeReplication","caller":"controllers/volumereplication_controller.go:164","msg":"adding finalizer to PersistentVolumeClaim object","Request.Name":"busybox-pvc-1","Request.Namespace":"busybox-workloads","Finalizer":"replication.storage.openshift.io/pvc-protection"}
>{"level":"info","timestamp":"2021-11-02T11:39:20.150Z","logger":"controllers.VolumeReplication","caller":"controllers/volumereplication_controller.go:210","msg":"force promoting volume due to known grpc error","Request.Name":"busybox-pvc-1","Request.Namespace":"busybox-workloads","error":"rpc error: code = FailedPrecondition desc = failed to promote image \"ocs-storagecluster-cephblockpool/csi-vol-f0aca7cb-389c-11ec-bca9-0a580a800241\" with error: rbd: ret=-16, Device or resource busy"}
>{"level":"info","timestamp":"2021-11-02T11:39:24.007Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume is marked primary","Request.Name":"busybox-pvc-1","Request.Namespace":"busybox-workloads"}
>{"level":"info","timestamp":"2021-11-02T11:39:24.007Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume handle","Request.Name":"busybox-pvc","Request.Namespace":"busybox-workloads","VolumeHandleName":"0001-0011-openshift-storage-0000000000000001-f126f53d-389c-11ec-bca9-0a580a800241"}
>{"level":"info","timestamp":"2021-11-02T11:39:24.007Z","logger":"controllers.VolumeReplication","caller":"controllers/volumereplication_controller.go:160","msg":"adding finalizer to volumeReplication object","Request.Name":"busybox-pvc","Request.Namespace":"busybox-workloads","Finalizer":"replication.storage.openshift.io"}
>{"level":"info","timestamp":"2021-11-02T11:39:24.019Z","logger":"controllers.VolumeReplication","caller":"controllers/volumereplication_controller.go:164","msg":"adding finalizer to PersistentVolumeClaim object","Request.Name":"busybox-pvc","Request.Namespace":"busybox-workloads","Finalizer":"replication.storage.openshift.io/pvc-protection"}
>{"level":"info","timestamp":"2021-11-02T11:39:24.233Z","logger":"controllers.VolumeReplication","caller":"controllers/volumereplication_controller.go:210","msg":"force promoting volume due to known grpc error","Request.Name":"busybox-pvc","Request.Namespace":"busybox-workloads","error":"rpc error: code = FailedPrecondition desc = failed to promote image \"ocs-storagecluster-cephblockpool/csi-vol-f126f53d-389c-11ec-bca9-0a580a800241\" with error: rbd: ret=-16, Device or resource busy"}
>{"level":"info","timestamp":"2021-11-02T11:39:28.094Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume is marked primary","Request.Name":"busybox-pvc","Request.Namespace":"busybox-workloads"}
>{"level":"info","timestamp":"2021-11-02T11:39:28.094Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume handle","Request.Name":"busybox-pvc-2","Request.Namespace":"busybox-workloads","VolumeHandleName":"0001-0011-openshift-storage-0000000000000001-f0419a11-389c-11ec-bca9-0a580a800241"}
>{"level":"info","timestamp":"2021-11-02T11:39:28.094Z","logger":"controllers.VolumeReplication","caller":"controllers/volumereplication_controller.go:160","msg":"adding finalizer to volumeReplication object","Request.Name":"busybox-pvc-2","Request.Namespace":"busybox-workloads","Finalizer":"replication.storage.openshift.io"}
>{"level":"info","timestamp":"2021-11-02T11:39:28.104Z","logger":"controllers.VolumeReplication","caller":"controllers/volumereplication_controller.go:164","msg":"adding finalizer to PersistentVolumeClaim object","Request.Name":"busybox-pvc-2","Request.Namespace":"busybox-workloads","Finalizer":"replication.storage.openshift.io/pvc-protection"}
>{"level":"info","timestamp":"2021-11-02T11:39:28.319Z","logger":"controllers.VolumeReplication","caller":"controllers/volumereplication_controller.go:210","msg":"force promoting volume due to known grpc error","Request.Name":"busybox-pvc-2","Request.Namespace":"busybox-workloads","error":"rpc error: code = FailedPrecondition desc = failed to promote image \"ocs-storagecluster-cephblockpool/csi-vol-f0419a11-389c-11ec-bca9-0a580a800241\" with error: rbd: ret=-16, Device or resource busy"}
>{"level":"info","timestamp":"2021-11-02T11:39:37.268Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume is marked primary","Request.Name":"busybox-pvc-2","Request.Namespace":"busybox-workloads"}
>{"level":"info","timestamp":"2021-11-02T13:56:21.002Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume handle","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","VolumeHandleName":"0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241"}
>{"level":"info","timestamp":"2021-11-02T13:56:22.384Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume is not ready to use","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads"}
>{"level":"info","timestamp":"2021-11-02T13:56:37.399Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume handle","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","VolumeHandleName":"0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241"}
>{"level":"error","timestamp":"2021-11-02T13:56:37.875Z","logger":"controllers.VolumeReplication","caller":"controllers/volumereplication_controller.go:236","msg":"failed to resync volume","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","error":"rpc error: code = Unavailable desc = awaiting initial resync due to split brain"}
>{"level":"error","timestamp":"2021-11-02T13:56:37.875Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"failed to Replicate","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","ReplicationState":"secondary","error":"rpc error: code = Unavailable desc = awaiting initial resync due to split brain"}
>{"level":"info","timestamp":"2021-11-02T13:56:48.094Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume handle","Request.Name":"busybox-pvc","Request.Namespace":"busybox-workloads","VolumeHandleName":"0001-0011-openshift-storage-0000000000000001-f126f53d-389c-11ec-bca9-0a580a800241"}
>{"level":"info","timestamp":"2021-11-02T13:56:48.756Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume is not ready to use","Request.Name":"busybox-pvc","Request.Namespace":"busybox-workloads"}
>{"level":"info","timestamp":"2021-11-02T13:56:48.858Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume handle","Request.Name":"busybox-pvc-1","Request.Namespace":"busybox-workloads","VolumeHandleName":"0001-0011-openshift-storage-0000000000000001-f0aca7cb-389c-11ec-bca9-0a580a800241"}
>{"level":"info","timestamp":"2021-11-02T13:56:50.102Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume is not ready to use","Request.Name":"busybox-pvc-1","Request.Namespace":"busybox-workloads"}
>{"level":"info","timestamp":"2021-11-02T13:56:50.125Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume handle","Request.Name":"busybox-pvc-4","Request.Namespace":"busybox-workloads","VolumeHandleName":"0001-0011-openshift-storage-0000000000000001-f0cb7c21-389c-11ec-bca9-0a580a800241"}
>{"level":"info","timestamp":"2021-11-02T13:56:51.107Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume is not ready to use","Request.Name":"busybox-pvc-4","Request.Namespace":"busybox-workloads"}
>{"level":"info","timestamp":"2021-11-02T13:56:51.116Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume handle","Request.Name":"busybox-pvc-3","Request.Namespace":"busybox-workloads","VolumeHandleName":"0001-0011-openshift-storage-0000000000000001-f036de0c-389c-11ec-bca9-0a580a800241"}
>{"level":"info","timestamp":"2021-11-02T13:56:52.216Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume is not ready to use","Request.Name":"busybox-pvc-3","Request.Namespace":"busybox-workloads"}
>{"level":"info","timestamp":"2021-11-02T13:56:52.229Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume handle","Request.Name":"busybox-pvc-5","Request.Namespace":"busybox-workloads","VolumeHandleName":"0001-0011-openshift-storage-0000000000000001-f06f752a-389c-11ec-bca9-0a580a800241"}
>{"level":"info","timestamp":"2021-11-02T13:56:53.491Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume is not ready to use","Request.Name":"busybox-pvc-5","Request.Namespace":"busybox-workloads"}
>{"level":"info","timestamp":"2021-11-02T13:56:53.511Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume handle","Request.Name":"busybox-pvc-2","Request.Namespace":"busybox-workloads","VolumeHandleName":"0001-0011-openshift-storage-0000000000000001-f0419a11-389c-11ec-bca9-0a580a800241"}
>{"level":"info","timestamp":"2021-11-02T13:56:54.661Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume is not ready to use","Request.Name":"busybox-pvc-2","Request.Namespace":"busybox-workloads"}
>{"level":"info","timestamp":"2021-11-02T13:56:55.003Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume handle","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","VolumeHandleName":"0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241"}
>{"level":"error","timestamp":"2021-11-02T13:56:58.112Z","logger":"controllers.VolumeReplication","caller":"controllers/volumereplication_controller.go:236","msg":"failed to resync volume","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","error":"rpc error: code = Unavailable desc = awaiting initial resync due to split brain"}
>{"level":"error","timestamp":"2021-11-02T13:56:58.112Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"failed to Replicate","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","ReplicationState":"secondary","error":"rpc error: code = Unavailable desc = awaiting initial resync due to split brain"}
>{"level":"info","timestamp":"2021-11-02T13:57:03.859Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume handle","Request.Name":"busybox-pvc","Request.Namespace":"busybox-workloads","VolumeHandleName":"0001-0011-openshift-storage-0000000000000001-f126f53d-389c-11ec-bca9-0a580a800241"}
>{"level":"error","timestamp":"2021-11-02T13:57:04.049Z","logger":"controllers.VolumeReplication","caller":"controllers/volumereplication_controller.go:236","msg":"failed to resync volume","Request.Name":"busybox-pvc","Request.Namespace":"busybox-workloads","error":"rpc error: code = Internal desc = not resyncing. image is in \"stopped\" state"}
>{"level":"error","timestamp":"2021-11-02T13:57:04.049Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"failed to Replicate","Request.Name":"busybox-pvc","Request.Namespace":"busybox-workloads","ReplicationState":"secondary","error":"rpc error: code = Internal desc = not resyncing. image is in \"stopped\" state"}
>{"level":"info","timestamp":"2021-11-02T13:57:05.126Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume handle","Request.Name":"busybox-pvc-1","Request.Namespace":"busybox-workloads","VolumeHandleName":"0001-0011-openshift-storage-0000000000000001-f0aca7cb-389c-11ec-bca9-0a580a800241"}
>{"level":"info","timestamp":"2021-11-02T13:57:05.297Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume is not ready to use, requeuing for resync","Request.Name":"busybox-pvc-1","Request.Namespace":"busybox-workloads"}
>{"level":"info","timestamp":"2021-11-02T13:57:06.117Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume handle","Request.Name":"busybox-pvc-4","Request.Namespace":"busybox-workloads","VolumeHandleName":"0001-0011-openshift-storage-0000000000000001-f0cb7c21-389c-11ec-bca9-0a580a800241"}
>{"level":"info","timestamp":"2021-11-02T13:57:06.316Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume is marked secondary","Request.Name":"busybox-pvc-4","Request.Namespace":"busybox-workloads"}
>{"level":"info","timestamp":"2021-11-02T13:57:07.229Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume handle","Request.Name":"busybox-pvc-3","Request.Namespace":"busybox-workloads","VolumeHandleName":"0001-0011-openshift-storage-0000000000000001-f036de0c-389c-11ec-bca9-0a580a800241"}
>{"level":"info","timestamp":"2021-11-02T13:57:07.418Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume is marked secondary","Request.Name":"busybox-pvc-3","Request.Namespace":"busybox-workloads"}
>{"level":"info","timestamp":"2021-11-02T13:57:08.512Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume handle","Request.Name":"busybox-pvc-5","Request.Namespace":"busybox-workloads","VolumeHandleName":"0001-0011-openshift-storage-0000000000000001-f06f752a-389c-11ec-bca9-0a580a800241"}
>{"level":"info","timestamp":"2021-11-02T13:57:08.865Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume is marked secondary","Request.Name":"busybox-pvc-5","Request.Namespace":"busybox-workloads"}
>{"level":"info","timestamp":"2021-11-02T13:57:10.004Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume handle","Request.Name":"busybox-pvc-2","Request.Namespace":"busybox-workloads","VolumeHandleName":"0001-0011-openshift-storage-0000000000000001-f0419a11-389c-11ec-bca9-0a580a800241"}
>{"level":"info","timestamp":"2021-11-02T13:57:10.334Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume is marked secondary","Request.Name":"busybox-pvc-2","Request.Namespace":"busybox-workloads"}
>{"level":"info","timestamp":"2021-11-02T13:57:13.129Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume handle","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","VolumeHandleName":"0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241"}
>{"level":"error","timestamp":"2021-11-02T13:57:13.199Z","logger":"controllers.VolumeReplication","caller":"controllers/volumereplication_controller.go:198","msg":"failed to enable volume replication","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"error","timestamp":"2021-11-02T13:57:13.199Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"failed to enable replication","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"error","timestamp":"2021-11-02T13:57:13.215Z","logger":"controller-runtime.manager.controller.volumereplication","caller":"controller/controller.go:253","msg":"Reconciler error","reconciler group":"replication.storage.openshift.io","reconciler kind":"VolumeReplication","name":"mysql-pv-claim","namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"info","timestamp":"2021-11-02T13:57:13.221Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume handle","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","VolumeHandleName":"0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241"}
>{"level":"error","timestamp":"2021-11-02T13:57:13.269Z","logger":"controllers.VolumeReplication","caller":"controllers/volumereplication_controller.go:198","msg":"failed to enable volume replication","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"error","timestamp":"2021-11-02T13:57:13.269Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"failed to enable replication","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"error","timestamp":"2021-11-02T13:57:13.282Z","logger":"controller-runtime.manager.controller.volumereplication","caller":"controller/controller.go:253","msg":"Reconciler error","reconciler group":"replication.storage.openshift.io","reconciler kind":"VolumeReplication","name":"mysql-pv-claim","namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"info","timestamp":"2021-11-02T13:57:13.293Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume handle","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","VolumeHandleName":"0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241"}
>{"level":"error","timestamp":"2021-11-02T13:57:13.344Z","logger":"controllers.VolumeReplication","caller":"controllers/volumereplication_controller.go:198","msg":"failed to enable volume replication","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"error","timestamp":"2021-11-02T13:57:13.345Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"failed to enable replication","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"error","timestamp":"2021-11-02T13:57:13.359Z","logger":"controller-runtime.manager.controller.volumereplication","caller":"controller/controller.go:253","msg":"Reconciler error","reconciler group":"replication.storage.openshift.io","reconciler kind":"VolumeReplication","name":"mysql-pv-claim","namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"info","timestamp":"2021-11-02T13:57:13.380Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume handle","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","VolumeHandleName":"0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241"}
>{"level":"error","timestamp":"2021-11-02T13:57:13.429Z","logger":"controllers.VolumeReplication","caller":"controllers/volumereplication_controller.go:198","msg":"failed to enable volume replication","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"error","timestamp":"2021-11-02T13:57:13.429Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"failed to enable replication","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"error","timestamp":"2021-11-02T13:57:13.442Z","logger":"controller-runtime.manager.controller.volumereplication","caller":"controller/controller.go:253","msg":"Reconciler error","reconciler group":"replication.storage.openshift.io","reconciler kind":"VolumeReplication","name":"mysql-pv-claim","namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"info","timestamp":"2021-11-02T13:57:13.482Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume handle","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","VolumeHandleName":"0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241"}
>{"level":"error","timestamp":"2021-11-02T13:57:13.516Z","logger":"controllers.VolumeReplication","caller":"controllers/volumereplication_controller.go:198","msg":"failed to enable volume replication","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"error","timestamp":"2021-11-02T13:57:13.516Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"failed to enable replication","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"error","timestamp":"2021-11-02T13:57:13.526Z","logger":"controller-runtime.manager.controller.volumereplication","caller":"controller/controller.go:253","msg":"Reconciler error","reconciler group":"replication.storage.openshift.io","reconciler kind":"VolumeReplication","name":"mysql-pv-claim","namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"info","timestamp":"2021-11-02T13:57:13.607Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume handle","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","VolumeHandleName":"0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241"}
>{"level":"error","timestamp":"2021-11-02T13:57:13.663Z","logger":"controllers.VolumeReplication","caller":"controllers/volumereplication_controller.go:198","msg":"failed to enable volume replication","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"error","timestamp":"2021-11-02T13:57:13.663Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"failed to enable replication","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"error","timestamp":"2021-11-02T13:57:13.673Z","logger":"controller-runtime.manager.controller.volumereplication","caller":"controller/controller.go:253","msg":"Reconciler error","reconciler group":"replication.storage.openshift.io","reconciler kind":"VolumeReplication","name":"mysql-pv-claim","namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"info","timestamp":"2021-11-02T13:57:13.834Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume handle","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","VolumeHandleName":"0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241"}
>{"level":"error","timestamp":"2021-11-02T13:57:13.884Z","logger":"controllers.VolumeReplication","caller":"controllers/volumereplication_controller.go:198","msg":"failed to enable volume replication","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"error","timestamp":"2021-11-02T13:57:13.884Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"failed to enable replication","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"error","timestamp":"2021-11-02T13:57:13.896Z","logger":"controller-runtime.manager.controller.volumereplication","caller":"controller/controller.go:253","msg":"Reconciler error","reconciler group":"replication.storage.openshift.io","reconciler kind":"VolumeReplication","name":"mysql-pv-claim","namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"info","timestamp":"2021-11-02T13:57:14.217Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume handle","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","VolumeHandleName":"0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241"}
>{"level":"error","timestamp":"2021-11-02T13:57:14.271Z","logger":"controllers.VolumeReplication","caller":"controllers/volumereplication_controller.go:198","msg":"failed to enable volume replication","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"error","timestamp":"2021-11-02T13:57:14.271Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"failed to enable replication","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"error","timestamp":"2021-11-02T13:57:14.285Z","logger":"controller-runtime.manager.controller.volumereplication","caller":"controller/controller.go:253","msg":"Reconciler error","reconciler group":"replication.storage.openshift.io","reconciler kind":"VolumeReplication","name":"mysql-pv-claim","namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"info","timestamp":"2021-11-02T13:57:14.926Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume handle","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","VolumeHandleName":"0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241"}
>{"level":"error","timestamp":"2021-11-02T13:57:14.961Z","logger":"controllers.VolumeReplication","caller":"controllers/volumereplication_controller.go:198","msg":"failed to enable volume replication","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"error","timestamp":"2021-11-02T13:57:14.961Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"failed to enable replication","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"error","timestamp":"2021-11-02T13:57:14.972Z","logger":"controller-runtime.manager.controller.volumereplication","caller":"controller/controller.go:253","msg":"Reconciler error","reconciler group":"replication.storage.openshift.io","reconciler kind":"VolumeReplication","name":"mysql-pv-claim","namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"info","timestamp":"2021-11-02T13:57:16.252Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume handle","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","VolumeHandleName":"0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241"}
>{"level":"error","timestamp":"2021-11-02T13:57:16.278Z","logger":"controllers.VolumeReplication","caller":"controllers/volumereplication_controller.go:198","msg":"failed to enable volume replication","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"error","timestamp":"2021-11-02T13:57:16.278Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"failed to enable replication","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"error","timestamp":"2021-11-02T13:57:16.288Z","logger":"controller-runtime.manager.controller.volumereplication","caller":"controller/controller.go:253","msg":"Reconciler error","reconciler group":"replication.storage.openshift.io","reconciler kind":"VolumeReplication","name":"mysql-pv-claim","namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"info","timestamp":"2021-11-02T13:57:18.849Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume handle","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","VolumeHandleName":"0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241"}
>{"level":"error","timestamp":"2021-11-02T13:57:18.875Z","logger":"controllers.VolumeReplication","caller":"controllers/volumereplication_controller.go:198","msg":"failed to enable volume replication","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"error","timestamp":"2021-11-02T13:57:18.876Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"failed to enable replication","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"error","timestamp":"2021-11-02T13:57:18.883Z","logger":"controller-runtime.manager.controller.volumereplication","caller":"controller/controller.go:253","msg":"Reconciler error","reconciler group":"replication.storage.openshift.io","reconciler kind":"VolumeReplication","name":"mysql-pv-claim","namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"info","timestamp":"2021-11-02T13:57:19.060Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume handle","Request.Name":"busybox-pvc","Request.Namespace":"busybox-workloads","VolumeHandleName":"0001-0011-openshift-storage-0000000000000001-f126f53d-389c-11ec-bca9-0a580a800241"}
>{"level":"info","timestamp":"2021-11-02T13:57:19.284Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume is marked secondary","Request.Name":"busybox-pvc","Request.Namespace":"busybox-workloads"}
>{"level":"info","timestamp":"2021-11-02T13:57:24.004Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume handle","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","VolumeHandleName":"0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241"}
>{"level":"error","timestamp":"2021-11-02T13:57:24.033Z","logger":"controllers.VolumeReplication","caller":"controllers/volumereplication_controller.go:198","msg":"failed to enable volume replication","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"error","timestamp":"2021-11-02T13:57:24.033Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"failed to enable replication","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"error","timestamp":"2021-11-02T13:57:24.039Z","logger":"controller-runtime.manager.controller.volumereplication","caller":"controller/controller.go:253","msg":"Reconciler error","reconciler group":"replication.storage.openshift.io","reconciler kind":"VolumeReplication","name":"mysql-pv-claim","namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"info","timestamp":"2021-11-02T13:57:34.281Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume handle","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","VolumeHandleName":"0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241"}
>{"level":"error","timestamp":"2021-11-02T13:57:34.351Z","logger":"controllers.VolumeReplication","caller":"controllers/volumereplication_controller.go:198","msg":"failed to enable volume replication","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"error","timestamp":"2021-11-02T13:57:34.351Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"failed to enable replication","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"error","timestamp":"2021-11-02T13:57:34.364Z","logger":"controller-runtime.manager.controller.volumereplication","caller":"controller/controller.go:253","msg":"Reconciler error","reconciler group":"replication.storage.openshift.io","reconciler kind":"VolumeReplication","name":"mysql-pv-claim","namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"info","timestamp":"2021-11-02T13:57:35.317Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume handle","Request.Name":"busybox-pvc-1","Request.Namespace":"busybox-workloads","VolumeHandleName":"0001-0011-openshift-storage-0000000000000001-f0aca7cb-389c-11ec-bca9-0a580a800241"}
>{"level":"info","timestamp":"2021-11-02T13:57:35.513Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume is marked secondary","Request.Name":"busybox-pvc-1","Request.Namespace":"busybox-workloads"}
>{"level":"info","timestamp":"2021-11-02T13:57:54.845Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume handle","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","VolumeHandleName":"0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241"}
>{"level":"error","timestamp":"2021-11-02T13:57:54.876Z","logger":"controllers.VolumeReplication","caller":"controllers/volumereplication_controller.go:198","msg":"failed to enable volume replication","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"error","timestamp":"2021-11-02T13:57:54.876Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"failed to enable replication","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"error","timestamp":"2021-11-02T13:57:54.883Z","logger":"controller-runtime.manager.controller.volumereplication","caller":"controller/controller.go:253","msg":"Reconciler error","reconciler group":"replication.storage.openshift.io","reconciler kind":"VolumeReplication","name":"mysql-pv-claim","namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"info","timestamp":"2021-11-02T13:58:35.844Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume handle","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","VolumeHandleName":"0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241"}
>{"level":"error","timestamp":"2021-11-02T13:58:35.878Z","logger":"controllers.VolumeReplication","caller":"controllers/volumereplication_controller.go:198","msg":"failed to enable volume replication","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"error","timestamp":"2021-11-02T13:58:35.878Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"failed to enable replication","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"error","timestamp":"2021-11-02T13:58:35.888Z","logger":"controller-runtime.manager.controller.volumereplication","caller":"controller/controller.go:253","msg":"Reconciler error","reconciler group":"replication.storage.openshift.io","reconciler kind":"VolumeReplication","name":"mysql-pv-claim","namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"info","timestamp":"2021-11-02T13:59:57.808Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume handle","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","VolumeHandleName":"0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241"}
>{"level":"error","timestamp":"2021-11-02T13:59:57.841Z","logger":"controllers.VolumeReplication","caller":"controllers/volumereplication_controller.go:198","msg":"failed to enable volume replication","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"error","timestamp":"2021-11-02T13:59:57.841Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"failed to enable replication","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"error","timestamp":"2021-11-02T13:59:57.848Z","logger":"controller-runtime.manager.controller.volumereplication","caller":"controller/controller.go:253","msg":"Reconciler error","reconciler group":"replication.storage.openshift.io","reconciler kind":"VolumeReplication","name":"mysql-pv-claim","namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"info","timestamp":"2021-11-02T14:02:41.688Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume handle","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","VolumeHandleName":"0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241"}
>{"level":"error","timestamp":"2021-11-02T14:02:41.732Z","logger":"controllers.VolumeReplication","caller":"controllers/volumereplication_controller.go:198","msg":"failed to enable volume replication","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"error","timestamp":"2021-11-02T14:02:41.733Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"failed to enable replication","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"error","timestamp":"2021-11-02T14:02:41.741Z","logger":"controller-runtime.manager.controller.volumereplication","caller":"controller/controller.go:253","msg":"Reconciler error","reconciler group":"replication.storage.openshift.io","reconciler kind":"VolumeReplication","name":"mysql-pv-claim","namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"info","timestamp":"2021-11-02T14:08:09.422Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume handle","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","VolumeHandleName":"0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241"}
>{"level":"error","timestamp":"2021-11-02T14:08:09.466Z","logger":"controllers.VolumeReplication","caller":"controllers/volumereplication_controller.go:198","msg":"failed to enable volume replication","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"error","timestamp":"2021-11-02T14:08:09.466Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"failed to enable replication","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"error","timestamp":"2021-11-02T14:08:09.475Z","logger":"controller-runtime.manager.controller.volumereplication","caller":"controller/controller.go:253","msg":"Reconciler error","reconciler group":"replication.storage.openshift.io","reconciler kind":"VolumeReplication","name":"mysql-pv-claim","namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"info","timestamp":"2021-11-02T14:19:04.836Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume handle","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","VolumeHandleName":"0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241"}
>{"level":"error","timestamp":"2021-11-02T14:19:04.880Z","logger":"controllers.VolumeReplication","caller":"controllers/volumereplication_controller.go:198","msg":"failed to enable volume replication","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"error","timestamp":"2021-11-02T14:19:04.880Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"failed to enable replication","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"error","timestamp":"2021-11-02T14:19:04.889Z","logger":"controller-runtime.manager.controller.volumereplication","caller":"controller/controller.go:253","msg":"Reconciler error","reconciler group":"replication.storage.openshift.io","reconciler kind":"VolumeReplication","name":"mysql-pv-claim","namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"info","timestamp":"2021-11-02T14:35:44.889Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume handle","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","VolumeHandleName":"0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241"}
>{"level":"error","timestamp":"2021-11-02T14:35:44.946Z","logger":"controllers.VolumeReplication","caller":"controllers/volumereplication_controller.go:198","msg":"failed to enable volume replication","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"error","timestamp":"2021-11-02T14:35:44.946Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"failed to enable replication","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"error","timestamp":"2021-11-02T14:35:44.960Z","logger":"controller-runtime.manager.controller.volumereplication","caller":"controller/controller.go:253","msg":"Reconciler error","reconciler group":"replication.storage.openshift.io","reconciler kind":"VolumeReplication","name":"mysql-pv-claim","namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"info","timestamp":"2021-11-02T14:52:24.960Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume handle","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","VolumeHandleName":"0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241"}
>{"level":"error","timestamp":"2021-11-02T14:52:25.011Z","logger":"controllers.VolumeReplication","caller":"controllers/volumereplication_controller.go:198","msg":"failed to enable volume replication","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"error","timestamp":"2021-11-02T14:52:25.011Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"failed to enable replication","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"error","timestamp":"2021-11-02T14:52:25.020Z","logger":"controller-runtime.manager.controller.volumereplication","caller":"controller/controller.go:253","msg":"Reconciler error","reconciler group":"replication.storage.openshift.io","reconciler kind":"VolumeReplication","name":"mysql-pv-claim","namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"info","timestamp":"2021-11-02T15:09:05.021Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume handle","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","VolumeHandleName":"0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241"}
>{"level":"error","timestamp":"2021-11-02T15:09:05.071Z","logger":"controllers.VolumeReplication","caller":"controllers/volumereplication_controller.go:198","msg":"failed to enable volume replication","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"error","timestamp":"2021-11-02T15:09:05.071Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"failed to enable replication","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"error","timestamp":"2021-11-02T15:09:05.081Z","logger":"controller-runtime.manager.controller.volumereplication","caller":"controller/controller.go:253","msg":"Reconciler error","reconciler group":"replication.storage.openshift.io","reconciler kind":"VolumeReplication","name":"mysql-pv-claim","namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"info","timestamp":"2021-11-02T15:25:45.082Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume handle","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","VolumeHandleName":"0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241"}
>{"level":"error","timestamp":"2021-11-02T15:25:45.129Z","logger":"controllers.VolumeReplication","caller":"controllers/volumereplication_controller.go:198","msg":"failed to enable volume replication","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"error","timestamp":"2021-11-02T15:25:45.129Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"failed to enable replication","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"error","timestamp":"2021-11-02T15:25:45.141Z","logger":"controller-runtime.manager.controller.volumereplication","caller":"controller/controller.go:253","msg":"Reconciler error","reconciler group":"replication.storage.openshift.io","reconciler kind":"VolumeReplication","name":"mysql-pv-claim","namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"info","timestamp":"2021-11-02T15:42:25.142Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"volume handle","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","VolumeHandleName":"0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241"}
>{"level":"error","timestamp":"2021-11-02T15:42:25.193Z","logger":"controllers.VolumeReplication","caller":"controllers/volumereplication_controller.go:198","msg":"failed to enable volume replication","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"error","timestamp":"2021-11-02T15:42:25.193Z","logger":"controllers.VolumeReplication","caller":"controller/controller.go:298","msg":"failed to enable replication","Request.Name":"mysql-pv-claim","Request.Namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}
>{"level":"error","timestamp":"2021-11-02T15:42:25.202Z","logger":"controller-runtime.manager.controller.volumereplication","caller":"controller/controller.go:253","msg":"Reconciler error","reconciler group":"replication.storage.openshift.io","reconciler kind":"VolumeReplication","name":"mysql-pv-claim","namespace":"busybox-workloads","error":"rpc error: code = NotFound desc = volume 0001-0011-openshift-storage-0000000000000001-f038968b-389c-11ec-bca9-0a580a800241 not found"}

Actions: View

Attachments on bug 2019931: 1839887 | 1839888 | 1839889 | 1839890 | 1839891