oc image mirror --filename=Image_Mapping.txt --insecure=true -a mirror_docker.conf --skip-missing=true --skip-verification=true
  <--- skip --->
  openshift/ose-node-feature-discovery
    blobs:
      registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-node-feature-discovery sha256:455ea8ab06218495bbbcb14b750a0d644897b24f8c5dcf9e8698e27882583412 1.575KiB
      registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-node-feature-discovery sha256:03b97248285403d0a56af7ae9c0376e39b0d118abb375bb34f4ec36a2044b6e7 5.222KiB
      registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-node-feature-discovery sha256:935ce2f796a92f0b6fa7762d7bab7d252ddb59d03cd3afba42d3785b571854f1 3.333MiB
      registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-node-feature-discovery sha256:df57162bfd87083528c3e2b590603b13874e667c6dc215a4646dbff0dc44f746 7.842MiB
      registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-node-feature-discovery sha256:42889c802436a9a21aebb09223852633bc2ab1120eb43ee60f1e93e9a83c76c9 21.23MiB
      registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-node-feature-discovery sha256:bb13d92caffa705f32b8a7f9f661e07ddede310c6ccfa78fb53a49539740e29b 72.71MiB
    manifests:
      sha256:753b867ca6ff098de373806bee7b61903e942104fef8e261448e8c3b8b490c94
      sha256:90503115bcda5d3fb1ee18406c0389c87d4d456027709f644662e64e3f522b37
      sha256:753b867ca6ff098de373806bee7b61903e942104fef8e261448e8c3b8b490c94 -> v4.3.5
      sha256:753b867ca6ff098de373806bee7b61903e942104fef8e261448e8c3b8b490c94 -> v4.3.5-202003020549
  openshift/ose-oauth-proxy
    blobs:
      registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-oauth-proxy sha256:455ea8ab06218495bbbcb14b750a0d644897b24f8c5dcf9e8698e27882583412 1.575KiB
      registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-oauth-proxy sha256:636a1d06a983db155481c3ca5ef173316556df8046eef55767516a0143d361f2 4.762KiB
      registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-oauth-proxy sha256:935ce2f796a92f0b6fa7762d7bab7d252ddb59d03cd3afba42d3785b571854f1 3.333MiB
      registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-oauth-proxy sha256:c6fb2a4f58eaf5e85eedafe592ee77665ce384e3a6deb3416649cfcb1c23c9ef 16.89MiB
      registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-oauth-proxy sha256:bb13d92caffa705f32b8a7f9f661e07ddede310c6ccfa78fb53a49539740e29b 72.71MiB
    manifests:
      sha256:091de6d4ec3aa3e3e7aaa8dbb6ea456dbd6e784e185a3d3cee3f33b3d142b3f5
      sha256:e2cc56931552bee158ed77d2f0a7e75202aba9204612f370d08630ba3992ec77
      sha256:091de6d4ec3aa3e3e7aaa8dbb6ea456dbd6e784e185a3d3cee3f33b3d142b3f5 -> v4.3.5
      sha256:091de6d4ec3aa3e3e7aaa8dbb6ea456dbd6e784e185a3d3cee3f33b3d142b3f5 -> v4.3.5-202003020549
  openshift/ose-promtail
    blobs:
      registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-promtail sha256:4fbc3bafa3d4400bb97a733c1fe12f2f99bf38b9d5b913d5034f29798739654d 1.585KiB
      registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-promtail sha256:4627307add29bf4cf97a26b24c441c649c1f073a3f8938c0545287a150be4d50 4.851KiB
      registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-promtail sha256:00d3df4a6fcbbbb7041e6c108c2ad06c5fef0e66a1c242027ca1a02448aa91c6 3.336MiB
      registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-promtail sha256:8f4e4f5f8aeb803ded0c9ba9fc37f91f4f4fdbb0ddbe1112facf836d7900a414 7.844MiB
      registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-promtail sha256:4266eb1508527bd0fe65c45ffbf3581baedfd2ff194136b192e9313fcc1ff407 17.94MiB
      registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-promtail sha256:34971b2d1eb98e410c2802e6bb3a7f499f9c5bff1b870ed329089ecdb21cf856 72.71MiB
    manifests:
      sha256:0c9582db3078e926302273d1e2a09c77779414351bd5374fa0e38215ac37e443
      sha256:72f530e48a09e146af142b02a6e3000d56dda22d34be796489fb7adf5e87f908
      sha256:1264aa92ebc6cccf46da3a35fbb54421b806dda5640c7e9706e6e815d13f509d -> v4.3.0
      sha256:1264aa92ebc6cccf46da3a35fbb54421b806dda5640c7e9706e6e815d13f509d -> v4.3.0-202001081344
  openshift/ose-template-service-broker
    blobs:
      registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-template-service-broker sha256:455ea8ab06218495bbbcb14b750a0d644897b24f8c5dcf9e8698e27882583412 1.575KiB
      registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-template-service-broker sha256:d803d269fd60523fefea817170aa822ab49f02a20e19d5486fdd33cf2d169213 5.125KiB
      registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-template-service-broker sha256:935ce2f796a92f0b6fa7762d7bab7d252ddb59d03cd3afba42d3785b571854f1 3.333MiB
      registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-template-service-broker sha256:df57162bfd87083528c3e2b590603b13874e667c6dc215a4646dbff0dc44f746 7.842MiB
      registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-template-service-broker sha256:4670d9fffbe8f50d8bc91b6199aa6fee8e817c1202f1b87eeeb91f2b732a1e33 12.97MiB
      registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-template-service-broker sha256:bb13d92caffa705f32b8a7f9f661e07ddede310c6ccfa78fb53a49539740e29b 72.71MiB
    manifests:
      sha256:7d884c5df58e77a1438127efebedda5a2f23c7aeb038d0cd874152b786ca9361
      sha256:913cc575945660a12a436928c676bcc67e0123e1434580dcc8942f782f3e865d
      sha256:913cc575945660a12a436928c676bcc67e0123e1434580dcc8942f782f3e865d -> v4.3.5
      sha256:913cc575945660a12a436928c676bcc67e0123e1434580dcc8942f782f3e865d -> v4.3.5-202003020549
  openshift/ose-template-service-broker-operator
    blobs:
      registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-template-service-broker-operator sha256:455ea8ab06218495bbbcb14b750a0d644897b24f8c5dcf9e8698e27882583412 1.575KiB
      registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-template-service-broker-operator sha256:7116167babc16bf649aa600e0076eaee0f186bbe7bfaa2275f735b4b007aa0ed 6.91KiB
      registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-template-service-broker-operator sha256:fce12904b2e1bccb67b8f46d6ef7eba0397fb3a0ad8a5eeac3314a89572428a8 7.899KiB
      registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-template-service-broker-operator sha256:935ce2f796a92f0b6fa7762d7bab7d252ddb59d03cd3afba42d3785b571854f1 3.333MiB
      registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-template-service-broker-operator sha256:6bbe6e6d5a7692b6104c257b79b81f1ff639bb40b1eee00c400caa81d7e4fa97 33.69MiB
      registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-template-service-broker-operator sha256:bb13d92caffa705f32b8a7f9f661e07ddede310c6ccfa78fb53a49539740e29b 72.71MiB
      registry-proxy.engineering.redhat.com/rh-osbs/openshift-ose-template-service-broker-operator sha256:fa9639089b9b366a80e9aba732eaa5030cf9259aa6b37821a1733fb60c608a19 76.15MiB
    manifests:
      sha256:3eb7a73e127872ac3bbad1abae1a52b84f5b0071c4dd8719678925c24bd4b0bc
      sha256:a50ddb888b5fa0b04a95a1e37aeb533bc8f6c72535be5b55bcd286794214960e
      sha256:3eb7a73e127872ac3bbad1abae1a52b84f5b0071c4dd8719678925c24bd4b0bc -> v4.3.5
      sha256:3eb7a73e127872ac3bbad1abae1a52b84f5b0071c4dd8719678925c24bd4b0bc -> v4.3.5-202003020549
  stats: shared=9 unique=57 size=3.054GiB ratio=0.81

phase 0:
  default-route-openshift-image-registry.apps.qeci-3211.qe.gcp.devcluster.openshift.com openshift/ose-logging-kibana5                  blobs=8 mounts=0 manifests=4 shared=0
  default-route-openshift-image-registry.apps.qeci-3211.qe.gcp.devcluster.openshift.com openshift/ose-template-service-broker-operator blobs=7 mounts=0 manifests=4 shared=5
  default-route-openshift-image-registry.apps.qeci-3211.qe.gcp.devcluster.openshift.com openshift/ose-promtail                         blobs=6 mounts=0 manifests=4 shared=2
phase 1:
  default-route-openshift-image-registry.apps.qeci-3211.qe.gcp.devcluster.openshift.com openshift/ose-metering-ansible-operator       blobs=7 mounts=5 manifests=4 shared=5
  default-route-openshift-image-registry.apps.qeci-3211.qe.gcp.devcluster.openshift.com openshift/ose-ansible-service-broker-operator blobs=7 mounts=5 manifests=4 shared=5
  default-route-openshift-image-registry.apps.qeci-3211.qe.gcp.devcluster.openshift.com openshift/ose-local-storage-operator          blobs=6 mounts=3 manifests=4 shared=4
  default-route-openshift-image-registry.apps.qeci-3211.qe.gcp.devcluster.openshift.com openshift/ose-metering-hive                   blobs=6 mounts=3 manifests=4 shared=4
  default-route-openshift-image-registry.apps.qeci-3211.qe.gcp.devcluster.openshift.com openshift/ose-logging-eventrouter             blobs=5 mounts=3 manifests=3 shared=3
  default-route-openshift-image-registry.apps.qeci-3211.qe.gcp.devcluster.openshift.com openshift/ose-logging-fluentd                 blobs=5 mounts=3 manifests=4 shared=3
  default-route-openshift-image-registry.apps.qeci-3211.qe.gcp.devcluster.openshift.com openshift/ose-metering-presto                 blobs=5 mounts=3 manifests=4 shared=3
  default-route-openshift-image-registry.apps.qeci-3211.qe.gcp.devcluster.openshift.com openshift/ose-logging-curator5                blobs=5 mounts=3 manifests=4 shared=3
  default-route-openshift-image-registry.apps.qeci-3211.qe.gcp.devcluster.openshift.com openshift/ose-oauth-proxy                     blobs=5 mounts=3 manifests=4 shared=3
  default-route-openshift-image-registry.apps.qeci-3211.qe.gcp.devcluster.openshift.com openshift/ose-logging-elasticsearch5          blobs=6 mounts=2 manifests=4 shared=2
phase 2:
  default-route-openshift-image-registry.apps.qeci-3211.qe.gcp.devcluster.openshift.com openshift/ose-ghostunnel                       blobs=6 mounts=4 manifests=4 shared=4
  default-route-openshift-image-registry.apps.qeci-3211.qe.gcp.devcluster.openshift.com openshift/ose-template-service-broker          blobs=6 mounts=4 manifests=4 shared=4
  default-route-openshift-image-registry.apps.qeci-3211.qe.gcp.devcluster.openshift.com openshift/ose-ansible-service-broker           blobs=6 mounts=4 manifests=4 shared=4
  default-route-openshift-image-registry.apps.qeci-3211.qe.gcp.devcluster.openshift.com openshift/ose-local-storage-diskmaker          blobs=6 mounts=4 manifests=4 shared=4
  default-route-openshift-image-registry.apps.qeci-3211.qe.gcp.devcluster.openshift.com openshift/ose-metering-reporting-operator      blobs=6 mounts=4 manifests=4 shared=4
  default-route-openshift-image-registry.apps.qeci-3211.qe.gcp.devcluster.openshift.com openshift/ose-node-feature-discovery           blobs=6 mounts=4 manifests=4 shared=4
  default-route-openshift-image-registry.apps.qeci-3211.qe.gcp.devcluster.openshift.com openshift/ose-elasticsearch-operator           blobs=6 mounts=4 manifests=4 shared=4
  default-route-openshift-image-registry.apps.qeci-3211.qe.gcp.devcluster.openshift.com openshift/ose-local-storage-static-provisioner blobs=6 mounts=4 manifests=4 shared=4
  default-route-openshift-image-registry.apps.qeci-3211.qe.gcp.devcluster.openshift.com openshift/ose-cluster-logging-operator         blobs=6 mounts=4 manifests=4 shared=4
  default-route-openshift-image-registry.apps.qeci-3211.qe.gcp.devcluster.openshift.com openshift/ose-cluster-nfd-operator             blobs=6 mounts=4 manifests=4 shared=4
  default-route-openshift-image-registry.apps.qeci-3211.qe.gcp.devcluster.openshift.com openshift/ose-metering-hadoop                  blobs=5 mounts=4 manifests=4 shared=4

info: Planning completed in 4.58s
error: unable to push manifest to default-route-openshift-image-registry.apps.qeci-3211.qe.gcp.devcluster.openshift.com/openshift/ose-logging-kibana5: received unexpected HTTP status: 500 Internal Server Error
error: unable to push manifest to default-route-openshift-image-registry.apps.qeci-3211.qe.gcp.devcluster.openshift.com/openshift/ose-promtail: received unexpected HTTP status: 500 Internal Server Error
error: unable to push manifest to default-route-openshift-image-registry.apps.qeci-3211.qe.gcp.devcluster.openshift.com/openshift/ose-promtail: manifest invalid: manifest invalid
error: unable to push manifest to default-route-openshift-image-registry.apps.qeci-3211.qe.gcp.devcluster.openshift.com/openshift/ose-logging-kibana5:v4.3.5: manifest invalid: manifest invalid
error: unable to push manifest to default-route-openshift-image-registry.apps.qeci-3211.qe.gcp.devcluster.openshift.com/openshift/ose-logging-kibana5:v4.3.5-202003020549: manifest invalid: manifest invalid
error: unable to push manifest to default-route-openshift-image-registry.apps.qeci-3211.qe.gcp.devcluster.openshift.com/openshift/ose-promtail:v4.3.0: manifest invalid: manifest invalid
error: unable to push manifest to default-route-openshift-image-registry.apps.qeci-3211.qe.gcp.devcluster.openshift.com/openshift/ose-promtail:v4.3.0-202001081344: manifest invalid: manifest invalid
info: Mirroring completed in 1.3s (0B/s)
error: one or more errors occurred while uploading images

Works when using oc 4.3.4
oc434 image mirror --filename=Image_Mapping.txt --insecure=true --skip-missing=true --skip-verification=true
default-route-openshift-image-registry.apps.qeci-3214.0303-otz.qe.rhcloud.com/
  openshift/ose-pod
    blobs:
      registry.redhat.io/openshift4/ose-pod sha256:4fbc3bafa3d4400bb97a733c1fe12f2f99bf38b9d5b913d5034f29798739654d 1.585KiB
      registry.redhat.io/openshift4/ose-pod sha256:34a9a5f1c95b6e6fa2ccf45f2481170d145347d809df43d0396afca43127ead5 4.887KiB
      registry.redhat.io/openshift4/ose-pod sha256:b8a1d56473c3a0983fb4a68f7ecf81ccf0d58cc8083f41dc84ba3f9b85a8cb9b 698.6KiB
      registry.redhat.io/openshift4/ose-pod sha256:8d4203411daf0c5f64ba4820c5cfca11c859871429f7ca8b02eebf810962ea7d 3.336MiB
      registry.redhat.io/openshift4/ose-pod sha256:e232ae6317c2547a0c0941579360e4600a08bdd3398fc22ad2160e187256e94f 7.842MiB
      registry.redhat.io/openshift4/ose-pod sha256:34971b2d1eb98e410c2802e6bb3a7f499f9c5bff1b870ed329089ecdb21cf856 72.71MiB
    manifests:
      sha256:2dc3bdcb2b0bf1d6c6ae749be0163e6d7ca813ecfba5e5f5d88970c73a9d12a9 -> v4.3.0
  stats: shared=0 unique=6 size=84.58MiB ratio=1.00

phase 0:
  default-route-openshift-image-registry.apps.qeci-3214.0303-otz.qe.rhcloud.com openshift/ose-pod blobs=6 mounts=0 manifests=1 shared=0

info: Planning completed in 1.8s
sha256:2dc3bdcb2b0bf1d6c6ae749be0163e6d7ca813ecfba5e5f5d88970c73a9d12a9 default-route-openshift-image-registry.apps.qeci-3214.0303-otz.qe.rhcloud.com/openshift/ose-pod:v4.3.0
info: Mirroring completed in 580ms (0B/s)


Doesn't work using 4.3.5
[anli@preserve-docker-slave 83568]$ oc435 image mirror --filename=Image_Mapping.txt --insecure=true --skip-missing=true --skip-verification=true
default-route-openshift-image-registry.apps.qeci-3214.0303-otz.qe.rhcloud.com/
  openshift/ose-pod
    blobs:
      registry.redhat.io/openshift4/ose-pod sha256:4fbc3bafa3d4400bb97a733c1fe12f2f99bf38b9d5b913d5034f29798739654d 1.585KiB
      registry.redhat.io/openshift4/ose-pod sha256:34a9a5f1c95b6e6fa2ccf45f2481170d145347d809df43d0396afca43127ead5 4.887KiB
      registry.redhat.io/openshift4/ose-pod sha256:b8a1d56473c3a0983fb4a68f7ecf81ccf0d58cc8083f41dc84ba3f9b85a8cb9b 698.6KiB
      registry.redhat.io/openshift4/ose-pod sha256:8d4203411daf0c5f64ba4820c5cfca11c859871429f7ca8b02eebf810962ea7d 3.336MiB
      registry.redhat.io/openshift4/ose-pod sha256:e232ae6317c2547a0c0941579360e4600a08bdd3398fc22ad2160e187256e94f 7.842MiB
      registry.redhat.io/openshift4/ose-pod sha256:34971b2d1eb98e410c2802e6bb3a7f499f9c5bff1b870ed329089ecdb21cf856 72.71MiB
    manifests:
      sha256:2dc3bdcb2b0bf1d6c6ae749be0163e6d7ca813ecfba5e5f5d88970c73a9d12a9
      sha256:d1a0cf18fab1e822040966c9aecafaba8c64a3e1fe65aeb87084e407953d4a49
      sha256:d1a0cf18fab1e822040966c9aecafaba8c64a3e1fe65aeb87084e407953d4a49 -> v4.3.0
  stats: shared=0 unique=6 size=84.58MiB ratio=1.00

phase 0:
  default-route-openshift-image-registry.apps.qeci-3214.0303-otz.qe.rhcloud.com openshift/ose-pod blobs=6 mounts=0 manifests=3 shared=0

info: Planning completed in 1.84s
error: unable to push manifest to default-route-openshift-image-registry.apps.qeci-3214.0303-otz.qe.rhcloud.com/openshift/ose-pod: received unexpected HTTP status: 500 Internal Server Error
error: unable to push manifest to default-route-openshift-image-registry.apps.qeci-3214.0303-otz.qe.rhcloud.com/openshift/ose-pod:v4.3.0: manifest invalid: manifest invalid
info: Mirroring completed in 640ms (0B/s)
error: one or more errors occurred while uploading images

A release block to 4.3.5. set to urgent

Same client version works to mirror image from quay.io to intergrated registry. All failed to mirror image from registry.redhat.io, registry-proxy.engineering.redhat.com.
And it's 100% reproduced

QE 3rd Level Automation testing require images to be mirrorred to intenral registry. That bugs is blocking QE automation testing.
Disconnect Cluster also need to mirror images to external registry. That may block the custom The images need to be mirror the images.

Looks like the fix from https://github.com/openshift/oc/pull/298 might be the cause of this problem, moving to OLM folks for further investigation.

Fix is not available yet at the latest nightly build. Moving to MODIFIED,  https://github.com/openshift/oc/pull/339 seems to add the fix into master. @Evan, do we have a PR for 4.3?

oc version
Client Version: 4.3.0-0.nightly-2020-03-09-200240

Options:
      --dir='': The directory on disk that file:// images will be copied under.
      --dry-run=false: Print the actions that would be taken and exit without writing to the destinations.
  -f, --filename=[]: One or more files to read SRC=DST or SRC DST [DST ...] mappings from.
      --filter-by-os='': A regular expression to control which images are considered when multiple variants are
available. Images will be passed as '<platform>/<architecture>[/<variant>]'.
      --force=false: Attempt to write all layers and manifests even if they exist in the remote repository.
      --from-dir='': The directory on disk that file:// images will be read from. Overrides --dir
      --insecure=false: Allow push and pull operations to registries to be made over HTTP
      --max-per-registry=6: Number of concurrent requests allowed per registry.
      --max-registry=4: Number of concurrent registries to connect to at any one time.
  -a, --registry-config='': Path to your registry credentials (defaults to ~/.docker/config.json)
      --s3-source-bucket=[]: A list of bucket/path locations on S3 that may contain already uploaded blobs. Add [store]
to the end to use the container image registry path convention.
      --skip-missing=false: If an input image is not found, skip them.
      --skip-mount=false: Always push layers instead of cross-mounting them
      --skip-multiple-scopes=false: Some registries do not support multiple scopes passed to the registry login.
      --skip-verification=false: Skip verifying the integrity of the retrieved content. This is not recommended, but may
be necessary when importing images from older image registries. Only bypass verification if the registry is known to be
trustworthy

Since the problem described in this bug report should be
resolved in a recent advisory, it has been closed with a
resolution of ERRATA.

For information on the advisory, and where to find the updated
files, follow the link below.

If the solution does not work for you, open a new bug report.

https://access.redhat.com/errata/RHBA-2020:0858