Description of problem: Storageclass conversion is failing to update the PVC references on job resources.  


Version-Release number of selected component (if applicable):
SOURCE CLUSTER: GCP 4.6 MTC 1.7.0
TARGET CLUSTER:  GCP 4.9 MTC 1.7.0
OADP 5.5.0


How reproducible: Always


Steps to Reproduce:
1.  Deploy an application in host cluster

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  namespace: ocp-job
  name: test-data
spec:
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 1Gi

---
apiVersion:  batch/v1
kind: Job
metadata:
  labels:
    app: analytic-warn-test
  name: analytic-warn-test
  namespace: ocp-job
spec:
  replicas:  1
  template:
    metadata:
      labels:
        app: analytic-warn-test
    spec:
      restartPolicy: OnFailure
      containers:
      - name: podtest
        imagePullPolicy: Always
        image: quay.io/openshifttest/alpine
        command: [ "/bin/sh", "-c", "--" ]
        args:
        - dd if=/dev/zero of=/data/test/file_1 bs=20M count=10;
          dd if=/dev/zero of=/data/test/file_2 bs=20M count=10;
          dd if=/dev/zero of=/data/test/file_3 bs=20M count=10;
          echo "Files created";
        restartPolicy: Never
        volumeMounts:
        - name: testvolume
          mountPath: /data/test
      volumes:
      - name: testvolume
        persistentVolumeClaim:
          claimName: test-data

2. Create a migplan and select migration type as a storage class conversion.
3. Execute cutover 

Actual results: Storage class conversion is failing to update the PVC references


Expected results: Completed jobs should never be updated


Additional info:

Controller Logs:
{"level":"error","ts":1643633678.1548114,"logger":"migration","msg":"failed updating jobs","migMigration":"migration-e1f82","phase":"SwapPVCReferences","namespace":"ocp-job","job":"analytic-warn-test","error":"Job.batch \"analytic-warn-test\" is invalid: spec.template: Invalid value: core.PodTemplateSpec{ObjectMeta:v1.ObjectMeta{Name:\"\", GenerateName:\"\", Namespace:\"\", SelfLink:\"\", UID:\"\", ResourceVersion:\"\", Generation:0, CreationTimestamp:v1.Time{Time:time.Time{wall:0x0, ext:0, loc:(*time.Location)(nil)}}, DeletionTimestamp:(*v1.Time)(nil), DeletionGracePeriodSeconds:(*int64)(nil), Labels:map[string]string{\"app\":\"analytic-warn-test\", \"controller-uid\":\"b4ca7a3b-7913-4810-868b-be0231604526\", \"job-name\":\"analytic-warn-test\"}, Annotations:map[string]string(nil), OwnerReferences:[]v1.OwnerReference(nil), Finalizers:[]string(nil), ClusterName:\"\", ManagedFields:[]v1.ManagedFieldsEntry(nil)}, Spec:core.PodSpec{Volumes:[]core.Volume{core.Volume{Name:\"testvolume\", VolumeSource:core.VolumeSource{HostPath:(*core.HostPathVolumeSource)(nil), EmptyDir:(*core.EmptyDirVolumeSource)(nil), GCEPersistentDisk:(*core.GCEPersistentDiskVolumeSource)(nil), AWSElasticBlockStore:(*core.AWSElasticBlockStoreVolumeSource)(nil), GitRepo:(*core.GitRepoVolumeSource)(nil), Secret:(*core.SecretVolumeSource)(nil), NFS:(*core.NFSVolumeSource)(nil), ISCSI:(*core.ISCSIVolumeSource)(nil), Glusterfs:(*core.GlusterfsVolumeSource)(nil), PersistentVolumeClaim:(*core.PersistentVolumeClaimVolumeSource)(0xc02a1b7368), RBD:(*core.RBDVolumeSource)(nil), Quobyte:(*core.QuobyteVolumeSource)(nil), FlexVolume:(*core.FlexVolumeSource)(nil), Cinder:(*core.CinderVolumeSource)(nil), CephFS:(*core.CephFSVolumeSource)(nil), Flocker:(*core.FlockerVolumeSource)(nil), DownwardAPI:(*core.DownwardAPIVolumeSource)(nil), FC:(*core.FCVolumeSource)(nil), AzureFile:(*core.AzureFileVolumeSource)(nil), ConfigMap:(*core.ConfigMapVolumeSource)(nil), VsphereVolume:(*core.VsphereVirtualDiskVolumeSource)(nil), AzureDisk:(*core.AzureDiskVolumeSource)(nil), PhotonPersistentDisk:(*core.PhotonPersistentDiskVolumeSource)(nil), Projected:(*core.ProjectedVolumeSource)(nil), PortworxVolume:(*core.PortworxVolumeSource)(nil), ScaleIO:(*core.ScaleIOVolumeSource)(nil), StorageOS:(*core.StorageOSVolumeSource)(nil), CSI:(*core.CSIVolumeSource)(nil), Ephemeral:(*core.EphemeralVolumeSource)(nil)}}}, InitContainers:[]core.Container(nil), Containers:[]core.Container{core.Container{Name:\"podtest\", Image:\"quay.io/openshifttest/alpine\", Command:[]string{\"/bin/sh\", \"-c\", \"--\"}, Args:[]string{\"dd if=/dev/zero of=/data/test/file_1 bs=20M count=10; dd if=/dev/zero of=/data/test/file_2 bs=20M count=10; dd if=/dev/zero of=/data/test/file_3 bs=20M count=10; echo \\\"Files created\\\";\"}, WorkingDir:\"\", Ports:[]core.ContainerPort(nil), EnvFrom:[]core.EnvFromSource(nil), Env:[]core.EnvVar(nil), Resources:core.ResourceRequirements{Limits:core.ResourceList(nil), Requests:core.ResourceList(nil)}, VolumeMounts:[]core.VolumeMount{core.VolumeMount{Name:\"testvolume\", ReadOnly:false, MountPath:\"/data/test\", SubPath:\"\", MountPropagation:(*core.MountPropagationMode)(nil), SubPathExpr:\"\"}}, VolumeDevices:[]core.VolumeDevice(nil), LivenessProbe:(*core.Probe)(nil), ReadinessProbe:(*core.Probe)(nil), StartupProbe:(*core.Probe)(nil), Lifecycle:(*core.Lifecycle)(nil), TerminationMessagePath:\"/dev/termination-log\", TerminationMessagePolicy:\"File\", ImagePullPolicy:\"Always\", SecurityContext:(*core.SecurityContext)(nil), Stdin:false, StdinOnce:false, TTY:false}}, EphemeralContainers:[]core.EphemeralContainer(nil), RestartPolicy:\"OnFailure\", TerminationGracePeriodSeconds:(*int64)(0xc044115a98), ActiveDeadlineSeconds:(*int64)(nil), DNSPolicy:\"ClusterFirst\", NodeSelector:map[string]string(nil), ServiceAccountName:\"\", AutomountServiceAccountToken:(*bool)(nil), NodeName:\"\", SecurityContext:(*core.PodSecurityContext)(0xc04ec96500), ImagePullSecrets:[]core.LocalObjectReference(nil), Hostname:\"\", Subdomain:\"\", SetHostnameAsFQDN:(*bool)(nil), Affinity:(*core.Affinity)(nil), SchedulerName:\"default-scheduler\", Tolerations:[]core.Toleration(nil), HostAliases:[]core.HostAlias(nil), PriorityClassName:\"\", Priority:(*int32)(nil), PreemptionPolicy:(*core.PreemptionPolicy)(nil), DNSConfig:(*core.PodDNSConfig)(nil), ReadinessGates:[]core.PodReadinessGate(nil), RuntimeClassName:(*string)(nil), Overhead:core.ResourceList(nil), EnableServiceLinks:(*bool)(nil), TopologySpreadConstraints:[]core.TopologySpreadConstraint(nil)}}: field is immutable","stacktrace":"github.com/konveyor/mig-controller/pkg/controller/migmigration.(*Task).swapJobsPVCRefs\n\t/remote-source/mig-controller/app/pkg/controller/migmigration/storage.go:381\ngithub.com/konveyor/mig-controller/pkg/controller/migmigration.(*Task).swapPVCReferences\n\t/remote-source/mig-controller/app/pkg/controller/migmigration/storage.go:95\ngithub.com/konveyor/mig-controller/pkg/controller/migmigration.(*Task).Run\n\t/remote-source/mig-controller/app/pkg/controller/migmigration/task.go:702\ngithub.com/konveyor/mig-controller/pkg/controller/migmigration.(*ReconcileMigMigration).migrate\n\t/remote-source/mig-controller/app/pkg/controller/migmigration/migrate.go:70\ngithub.com/konveyor/mig-controller/pkg/controller/migmigration.(*ReconcileMigMigration).Reconcile\n\t/remote-source/mig-controller/app/pkg/controller/migmigration/migmigration_controller.go:264\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/remote-source/mig-controller/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/remote-source/mig-controller/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:235\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func1.1\n\t/remote-source/mig-controller/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:198\nk8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext.func1\n\t/remote-source/mig-controller/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:185\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1\n\t/remote-source/mig-controller/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:155\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil\n\t/remote-source/mig-controller/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:156\nk8s.io/apimachinery/pkg/util/wait.JitterUntil\n\t/remote-source/mig-controller/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:133\nk8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext\n\t/remote-source/mig-controller/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:185\nk8s.io/apimachinery/pkg/util/wait.UntilWithContext\n\t/remote-source/mig-controller/app/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:99"}