Bug 2092937

Created attachment 1886120 [details]
master log

Created attachment 1886121 [details]
nbdb

Created attachment 1886122 [details]
sbdb

Created attachment 1886149 [details]
ovnkube-node worker 23

Created attachment 1886151 [details]
ovnkube-node worker 24

I0602 20:13:03.237637       1 transact.go:41] Configuring OVN: [{Op:update Table:Logical_Switch_Port Row:map[addresses:{GoSet:[unknown]} options:{GoMap:map[network_name:physnet]} tag_request:{GoSet:[0]} type:localnet] Rows:[] Columns:[] Mutations:[] Timeout:<nil> Where:[where column _uuid == {ef49b773-50d7-4125-8ac3-706ca8a54f0f}] Until: Durable:<nil> Comment:<nil> Lock:<nil> UUIDName:} {Op:mutate Table:Logical_Switch Row:map[] Rows:[] Columns:[] Mutations:[{Column:ports Mutator:insert Value:{GoSet:[{GoUUID:ef49b773-50d7-4125-8ac3-706ca8a54f0f}]}}] Timeout:<nil> Where:[where column _uuid == {06d9ce45-9730-48cc-91a1-a9deb47fc604}] Until: Durable:<nil> Comment:<nil> Lock:<nil> UUIDName:}]
I0602 20:13:03.237704       1 transact.go:41] Configuring OVN: [{Op:update Table:Logical_Switch_Port Row:map[addresses:{GoSet:[f4:e9:d4:ee:84:6e]} options:{GoMap:map[router-port:rtoe-GR_master-2]} type:router] Rows:[] Columns:[] Mutations:[] Timeout:<nil> Where:[where column _uuid == {5cec15d7-3af7-43e9-9e0d-e7e91861fdec}] Until: Durable:<nil> Comment:<nil> Lock:<nil> UUIDName:} {Op:mutate Table:Logical_Switch Row:map[] Rows:[] Columns:[] Mutations:[{Column:ports Mutator:insert Value:{GoSet:[{GoUUID:5cec15d7-3af7-43e9-9e0d-e7e91861fdec}]}}] Timeout:<nil> Where:[where column _uuid == {8ddb2b1c-475e-4c0c-ad55-871e5ee529d2}] Until: Durable:<nil> Comment:<nil> Lock:<nil> UUIDName:}]
ovsset supports only go slice/string/numbers/uuid or pointers to those typespanic: reflect: call of reflect.Value.Interface on zero Value




goroutine 455 [running]:
reflect.valueInterface({0x0, 0x0, 0x1bb02b2}, 0x4c)
        /usr/local/go/src/reflect/value.go:1356 +0x10e
reflect.Value.Interface(...)
        /usr/local/go/src/reflect/value.go:1351
github.com/ovn-org/libovsdb/ovsdb.NewOvsSet({0x17b0680, 0x0})
        /home/surya/go/src/github.com/openshift/ovn-kubernetes/go-controller/vendor/github.com/ovn-org/libovsdb/ovsdb/set.go:47 +0x4dd
github.com/ovn-org/libovsdb/ovsdb.NativeToOvs(0xc0004f0d20, {0x17b0680, 0x0})
        /home/surya/go/src/github.com/openshift/ovn-kubernetes/go-controller/vendor/github.com/ovn-org/libovsdb/ovsdb/bindings.go:279 +0x3ac
github.com/ovn-org/libovsdb/mapper.Mapper.NewRow({{{0xc000270570, 0xe}, {0xc000270580, 0x6}, 0xc0002d0930}}, 0xc00330a2e8, {0xc002d4b280, 0x4, 0x0})
        /home/surya/go/src/github.com/openshift/ovn-kubernetes/go-controller/vendor/github.com/ovn-org/libovsdb/mapper/mapper.go:121 +0x305
github.com/ovn-org/libovsdb/client.api.Update({0xc000c92160, {0x1dcbf78, 0xc0030db170}, 0xc00020af90}, {0x19a61e0, 0xc000fee600}, {0xc002d4b280, 0xc000fee6c0, 0x4})
        /home/surya/go/src/github.com/openshift/ovn-kubernetes/go-controller/vendor/github.com/ovn-org/libovsdb/client/api.go:358 +0x43a
github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdbops.(*ModelClient).update(0x496371, {0x19a61e0, 0xc000fee900}, 0xc002c59580)
        /home/surya/go/src/github.com/openshift/ovn-kubernetes/go-controller/pkg/libovsdbops/model_client.go:318 +0x7c
github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdbops.(*ModelClient).CreateOrUpdateOps.func1({0x19a61e0, 0xc000fee900}, 0xc000fee900)
        /home/surya/go/src/github.com/openshift/ovn-kubernetes/go-controller/pkg/libovsdbops/model_client.go:163 +0x85
github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdbops.(*ModelClient).buildOps.func1({0x19a61e0, 0xc000fee900})
        /home/surya/go/src/github.com/openshift/ovn-kubernetes/go-controller/pkg/libovsdbops/model_client.go:240 +0x71
github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdbops.onModels({0x17a2a80, 0xc00320ad98}, 0xc00330a870)
        /home/surya/go/src/github.com/openshift/ovn-kubernetes/go-controller/pkg/libovsdbops/model.go:266 +0x21c
github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdbops.(*ModelClient).buildOps(0x7f747d05ef18, 0xc00330a8e8, 0xc00330a8d8, {0xc00330ae90, 0x2, 0x7f7454074108})
        /home/surya/go/src/github.com/openshift/ovn-kubernetes/go-controller/pkg/libovsdbops/model_client.go:235 +0x37b
github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdbops.(*ModelClient).CreateOrUpdateOps(0x0, {0xc00330ae90, 0xc00330a940, 0x40edc7})
        /home/surya/go/src/github.com/openshift/ovn-kubernetes/go-controller/pkg/libovsdbops/model_client.go:172 +0x5f
github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdbops.(*ModelClient).CreateOrUpdate(0xc000c40f30, {0xc00330ae90, 0x1b3e958, 0xc})
        /home/surya/go/src/github.com/openshift/ovn-kubernetes/go-controller/pkg/libovsdbops/model_client.go:153 +0x25
github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn.(*Controller).addExternalSwitch(0xc000c40d80, {0x1b33e2a, 0x5}, {0xc002e8dfb0, 0xf}, {0xc00293ea68, 0x8}, {0xc0030040b0, 0x7f74540f6760}, {0xc002a419f8, ...}, ...)
        /home/surya/go/src/github.com/openshift/ovn-kubernetes/go-controller/pkg/ovn/gateway_init.go:544 +0x80d
github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn.(*Controller).gatewayInit(0xc000c40d80, {0xc00293ea68, 0x8}, {0xc000e82b60, 0x2, 0xc003000820}, {0xc0005da900, 0x2, 0x18c7140}, 0xc003000750, ...)
        /home/surya/go/src/github.com/openshift/ovn-kubernetes/go-controller/pkg/ovn/gateway_init.go:236 +0x222e
github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn.(*Controller).syncGatewayLogicalNetwork(0xc000c40d80, 0xc0029785a0, 0xc003000750, {0xc0005da900, 0x0, 0x2}, 0xc002f12d00)
        /home/surya/go/src/github.com/openshift/ovn-kubernetes/go-controller/pkg/ovn/master.go:668 +0x25d
github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn.(*Controller).syncNodeGateway(0xc000c40d80, 0xc0029785a0, {0xc0005da900, 0x2, 0x2})
        /home/surya/go/src/github.com/openshift/ovn-kubernetes/go-controller/pkg/ovn/ovn.go:1121 +0x228
github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn.(*Controller).WatchNodes.func1({0x1b10500, 0xc0029785a0})
        /home/surya/go/src/github.com/openshift/ovn-kubernetes/go-controller/pkg/ovn/ovn.go:1175 +0x757
k8s.io/client-go/tools/cache.ResourceEventHandlerFuncs.OnAdd(...)
        /home/surya/go/src/github.com/openshift/ovn-kubernetes/go-controller/vendor/k8s.io/client-go/tools/cache/controller.go:231
k8s.io/client-go/tools/cache.FilteringResourceEventHandler.OnAdd({0xc001bf9740, {0x1dd6bf8, 0xc001a3c360}}, {0x1b10500, 0xc0029785a0})
        /home/surya/go/src/github.com/openshift/ovn-kubernetes/go-controller/vendor/k8s.io/client-go/tools/cache/controller.go:264 +0x64
github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory.(*Handler).OnAdd(...)
        /home/surya/go/src/github.com/openshift/ovn-kubernetes/go-controller/pkg/factory/handler.go:40
github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory.newQueuedInformer.func1.1(0xc00075e8c0)
        /home/surya/go/src/github.com/openshift/ovn-kubernetes/go-controller/pkg/factory/handler.go:450 +0xba
created by github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory.newQueuedInformer.func1
        /home/surya/go/src/github.com/openshift/ovn-kubernetes/go-controller/pkg/factory/handler.go:443 +0x93

gave a temp fix for now:

	modelUpdates := []interface{}{
		&externalLogicalSwitchPort.Addresses,
		&externalLogicalSwitchPort.Type,
		&externalLogicalSwitchPort.Options,
	}
	if vlanID != nil {
		intVlanID := int(*vlanID)
		externalLogicalSwitchPort.TagRequest = &intVlanID
		modelUpdates = append(modelUpdates, &externalLogicalSwitchPort.TagRequest)
	}

	opModels = []libovsdbops.OperationModel{
		{
			Model: &externalLogicalSwitchPort,
			OnModelUpdates: modelUpdates,
			DoAfter: func() {
				externalLogicalSwitch.Ports = []string{externalLogicalSwitchPort.UUID}
			},
		},

Tim has reproduced this locally and is working on a fix. We probably still have a second problem here where the lr-routes for ecmp are not being created correctly. Need to figure out why.

Created attachment 1886446 [details]
ovnkube-lmaster logs

Since the problem described in this bug report should be
resolved in a recent advisory, it has been closed with a
resolution of ERRATA.

For information on the advisory (Important: OpenShift Container Platform 4.11.0 bug fix and security update), and where to find the updated
files, follow the link below.

If the solution does not work for you, open a new bug report.

https://access.redhat.com/errata/RHSA-2022:5069

The needinfo request[s] on this closed bug have been removed as they have been unresolved for 365 days