Bug 2119942

Summary: stuck ovn-controller
Product: Red Hat Enterprise Linux Fast Datapath Reporter: OVN Bot <ovn-bot>
Component: ovn22.06Assignee: Mohammad Heib <mheib>
Status: CLOSED ERRATA QA Contact: Jianlin Shi <jishi>
Severity: unspecified Docs Contact:
Priority: high    
Version: FDP 22.ACC: awalsh, ctrautma, jiji, jmelvin, ljozsa, mheib, mmichels, ralongi
Target Milestone: ---   
Target Release: ---   
Hardware: Unspecified   
OS: Unspecified   
Whiteboard:
Fixed In Version: ovn22.06-22.06.0-43.el8fdp Doc Type: If docs needed, set a value
Doc Text:
Story Points: ---
Clone Of: Environment:
Last Closed: 2022-10-11 12:25:32 UTC Type: ---
Regression: --- Mount Type: ---
Documentation: --- CRM:
Verified Versions: Category: ---
oVirt Team: --- RHEL 7.3 requirements from Atomic Host:
Cloudforms Team: --- Target Upstream Version:
Embargoed:

Description OVN Bot 2022-08-20 04:04:07 UTC
This is an automatically-generated clone of issue https://bugzilla.redhat.com/show_bug.cgi?id=2077078

Comment 3 Jianlin Shi 2022-09-21 09:53:33 UTC
tested with following script:

1. download src rpm for ovn
2. install the src rpm
3. apply the following patch to SOURCES/ovn-22.06.0.tar.gz 
diff ovn-22.06.0/controller/ovn-controller.c ovn-22.06.0-old/controller/ovn-controller.c                                                                                      
3956,3965d3955                                                                                        
<                         const struct local_datapath *ld;                                            
<                         HMAP_FOR_EACH (ld, hmap_node, &runtime_data->local_datapaths) {             
< 
<                             for (size_t i = 0; i < ld->n_peer_ports; i++) {                         
<                                 const struct sbrec_port_binding *peer = ld->peer_ports[i].remote;   
<                                 const struct sbrec_port_binding *mypb = ld->peer_ports[i].local;    
<                                 VLOG_INFO("PEER NAME = %s\n", peer->logical_port);                  
<                                 VLOG_INFO("pb NAME = %s\n", mypb->logical_port);                    
<                             }
<                         }

4. rebuild the rpm with: rpmbuild  -bb ovn22.06.spec
5. install the new packages from rpmbuild/RPMS
6. run following script:

systemctl start openvswitch
systemctl start ovn-northd 
ovn-nbctl set-connection ptcp:6641                                       
ovn-sbctl set-connection ptcp:6642                                                                    
ovs-vsctl set open . external_ids:system-id=hv1 external_ids:ovn-remote=tcp:1.1.40.25:6642 external_ids:ovn-encap-type=geneve external_ids:ovn-encap-ip=1.1.40.25
systemctl restart ovn-controller
                                                                                                      
ovn-nbctl ls-add sw0
ovn-nbctl lr-add ro0
ovn-nbctl lsp-add sw0 tmp
ovn-nbctl lsp-add sw0 lsp
ovn-nbctl lsp-set-type lsp router
ovn-nbctl lsp-set-options lsp router-port=lrp
ovn-nbctl lsp-set-addresses lsp  00:00:00:00:00:1
ovn-nbctl lrp-add ro0 lrp 00:00:00:00:00:1 aef0:0:0:0:0:0:0:1/64                                      
ovs-vsctl add-port br-int tmp -- set interface tmp type=internal -- set interface tmp external_ids:iface-id=tmp
ovn-nbctl set Logical_Router_Port lrp ipv6_ra_configs:send_periodic=true -- set Logical_Router_Port lrp ipv6_ra_configs:address_mode=slaac -- set Logical_Router_Port lrp ipv6_ra_configs:mtu=1280 -- set Logical_Router_Port lrp ipv6_ra_configs:max_interval=2 -- set Logical_Router_Port lrp ipv6_ra_configs:min_interval=1
ovn-nbctl --wait=hv sync
ovn-nbctl lsp-set-type lsp localnet
ovn-nbctl lsp-del lsp
ovn-nbctl --wait=hv sync
7. check /var/log/ovn/ovn-controller.log

reproduced on ovn22.06-22.06.0-27.el8:

grep -E "PEER|pb" /var/log/ovn/ovn-controller.log

......
2022-09-21T08:22:11.261Z|00121|main|INFO|PEER NAME = lsp
2022-09-21T08:22:11.261Z|00122|main|INFO|pb NAME = lrp
2022-09-21T08:22:11.261Z|00125|main|INFO|PEER NAME = lrp
2022-09-21T08:22:11.261Z|00126|main|INFO|pb NAME = þ $8 V
2022-09-21T08:22:11.261Z|00128|main|INFO|PEER NAME = þ $8 V
2022-09-21T08:22:11.261Z|00129|main|INFO|pb NAME = lrp
2022-09-21T08:22:11.270Z|00132|main|INFO|PEER NAME = lrp
2022-09-21T08:22:11.270Z|00133|main|INFO|pb NAME = þ $8 V
2022-09-21T08:22:11.271Z|00135|main|INFO|PEER NAME = þ $8 V

<=== the invalid memory pointer
2022-09-21T08:22:11.271Z|00136|main|INFO|pb NAME = lrp
......

Verified on ovn22.06-22.06.0-57.el8:

[root@dell-per740-69 bz2119942]# grep -E "PEER|pb" /var/log/ovn/ovn-controller.log                    
2022-09-21T09:47:51.596Z|00025|main|INFO|PEER NAME = lrp                                              
2022-09-21T09:47:51.596Z|00026|main|INFO|pb NAME = lsp                                                
2022-09-21T09:47:51.596Z|00027|main|INFO|PEER NAME = lsp                                              
2022-09-21T09:47:51.596Z|00028|main|INFO|pb NAME = lrp                                                
2022-09-21T09:47:51.597Z|00029|main|INFO|PEER NAME = lrp                                              
2022-09-21T09:47:51.597Z|00030|main|INFO|pb NAME = lsp                                                
2022-09-21T09:47:51.597Z|00031|main|INFO|PEER NAME = lsp                                              
2022-09-21T09:47:51.597Z|00032|main|INFO|pb NAME = lrp                                                
2022-09-21T09:47:51.597Z|00033|main|INFO|PEER NAME = lrp                                              
2022-09-21T09:47:51.597Z|00034|main|INFO|pb NAME = lsp                                                
2022-09-21T09:47:51.597Z|00035|main|INFO|PEER NAME = lsp                                              
2022-09-21T09:47:51.597Z|00036|main|INFO|pb NAME = lrp                                                
2022-09-21T09:47:51.612Z|00037|main|INFO|PEER NAME = lrp                                              
2022-09-21T09:47:51.612Z|00038|main|INFO|pb NAME = lsp                                                
2022-09-21T09:47:51.612Z|00039|main|INFO|PEER NAME = lsp                                              
2022-09-21T09:47:51.612Z|00040|main|INFO|pb NAME = lrp                                                
2022-09-21T09:47:51.613Z|00041|main|INFO|PEER NAME = lrp                                              
2022-09-21T09:47:51.613Z|00042|main|INFO|pb NAME = lsp                                                
2022-09-21T09:47:51.613Z|00043|main|INFO|PEER NAME = lsp                                              
2022-09-21T09:47:51.613Z|00044|main|INFO|pb NAME = lrp                                                
2022-09-21T09:47:51.613Z|00047|main|INFO|PEER NAME = lrp                                              
2022-09-21T09:47:51.613Z|00048|main|INFO|pb NAME = lsp                                                
2022-09-21T09:47:51.613Z|00049|main|INFO|PEER NAME = lsp                                              
2022-09-21T09:47:51.613Z|00050|main|INFO|pb NAME = lrp                                                
2022-09-21T09:47:51.614Z|00051|main|INFO|PEER NAME = lrp                                              
2022-09-21T09:47:51.614Z|00052|main|INFO|pb NAME = lsp                                                
2022-09-21T09:47:51.614Z|00053|main|INFO|PEER NAME = lsp                                              
2022-09-21T09:47:51.614Z|00054|main|INFO|pb NAME = lrp                                                
2022-09-21T09:47:51.614Z|00055|main|INFO|PEER NAME = lrp                                              
2022-09-21T09:47:51.614Z|00056|main|INFO|pb NAME = lsp                                                
2022-09-21T09:47:51.614Z|00057|main|INFO|PEER NAME = lsp                                              
2022-09-21T09:47:51.614Z|00058|main|INFO|pb NAME = lrp                                                
2022-09-21T09:47:51.614Z|00059|main|INFO|PEER NAME = lrp                                              
2022-09-21T09:47:51.614Z|00060|main|INFO|pb NAME = lsp                                                
2022-09-21T09:47:51.614Z|00061|main|INFO|PEER NAME = lsp                                              
2022-09-21T09:47:51.614Z|00062|main|INFO|pb NAME = lrp                                                
2022-09-21T09:47:51.614Z|00063|main|INFO|PEER NAME = lrp                                              
2022-09-21T09:47:51.614Z|00064|main|INFO|pb NAME = lsp                                                
2022-09-21T09:47:51.614Z|00065|main|INFO|PEER NAME = lsp                                              
2022-09-21T09:47:51.614Z|00066|main|INFO|pb NAME = lrp

<=== there is no invalid memory pointer

Comment 4 Jianlin Shi 2022-09-22 04:06:22 UTC
the added code is as follows:

                        stopwatch_start(PINCTRL_RUN_STOPWATCH_NAME,
                                        time_msec());                                                 
+                        const struct local_datapath *ld;
+                        HMAP_FOR_EACH (ld, hmap_node, &runtime_data->local_datapaths) {               
+
+                            for (size_t i = 0; i < ld->n_peer_ports; i++) {                           
+                                const struct sbrec_port_binding *peer = ld->peer_ports[i].remote;     
+                                const struct sbrec_port_binding *mypb = ld->peer_ports[i].local;      
+                                VLOG_INFO("PEER NAME = %s\n", peer->logical_port);
+                                VLOG_INFO("pb NAME = %s\n", mypb->logical_port);
+                            }
+                        }
                        pinctrl_run(ovnsb_idl_txn,

Comment 6 errata-xmlrpc 2022-10-11 12:25:32 UTC
Since the problem described in this bug report should be
resolved in a recent advisory, it has been closed with a
resolution of ERRATA.

For information on the advisory (ovn22.06 bug fix and enhancement update), and where to find the updated
files, follow the link below.

If the solution does not work for you, open a new bug report.

https://access.redhat.com/errata/RHBA-2022:6871