Kernel: kernel-4.18.0-305.el8

The Openvswitch module queue_userspace_packet() routine calls kfree_skb() when it should be calling __kfree_skb() under no error condition. This skews dropwatch results and makes it look like there was packet drop when in fact there was none:

git/rhel-8/net/openvswitch/datapath.c

 412 static int queue_userspace_packet(struct datapath *dp, struct sk_buff *skb,↩
 413 ▹       ▹       ▹       ▹         const struct sw_flow_key *key,↩
 414 ▹       ▹       ▹       ▹         const struct dp_upcall_info *upcall_info,↩
 415 ▹       ▹       ▹       ▹         uint32_t cutlen)↩
 416 {↩
.
.

 560 ▹       err = genlmsg_unicast(ovs_dp_get_net(dp), user_skb, upcall_info->portid);↩ <---- this returns 0
 561 ▹       user_skb = NULL;↩
 562 out:↩
 563 ▹       if (err)↩
 564 ▹       ▹       skb_tx_error(skb);↩
 565 ▹       kfree_skb(user_skb);↩
 566 ▹       kfree_skb(nskb);↩ <---- and it call kfree_skb() instead of __kfree_skb() here incorrectly (this is a minor bug)
 567 ▹       return err;↩
 568 }↩

From dropwatch2.stp:

#! /usr/bin/env stap
### dropwatch2.stp - logs places where the kernel has called kfree_skb ###
# usage: stap dropwatch2.stp

# Array to hold the list of drop points we find
global locations

# Note when we turn the monitor on and off
probe begin { printf("Monitoring for dropped packets...\n") }
probe end { printf("Stopping dropped packet monitor.\n") }

# increment a drop counter for every location we drop at
probe kernel.trace("kfree_skb") { locations[$location] <<< 1 }

# Every 5 seconds report our drop locations
probe timer.sec(5)
{
  printf("\n")
  printf("%s\n", ctime(gettimeofday_s()))
  foreach (l in locations-) {
    printf("%d packets dropped at %s\n",
           @count(locations[l]), symname(l))
  }
  delete locations
}
### dropwatch2.stp ends ###

Result:

18 packets dropped at queue_userspace_packet