Bug 1475387 - patch port mysteriously deleted sporadically on a provider network of a br-ex type
patch port mysteriously deleted sporadically on a provider network of a br-e...
Status: POST
Product: Red Hat OpenStack
Classification: Red Hat
Component: opendaylight (Show other bugs)
unspecified
Unspecified Unspecified
high Severity high
: ---
: 12.0 (Pike)
Assigned To: Josh Hershberg
Itzik Brown
: Triaged, ZStream
Depends On:
Blocks:
  Show dependency treegraph
 
Reported: 2017-07-26 10:35 EDT by Josh Hershberg
Modified: 2017-12-14 09:23 EST (History)
5 users (show)

See Also:
Fixed In Version:
Doc Type: If docs needed, set a value
Doc Text:
Story Points: ---
Clone Of:
Environment:
Last Closed:
Type: Bug
Regression: ---
Mount Type: ---
Documentation: ---
CRM:
Verified Versions:
Category: ---
oVirt Team: ---
RHEL 7.3 requirements from Atomic Host:
Cloudforms Team: ---


Attachments (Terms of Use)


External Trackers
Tracker ID Priority Status Summary Last Updated
OpenDaylight Bug 8893 None None None 2017-07-26 10:39 EDT
OpenDaylight gerrit 62683 None None None 2017-09-18 14:31 EDT

  None (edit)
Description Josh Hershberg 2017-07-26 10:35:22 EDT
running netvirt stable/carbon w/ devstack and  a provider network on a br-ex type (I called it extnet) bridge and netvirt creates patch ports between br-int to extnet as expected. However, almost immediately the br-int side patch port is erased. The sporadic nature of this seems to be effected by logging because when I turned on logging, I had to rerun three times to get the failure. It should also be noted that disconnecting and then reconnecting OVS seems to fix the issue, the patch port gets configured. The following extracts from the ovsdb tcp conn and md-sal show the diagnosis. Since the patch port is in config but not operational, this would imply the issue lies in the ovsdb component somewhere. I've also attached a karaf.log with ovsdb southbound logs set to TRACE but note that it is from a different run than the following materials in this description:

[1] ovsdb request where the br-int side patch port is created:
{
  "id":"51c8c403-fdb0-43a9-a918-8137a7ae536d",
  "method":"transact",
  "params":[
    "Open_vSwitch",
    {
      "op":"insert",
      "row":{
        "name":"extnet-patch",
        "options":[
          "map",
          [
            [
              "peer",
              "extnet-int-patch"
            ]
          ]
        ],
        "type":"patch"
      },
      "uuid-name":"Interface_Random_2c5bea2ba983428a93d666ea80b60d76",
      "table":"Interface"
    },
    {
      "op":"mutate",
      "where":[
        [
          "name",
          "==",
          "extnet-patch"
        ]
      ],
      "mutations":[
        [
          "external_ids",
          "delete",
          [
            "set",
            [
              "opendaylight-iid"
            ]
          ]
        ],
        [
          "external_ids",
          "insert",
          [
            "map",
            [
              [
                "opendaylight-iid",
                "/network-topology:network-topology/network-topology:topology[
                  network-topology:topology-id='ovsdb:1'
                ]
                /network-topology:node[
                  network-topology:node-id='ovsdb://uuid/d6b942f3-3d84-4403-b840-269bf6a8e867/bridge/br-int'
                ]
                /network-topology:termination-point[
                  network-topology:tp-id='extnet-patch'
                ]
                "
              ]
            ]
          ]
        ]
      ],
      "table":"Port"
    },
    {
      "op":"insert",
      "row":{
        "interfaces":[
          "set",
          [
            [
              "named-uuid",
              "Interface_Random_2c5bea2ba983428a93d666ea80b60d76"
            ]
          ]
        ],
        "name":"extnet-patch"
      },
      "uuid-name":"Port_Random_409734ea329b41b08fef2f5d1ed93ad0",
      "table":"Port"
    },
    {
      "op":"mutate",
      "where":[
        [
          "name",
          "==",
          "br-int"
        ]
      ],
      "mutations":[
        [
          "ports",
          "insert",
          [
            "set",
            [
              [
                "named-uuid",
                "Port_Random_409734ea329b41b08fef2f5d1ed93ad0"
              ]
            ]
          ]
        ]
      ],
      "table":"Bridge"
    },
    {
      "op":"update",
      "row":{
        "options":[
          "map",
          [
            [
              "peer",
              "extnet-int-patch"
            ]
          ]
        ]
      },
      "where":[
        [
          "name",
          "==",
          "extnet-patch"
        ]
      ],
      "table":"Interface"
    },
    {
      "op":"mutate",
      "where":[
        [
          "name",
          "==",
          "extnet-patch"
        ]
      ],
      "mutations":[
        [
          "external_ids",
          "delete",
          [
            "set",
            [
              "opendaylight-iid"
            ]
          ]
        ],
        [
          "external_ids",
          "insert",
          [
            "map",
            [
              [
                "opendaylight-iid",
                "/network-topology:network-topology/network-topology:topology[
                  network-topology:topology-id='ovsdb:1'
                ]
                /network-topology:node[
                  network-topology:node-id='ovsdb://uuid/d6b942f3-3d84-4403-b840-269bf6a8e867/bridge/br-int'
                ]
                /network-topology:termination-point[
                  network-topology:tp-id='extnet-patch'
                ]
                "
              ]
            ]
          ]
        ]
      ],
      "table":"Port"
    },
    {
      "op":"update",
      "row":{
        "qos":[
          "set",
          []
        ]
      },
      "where":[
        [
          "name",
          "==",
          "extnet-patch"
        ]
      ],
      "table":"Port"
    }
  ]
}

[2] ovsdb request where it's deleted:
{
  "id":"1c35c7ba-c5eb-42c5-8593-cb565828138c",
  "method":"transact",
  "params":[
    "Open_vSwitch",
    {
      "op":"update",
      "row":{
        "other_config":[
          "map",
          [
            [
              "disable-in-band",
              "true"
            ],
            [
              "datapath-id",
              "0000960cbf79f448"
            ]
          ]
        ],
        "fail_mode":[
          "set",
          [
            "secure"
          ]
        ],
        "external_ids":[
          "map",
          [
            [
              "opendaylight-iid",
              "/network-topology:network-topology/network-topology:topology[
                network-topology:topology-id='ovsdb:1'
              ]
              /network-topology:node[
                network-topology:node-id='ovsdb://uuid/d6b942f3-3d84-4403-b840-269bf6a8e867/bridge/br-int'
              ]
              "
            ]
          ]
        ]
      },
      "where":[
        [
          "name",
          "==",
          "br-int"
        ]
      ],
      "table":"Bridge"
    },
    {
      "op":"mutate",
      "where":[
        [
          "name",
          "==",
          "br-int"
        ]
      ],
      "mutations":[
        [
          "external_ids",
          "delete",
          [
            "set",
            [
              "opendaylight-iid"
            ]
          ]
        ],
        [
          "external_ids",
          "insert",
          [
            "map",
            [
              [
                "opendaylight-iid",
                "/network-topology:network-topology/network-topology:topology[
                  network-topology:topology-id='ovsdb:1'
                ]
                /network-topology:node[
                  network-topology:node-id='ovsdb://uuid/d6b942f3-3d84-4403-b840-269bf6a8e867/bridge/br-int'
                ]
                "
              ]
            ]
          ]
        ]
      ],
      "table":"Bridge"
    },
    {
      "op":"insert",
      "row":{
        "target":"tcp:10.9.8.1:6653"
      },
      "uuid-name":"Random_a61e787726874803a55e91c08d748972",
      "table":"Controller"
    },
    {
      "op":"mutate",
      "where":[
        [
          "name",
          "==",
          "br-int"
        ]
      ],
      "mutations":[
        [
          "controller",
          "insert",
          [
            "set",
            [
              [
                "named-uuid",
                "Random_a61e787726874803a55e91c08d748972"
              ]
            ]
          ]
        ]
      ],
      "table":"Bridge"
    },
    {
      "op":"mutate",
      "where":[
        [
          "name",
          "==",
          "br-int"
        ]
      ],
      "mutations":[
        [
          "protocols",
          "insert",
          [
            "set",
            [
              "OpenFlow13"
            ]
          ]
        ]
      ],
      "table":"Bridge"
    },
    {
      "op":"delete",
      "where":[
        [
          "_uuid",
          "==",
          [
            "uuid",
            "32b896c1-7ccf-42b7-a0bb-6244125b8c3b"
          ]
        ]
      ],
      "table":"Port"
    },
    {
      "op":"comment",
      "comment":"Port: Deleting extnet-patch attached to br-int"
    },
    {
      "op":"mutate",
      "where":[
        [
          "name",
          "==",
          "br-int"
        ]
      ],
      "mutations":[
        [
          "ports",
          "delete",
          [
            "set",
            [
              [
                "uuid",
                "32b896c1-7ccf-42b7-a0bb-6244125b8c3b"
              ]
            ]
          ]
        ]
      ],
      "table":"Bridge"
    },
    {
      "op":"comment",
      "comment":"Bridge: Mutating br-int to remove port 32b896c1-7ccf-42b7-a0bb-6244125b8c3b"
    }
  ]
}

[3] operational DS (no extnet-patch)
{
            "node-id": "ovsdb://uuid/d6b942f3-3d84-4403-b840-269bf6a8e867/bridge/br-int",
            "ovsdb:managed-by": "/network-topology:network-topology/network-topology:topology[network-topology:topology-id='ovsdb:1']/network-topology:node[network-topology:node-id='ovsdb://uuid/d6b942f3-3d84-4403-b840-269bf6a8e867']",
            "ovsdb:bridge-uuid": "79bf0c97-946c-48f4-bac8-bdd926916eab",
            "ovsdb:stp_enable": false,
            "ovsdb:datapath-id": "00:00:96:0c:bf:79:f4:48",
            "ovsdb:bridge-name": "br-int",
            "ovsdb:datapath-type": "ovsdb:datapath-type-system",
            "ovsdb:protocol-entry": [
              {
                "protocol": "ovsdb:ovsdb-bridge-protocol-openflow-13"
              }
            ],
            "ovsdb:fail-mode": "ovsdb:ovsdb-fail-mode-secure",
            "ovsdb:bridge-external-ids": [
              {
                "bridge-external-id-key": "opendaylight-iid",
                "bridge-external-id-value": "/network-topology:network-topology/network-topology:topology[network-topology:topology-id='ovsdb:1']/network-topology:node[network-topology:node-id='ovsdb://uuid/d6b942f3-3d84-4403-b840-269bf6a8e867/bridge/br-int']"
              }
            ],
            "ovsdb:controller-entry": [
              {
                "target": "tcp:10.9.8.1:6653",
                "is-connected": true,
                "controller-uuid": "7ac787bf-2c25-41ff-98bb-c11b44709885"
              }
            ],
            "ovsdb:bridge-other-configs": [
              {
                "bridge-other-config-key": "disable-in-band",
                "bridge-other-config-value": "true"
              },
              {
                "bridge-other-config-key": "datapath-id",
                "bridge-other-config-value": "0000960cbf79f448"
              }
            ],
            "termination-point": [
              {
                "tp-id": "br-int",
                "ovsdb:interface-uuid": "790a4870-067e-4443-8809-31f8989b9488",
                "ovsdb:ingress-policing-burst": 0,
                "ovsdb:name": "br-int",
                "ovsdb:ofport": 65534,
                "ovsdb:port-uuid": "b3b02fcd-2a9c-465c-8a39-bee297a87918",
                "ovsdb:ifindex": 18,
                "ovsdb:ingress-policing-rate": 0,
                "ovsdb:interface-type": "ovsdb:interface-type-internal"
              }
            ]
          }
[4] config DS (w/ extnet-patch)
{
            "node-id": "ovsdb://uuid/d6b942f3-3d84-4403-b840-269bf6a8e867/bridge/br-int",
            "ovsdb:protocol-entry": [
              {
                "protocol": "ovsdb:ovsdb-bridge-protocol-openflow-13"
              }
            ],
            "ovsdb:fail-mode": "ovsdb:ovsdb-fail-mode-secure",
            "ovsdb:managed-by": "/network-topology:network-topology/network-topology:topology[network-topology:topology-id='ovsdb:1']/network-topology:node[network-topology:node-id='ovsdb://uuid/d6b942f3-3d84-4403-b840-269bf6a8e867']",
            "ovsdb:controller-entry": [
              {
                "target": "tcp:10.9.8.1:6653"
              }
            ],
            "ovsdb:bridge-name": "br-int",
            "ovsdb:bridge-other-configs": [
              {
                "bridge-other-config-key": "disable-in-band",
                "bridge-other-config-value": "true"
              },
              {
                "bridge-other-config-key": "datapath-id",
                "bridge-other-config-value": "0000960cbf79f448"
              }
            ],
            "termination-point": [
              {
                "tp-id": "extnet-patch",
                "ovsdb:name": "extnet-patch",
                "ovsdb:interface-type": "ovsdb:interface-type-patch",
                "ovsdb:options": [
                  {
                    "option": "peer",
                    "value": "extnet-int-patch"
                  }
                ]
              }
            ]
          }​
Comment 1 Josh Hershberg 2017-07-26 10:35:52 EDT
Upstream bug: https://bugs.opendaylight.org/show_bug.cgi?id=8893
Comment 2 Josh Hershberg 2017-07-26 10:38:55 EDT
Tim, could you please try this with tripleo as discussed at the weekly meeting? Thanks.
Comment 3 Tim Rozet 2017-09-18 14:30:45 EDT
I do not see this behavior.  We do wait quite a bit of time before we connect OVS to ODL, which may be why.  I see there is now a fix committed upstream though:
https://git.opendaylight.org/gerrit/#/c/62347/

Note You need to log in before you can comment on or make changes to this bug.