Attachment 1489474 Details for Bug 1635242 – neutron-dhcp-agent consume 50% memory.txt

neutron-dhcp-agent consume 50% memory.txt

neutron-dhcp-agent consume 50% memory.txt (text/plain), 49.22 KB, created by Noam Manos on 2018-10-02 13:01:13 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Noam Manos

Created: 2018-10-02 13:01:13 UTC

Size: 49.22 KB

patch

obsolete

>[root@titan08 ~]# ps aux --sort -rss | head -11
>USER       PID %CPU %MEM    VSZ   RSS TTY      STAT START   TIME COMMAND
>qemu     14363  408 25.2 34645588 33201776 ?   Sl   Sep20 70600:48 /usr/libexec/qemu-kvm -name controller-0 -S -machine pc-i440fx-rhel7.0.0,accel=kvm,usb=off,dump-guest-core=off -cpu host -m 32768 -realtime mlock=off -smp 8,sockets=8,cores=1,threads=1 -uuid abb428b2-0232-4396-b2c5-4bbf801ddcf4 -no-user-config -nodefaults -chardev socket,id=charmonitor,path=/var/lib/libvirt/qemu/domain-17-controller-0/monitor.sock,server,nowait -mon chardev=charmonitor,id=monitor,mode=control -rtc base=utc,driftfix=slew -global kvm-pit.lost_tick_policy=delay -no-hpet -no-shutdown -global PIIX4_PM.disable_s3=1 -global PIIX4_PM.disable_s4=1 -boot strict=on -device ich9-usb-ehci1,id=usb,bus=pci.0,addr=0x6.0x7 -device ich9-usb-uhci1,masterbus=usb.0,firstport=0,bus=pci.0,multifunction=on,addr=0x6 -device ich9-usb-uhci2,masterbus=usb.0,firstport=2,bus=pci.0,addr=0x6.0x1 -device ich9-usb-uhci3,masterbus=usb.0,firstport=4,bus=pci.0,addr=0x6.0x2 -device virtio-serial-pci,id=virtio-serial0,bus=pci.0,addr=0x7 -drive file=/var/lib/libvirt/images/controller-0-disk1.qcow2,format=qcow2,if=none,id=drive-virtio-disk0,cache=unsafe -device virtio-blk-pci,scsi=off,bus=pci.0,addr=0x8,drive=drive-virtio-disk0,id=virtio-disk0,bootindex=1 -netdev tap,fd=27,id=hostnet0,vhost=on,vhostfd=30 -device virtio-net-pci,netdev=hostnet0,id=net0,mac=52:54:00:02:db:e0,bus=pci.0,addr=0x3 -netdev tap,fd=31,id=hostnet1,vhost=on,vhostfd=32 -device virtio-net-pci,netdev=hostnet1,id=net1,mac=52:54:00:92:f4:81,bus=pci.0,addr=0x4 -netdev tap,fd=33,id=hostnet2,vhost=on,vhostfd=34 -device virtio-net-pci,netdev=hostnet2,id=net2,mac=52:54:00:0f:5b:c8,bus=pci.0,addr=0x5 -chardev pty,id=charserial0 -device isa-serial,chardev=charserial0,id=serial0 -chardev socket,id=charchannel0,path=/var/lib/libvirt/qemu/channel/target/domain-17-controller-0/org.qemu.guest_agent.0,server,nowait -device virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0,id=channel0,name=org.qemu.guest_agent.0 -device usb-tablet,id=input0,bus=usb.0,port=1 -vnc 127.0.0.1:0 -vga cirrus -device virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x9 -object rng-random,id=objrng0,filename=/dev/urandom -device virtio-rng-pci,rng=objrng0,id=rng0,bus=pci.0,addr=0xa -msg timestamp=on
>qemu     14464  411 25.1 34694764 33194752 ?   Sl   Sep20 71139:15 /usr/libexec/qemu-kvm -name controller-1 -S -machine pc-i440fx-rhel7.0.0,accel=kvm,usb=off,dump-guest-core=off -cpu host -m 32768 -realtime mlock=off -smp 8,sockets=8,cores=1,threads=1 -uuid 18312ed2-b554-463c-974c-3fa3000cd210 -no-user-config -nodefaults -chardev socket,id=charmonitor,path=/var/lib/libvirt/qemu/domain-18-controller-1/monitor.sock,server,nowait -mon chardev=charmonitor,id=monitor,mode=control -rtc base=utc,driftfix=slew -global kvm-pit.lost_tick_policy=delay -no-hpet -no-shutdown -global PIIX4_PM.disable_s3=1 -global PIIX4_PM.disable_s4=1 -boot strict=on -device ich9-usb-ehci1,id=usb,bus=pci.0,addr=0x6.0x7 -device ich9-usb-uhci1,masterbus=usb.0,firstport=0,bus=pci.0,multifunction=on,addr=0x6 -device ich9-usb-uhci2,masterbus=usb.0,firstport=2,bus=pci.0,addr=0x6.0x1 -device ich9-usb-uhci3,masterbus=usb.0,firstport=4,bus=pci.0,addr=0x6.0x2 -device virtio-serial-pci,id=virtio-serial0,bus=pci.0,addr=0x7 -drive file=/var/lib/libvirt/images/controller-1-disk1.qcow2,format=qcow2,if=none,id=drive-virtio-disk0,cache=unsafe -device virtio-blk-pci,scsi=off,bus=pci.0,addr=0x8,drive=drive-virtio-disk0,id=virtio-disk0,bootindex=1 -netdev tap,fd=29,id=hostnet0,vhost=on,vhostfd=31 -device virtio-net-pci,netdev=hostnet0,id=net0,mac=52:54:00:7b:1f:cd,bus=pci.0,addr=0x3 -netdev tap,fd=32,id=hostnet1,vhost=on,vhostfd=33 -device virtio-net-pci,netdev=hostnet1,id=net1,mac=52:54:00:13:db:63,bus=pci.0,addr=0x4 -netdev tap,fd=34,id=hostnet2,vhost=on,vhostfd=35 -device virtio-net-pci,netdev=hostnet2,id=net2,mac=52:54:00:b0:25:46,bus=pci.0,addr=0x5 -chardev pty,id=charserial0 -device isa-serial,chardev=charserial0,id=serial0 -chardev socket,id=charchannel0,path=/var/lib/libvirt/qemu/channel/target/domain-18-controller-1/org.qemu.guest_agent.0,server,nowait -device virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0,id=channel0,name=org.qemu.guest_agent.0 -device usb-tablet,id=input0,bus=usb.0,port=1 -vnc 127.0.0.1:1 -vga cirrus -device virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x9 -object rng-random,id=objrng0,filename=/dev/urandom -device virtio-rng-pci,rng=objrng0,id=rng0,bus=pci.0,addr=0xa -msg timestamp=on
>qemu     30028  232 12.1 17588132 15973208 ?   Sl   Sep20 40453:08 /usr/libexec/qemu-kvm -name undercloud-0 -S -machine pc-i440fx-rhel7.0.0,accel=kvm,usb=off,dump-guest-core=off -cpu host -m 16368 -realtime mlock=off -smp 4,sockets=4,cores=1,threads=1 -uuid c6998482-c718-482a-89f3-59d579567e2d -no-user-config -nodefaults -chardev socket,id=charmonitor,path=/var/lib/libvirt/qemu/domain-4-undercloud-0/monitor.sock,server,nowait -mon chardev=charmonitor,id=monitor,mode=control -rtc base=utc,driftfix=slew -global kvm-pit.lost_tick_policy=delay -no-hpet -no-shutdown -global PIIX4_PM.disable_s3=1 -global PIIX4_PM.disable_s4=1 -boot strict=on -device ich9-usb-ehci1,id=usb,bus=pci.0,addr=0x6.0x7 -device ich9-usb-uhci1,masterbus=usb.0,firstport=0,bus=pci.0,multifunction=on,addr=0x6 -device ich9-usb-uhci2,masterbus=usb.0,firstport=2,bus=pci.0,addr=0x6.0x1 -device ich9-usb-uhci3,masterbus=usb.0,firstport=4,bus=pci.0,addr=0x6.0x2 -device virtio-serial-pci,id=virtio-serial0,bus=pci.0,addr=0x7 -drive file=/var/lib/libvirt/images/undercloud-0-disk1.qcow2,format=qcow2,if=none,id=drive-virtio-disk0,cache=unsafe -device virtio-blk-pci,scsi=off,bus=pci.0,addr=0x8,drive=drive-virtio-disk0,id=virtio-disk0,bootindex=1 -netdev tap,fd=29,id=hostnet0,vhost=on,vhostfd=31 -device virtio-net-pci,netdev=hostnet0,id=net0,mac=52:54:00:29:88:55,bus=pci.0,addr=0x3 -netdev tap,fd=32,id=hostnet1,vhost=on,vhostfd=33 -device virtio-net-pci,netdev=hostnet1,id=net1,mac=52:54:00:3d:55:d2,bus=pci.0,addr=0x4 -netdev tap,fd=34,id=hostnet2,vhost=on,vhostfd=35 -device virtio-net-pci,netdev=hostnet2,id=net2,mac=52:54:00:90:17:f1,bus=pci.0,addr=0x5 -chardev pty,id=charserial0 -device isa-serial,chardev=charserial0,id=serial0 -chardev socket,id=charchannel0,path=/var/lib/libvirt/qemu/channel/target/domain-4-undercloud-0/org.qemu.guest_agent.0,server,nowait -device virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0,id=channel0,name=org.qemu.guest_agent.0 -device usb-tablet,id=input0,bus=usb.0,port=1 -vnc 127.0.0.1:3 -vga cirrus -device virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x9 -object rng-random,id=objrng0,filename=/dev/urandom -device virtio-rng-pci,rng=objrng0,id=rng0,bus=pci.0,addr=0xa -msg timestamp=on
>qemu     14664 11.1  6.3 17849532 8362080 ?    Sl   Sep20 1920:54 /usr/libexec/qemu-kvm -name compute-0 -S -machine pc-i440fx-rhel7.0.0,accel=kvm,usb=off,dump-guest-core=off -cpu host -m 16368 -realtime mlock=off -smp 2,sockets=2,cores=1,threads=1 -uuid adba5c76-4368-496b-9c83-b5ecc99739d1 -no-user-config -nodefaults -chardev socket,id=charmonitor,path=/var/lib/libvirt/qemu/domain-20-compute-0/monitor.sock,server,nowait -mon chardev=charmonitor,id=monitor,mode=control -rtc base=utc,driftfix=slew -global kvm-pit.lost_tick_policy=delay -no-hpet -no-shutdown -global PIIX4_PM.disable_s3=1 -global PIIX4_PM.disable_s4=1 -boot strict=on -device ich9-usb-ehci1,id=usb,bus=pci.0,addr=0x6.0x7 -device ich9-usb-uhci1,masterbus=usb.0,firstport=0,bus=pci.0,multifunction=on,addr=0x6 -device ich9-usb-uhci2,masterbus=usb.0,firstport=2,bus=pci.0,addr=0x6.0x1 -device ich9-usb-uhci3,masterbus=usb.0,firstport=4,bus=pci.0,addr=0x6.0x2 -device virtio-serial-pci,id=virtio-serial0,bus=pci.0,addr=0x7 -drive file=/var/lib/libvirt/images/compute-0-disk1.qcow2,format=qcow2,if=none,id=drive-virtio-disk0,cache=unsafe -device virtio-blk-pci,scsi=off,bus=pci.0,addr=0x8,drive=drive-virtio-disk0,id=virtio-disk0,bootindex=1 -netdev tap,fd=36,id=hostnet0,vhost=on,vhostfd=38 -device virtio-net-pci,netdev=hostnet0,id=net0,mac=52:54:00:b4:ed:8b,bus=pci.0,addr=0x3 -netdev tap,fd=39,id=hostnet1,vhost=on,vhostfd=40 -device virtio-net-pci,netdev=hostnet1,id=net1,mac=52:54:00:0f:5a:48,bus=pci.0,addr=0x4 -netdev tap,fd=41,id=hostnet2,vhost=on,vhostfd=42 -device virtio-net-pci,netdev=hostnet2,id=net2,mac=52:54:00:a0:b8:4a,bus=pci.0,addr=0x5 -chardev pty,id=charserial0 -device isa-serial,chardev=charserial0,id=serial0 -chardev socket,id=charchannel0,path=/var/lib/libvirt/qemu/channel/target/domain-20-compute-0/org.qemu.guest_agent.0,server,nowait -device virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0,id=channel0,name=org.qemu.guest_agent.0 -device usb-tablet,id=input0,bus=usb.0,port=1 -vnc 127.0.0.1:4 -vga cirrus -device virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x9 -object rng-random,id=objrng0,filename=/dev/urandom -device virtio-rng-pci,rng=objrng0,id=rng0,bus=pci.0,addr=0xa -msg timestamp=on
>qemu     14755 13.8  4.8 17357776 6386432 ?    Sl   Sep20 2387:36 /usr/libexec/qemu-kvm -name compute-1 -S -machine pc-i440fx-rhel7.0.0,accel=kvm,usb=off,dump-guest-core=off -cpu host -m 16368 -realtime mlock=off -smp 2,sockets=2,cores=1,threads=1 -uuid 48a2a1b9-33ba-49f2-942e-cd4850b97baf -no-user-config -nodefaults -chardev socket,id=charmonitor,path=/var/lib/libvirt/qemu/domain-21-compute-1/monitor.sock,server,nowait -mon chardev=charmonitor,id=monitor,mode=control -rtc base=utc,driftfix=slew -global kvm-pit.lost_tick_policy=delay -no-hpet -no-shutdown -global PIIX4_PM.disable_s3=1 -global PIIX4_PM.disable_s4=1 -boot strict=on -device ich9-usb-ehci1,id=usb,bus=pci.0,addr=0x6.0x7 -device ich9-usb-uhci1,masterbus=usb.0,firstport=0,bus=pci.0,multifunction=on,addr=0x6 -device ich9-usb-uhci2,masterbus=usb.0,firstport=2,bus=pci.0,addr=0x6.0x1 -device ich9-usb-uhci3,masterbus=usb.0,firstport=4,bus=pci.0,addr=0x6.0x2 -device virtio-serial-pci,id=virtio-serial0,bus=pci.0,addr=0x7 -drive file=/var/lib/libvirt/images/compute-1-disk1.qcow2,format=qcow2,if=none,id=drive-virtio-disk0,cache=unsafe -device virtio-blk-pci,scsi=off,bus=pci.0,addr=0x8,drive=drive-virtio-disk0,id=virtio-disk0,bootindex=1 -netdev tap,fd=32,id=hostnet0,vhost=on,vhostfd=34 -device virtio-net-pci,netdev=hostnet0,id=net0,mac=52:54:00:ce:b6:e6,bus=pci.0,addr=0x3 -netdev tap,fd=35,id=hostnet1,vhost=on,vhostfd=37 -device virtio-net-pci,netdev=hostnet1,id=net1,mac=52:54:00:66:57:39,bus=pci.0,addr=0x4 -netdev tap,fd=38,id=hostnet2,vhost=on,vhostfd=39 -device virtio-net-pci,netdev=hostnet2,id=net2,mac=52:54:00:93:0f:a5,bus=pci.0,addr=0x5 -chardev pty,id=charserial0 -device isa-serial,chardev=charserial0,id=serial0 -chardev socket,id=charchannel0,path=/var/lib/libvirt/qemu/channel/target/domain-21-compute-1/org.qemu.guest_agent.0,server,nowait -device virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0,id=channel0,name=org.qemu.guest_agent.0 -device usb-tablet,id=input0,bus=usb.0,port=1 -vnc 127.0.0.1:5 -vga cirrus -device virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x9 -object rng-random,id=objrng0,filename=/dev/urandom -device virtio-rng-pci,rng=objrng0,id=rng0,bus=pci.0,addr=0xa -msg timestamp=on
>root      8551  0.0  0.0 587620 92896 ?        Sl   Sep20   3:29 /usr/bin/python /usr/bin/vbmc stop controller-2
>root      8626  0.0  0.0 587412 91680 ?        Sl   Sep20   3:25 /usr/bin/python /usr/bin/vbmc stop controller-2
>root      8700  0.0  0.0 587312 91564 ?        Sl   Sep20   3:30 /usr/bin/python /usr/bin/vbmc stop controller-2
>root      8850  0.0  0.0 587776 90524 ?        Sl   Sep20   4:25 /usr/bin/python /usr/bin/vbmc stop controller-2
>root      8774  0.0  0.0 587512 90460 ?        Sl   Sep20   3:26 /usr/bin/python /usr/bin/vbmc stop controller-2
>
>
>[root@titan08 ~]# ssh stack@undercloud-0
>Last login: Tue Oct  2 06:00:42 2018 from 172.16.0.1
>[stack@undercloud-0 ~]$ urc
>+--------------------------------------+--------------+---------+------------------------+----------------+------------+
>| ID                                   | Name         | Status  | Networks               | Image          | Flavor     |
>+--------------------------------------+--------------+---------+------------------------+----------------+------------+
>| d2ed189d-542d-47c8-b6de-551bfed8a0cf | controller-2 | ACTIVE  | ctlplane=192.168.24.21 | overcloud-full | controller |
>| edf3cda3-04b2-4ca5-b31a-77142e85363d | controller-0 | SHUTOFF | ctlplane=192.168.24.18 | overcloud-full | controller |
>| 88e572bd-abbd-42c7-b1c9-4a9efa7cc585 | controller-1 | ACTIVE  | ctlplane=192.168.24.13 | overcloud-full | controller |
>| e11d237f-eebb-4a3e-a005-d4956ae6c6ef | compute-1    | ACTIVE  | ctlplane=192.168.24.14 | overcloud-full | compute    |
>| 9f41767a-b8e8-4c6b-a84a-5decccfe1cb2 | compute-0    | ACTIVE  | ctlplane=192.168.24.8  | overcloud-full | compute    |
>+--------------------------------------+--------------+---------+------------------------+----------------+------------+
>
>
>
>
>
>(undercloud) [stack@undercloud-0 ~]$ ssh heat-admin@192.168.24.13
>Last login: Tue Oct  2 11:03:31 2018 from 192.168.24.1
>[heat-admin@controller-1 ~]$
>[heat-admin@controller-1 ~]$ ps aux --sort -rss | head -11
>USER         PID %CPU %MEM    VSZ   RSS TTY      STAT START   TIME COMMAND
>42435     112673  2.5 50.4 16838472 16521656 ?   Ss   Sep20 440:49 /usr/bin/python2 /usr/bin/neutron-dhcp-agent --config-file /usr/share/neutron/neutron-dist.conf --config-file /etc/neutron/neutron.conf --config-file /etc/neutron/dhcp_agent.ini --config-dir /etc/neutron/conf.d/common --config-dir /etc/neutron/conf.d/neutron-dhcp-agent --log-file=/var/log/neutron/dhcp-agent.log
>42462      42016 11.2  6.4 8629968 2113492 ?     Sl   Sep20 1945:21 /usr/bin/java -Djava.security.properties=/opt/opendaylight/etc/odl.java.security -Xms128M -Xmx2048m -XX:+UnlockDiagnosticVMOptions -XX:+HeapDumpOnOutOfMemoryError -Dcom.sun.management.jmxremote -Djava.net.preferIPv4Stack=true -Djava.security.egd=file:/dev/./urandom -Djava.endorsed.dirs=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.181-3.b13.el7_5.x86_64/jre/jre/lib/endorsed:/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.181-3.b13.el7_5.x86_64/jre/lib/endorsed:/opt/opendaylight/lib/endorsed -Djava.ext.dirs=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.181-3.b13.el7_5.x86_64/jre/jre/lib/ext:/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.181-3.b13.el7_5.x86_64/jre/lib/ext:/opt/opendaylight/lib/ext -Dkaraf.instances=/opt/opendaylight/instances -Dkaraf.home=/opt/opendaylight -Dkaraf.base=/opt/opendaylight -Dkaraf.data=/opt/opendaylight/data -Dkaraf.etc=/opt/opendaylight/etc -Dkaraf.restart.jvm.supported=true -Djava.io.tmpdir=/opt/opendaylight/data/tmp -Djava.util.logging.config.file=/opt/opendaylight/etc/java.util.logging.properties -Dkaraf.startLocalConsole=false -Dkaraf.startRemoteShell=true -classpath /opt/opendaylight/lib/boot/org.apache.karaf.diagnostic.boot-4.1.5.jar:/opt/opendaylight/lib/boot/org.apache.karaf.jaas.boot-4.1.5.jar:/opt/opendaylight/lib/boot/org.apache.karaf.main-4.1.5.jar:/opt/opendaylight/lib/boot/org.osgi.core-6.0.0.jar org.apache.karaf.main.Main
>42434      59731  1.7  1.2 5004924 425476 ?      Sl   Sep20 301:49 /usr/libexec/mysqld --defaults-file=/etc/my.cnf --basedir=/usr --datadir=/var/lib/mysql --plugin-dir=/usr/lib64/mysql/plugin --user=mysql --wsrep_on=ON --wsrep_provider=/usr/lib64/galera/libgalera_smm.so --wsrep-cluster-address=gcomm://controller-0.internalapi.localdomain,controller-1.internalapi.localdomain,controller-2.internalapi.localdomain --log-error=/var/log/mysql/mysqld.log --open-files-limit=16384 --pid-file=/var/run/mysql/mysqld.pid --socket=/var/lib/mysql/mysql.sock --port=3306 --wsrep_start_position=00000000-0000-0000-0000-000000000000:-1
>root       15044  0.8  0.9 3373424 312996 ?      Ssl  Sep20 147:02 /usr/bin/docker-containerd-current -l unix:///var/run/docker/libcontainerd/docker-containerd.sock --metrics-interval=0 --start-timeout 2m --state-dir /var/run/docker/libcontainerd/containerd --shim docker-containerd-shim --runtime docker-runc --debug --runtime-args --systemd-cgroup=true
>42439      53019  7.8  0.7 4670288 242984 ?      Sl   Sep20 1350:23 /usr/lib64/erlang/erts-7.3.1.4/bin/beam.smp -W w -A 128 -K true -P 1048576 -K true -B i -- -root /usr/lib64/erlang -progname erl -- -home /var/lib/rabbitmq -- -pa /usr/lib/rabbitmq/lib/rabbitmq_server-3.6.16/ebin -noshell -noinput -s rabbit boot -sname rabbit@controller-1 -boot start_sasl -config /etc/rabbitmq/rabbitmq -kernel inet_default_connect_options [{nodelay,true}] -kernel inet_default_connect_options [{nodelay,true}] -sasl errlog_type error -sasl sasl_error_logger false -rabbit error_logger {file,"/var/log/rabbitmq/rabbit@controller-1.log"} -rabbit sasl_error_logger {file,"/var/log/rabbitmq/rabbit@controller-1-sasl.log"} -rabbit enabled_plugins_file "/etc/rabbitmq/enabled_plugins" -rabbit plugins_dir "/usr/lib/rabbitmq/plugins:/usr/lib/rabbitmq/lib/rabbitmq_server-3.6.16/plugins" -rabbit plugins_expand_dir "/var/lib/rabbitmq/mnesia/rabbit@controller-1-plugins-expand" -os_mon start_cpu_sup false -os_mon start_disksup false -os_mon start_memsup false -mnesia dir "/var/lib/rabbitmq/mnesia/rabbit@controller-1"
>42436     112266  0.5  0.6 1013504 197212 ?      Sl   Sep20  91:44 nova_api_wsgi   -DFOREGROUND
>42436     112264  0.5  0.5 1086980 179460 ?      Sl   Sep20  91:02 nova_api_wsgi   -DFOREGROUND
>42436     112265  0.5  0.5 1012224 178288 ?      Sl   Sep20  88:56 nova_api_wsgi   -DFOREGROUND
>42436     112263  0.4  0.5 1082516 168080 ?      Sl   Sep20  85:38 nova_api_wsgi   -DFOREGROUND
>42407     108096  0.3  0.4 1056636 142132 ?      Sl   Sep20  57:26 cinder_wsgi     -DFOREGROUND
>
>
>
>[heat-admin@controller-1 ~]$ tail -100 /var/log/containers/neutron/dhcp-agent.log
>2018-10-02 11:14:12.799 112673 ERROR neutron.agent.dhcp.agent     message = self.waiters.get(msg_id, timeout=timeout)
>2018-10-02 11:14:12.799 112673 ERROR neutron.agent.dhcp.agent   File "/usr/lib/python2.7/site-packages/oslo_messaging/_drivers/amqpdriver.py", line 336, in get
>2018-10-02 11:14:12.799 112673 ERROR neutron.agent.dhcp.agent     'to message ID %s' % msg_id)
>2018-10-02 11:14:12.799 112673 ERROR neutron.agent.dhcp.agent MessagingTimeout: Timed out waiting for a reply to message ID 9081b14e95b14d52a72282389243bbe4
>2018-10-02 11:14:12.799 112673 ERROR neutron.agent.dhcp.agent
>2018-10-02 11:14:12.801 112673 WARNING oslo.service.loopingcall [req-2246a5f9-45fb-49e5-9809-24a86bf2e8ec - - - - -] Function 'neutron.agent.dhcp.agent.DhcpAgentWithStateReport._report_state' run outlasted interval by 30.00 sec
>2018-10-02 11:14:46.421 112673 DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process._check_child_processes" :: waited 0.000s inner /usr/lib/python2.7/site-packages/oslo_concurrency/lockutils.py:273
>2018-10-02 11:14:46.421 112673 DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" released by "neutron.agent.linux.external_process._check_child_processes" :: held 0.001s inner /usr/lib/python2.7/site-packages/oslo_concurrency/lockutils.py:285
>2018-10-02 11:15:12.804 112673 ERROR neutron.agent.dhcp.agent [req-661d95c6-30c3-40bc-ade4-4db9f42a1afb - - - - -] Failed reporting state!: MessagingTimeout: Timed out waiting for a reply to message ID 14fb44a23a8c4f97b049c5337a5f20d4
>2018-10-02 11:15:12.804 112673 ERROR neutron.agent.dhcp.agent Traceback (most recent call last):
>2018-10-02 11:15:12.804 112673 ERROR neutron.agent.dhcp.agent   File "/usr/lib/python2.7/site-packages/neutron/agent/dhcp/agent.py", line 762, in _report_state
>2018-10-02 11:15:12.804 112673 ERROR neutron.agent.dhcp.agent     ctx, self.agent_state, True)
>2018-10-02 11:15:12.804 112673 ERROR neutron.agent.dhcp.agent   File "/usr/lib/python2.7/site-packages/neutron/agent/rpc.py", line 97, in report_state
>2018-10-02 11:15:12.804 112673 ERROR neutron.agent.dhcp.agent     return method(context, 'report_state', **kwargs)
>2018-10-02 11:15:12.804 112673 ERROR neutron.agent.dhcp.agent   File "/usr/lib/python2.7/site-packages/oslo_messaging/rpc/client.py", line 179, in call
>2018-10-02 11:15:12.804 112673 ERROR neutron.agent.dhcp.agent     retry=self.retry)
>2018-10-02 11:15:12.804 112673 ERROR neutron.agent.dhcp.agent   File "/usr/lib/python2.7/site-packages/oslo_messaging/transport.py", line 133, in _send
>2018-10-02 11:15:12.804 112673 ERROR neutron.agent.dhcp.agent     retry=retry)
>2018-10-02 11:15:12.804 112673 ERROR neutron.agent.dhcp.agent   File "/usr/lib/python2.7/site-packages/oslo_messaging/_drivers/amqpdriver.py", line 584, in send
>2018-10-02 11:15:12.804 112673 ERROR neutron.agent.dhcp.agent     call_monitor_timeout, retry=retry)
>2018-10-02 11:15:12.804 112673 ERROR neutron.agent.dhcp.agent   File "/usr/lib/python2.7/site-packages/oslo_messaging/_drivers/amqpdriver.py", line 573, in _send
>2018-10-02 11:15:12.804 112673 ERROR neutron.agent.dhcp.agent     call_monitor_timeout)
>2018-10-02 11:15:12.804 112673 ERROR neutron.agent.dhcp.agent   File "/usr/lib/python2.7/site-packages/oslo_messaging/_drivers/amqpdriver.py", line 459, in wait
>2018-10-02 11:15:12.804 112673 ERROR neutron.agent.dhcp.agent     message = self.waiters.get(msg_id, timeout=timeout)
>2018-10-02 11:15:12.804 112673 ERROR neutron.agent.dhcp.agent   File "/usr/lib/python2.7/site-packages/oslo_messaging/_drivers/amqpdriver.py", line 336, in get
>2018-10-02 11:15:12.804 112673 ERROR neutron.agent.dhcp.agent     'to message ID %s' % msg_id)
>2018-10-02 11:15:12.804 112673 ERROR neutron.agent.dhcp.agent MessagingTimeout: Timed out waiting for a reply to message ID 14fb44a23a8c4f97b049c5337a5f20d4
>2018-10-02 11:15:12.804 112673 ERROR neutron.agent.dhcp.agent
>2018-10-02 11:15:12.806 112673 WARNING oslo.service.loopingcall [req-661d95c6-30c3-40bc-ade4-4db9f42a1afb - - - - -] Function 'neutron.agent.dhcp.agent.DhcpAgentWithStateReport._report_state' run outlasted interval by 30.00 sec
>2018-10-02 11:15:46.424 112673 DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process._check_child_processes" :: waited 0.001s inner /usr/lib/python2.7/site-packages/oslo_concurrency/lockutils.py:273
>2018-10-02 11:15:46.425 112673 DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" released by "neutron.agent.linux.external_process._check_child_processes" :: held 0.002s inner /usr/lib/python2.7/site-packages/oslo_concurrency/lockutils.py:285
>2018-10-02 11:16:12.810 112673 ERROR neutron.agent.dhcp.agent [req-cd3b7c33-968e-4159-a2f8-1c7805f565df - - - - -] Failed reporting state!: MessagingTimeout: Timed out waiting for a reply to message ID 0bca13f55ff94038a143a29ca89a2c3b
>2018-10-02 11:16:12.810 112673 ERROR neutron.agent.dhcp.agent Traceback (most recent call last):
>2018-10-02 11:16:12.810 112673 ERROR neutron.agent.dhcp.agent   File "/usr/lib/python2.7/site-packages/neutron/agent/dhcp/agent.py", line 762, in _report_state
>2018-10-02 11:16:12.810 112673 ERROR neutron.agent.dhcp.agent     ctx, self.agent_state, True)
>2018-10-02 11:16:12.810 112673 ERROR neutron.agent.dhcp.agent   File "/usr/lib/python2.7/site-packages/neutron/agent/rpc.py", line 97, in report_state
>2018-10-02 11:16:12.810 112673 ERROR neutron.agent.dhcp.agent     return method(context, 'report_state', **kwargs)
>2018-10-02 11:16:12.810 112673 ERROR neutron.agent.dhcp.agent   File "/usr/lib/python2.7/site-packages/oslo_messaging/rpc/client.py", line 179, in call
>2018-10-02 11:16:12.810 112673 ERROR neutron.agent.dhcp.agent     retry=self.retry)
>2018-10-02 11:16:12.810 112673 ERROR neutron.agent.dhcp.agent   File "/usr/lib/python2.7/site-packages/oslo_messaging/transport.py", line 133, in _send
>2018-10-02 11:16:12.810 112673 ERROR neutron.agent.dhcp.agent     retry=retry)
>2018-10-02 11:16:12.810 112673 ERROR neutron.agent.dhcp.agent   File "/usr/lib/python2.7/site-packages/oslo_messaging/_drivers/amqpdriver.py", line 584, in send
>2018-10-02 11:16:12.810 112673 ERROR neutron.agent.dhcp.agent     call_monitor_timeout, retry=retry)
>2018-10-02 11:16:12.810 112673 ERROR neutron.agent.dhcp.agent   File "/usr/lib/python2.7/site-packages/oslo_messaging/_drivers/amqpdriver.py", line 573, in _send
>2018-10-02 11:16:12.810 112673 ERROR neutron.agent.dhcp.agent     call_monitor_timeout)
>2018-10-02 11:16:12.810 112673 ERROR neutron.agent.dhcp.agent   File "/usr/lib/python2.7/site-packages/oslo_messaging/_drivers/amqpdriver.py", line 459, in wait
>2018-10-02 11:16:12.810 112673 ERROR neutron.agent.dhcp.agent     message = self.waiters.get(msg_id, timeout=timeout)
>2018-10-02 11:16:12.810 112673 ERROR neutron.agent.dhcp.agent   File "/usr/lib/python2.7/site-packages/oslo_messaging/_drivers/amqpdriver.py", line 336, in get
>2018-10-02 11:16:12.810 112673 ERROR neutron.agent.dhcp.agent     'to message ID %s' % msg_id)
>2018-10-02 11:16:12.810 112673 ERROR neutron.agent.dhcp.agent MessagingTimeout: Timed out waiting for a reply to message ID 0bca13f55ff94038a143a29ca89a2c3b
>2018-10-02 11:16:12.810 112673 ERROR neutron.agent.dhcp.agent
>2018-10-02 11:16:12.812 112673 WARNING oslo.service.loopingcall [req-cd3b7c33-968e-4159-a2f8-1c7805f565df - - - - -] Function 'neutron.agent.dhcp.agent.DhcpAgentWithStateReport._report_state' run outlasted interval by 30.01 sec
>2018-10-02 11:16:46.421 112673 DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process._check_child_processes" :: waited 0.000s inner /usr/lib/python2.7/site-packages/oslo_concurrency/lockutils.py:273
>2018-10-02 11:16:46.422 112673 DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" released by "neutron.agent.linux.external_process._check_child_processes" :: held 0.001s inner /usr/lib/python2.7/site-packages/oslo_concurrency/lockutils.py:285
>2018-10-02 11:17:12.819 112673 ERROR neutron.agent.dhcp.agent [req-a2369e47-3079-4b00-b59e-3ca690a4d046 - - - - -] Failed reporting state!: MessagingTimeout: Timed out waiting for a reply to message ID e7d44f0e43ae4b9baedc6ee26744e1ee
>2018-10-02 11:17:12.819 112673 ERROR neutron.agent.dhcp.agent Traceback (most recent call last):
>2018-10-02 11:17:12.819 112673 ERROR neutron.agent.dhcp.agent   File "/usr/lib/python2.7/site-packages/neutron/agent/dhcp/agent.py", line 762, in _report_state
>2018-10-02 11:17:12.819 112673 ERROR neutron.agent.dhcp.agent     ctx, self.agent_state, True)
>2018-10-02 11:17:12.819 112673 ERROR neutron.agent.dhcp.agent   File "/usr/lib/python2.7/site-packages/neutron/agent/rpc.py", line 97, in report_state
>2018-10-02 11:17:12.819 112673 ERROR neutron.agent.dhcp.agent     return method(context, 'report_state', **kwargs)
>2018-10-02 11:17:12.819 112673 ERROR neutron.agent.dhcp.agent   File "/usr/lib/python2.7/site-packages/oslo_messaging/rpc/client.py", line 179, in call
>2018-10-02 11:17:12.819 112673 ERROR neutron.agent.dhcp.agent     retry=self.retry)
>2018-10-02 11:17:12.819 112673 ERROR neutron.agent.dhcp.agent   File "/usr/lib/python2.7/site-packages/oslo_messaging/transport.py", line 133, in _send
>2018-10-02 11:17:12.819 112673 ERROR neutron.agent.dhcp.agent     retry=retry)
>2018-10-02 11:17:12.819 112673 ERROR neutron.agent.dhcp.agent   File "/usr/lib/python2.7/site-packages/oslo_messaging/_drivers/amqpdriver.py", line 584, in send
>2018-10-02 11:17:12.819 112673 ERROR neutron.agent.dhcp.agent     call_monitor_timeout, retry=retry)
>2018-10-02 11:17:12.819 112673 ERROR neutron.agent.dhcp.agent   File "/usr/lib/python2.7/site-packages/oslo_messaging/_drivers/amqpdriver.py", line 573, in _send
>2018-10-02 11:17:12.819 112673 ERROR neutron.agent.dhcp.agent     call_monitor_timeout)
>2018-10-02 11:17:12.819 112673 ERROR neutron.agent.dhcp.agent   File "/usr/lib/python2.7/site-packages/oslo_messaging/_drivers/amqpdriver.py", line 459, in wait
>2018-10-02 11:17:12.819 112673 ERROR neutron.agent.dhcp.agent     message = self.waiters.get(msg_id, timeout=timeout)
>2018-10-02 11:17:12.819 112673 ERROR neutron.agent.dhcp.agent   File "/usr/lib/python2.7/site-packages/oslo_messaging/_drivers/amqpdriver.py", line 336, in get
>2018-10-02 11:17:12.819 112673 ERROR neutron.agent.dhcp.agent     'to message ID %s' % msg_id)
>2018-10-02 11:17:12.819 112673 ERROR neutron.agent.dhcp.agent MessagingTimeout: Timed out waiting for a reply to message ID e7d44f0e43ae4b9baedc6ee26744e1ee
>2018-10-02 11:17:12.819 112673 ERROR neutron.agent.dhcp.agent
>2018-10-02 11:17:12.821 112673 WARNING oslo.service.loopingcall [req-a2369e47-3079-4b00-b59e-3ca690a4d046 - - - - -] Function 'neutron.agent.dhcp.agent.DhcpAgentWithStateReport._report_state' run outlasted interval by 30.01 sec
>2018-10-02 11:17:46.422 112673 DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process._check_child_processes" :: waited 0.000s inner /usr/lib/python2.7/site-packages/oslo_concurrency/lockutils.py:273
>2018-10-02 11:17:46.422 112673 DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" released by "neutron.agent.linux.external_process._check_child_processes" :: held 0.001s inner /usr/lib/python2.7/site-packages/oslo_concurrency/lockutils.py:285
>2018-10-02 11:18:12.827 112673 ERROR neutron.agent.dhcp.agent [req-5faa87d2-ad50-4fdd-b166-94f915ef347d - - - - -] Failed reporting state!: MessagingTimeout: Timed out waiting for a reply to message ID 3da35cce92f74e59b0bddcff1da9f8a5
>2018-10-02 11:18:12.827 112673 ERROR neutron.agent.dhcp.agent Traceback (most recent call last):
>2018-10-02 11:18:12.827 112673 ERROR neutron.agent.dhcp.agent   File "/usr/lib/python2.7/site-packages/neutron/agent/dhcp/agent.py", line 762, in _report_state
>2018-10-02 11:18:12.827 112673 ERROR neutron.agent.dhcp.agent     ctx, self.agent_state, True)
>2018-10-02 11:18:12.827 112673 ERROR neutron.agent.dhcp.agent   File "/usr/lib/python2.7/site-packages/neutron/agent/rpc.py", line 97, in report_state
>2018-10-02 11:18:12.827 112673 ERROR neutron.agent.dhcp.agent     return method(context, 'report_state', **kwargs)
>2018-10-02 11:18:12.827 112673 ERROR neutron.agent.dhcp.agent   File "/usr/lib/python2.7/site-packages/oslo_messaging/rpc/client.py", line 179, in call
>2018-10-02 11:18:12.827 112673 ERROR neutron.agent.dhcp.agent     retry=self.retry)
>2018-10-02 11:18:12.827 112673 ERROR neutron.agent.dhcp.agent   File "/usr/lib/python2.7/site-packages/oslo_messaging/transport.py", line 133, in _send
>2018-10-02 11:18:12.827 112673 ERROR neutron.agent.dhcp.agent     retry=retry)
>2018-10-02 11:18:12.827 112673 ERROR neutron.agent.dhcp.agent   File "/usr/lib/python2.7/site-packages/oslo_messaging/_drivers/amqpdriver.py", line 584, in send
>2018-10-02 11:18:12.827 112673 ERROR neutron.agent.dhcp.agent     call_monitor_timeout, retry=retry)
>2018-10-02 11:18:12.827 112673 ERROR neutron.agent.dhcp.agent   File "/usr/lib/python2.7/site-packages/oslo_messaging/_drivers/amqpdriver.py", line 573, in _send
>2018-10-02 11:18:12.827 112673 ERROR neutron.agent.dhcp.agent     call_monitor_timeout)
>2018-10-02 11:18:12.827 112673 ERROR neutron.agent.dhcp.agent   File "/usr/lib/python2.7/site-packages/oslo_messaging/_drivers/amqpdriver.py", line 459, in wait
>2018-10-02 11:18:12.827 112673 ERROR neutron.agent.dhcp.agent     message = self.waiters.get(msg_id, timeout=timeout)
>2018-10-02 11:18:12.827 112673 ERROR neutron.agent.dhcp.agent   File "/usr/lib/python2.7/site-packages/oslo_messaging/_drivers/amqpdriver.py", line 336, in get
>2018-10-02 11:18:12.827 112673 ERROR neutron.agent.dhcp.agent     'to message ID %s' % msg_id)
>2018-10-02 11:18:12.827 112673 ERROR neutron.agent.dhcp.agent MessagingTimeout: Timed out waiting for a reply to message ID 3da35cce92f74e59b0bddcff1da9f8a5
>2018-10-02 11:18:12.827 112673 ERROR neutron.agent.dhcp.agent
>2018-10-02 11:18:12.829 112673 WARNING oslo.service.loopingcall [req-5faa87d2-ad50-4fdd-b166-94f915ef347d - - - - -] Function 'neutron.agent.dhcp.agent.DhcpAgentWithStateReport._report_state' run outlasted interval by 30.01 sec
>2018-10-02 11:18:46.422 112673 DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" acquired by "neutron.agent.linux.external_process._check_child_processes" :: waited 0.000s inner /usr/lib/python2.7/site-packages/oslo_concurrency/lockutils.py:273
>2018-10-02 11:18:46.422 112673 DEBUG oslo_concurrency.lockutils [-] Lock "_check_child_processes" released by "neutron.agent.linux.external_process._check_child_processes" :: held 0.001s inner /usr/lib/python2.7/site-packages/oslo_concurrency/lockutils.py:285
>
>
>
>
>[heat-admin@controller-1 ~]$ sudo docker ps | grep neu
>3b34b38682fd        192.168.24.1:8787/rhosp14/openstack-neutron-metadata-agent:2018-09-06.1        "kolla_start"            12 days ago         Up 12 days (healthy)                         neutron_metadata_agent
>13d96f96c91e        192.168.24.1:8787/rhosp14/openstack-neutron-dhcp-agent:2018-09-06.1            "kolla_start"            12 days ago         Up 12 days (healthy)                         neutron_dhcp
>3103e7e5ce03        192.168.24.1:8787/rhosp14/openstack-neutron-server-opendaylight:2018-09-06.1   "kolla_start"            12 days ago         Up 11 days (unhealthy)                       neutron_api
>
>
>[heat-admin@controller-1 ~]$ ll -t /var/log/containers/neutron/
>total 14632
>-rw-r--r--. 1 42435 42435 2668800 Oct  2 13:20 dhcp-agent.log
>-rw-r--r--. 1 42435 42435 4803840 Oct  2 00:00 dhcp-agent.log.1
>-rw-r--r--. 1 42435 42435  220841 Oct  1 00:00 dhcp-agent.log.2.gz
>-rw-r--r--. 1 42435 42435   13690 Sep 30 00:00 metadata-agent.log.1
>-rw-r--r--. 1 42435 42435  222493 Sep 29 23:59 dhcp-agent.log.3.gz
>-rw-r--r--. 1 42435 42435     685 Sep 29 02:01 metadata-agent.log.2.gz
>-rw-r--r--. 1 42435 42435     818 Sep 29 01:59 metadata-agent.log.3.gz
>-rw-r--r--. 1 42435 42435  228563 Sep 29 00:00 dhcp-agent.log.4.gz
>-rw-r--r--. 1 42435 42435  225635 Sep 27 23:59 dhcp-agent.log.5.gz
>-rw-r--r--. 1 42435 42435  225403 Sep 27 00:00 dhcp-agent.log.6.gz
>-rw-r--r--. 1 42435 42435  223989 Sep 26 00:01 dhcp-agent.log.7.gz
>-rw-r--r--. 1 42435 42435  222977 Sep 25 00:00 dhcp-agent.log.8.gz
>-rw-r--r--. 1 42435 42435  223793 Sep 23 23:59 dhcp-agent.log.9.gz
>-rw-r--r--. 1 42435 42435  228109 Sep 23 00:00 dhcp-agent.log.10.gz
>-rw-r--r--. 1 42435 42435  135393 Sep 22 00:00 dhcp-agent.log.11.gz
>-rw-r--r--. 1 42435 42435   13590 Sep 21 10:01 server.log.1
>-rw-r--r--. 1 42435 42435  773830 Sep 21 10:01 server.log.2.gz
>-rw-r--r--. 1 42435 42435  766669 Sep 21 05:01 server.log.3.gz
>-rw-r--r--. 1 42435 42435  133110 Sep 21 00:00 server.log.4.gz
>-rw-r--r--. 1 42435 42435  329861 Sep 20 23:59 dhcp-agent.log.12.gz
>-rw-r--r--. 1 42435 42435  835787 Sep 20 23:00 server.log.5.gz
>-rw-r--r--. 1 42435 42435  931343 Sep 20 17:00 server.log.6.gz
>-rw-r--r--. 1 42435 42435   39192 Sep 20 14:48 metadata-agent.log.4.gz
>-rw-r--r--. 1 42435 42435 1240458 Sep 20 14:00 server.log.7.gz
>-rw-r--r--. 1 42435 42435       0 Sep 20 12:06 privsep-helper.log
>[heat-admin@controller-1 ~]$
>
>[heat-admin@controller-1 ~]$ tail -100 /var/log/containers/neutron/server.log.1
>2018-09-21 10:01:09.285 9238 DEBUG neutron.wsgi [-] (9238) accepted ('172.17.1.31', 43832) server /usr/lib/python2.7/site-packages/eventlet/wsgi.py:883
>2018-09-21 10:01:09.290 9238 INFO neutron.wsgi [-] 172.17.1.31 "OPTIONS / HTTP/1.0" status: 200  len: 248 time: 0.0021241
>2018-09-21 10:01:09.538 9238 DEBUG neutron.wsgi [-] (9238) accepted ('172.17.1.16', 56830) server /usr/lib/python2.7/site-packages/eventlet/wsgi.py:883
>2018-09-21 10:01:09.541 9238 INFO neutron.wsgi [-] 172.17.1.16 "OPTIONS / HTTP/1.0" status: 200  len: 248 time: 0.0013981
>2018-09-21 10:01:10.166 9256 DEBUG neutron_lib.callbacks.manager [req-f5789b3f-119b-4fe0-b657-d25a58ad72d7 - - - - -] Notify callbacks ['neutron.services.segments.db._update_segment_host_mapping_for_agent-8726361506007'] for agent, after_update _notify_loop /usr/lib/python2.7/site-packages/neutron_lib/callbacks/manager.py:193
>2018-09-21 10:01:10.327 9238 DEBUG neutron.wsgi [-] (9238) accepted ('172.17.1.14', 43616) server /usr/lib/python2.7/site-packages/eventlet/wsgi.py:883
>2018-09-21 10:01:10.331 9238 INFO neutron.wsgi [-] 172.17.1.14 "OPTIONS / HTTP/1.0" status: 200  len: 248 time: 0.0018580
>2018-09-21 10:01:10.802 9241 DEBUG networking_odl.journal.journal [req-59abaab0-f541-4333-8d40-5cfcc491aa88 - - - - -] Start processing journal entries sync_pending_entries /usr/lib/python2.7/site-packages/networking_odl/journal/journal.py:250
>2018-09-21 10:01:10.825 9241 DEBUG networking_odl.journal.journal [req-59abaab0-f541-4333-8d40-5cfcc491aa88 - - - - -] No journal entries to process sync_pending_entries /usr/lib/python2.7/site-packages/networking_odl/journal/journal.py:254
>2018-09-21 10:01:11.292 9239 DEBUG neutron.wsgi [-] (9239) accepted ('172.17.1.31', 44972) server /usr/lib/python2.7/site-packages/eventlet/wsgi.py:883
>2018-09-21 10:01:11.296 9239 INFO neutron.wsgi [-] 172.17.1.31 "OPTIONS / HTTP/1.0" status: 200  len: 248 time: 0.0016930
>2018-09-21 10:01:11.543 9238 DEBUG neutron.wsgi [-] (9238) accepted ('172.17.1.16', 57264) server /usr/lib/python2.7/site-packages/eventlet/wsgi.py:883
>2018-09-21 10:01:11.547 9238 INFO neutron.wsgi [-] 172.17.1.16 "OPTIONS / HTTP/1.0" status: 200  len: 248 time: 0.0019832
>2018-09-21 10:01:11.748 1 INFO oslo_service.service [-] Caught SIGHUP, stopping children
>2018-09-21 10:01:11.791 1 DEBUG oslo_concurrency.lockutils [-] Acquired semaphore "singleton_lock" lock /usr/lib/python2.7/site-packages/oslo_concurrency/lockutils.py:212
>2018-09-21 10:01:11.792 1 DEBUG oslo_concurrency.lockutils [-] Releasing semaphore "singleton_lock" lock /usr/lib/python2.7/site-packages/oslo_concurrency/lockutils.py:228
>2018-09-21 10:01:11.870 1 INFO neutron.common.config [-] Logging enabled!
>2018-09-21 10:01:11.871 1 INFO neutron.common.config [-] /usr/bin/neutron-server version 13.0.1.dev1
>2018-09-21 10:01:11.871 1 DEBUG neutron.common.config [-] command line: /usr/bin/neutron-server --config-file /usr/share/neutron/neutron-dist.conf --config-dir /usr/share/neutron/server --config-file /etc/neutron/neutron.conf --config-file /etc/neutron/plugin.ini --config-dir /etc/neutron/conf.d/common --config-dir /etc/neutron/conf.d/neutron-server --log-file=/var/log/neutron/server.log setup_logging /usr/lib/python2.7/site-packages/neutron/common/config.py:103
>2018-09-21 10:01:11.876 1 DEBUG oslo_policy._cache_handler [-] Reloading cached file /etc/neutron/policy.json read_cached_file /usr/lib/python2.7/site-packages/oslo_policy/_cache_handler.py:40
>2018-09-21 10:01:11.916 1 DEBUG oslo_policy.policy [-] Reloaded policy file: /etc/neutron/policy.json _load_policy_file /usr/lib/python2.7/site-packages/oslo_policy/policy.py:770
>2018-09-21 10:01:11.918 1 DEBUG oslo_policy._cache_handler [-] Reloading cached file /etc/neutron/policy.d/bgpvpn.conf read_cached_file /usr/lib/python2.7/site-packages/oslo_policy/_cache_handler.py:40
>2018-09-21 10:01:11.930 1 DEBUG oslo_policy.policy [-] Reloaded policy file: /etc/neutron/policy.d/bgpvpn.conf _load_policy_file /usr/lib/python2.7/site-packages/oslo_policy/policy.py:770
>2018-09-21 10:01:11.940 9238 INFO neutron.wsgi [-] (9238) wsgi exited, is_accepting=True
>2018-09-21 10:01:11.940 9240 INFO neutron.wsgi [-] (9240) wsgi exited, is_accepting=True
>2018-09-21 10:01:11.938 1 INFO oslo_service.service [-] Wait called after thread killed. Cleaning up.
>2018-09-21 10:01:11.941 9239 INFO neutron.wsgi [-] (9239) wsgi exited, is_accepting=True
>2018-09-21 10:01:11.941 1 DEBUG oslo_service.service [-] Stop services. stop /usr/lib/python2.7/site-packages/oslo_service/service.py:697
>2018-09-21 10:01:11.942 1 DEBUG neutron.service [-] calling RpcWorker stop() stop /usr/lib/python2.7/site-packages/neutron/service.py:136
>2018-09-21 10:01:11.943 1 DEBUG neutron.service [-] calling RpcWorker stop() stop /usr/lib/python2.7/site-packages/neutron/service.py:136
>2018-09-21 10:01:11.943 9240 DEBUG oslo_concurrency.lockutils [-] Acquired semaphore "singleton_lock" lock /usr/lib/python2.7/site-packages/oslo_concurrency/lockutils.py:212
>2018-09-21 10:01:11.944 1 DEBUG oslo_service.service [-] Killing children. stop /usr/lib/python2.7/site-packages/oslo_service/service.py:702
>2018-09-21 10:01:11.944 9240 DEBUG oslo_concurrency.lockutils [-] Releasing semaphore "singleton_lock" lock /usr/lib/python2.7/site-packages/oslo_concurrency/lockutils.py:228
>2018-09-21 10:01:11.944 9238 DEBUG oslo_concurrency.lockutils [-] Acquired semaphore "singleton_lock" lock /usr/lib/python2.7/site-packages/oslo_concurrency/lockutils.py:212
>2018-09-21 10:01:11.945 9239 DEBUG oslo_concurrency.lockutils [-] Acquired semaphore "singleton_lock" lock /usr/lib/python2.7/site-packages/oslo_concurrency/lockutils.py:212
>2018-09-21 10:01:11.945 1 INFO oslo_service.service [-] Waiting on 9 children to exit
>2018-09-21 10:01:11.945 9238 DEBUG oslo_concurrency.lockutils [-] Releasing semaphore "singleton_lock" lock /usr/lib/python2.7/site-packages/oslo_concurrency/lockutils.py:228
>2018-09-21 10:01:11.945 9239 DEBUG oslo_concurrency.lockutils [-] Releasing semaphore "singleton_lock" lock /usr/lib/python2.7/site-packages/oslo_concurrency/lockutils.py:228
>2018-09-21 10:01:11.948 9242 DEBUG oslo_concurrency.lockutils [-] Acquired semaphore "singleton_lock" lock /usr/lib/python2.7/site-packages/oslo_concurrency/lockutils.py:212
>2018-09-21 10:01:11.949 9242 DEBUG oslo_concurrency.lockutils [-] Releasing semaphore "singleton_lock" lock /usr/lib/python2.7/site-packages/oslo_concurrency/lockutils.py:228
>2018-09-21 10:01:11.951 9241 DEBUG networking_odl.journal.worker [-] JournalPeriodicProcessor stopping stop /usr/lib/python2.7/site-packages/networking_odl/journal/worker.py:99
>2018-09-21 10:01:11.951 9241 DEBUG networking_odl.journal.journal [-] Stopping the sync thread stop /usr/lib/python2.7/site-packages/networking_odl/journal/journal.py:211
>2018-09-21 10:01:11.952 9256 DEBUG neutron.service [-] calling RpcWorker stop() stop /usr/lib/python2.7/site-packages/neutron/service.py:136
>2018-09-21 10:01:11.952 9244 DEBUG neutron.service [-] calling RpcWorker stop() stop /usr/lib/python2.7/site-packages/neutron/service.py:136
>2018-09-21 10:01:11.953 9256 DEBUG neutron.service [-] calling stop on <oslo_messaging.rpc.server.RPCServer object at 0x7efc3a3de750> stop /usr/lib/python2.7/site-packages/neutron/service.py:139
>2018-09-21 10:01:11.952 9241 DEBUG networking_odl.journal.journal [req-59abaab0-f541-4333-8d40-5cfcc491aa88 - - - - -] Start processing journal entries sync_pending_entries /usr/lib/python2.7/site-packages/networking_odl/journal/journal.py:250
>2018-09-21 10:01:11.953 9244 DEBUG neutron.service [-] calling stop on <oslo_messaging.rpc.server.RPCServer object at 0x7efc3a3de750> stop /usr/lib/python2.7/site-packages/neutron/service.py:139
>2018-09-21 10:01:11.968 9243 DEBUG oslo_concurrency.lockutils [-] Acquired semaphore "singleton_lock" lock /usr/lib/python2.7/site-packages/oslo_concurrency/lockutils.py:212
>2018-09-21 10:01:11.969 9243 DEBUG oslo_concurrency.lockutils [-] Releasing semaphore "singleton_lock" lock /usr/lib/python2.7/site-packages/oslo_concurrency/lockutils.py:228
>2018-09-21 10:01:11.984 1 INFO oslo_service.service [-] Child 9238 exited with status 0
>2018-09-21 10:01:11.985 1 WARNING oslo_service.service [-] pid 9238 not in child list: OSError: [Errno 3] No such process
>2018-09-21 10:01:12.004 1 INFO oslo_service.service [-] Child 9239 exited with status 0
>2018-09-21 10:01:12.005 1 WARNING oslo_service.service [-] pid 9239 not in child list: OSError: [Errno 3] No such process
>2018-09-21 10:01:12.009 1 INFO oslo_service.service [-] Child 9242 exited with status 0
>2018-09-21 10:01:12.011 1 INFO oslo_service.service [-] Child 9240 exited with status 0
>2018-09-21 10:01:12.012 1 WARNING oslo_service.service [-] pid 9240 not in child list: OSError: [Errno 3] No such process
>2018-09-21 10:01:12.032 9241 DEBUG networking_odl.journal.journal [req-59abaab0-f541-4333-8d40-5cfcc491aa88 - - - - -] No journal entries to process sync_pending_entries /usr/lib/python2.7/site-packages/networking_odl/journal/journal.py:254
>2018-09-21 10:01:12.054 1 INFO oslo_service.service [-] Child 9243 exited with status 0
>2018-09-21 10:01:12.060 9241 DEBUG oslo_concurrency.lockutils [-] Acquired semaphore "singleton_lock" lock /usr/lib/python2.7/site-packages/oslo_concurrency/lockutils.py:212
>2018-09-21 10:01:12.061 9241 DEBUG oslo_concurrency.lockutils [-] Releasing semaphore "singleton_lock" lock /usr/lib/python2.7/site-packages/oslo_concurrency/lockutils.py:228
>2018-09-21 10:01:12.087 1 INFO oslo_service.service [-] Child 9241 exited with status 0
>2018-09-21 10:01:14.204 9244 DEBUG neutron_lib.callbacks.manager [req-1805a3a2-abed-4b92-8179-2d8a3f692575 - - - - -] Notify callbacks ['neutron.services.segments.db._update_segment_host_mapping_for_agent-8726361506007'] for agent, after_update _notify_loop /usr/lib/python2.7/site-packages/neutron_lib/callbacks/manager.py:193
>2018-09-21 10:01:22.566 9244 DEBUG neutron.service [-] calling stop on <oslo_messaging.rpc.server.RPCServer object at 0x7efc3a36fd10> stop /usr/lib/python2.7/site-packages/neutron/service.py:139
>2018-09-21 10:01:22.584 9244 DEBUG neutron.service [-] calling stop on <oslo_messaging.rpc.server.RPCServer object at 0x7efc3a3d6110> stop /usr/lib/python2.7/site-packages/neutron/service.py:139
>2018-09-21 10:01:25.496 9256 DEBUG neutron.service [-] calling stop on <oslo_messaging.rpc.server.RPCServer object at 0x7efc3a36fd10> stop /usr/lib/python2.7/site-packages/neutron/service.py:139
>2018-09-21 10:01:25.610 9256 DEBUG neutron.service [-] calling stop on <oslo_messaging.rpc.server.RPCServer object at 0x7efc3a3d6110> stop /usr/lib/python2.7/site-packages/neutron/service.py:139
>2018-09-21 10:01:29.162 9244 DEBUG oslo_concurrency.lockutils [-] Acquired semaphore "singleton_lock" lock /usr/lib/python2.7/site-packages/oslo_concurrency/lockutils.py:212
>2018-09-21 10:01:29.162 9244 DEBUG oslo_concurrency.lockutils [-] Releasing semaphore "singleton_lock" lock /usr/lib/python2.7/site-packages/oslo_concurrency/lockutils.py:228
>2018-09-21 10:01:29.162 9244 DEBUG neutron.service [-] calling RpcWorker wait() _wait /usr/lib/python2.7/site-packages/neutron/service.py:126
>2018-09-21 10:01:29.163 9244 DEBUG neutron.service [-] calling wait on <oslo_messaging.rpc.server.RPCServer object at 0x7efc3a3de750> _wait /usr/lib/python2.7/site-packages/neutron/service.py:129
>2018-09-21 10:01:29.201 9244 DEBUG neutron.service [-] calling wait on <oslo_messaging.rpc.server.RPCServer object at 0x7efc3a36fd10> _wait /usr/lib/python2.7/site-packages/neutron/service.py:129
>2018-09-21 10:01:29.240 9244 DEBUG neutron.service [-] calling wait on <oslo_messaging.rpc.server.RPCServer object at 0x7efc3a3d6110> _wait /usr/lib/python2.7/site-packages/neutron/service.py:129
>2018-09-21 10:01:29.282 9244 DEBUG neutron.service [-] returning from RpcWorker wait() _wait /usr/lib/python2.7/site-packages/neutron/service.py:133
>2018-09-21 10:01:29.304 1 INFO oslo_service.service [-] Child 9244 exited with status 0
>2018-09-21 10:01:35.444 9256 DEBUG oslo_concurrency.lockutils [-] Acquired semaphore "singleton_lock" lock /usr/lib/python2.7/site-packages/oslo_concurrency/lockutils.py:212
>2018-09-21 10:01:35.444 9256 DEBUG oslo_concurrency.lockutils [-] Releasing semaphore "singleton_lock" lock /usr/lib/python2.7/site-packages/oslo_concurrency/lockutils.py:228
>2018-09-21 10:01:35.445 9256 DEBUG neutron.service [-] calling RpcWorker wait() _wait /usr/lib/python2.7/site-packages/neutron/service.py:126
>2018-09-21 10:01:35.445 9256 DEBUG neutron.service [-] calling wait on <oslo_messaging.rpc.server.RPCServer object at 0x7efc3a3de750> _wait /usr/lib/python2.7/site-packages/neutron/service.py:129
>2018-09-21 10:01:35.482 9256 DEBUG neutron.service [-] calling wait on <oslo_messaging.rpc.server.RPCServer object at 0x7efc3a36fd10> _wait /usr/lib/python2.7/site-packages/neutron/service.py:129
>2018-09-21 10:01:35.488 9256 DEBUG neutron_lib.callbacks.manager [req-c296ea96-b1fb-4d5d-97ee-6ce6681914dc - - - - -] Notify callbacks ['neutron.services.segments.db._update_segment_host_mapping_for_agent-8726361506007'] for agent, after_update _notify_loop /usr/lib/python2.7/site-packages/neutron_lib/callbacks/manager.py:193
>2018-09-21 10:01:35.522 9256 DEBUG neutron.service [-] calling wait on <oslo_messaging.rpc.server.RPCServer object at 0x7efc3a3d6110> _wait /usr/lib/python2.7/site-packages/neutron/service.py:129
>2018-09-21 10:01:35.558 9256 DEBUG neutron.service [-] returning from RpcWorker wait() _wait /usr/lib/python2.7/site-packages/neutron/service.py:133
>2018-09-21 10:01:35.573 1 INFO oslo_service.service [-] Child 9256 exited with status 0
>[heat-admin@controller-1 ~]$
>
>
>
>
>[heat-admin@controller-1 ~]$
>[heat-admin@controller-1 ~]$ zgrep 'SIGHUP' /var/log/containers/neutron/*
>/var/log/containers/neutron/metadata-agent.log.1:2018-09-30 00:00:16.376 113462 INFO oslo_service.service [req-32e418d5-50fb-435c-a88b-d90830f6060f - - - - -] Child caught SIGHUP, exiting
>/var/log/containers/neutron/metadata-agent.log.1:2018-09-30 00:00:16.378 113472 INFO oslo_service.service [req-32e418d5-50fb-435c-a88b-d90830f6060f - - - - -] Child caught SIGHUP, exiting
>/var/log/containers/neutron/metadata-agent.log.1:2018-09-30 00:00:16.564 113461 INFO oslo_service.service [req-32e418d5-50fb-435c-a88b-d90830f6060f - - - - -] Child caught SIGHUP, exiting
>/var/log/containers/neutron/metadata-agent.log.1:2018-09-30 00:00:16.912 112825 INFO oslo_service.service [req-f5789b3f-119b-4fe0-b657-d25a58ad72d7 - - - - -] Caught SIGHUP, stopping children
>/var/log/containers/neutron/metadata-agent.log.2.gz:2018-09-29 02:01:21.060 113464 INFO oslo_service.service [req-32e418d5-50fb-435c-a88b-d90830f6060f - - - - -] Child caught SIGHUP, exiting
>/var/log/containers/neutron/server.log.1:2018-09-21 10:01:11.748 1 INFO oslo_service.service [-] Caught SIGHUP, stopping children
>/var/log/containers/neutron/server.log.3.gz:2018-09-21 00:00:27.813 1 INFO oslo_service.service [-] Caught SIGHUP, stopping children
>/var/log/containers/neutron/server.log.3.gz:2018-09-21 00:00:27.964 1 INFO oslo_service.service [-] Caught SIGHUP, stopping children
>/var/log/containers/neutron/server.log.5.gz:2018-09-20 17:00:11.266 39 INFO oslo_service.service [req-812a5a18-8f55-499b-b64e-d7b55fa9cf30 - - - - -] Child caught SIGHUP, exiting
>/var/log/containers/neutron/server.log.5.gz:2018-09-20 17:00:11.871 1 INFO oslo_service.service [-] Caught SIGHUP, stopping children
>/var/log/containers/neutron/server.log.5.gz:2018-09-20 17:00:11.975 1 INFO oslo_service.service [-] Caught SIGHUP, stopping children

Actions: View

Attachments on bug 1635242: 1489474