Attachment 902566 Details for Bug 1105195 – reproducer bash script

reproducer bash script

reproducer.sh (text/plain), 3.95 KB, created by mkovacik on 2014-06-05 14:53:05 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: mkovacik

Created: 2014-06-05 14:53:05 UTC

Size: 3.95 KB

patch

obsolete

>#!/bin/bash
>#set -x
>
>ITERATION=${ITERATION:-1}
>echo "### $0 iteration: $ITERATION"
>
>### About
># demonstrates pulp_worker processes leaking tasks in case of abnormal termination
># see also: 
>
>function services() {
>	# print service names filtered by given pattern
>	local pattern=$1
>	shift
>	systemctl list-unit-files | egrep "${pattern}" | sed 's,\s\+.*,,'
>}
>
>function collection_content() {
>	local database=$1
>	local collection=$2
>	echo "# mongo ${database}.${collection}:"
>	mongo $database --eval "db.${collection}.find().forEach(function(x){printjson(x)})" --quiet 
>	echo 
>}
>
>function _sleep() {
>	local seconds=$1
>	local reason=${2:-no}
>	date; echo sleeping $seconds seconds for $reason reason; sleep $seconds ; date
>}
>
># patch WorkerTimeoutMonitor Thread in order to detect dead workers 10x faster
>patch -s --forward /usr/lib/python2.7/site-packages/pulp/server/async/scheduler.py <<__WORKER_TIMEOUT_MONITOR_PATCH
>193c193
><     WORKER_TIMEOUT_SECONDS = 300
>---
>>     WORKER_TIMEOUT_SECONDS = 30
>196c196
><     FREQUENCY = 60
>---
>>     FREQUENCY = 6
>__WORKER_TIMEOUT_MONITOR_PATCH
>
># start clean
>systemctl stop `services 'pulp|httpd'`
>mongo pulp_database --eval "db.dropDatabase()" || exit $?
>sudo -u apache pulp-manage-db > /dev/null || exit $?
>systemctl start `services 'pulp|httpd'`
>
># allow pulp to settle down
>_sleep 3 pulp-settling-down
>
># create repo
>pulp-admin -u admin -p admin rpm repo create --repo-id zoo --feed http://repos.fedorapeople.org/repos/pulp/pulp/demo_repos/zoo/
>
># kill all worker processes to induce abnormal exit
>kill -9 `ps -eo pid,command | grep [r]eserved_resource_worker | sed 's,\s*$[[:digit:]]\+$.*,\1,' `
>
># 3 seconds till the queues disappear from qpid
>#_sleep 6 qpid-settling-down
># show that although the worker queues are gone, pulp keeps track of them
>systemctl status `services pulp_worker-`
>qpid-stat -q | grep reserved_resource_worker
>for service in `services pulp_worker- ` ; do
>	# exit if queues absent
>	queue=reserved_resource_worker-${service//[^[:digit:]]/}
>	collection_content pulp_database available_queues | grep ${queue} || {
>		# restart
>		[ $ITERATION -ge 10 ] && { echo too many reproducer iterations, exiting ; exit 1 ; }
>		ITERATION=$[ITERATION + 1]
>		collection_content pulp_database available_queues
>		echo missing queue: $queue, restarting
>		# kinda tail-recursion--loop
>		$0 &
>		exit `wait $!`;
>	}
>	
>done
>
>collection_content pulp_database available_queues
>collection_content pulp_database reserved_resources
>collection_content pulp_database task_status
>
># induce an async task
>pulp-admin -u admin -p admin rpm repo sync run --repo-id zoo &
>_sleep 3 pulp-sync-run
>kill -15 $! # kill the pending sync call
>
># show the task was assigned to a non-existing resource
>collection_content pulp_database available_queues
>collection_content pulp_database reserved_resources
>collection_content pulp_database task_status
>
># wait till pulp.server.async.scheduler: WorkerTimeoutMonitor
># cleans the pulp_worker queues
>_sleep 60 WorkerTimeoutMonitor-pick-up-to-happen
>
>collection_content pulp_database available_queues
>collection_content pulp_database reserved_resources
>collection_content pulp_database task_status
>
># in case no cleanup was performed, this should have finished the pending task
>systemctl restart  `services pulp_worker-`
>
>collection_content pulp_database available_queues
>collection_content pulp_database reserved_resources
>collection_content pulp_database task_status
>
># there's no way how to check all services work with either is-active/is-failed with systemctl
># following creates a command string:
>#    true && is-active pulp_worker-1\\n && is-active pulp_worker-2\\n...\
># to evaluate and check for all pulp-worker services are up and running
> 
>eval "true `services pulp_worker- | sed 's,$.*$,\&\& systemctl is-active \1\\\\,'`" && {
>	_sleep 60 pending-tasks-settling-down
>	# show the sync task is still pending
>	collection_content pulp_database available_queues
>	collection_content pulp_database reserved_resources;
>	collection_content pulp_database task_status
>	date;
>}

Actions: View

Attachments on bug 1105195: 902566 | 902567 | 917436 | 918412