Bug 1304106

Created attachment 1120575 [details]
C++ source code that reproduces the bug

Description of problem:
dlopen() deadlocks when a prior dlopen() call is cancelled via pthread_cancel()

Version-Release number of selected component (if applicable):
All versions

How reproducible:
Always (see reproducer, which is attached)

#include <iostream>
#include <unistd.h>
#include <pthread.h>
#include <dlfcn.h>

using namespace std;

// to reproduce (copy contents to dl_deadlock.cpp)
//
// $ g++ -D FOO_SHARE_LIB -shared -o libfoo.so -fPIC dl_deadlock.cpp
// $ g++ -g dl_deadlock.cpp -ldl -lpthread
// $ ./a.out

#ifdef FOO_SHARE_LIB
class Foo
{
  public:
  Foo() {
    cout << "construct foo" << endl;
    usleep(500000);
    cout << "done construct foo" << endl;
  }
};
Foo g_foo;

#else

extern "C" void *thrfxn_init(void *arg)
{ usleep(500000); return NULL; }

extern "C" void *thrfxn(void *arg)
{
  cout << "before dlopen" << endl;
  void *h = dlopen("./libfoo.so",  RTLD_GLOBAL | RTLD_NOW);
  cout << "after dlopen" << endl;
  if (h);
    dlclose(h);
}

int main()
{
  pthread_t thr;

  // need to call cancel so pthread_cancel_init is called,
  // which calls into libdl (needed since first pthread_cancel waits on dlopen)
  pthread_create(&thr, NULL, thrfxn_init, NULL);
  pthread_cancel(thr);
  pthread_join(thr, NULL);
  
  cout << "before thread create" << endl;
  pthread_create(&thr, NULL, thrfxn, NULL);
  cout << "after thread create" << endl;

  // sleep 5ms
  usleep(5000);

  // cancel thread that is waiting in dlopen
  cout << "before cancel" << endl;
  pthread_cancel(thr);
  cout << "after cancel" << endl;

  // this dlopen will now deadlock
  void *h = dlopen("/lib64/libc.so.6", RTLD_GLOBAL | RTLD_NOW);
  if (h)
    dlclose(h);

  //find_symbol("PA_REPORT_CALC_DISPATCH", PA_REPORT_CALC_DISPATCH);
  //FDSA_FQL_SERVICE, CREATE_FDSA_FQL_SERVICE;

  cout << "before join" << endl;
  pthread_join(thr, NULL);
  cout << "after join" << endl;

  return 0;
}

#endif

Steps to Reproduce:
1. g++ -D FOO_SHARE_LIB -shared -o libfoo.so -fPIC dl_deadlock.cpp
2. g++ -g dl_deadlock.cpp -ldl -lpthread
3. ./a.out

Actual results:
before thread create
after thread create
before dlopen
construct foo
before cancel
after cancel
-- deadlock --

Expected results:
before thread create
after thread create
before dlopen
construct foo
before cancel
after cancel
done construct foo
after dlopen
before join
-- program completion --

Additional info: