├── .gitignore ├── gdbcommands ├── displayDeadlock.sh ├── LICENSE ├── deadlockExample.c ├── gdbDisplayLockedThreads.py └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | core 2 | deadlockExample 3 | __pycache__ 4 | -------------------------------------------------------------------------------- /gdbcommands: -------------------------------------------------------------------------------- 1 | python 2 | import sys 3 | sys.path.append('.') 4 | import gdbDisplayLockedThreads 5 | end 6 | 7 | thread apply all bt 8 | 9 | blocked 10 | -------------------------------------------------------------------------------- /displayDeadlock.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | # 3 | # displayDeadlock.sh 4 | # Copyright (C) 2017 Damian Ziobro 5 | 6 | gdb -c core ./deadlockExample -x gdbcommands -batch 7 | 8 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Damian Ziobro 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /deadlockExample.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | pthread_mutex_t read_mutex; 7 | pthread_mutex_t write_mutex; 8 | 9 | void * writeTest(void *temp) { 10 | char *ret; 11 | FILE *file1; 12 | char *str; 13 | pthread_mutex_lock(&write_mutex); 14 | sleep(5); 15 | pthread_mutex_lock(&read_mutex); 16 | printf("\nFile locked, please enter the message \n"); 17 | str=(char *)malloc(10*sizeof(char)); 18 | file1=fopen("temp","w"); 19 | scanf("%s",str); 20 | fprintf(file1,"%s",str); 21 | fclose(file1); 22 | pthread_mutex_unlock(&read_mutex); 23 | pthread_mutex_unlock(&write_mutex); 24 | printf("\nUnlocked the file you can read it now \n"); 25 | return ret; 26 | } 27 | 28 | 29 | void * readTest(void *temp) { 30 | char *ret; 31 | FILE *file1; 32 | char *str; 33 | pthread_mutex_lock(&read_mutex); 34 | sleep(5); 35 | pthread_mutex_lock(&write_mutex); 36 | printf("\n Opening file \n"); 37 | file1=fopen("temp","r"); 38 | str=(char *)malloc(10*sizeof(char)); 39 | fscanf(file1,"%s",str); 40 | printf("\n Message from file is %s \n",str); 41 | 42 | fclose(file1); 43 | 44 | pthread_mutex_unlock(&write_mutex); 45 | pthread_mutex_unlock(&read_mutex); 46 | return ret; 47 | } 48 | 49 | 50 | 51 | 52 | int main() { 53 | pthread_t thread_id,thread_id1; 54 | pthread_attr_t attr; 55 | int ret; 56 | void *res; 57 | ret=pthread_create(&thread_id,NULL,&writeTest,NULL); 58 | ret=pthread_create(&thread_id1,NULL,&readTest,NULL); 59 | printf("\n Created thread"); 60 | pthread_join(thread_id,&res); 61 | pthread_join(thread_id1,&res); 62 | } 63 | -------------------------------------------------------------------------------- /gdbDisplayLockedThreads.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright © 2017 Damian Ziobro 5 | 6 | """ 7 | This script shows blocking C/C++ threads in gdb based on data from core file. 8 | 9 | Author: Damian Ziobro 10 | 11 | Instruction: 12 | 1) Add this to your file 'gdbcommands' (or your .gdbinit file) 13 | ' 14 | python 15 | import gdbDisplayLockedThreads 16 | end 17 | 18 | thread apply all bt 19 | 20 | blocked 21 | ' 22 | 2. Run this command: 23 | gdb -c core /path/to/exec -x gdbcommands -batch 24 | 25 | 3. You should see all backtraces from your process followd by additionally info of 26 | which threads are waiting for mutexes holding by which other threads ex. 27 | 28 | Thread: 27184 waits for thread: 27185 AND DEADLOCKED 29 | Thread: 27185 waits for thread: 27184 AND DEADLOCKED 30 | 31 | It means that thread 27184 waits for mutex holded by thread 27185 AND thread 27185 32 | waits for mutex holded by thread 27184 - so threas 27185 and 27184 are deadlocked 33 | 34 | """ 35 | 36 | import gdb 37 | 38 | class Thread(): 39 | def __init__(self): 40 | self.frames = [] 41 | self.waitOnThread = None 42 | self.threadId = None 43 | 44 | def __getitem__(self): 45 | return self.waitOnThread 46 | 47 | 48 | class DisplayLockedThreads(gdb.Command): 49 | """custom command => blocked - command show how threads blocks themselves waiting on mutexes""" 50 | def __init__(self): 51 | super (DisplayLockedThreads, self).__init__("blocked", gdb.COMMAND_SUPPORT,gdb.COMPLETE_NONE,True) 52 | print (self.__doc__) 53 | 54 | def invoke(self, arg, from_tty): 55 | print ("\n********************************************************************************") 56 | print ("Displaying blocking threads using 'blocked' command") 57 | threads = {} 58 | for process in gdb.inferiors(): 59 | for thread in process.threads(): 60 | trd = Thread() 61 | trd.threadId = thread.ptid[1] #[1] - threadId; [0] - process pid 62 | #print ("Thread: {0}".format(threads[-1].threadId)) 63 | thread.switch() 64 | frame = gdb.selected_frame() 65 | while frame: 66 | frame.select() 67 | name = frame.name() 68 | if name is None: 69 | name = "??" 70 | if "pthread_mutex_lock" in name: 71 | trd.waitOnThread = int(gdb.execute("print mutex.__data.__owner", to_string=True).split()[2]) 72 | #print(threads[-1].waitOnThread) 73 | trd.frames.append(name) 74 | frame = frame.older() 75 | threads[trd.threadId] = trd 76 | 77 | for (tid,thread) in threads.items(): 78 | if thread.waitOnThread: 79 | if thread.waitOnThread in threads and threads[thread.waitOnThread].waitOnThread == thread.threadId: 80 | deadlockedText = "AND DEADLOCKED" 81 | else: 82 | deadlockedText = "" 83 | print ("Thread: {0} waits for thread: {1} {2}".format(thread.threadId, thread.waitOnThread, deadlockedText)) 84 | print ("********************************************************************************") 85 | 86 | DisplayLockedThreads() 87 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | gdb-automatic-deadlock-detector 3 | ======= 4 | 5 | Script adds new GDB command which automatically detects C/C++ thread lockups and deadlocks in GDB debugger. 6 | 7 | **It automates process of deadlock detection described in this instruction**: 8 | https://en.wikibooks.org/wiki/Linux_Applications_Debugging_Techniques/Deadlocks 9 | 10 | Automation is written as GDB Python script based on Python GBD API: https://sourceware.org/gdb/onlinedocs/gdb/Python-API.html 11 | 12 | **This script adds new 'blocked' command to GDB**. 'blocked' command shows which threads 13 | are blocking other threads (or are deadlocked each other) ex. 14 | ``` 15 | (gdb) blocked 16 | ******************************************************************************** 17 | Displaying blocking threads using 'blocked' command 18 | Thread: 7960 waits for thread: 7959 AND DEADLOCKED 19 | Thread: 7959 waits for thread: 7960 AND DEADLOCKED 20 | ******************************************************************************** 21 | ``` 22 | It means that thread 7960 waits for mutex holded by thread 7959 AND thread 7959 23 | waits for mutex holded by thread 7960 - so threas 7959 and 7960 are deadlocked 24 | 25 | 26 | Preparing sample deadlocked app 27 | ----- 28 | 1. Compile sample application containing thread deadlock. 29 | ``` 30 | gcc -pthread deadlockExample.c -o deadlockExample 31 | ``` 32 | 2. Make sure that your Linux OS can dump core files from the processes. Set it 33 | up by: 34 | ``` 35 | ulimit -c unlimited 36 | ``` 37 | 3. Run compiled application 38 | ``` 39 | ./deadlockExample 40 | ``` 41 | 4. Notice that this application stucks (because of deadlock). Run another 42 | terminal and send SIGABRT signal (signal nr 6) to the running app: 43 | ``` 44 | kill -6 $(pidof deadlockExample) 45 | ``` 46 | 5. You should have core file outputed to your PWD dir. 47 | 6. Also you need to have your GDB compiled with Python support. Check whether 48 | you can run 'python' command in your GDB. If not the recompile your GDB 49 | using this instruction: https://askubuntu.com/questions/513626/cannot-compile-gdb7-8-with-python-support 50 | 51 | Usage 52 | ----- 53 | 1. See content of the 'gdbcommands' file - it contains commands wilchi will be 54 | invoked in GDB to run this to detect deadlock automatically 55 | (**notice import our Python script gdbDisplayLockedThreads and invoking 'blocked' command**): 56 | ``` 57 | python 58 | import sys 59 | sys.path.append('.') 60 | import gdbDisplayLockedThreads 61 | end 62 | thread apply all bt 63 | blocked 64 | ``` 65 | 66 | 2. Run gdb and invoke commands from 'gdbcommands' file automatically: 67 | ``` 68 | gdb -c core ./deadlockExample -x ./gdbcommands -batch 69 | ``` 70 | 71 | 3. **Output shows all backtraces of threads from deadlockExample followed by additional info of 72 | which threads are waiting for mutexes holding by other threads** ex. 73 | ``` 74 | Thread 3 (Thread 0x7efc4958f700 (LWP 7960)) 75 | #0 __lll_lock_wait () at ../sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135 76 | #1 0x00007efc4a164cfd in __GI___pthread_mutex_lock (mutex=0x601300 ) at ../nptl/pthread_mutex_lock.c:80 77 | #2 0x0000000000400aae in readTest () 78 | #3 0x00007efc4a1626aa in start_thread (arg=0x7efc4958f700) at pthread_create.c:333 79 | #4 0x00007efc49e97eed in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 80 | 81 | Thread 2 (Thread 0x7efc49d90700 (LWP 7959)): 82 | #0 __lll_lock_wait () at ../sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135 83 | #1 0x00007efc4a164cfd in __GI___pthread_mutex_lock (mutex=0x6012c0 ) at ../nptl/pthread_mutex_lock.c:80 84 | #2 0x0000000000400a00 in writeTest () 85 | #3 0x00007efc4a1626aa in start_thread (arg=0x7efc49d90700) at pthread_create.c:333 86 | #4 0x00007efc49e97eed in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 87 | 88 | Thread 1 (Thread 0x7efc4a564700 (LWP 7958)): 89 | #0 0x00007efc4a1638ed in pthread_join (threadid=139622035818960, thread_return=0x0) at pthread_join.c:90 90 | #1 0x0000000000400b95 in main () 91 | 92 | ******************************************************************************** 93 | Displaying blocking threads using 'blocked' command 94 | Thread: 7960 waits for thread: 7959 AND DEADLOCKED 95 | Thread: 7959 waits for thread: 7960 AND DEADLOCKED 96 | ******************************************************************************** 97 | ``` 98 | 99 | It means that thread 7960 waits for mutex holded by thread 7959 AND thread 7959 100 | waits for mutex holded by thread 7960 - so threas 7959 and 7960 are deadlocked 101 | 102 | 4. Now you can detect deadlock for any core using this quick gdb command from 103 | point 2 or 'blocked' command from gdb. 104 | 105 | Enjoy your deadlock detections ! 106 | ----- 107 | --------------------------------------------------------------------------------