Welcome to the Linux Foundation Forum!

Circular debugging using ptrace results in deadlock due to race condition?

Posts: 1
edited June 2017 in Software Development

Hi guys,

As part of my personal research I am facing a challenging problem.

I am trying to let two processes be each other's debuggers using the ptrace syscall. However, my proof-of-concept implementation always results in a deadlock state (both processes get stuck in 't+' state as shown by 'ps aux').

Here is my code, it's pretty simple:

 


  1. /* C standard headers */
  2. #include <errno.h>
  3. #include <inttypes.h>
  4. #include <setjmp.h>
  5. #include <stdbool.h>
  6. #include <stddef.h>
  7. #include <stdint.h>
  8. #include <stdio.h>
  9. #include <stdlib.h>
  10. #include <string.h>
  12. /* Linux headers */
  13. #include <dirent.h>
  14. #include <fcntl.h>
  15. #include <sys/mman.h>
  16. #include <sys/ptrace.h>
  17. #include <sys/types.h>
  18. #include <sys/wait.h>
  19. #include <unistd.h>
  20. #include <sys/user.h>
  21. /* Architecture-specific headers */
  22. #include <asm/ptrace.h>
  23. #include <asm/unistd.h>
  25. #include <signal.h>
  26. #include <assert.h>
  28. typedef void fun_moved_from_context();
  29. // using namespace std;
  30. void attachTo(pid_t pid, char* id) {
  31.   long ret = ptrace(PTRACE_ATTACH, pid, NULL, NULL);
  32.   printf("\t%s\tattachTo: %ld\n", id, ret);
  33.   if (ret == -1) perror("err: ");
  34. }
  35. void seizeTo(pid_t pid, char* id) {
  36.   long ret = ptrace(PTRACE_SEIZE, pid, NULL, NULL);
  37.   assert(ret > 0);
  38.   printf("\t%s\tseizeTo: %ld\n", id, ret);
  39. }
  40. void detachFrom(pid_t pid, char* id) {
  41.   long ret = ptrace(PTRACE_DETACH, pid, NULL, NULL);
  42.   printf("\t%s\tdetachFrom: %ld\n", id, ret);
  43. }
  44. void setOptions(pid_t pid, char* id) {
  45.   long ret = ptrace(
  46.       PTRACE_SETOPTIONS, pid, NULL,
  47.       (void*)(PTRACE_O_TRACECLONE | PTRACE_O_TRACEEXEC | PTRACE_O_TRACEEXIT |
  48.               PTRACE_O_TRACEFORK | PTRACE_O_TRACEVFORK));
  49.   printf("\t%s\tsetOptions: %ld\n", id, ret);
  50. }
  51. void setVarData(pid_t pid, volatile bool* can_run, void* data, char* id) {
  52.   long ret = ptrace(PTRACE_POKEDATA, pid, (void*)can_run, (void*)data);
  53.   printf("\t%s\tsetVarData: %ld\n", id, ret);
  54. }
  55. void cont(pid_t pid, char* id) {
  56.   long ret = ptrace(PTRACE_CONT, pid, NULL, NULL);
  57.   printf("\t%s\tcont: %ld\n", id, ret);
  58. }
  59. void interrupt(pid_t pid, char* id) {
  60.   long ret = ptrace(PTRACE_INTERRUPT, pid, NULL, NULL);
  61.   printf("\t%s\tinterrupt: %ld\n", id, ret);
  62. }
  63. void debug(int id) {
  64.   int status;
  65.   while (true) {
  66.     printf("\t%s\twhile\n", id);
  67.     sleep(1);
  68.     struct user_regs regs;
  69.     pid_t recv = wait(&status);
  70.     if (recv == -1) {
  71.       printf("\t%s\tDebugger exiting\n", id);
  72.       return 0;
  73.     } else {
  74.       if (WIFSTOPPED(status)) {
  75.         int signal = WSTOPSIG(status);
  76.         switch (signal) {
  77.           case SIGTRAP: {
  78.             int event_code = (status >> 8) ^ SIGTRAP;
  79.             switch (event_code) {
  80.               case PTRACE_EVENT_FORK << 8:
  81.                 printf("\t%s\tFORK EVENT.\n", id);
  82.                 cont(recv, 2);
  83.                 break;
  84.               case PTRACE_EVENT_EXIT << 8:
  85.                 printf("\t%s\t%li exited.\n", id, recv);
  86.                 return 0;
  87.                 break;
  88.               default: {
  89.                 printf("\t%s\trecv: %i ; status: %i\n", id, recv, status);
  90.                 long ret = ptrace(PTRACE_GETREGS, recv, NULL, &regs);
  91.                 regs.uregs[15] += 2;  // addr_size;
  92.                 printf("\t%s\tp: new PC: %lx\n", id, regs.uregs[15]);
  93.                 ptrace(PTRACE_SETREGS, recv, NULL, &regs);
  94.                 cont(recv, id);
  95.               }
  96.             }
  97.           }
  98.         }
  99.       }
  100.     }
  101.   }
  102. }
  104. int main() {
  105.   volatile bool can_runA = false, can_runB = false;
  106.   pid_t procA = getpid();
  107.   volatile pid_t procB = 0;
  109.   if (fork() > 0) {  // process A
  110.     while (!can_runA) {
  111.       printf("\tA\twaiting to continue...\n");
  112.       sleep(1);
  113.     }
  114.     attachTo(procB, "A");
  115.     waitpid(procB, NULL, __WALL);
  116.     setOptions(procB, "A");
  117.     setVarData(procB, &can_runB, 1, "A");
  118.     cont(procB, "A");
  119.     printf("\tA\tfinished\n");
  120.   } else {  // process B
  121.     procB = getpid();
  122.     attachTo(procA, "B");
  123.     waitpid(procA, NULL, __WALL);
  124.     setOptions(procA, "B");
  125.     setVarData(procA, &can_runA, 1, "B");
  126.     setVarData(procA, &procB, procB, "B");
  127.     cont(procA, "B");
  128.     while (!can_runB) {
  129.       printf("\tB\twaiting to continue...\n");
  130.       sleep(1);
  131.     }
  132.     printf("\tB\tfinished\n");
  133.   }  
  134.   return 0;
  135. }

I have compiled and run this on an ARMv7 developer board with kernel version 3.0.35 (Linaro 13.08).

The output of the above code is this:


  1.   A    waiting to continue...
  2.   B    attachTo: 0
  3.   B    setOptions: 0
  4.   B    setVarData: 0
  5.   B    setVarData: 0
  6.   B    cont: 0
  7.   B    waiting to continue...
  8.   B    waiting to continue...
  9.   A    attachTo: 0

As you can see it never reaches the "finished" printf code, and gets stuck as soon as the other process attempts to attach to the debugger.

I have done a similar experiment for 3 processes, such that each one attempts to attach to the other in a circular fashion: A -> B -> C -> A

The result in this case was exactly the same. However, here I was able to detect a race condition, because sometimes the code executed properly without getting stuck in a deadlock (but it's hard to reproduce).

If you wish, you can test this by using a lightweight debugger I've developed and three console terminals. Here's the code:


  1. /* C standard headers */
  2. #include <errno.h>
  3. #include <inttypes.h>
  4. #include <setjmp.h>
  5. #include <stdbool.h>
  6. #include <stddef.h>
  7. #include <stdint.h>
  8. #include <stdio.h>
  9. #include <stdlib.h>
  10. #include <string.h>
  12. /* Linux headers */
  13. #include <dirent.h>
  14. #include <fcntl.h>
  15. #include <sys/mman.h>
  16. #include <sys/ptrace.h>
  17. #include <sys/types.h>
  18. #include <sys/wait.h>
  19. #include <unistd.h>
  20. #include <sys/user.h>
  21. /* Architecture-specific headers */
  22. #include <asm/ptrace.h>
  23. #include <asm/unistd.h>
  25. #include <signal.h>
  26. #include <assert.h>
  28. typedef void fun_moved_from_context();
  29. //using namespace std;
  30. void attachTo(pid_t pid, int id) {
  31.     long ret = ptrace (PTRACE_ATTACH, pid, NULL, NULL);        
  32.     printf("%i  attachTo: %ld\n", id, ret);
  33. }
  34. void seizeTo(pid_t pid, int id) {
  35.     long ret = ptrace (PTRACE_SEIZE, pid, NULL, NULL);       
  36.     //assert(ret > 0);
  37.     printf("%i  seizeTo: %ld\n", id, ret);        
  38. }
  39. void detachFrom(pid_t pid, int id) {    
  40.     long ret = ptrace (PTRACE_DETACH, pid, NULL, NULL);        
  41.     printf("%i  detachFrom: %ld\n", id, ret);    
  42. }
  43. void setOptions(pid_t pid, int id) {    
  44.     long ret = ptrace(PTRACE_SETOPTIONS, pid, NULL, (void*) (PTRACE_O_TRACECLONE | PTRACE_O_TRACEEXEC | PTRACE_O_TRACEEXIT | PTRACE_O_TRACEFORK | PTRACE_O_TRACEVFORK));   
  45.     printf("%i  setOptions: %ld\n", id, ret);    
  46. }
  47. void setVarData(pid_t pid, volatile bool* can_run, void* data, int id) {
  48.     long ret = ptrace(PTRACE_POKEDATA, pid, (void*)can_run, (void*)data);
  49.     printf("%i  setVarData: %ld\n", id, ret);    
  50. }
  51. void cont(pid_t pid, int id) {
  52.     long ret = ptrace (PTRACE_CONT, pid, NULL, NULL);
  53.     printf("%i  cont: %ld\n", id, ret);    
  54. }
  55. void interrupt(pid_t pid, int id) {
  56.     long ret = ptrace (PTRACE_INTERRUPT, pid, NULL, NULL);
  57.     printf("%i  interrupt: %ld\n", id, ret);    
  58. }
  59. void debug(int id) {
  60.     int status;  
  61.     while (true) {          
  62.         printf("%i  while\n", id);
  63.         sleep(1);
  64.         struct user_regs regs;    
  65.         pid_t recv = wait(&status);   
  66.         if (recv == -1) {
  67.             printf("%i  Debugger exiting\n", id);
  68.             return 0;
  69.         } else {
  70.             if (WIFSTOPPED(status)) {
  71.                 int signal = WSTOPSIG(status);    
  72.                 printf("%i  signal: %i\n",id,signal);
  73.                 switch (signal) {
  74.                     case SIGTRAP: {                 
  75.                         int event_code = (status >> 8) ^ SIGTRAP;
  76.                         printf("%i  event_code: %i\n",id,event_code);
  77.                         switch (event_code) {
  78.                             case PTRACE_EVENT_FORK << 8:
  79.                                 printf("%i  FORK EVENT.\n", id);
  80.                                 cont(recv,id);
  81.                                 break;
  82.                             case PTRACE_EVENT_EXIT << 8:
  83.                                 printf("%i  %li exited.\n", id, recv);
  84.                                 return 0;
  85.                                 break;
  86.                             default: {
  87.                                 printf("%i  recv: %i ; status: %i\n", id, recv, status);
  88.                                 long ret=ptrace (PTRACE_GETREGS, recv, NULL, &regs);
  89.                                 regs.uregs[15] += 2;//addr_size;
  90.                                 printf("%i  p: new PC: %lx\n", id, regs.uregs[15]);
  91.                                 ptrace (PTRACE_SETREGS, recv, NULL, &regs);                
  92.                                 cont(recv,id);
  93.                             }
  94.                         }  
  95.                         break;                      
  96.                     }   
  97.                     default: {
  98.                         cont(recv,id);
  99.                         break;
  100.                     }                
  101.                 }
  102.             }
  104.         }         
  105.     }
  106. }
  108. int main() {    
  109.     int pid;
  110.     int me = getpid();
  111.     printf("Hello, I am %d\n", me);
  112.     printf("pid:");
  113.     scanf("%d",&pid);
  114.     if (pid == 0) {
  115.         printf("bkpt asm\n");
  116.         asm("bkpt");
  117.     } else {
  118.         attachTo(pid, me);
  119.         printf("start waitpid\n");
  120.         waitpid(pid, NULL, __WALL);
  121.         printf("end waitpid\n");
  122.         setOptions(pid, me);  
  123.         cont(pid,me);   
  124.         debug(me);
  125.     }
  126.     return 0;
  127. }



Once you've compiled the above code, you simply run the binary on each console and enter the PID of another process to establish a 3-way circle.



I am far from an expert on the kernel, but I did have a look at the ARM specific kernel implementation which left me puzzled. I couldn't find where/how/why this code does not work.

Now I'm wondering if it's possible at all to make this work without a deadlock occurring? Does anyone have any experience with this, or can provide some clues/feedback?

Thank you greatly for your time, attention and effort! :)

Ilya

Welcome!

It looks like you're new here. Sign in or register to get started.
Sign In

Welcome!

It looks like you're new here. Sign in or register to get started.
Sign In

Quick Links

Categories

Upcoming Training