I have a program called CLARK. It has a bunch of scripts that setup the actual execution of its binary.
Hence a typical run of CLARK would be:
set_targets.sh ...
classify_metagenome.sh ...
Eventually classify_metagenome.sh would invoke the c++ compiled binary CLARK.
My problem is the following, when I run CLARK like this:
host$ docker exec -it CONTAINERNAME /bin/bash
CONTAINERNAME$ set_targets.sh ...
CONTAINERNAME$ classify_metagenome.sh ...
Everything works out just fine.
But when I run it like this:
host$ docker exec -it CONTAINERNAME "set_targets.sh ... && classify_metagenome.sh ..."
The program SEGFAULTs
I have tried to chase this error far and wide and I believe it has something to do with what resources are available for the container, because in another test, if I run this for instance with 8 threads, it works fine, but if I run it with the max the machine has (14) it SEGFAULTs again.
I am not sure how to tackle this problem. Any help would be appreciated.
This is the strace of the SEGFAULT:
...
pid 870] set_robust_list(0x1087f6df39a0, 24 <unfinished ...>
[pid 857] futex(0x5e2cb2b35174, FUTEX_WAIT_PRIVATE, 0, NULL <unfinished ...>
[pid 870] <... set_robust_list resumed>) = 0
[pid 870] rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0
[pid 870] futex(0x5e2cb2b35174, FUTEX_WAKE_PRIVATE, 2147483647 <unfinished ...>
[pid 864] <... futex resumed>) = 0
[pid 865] <... futex resumed>) = 0
[pid 867] <... futex resumed>) = 0
[pid 870] <... futex resumed>) = 13
[pid 866] <... futex resumed>) = 0
[pid 868] <... futex resumed>) = 0
[pid 863] <... futex resumed>) = 0
[pid 862] <... futex resumed>) = 0
[pid 861] <... futex resumed>) = 0
[pid 860] <... futex resumed>) = 0
[pid 869] <... futex resumed>) = 0
[pid 859] <... futex resumed>) = 0
[pid 858] <... futex resumed>) = 0
[pid 857] <... futex resumed>) = 0
[pid 865] mmap(NULL, 134217728, PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x1087eea00000
[pid 865] munmap(0x1087eea00000, 23068672) = 0
[pid 865] munmap(0x1087f4000000, 44040192) = 0
[pid 865] mprotect(0x1087f0000000, 135168, PROT_READ|PROT_WRITE <unfinished ...>
[pid 858] mmap(NULL, 134217728, PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0 <unfinished ...>
[pid 865] <... mprotect resumed>) = 0
[pid 858] <... mmap resumed>) = 0x1087e8000000
[pid 858] munmap(0x1087ec000000, 67108864 <unfinished ...>
[pid 862] mmap(0x1087ec000000, 67108864, PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0 <unfinished ...>
[pid 858] <... munmap resumed>) = 0
[pid 862] <... mmap resumed>) = 0x1087ec000000
[pid 858] mprotect(0x1087e8000000, 135168, PROT_READ|PROT_WRITE <unfinished ...>
[pid 862] mprotect(0x1087ec000000, 135168, PROT_READ|PROT_WRITE) = 0
[pid 858] <... mprotect resumed>) = 0
[pid 866] mmap(NULL, 134217728, PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x1087e0000000
[pid 866] munmap(0x1087e4000000, 67108864) = 0
[pid 866] mprotect(0x1087e0000000, 135168, PROT_READ|PROT_WRITE) = 0
[pid 861] mmap(0x1087e4000000, 67108864, PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x1087e4000000
[pid 861] mprotect(0x1087e4000000, 135168, PROT_READ|PROT_WRITE) = 0
[pid 864] mmap(NULL, 134217728, PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x1087d8000000
[pid 864] munmap(0x1087dc000000, 67108864) = 0
[pid 864] mprotect(0x1087d8000000, 135168, PROT_READ|PROT_WRITE) = 0
[pid 869] mmap(0x1087dc000000, 67108864, PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x1087dc000000
[pid 869] mprotect(0x1087dc000000, 135168, PROT_READ|PROT_WRITE) = 0
[pid 867] mmap(NULL, 134217728, PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x1087d0000000
[pid 867] munmap(0x1087d4000000, 67108864) = 0
[pid 867] mprotect(0x1087d0000000, 135168, PROT_READ|PROT_WRITE) = 0
[pid 863] openat(AT_FDCWD, "/sys/devices/system/cpu/online", O_RDONLY|O_CLOEXEC) = 5
[pid 863] read(5, "0-19\n", 1024) = 5
[pid 863] close(5) = 0
[pid 863] mmap(0x1087d4000000, 67108864, PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x1087d4000000
[pid 863] mprotect(0x1087d4000000, 135168, PROT_READ|PROT_WRITE) = 0
[pid 870] --- SIGSEGV {si_signo=SIGSEGV, si_code=SEGV_MAPERR, si_addr=0x1087f9bbc000} ---
[pid 869] +++ killed by SIGSEGV (core dumped) +++
[pid 870] +++ killed by SIGSEGV (core dumped) +++
[pid 866] +++ killed by SIGSEGV (core dumped) +++
[pid 865] +++ killed by SIGSEGV (core dumped) +++
[pid 863] +++ killed by SIGSEGV (core dumped) +++
[pid 862] +++ killed by SIGSEGV (core dumped) +++
[pid 861] +++ killed by SIGSEGV (core dumped) +++
[pid 860] +++ killed by SIGSEGV (core dumped) +++
[pid 859] +++ killed by SIGSEGV (core dumped) +++
[pid 858] +++ killed by SIGSEGV (core dumped) +++
[pid 868] +++ killed by SIGSEGV (core dumped) +++
[pid 864] +++ killed by SIGSEGV (core dumped) +++
[pid 867] +++ killed by SIGSEGV (core dumped) +++
[pid 857] +++ killed by SIGSEGV (core dumped) +++
<... wait4 resumed>, [{WIFSIGNALED(s) && WTERMSIG(s) == SIGSEGV && WCOREDUMP(s)}], 0, NULL) = 857
rt_sigaction(SIGINT, {sa_handler=0x5d6023506400, sa_mask=[], sa_flags=SA_RESTORER, sa_restorer=0x5dce4957cb0}, {sa_handler=0x5d60234d8400, sa_mask=[], sa_flags=SA_RESTORER, sa_restorer=0x5dce4957cb0}, 8) = 0
ioctl(2, TIOCGWINSZ, 0x7ffff64c2880) = -1 ENOTTY (Inappropriate ioctl for device)
fstat(2, {st_mode=S_IFIFO|0600, st_size=0, ...}) = 0
write(2, "/opt/CLARK/scripts/classify_meta"..., 128/opt/CLARK/scripts/classify_metagenome.sh: line 181: 857 Segmentation fault (core dumped) "$CLARK_EXE" "${PARAMS[@]}"
) = 128
rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0
--- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_KILLED, si_pid=857, si_uid=0, si_status=SIGSEGV, si_utime=1847 /* 18.47 s */, si_stime=3161 /* 31.61 s */} ---
wait4(-1, 0x7ffff64c1a94, WNOHANG, NULL) = -1 ECHILD (No child processes)
rt_sigreturn({mask=[]}) = 0
rt_sigprocmask(SIG_BLOCK, NULL, [], 8) = 0
exit_group(139) = ?
+++ exited with 139 +++
Also the bt from GDB:
Thread 14 "CLARK" received signal SIGSEGV, Segmentation fault.
[Switching to Thread 0x757459f116c0 (LWP 2327)]
0x0000757a8b5cd16f in std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_replace(unsigned long, unsigned long, char const*, unsigned long) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
(gdb) bt
#0 0x0000757a8b5cd16f in std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_replace(unsigned long, unsigned long, char const*, unsigned long) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#1 0x000062b2ec8da03f in std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::assign (this=0x62b84969a070, __s=0x62b2ec94e534 "")
at /usr/include/c++/15/bits/basic_string.h:1163
#2 std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::operator= (this=0x62b84969a070, __s=0x62b2ec94e534 "")
at /usr/include/c++/15/bits/basic_string.h:916
#3 FileHandlerA::GetRead (this=0x62b328a99420, i_cpu=@0x757459f10d38: 13,
out=0x7574614da010 "", size=@0x757459f10d3c: 0,
id=<error: Cannot access memory at address 0x4347415441434354>)
at build/default/FileHandlerA.cc:78
#4 0x000062b2ec8f293d in _ZN5CLARKIjE25getObjectsDataComputeFullERKbPKcS4_._omp_fn.1(void) () at /usr/include/c++/15/bits/stl_vector.h:1263
#5 0x0000757a8b418b5f in ?? () from /usr/lib/x86_64-linux-gnu/libgomp.so.1
#6 0x0000757a8b24a0da in start_thread (arg=<optimized out>)
at ./nptl/pthread_create.c:454
#7 0x0000757a8b2dd5b4 in clone ()
at ../sysdeps/unix/sysv/linux/x86_64/clone.S:100
(gdb)