Hello!
My Kamailio 5.4.7 on CentOS 7
[root@pbxphone-0866-fe-1 kamailio_crash]# kamailio -V
version: kamailio 5.4.7 (x86_64/linux) a19fe1
flags: USE_TCP, USE_TLS, USE_SCTP, TLS_HOOKS, USE_RAW_SOCKS, DISABLE_NAGLE, USE_MCAST, DNS_IP_HACK, SHM_MMAP, PKG_MALLOC, Q_MALLOC, F_MALLOC, TLSF_MALLOC, DBG_SR_MEMORY, USE_FUTEX, FAST_LOCK-ADAPTIVE_WAIT, USE_DNS_CACHE, USE_DNS_FAILOVER, USE_NAPTR, USE_DST_BLACKLIST, HAVE_RESOLV_RES
ADAPTIVE_WAIT_LOOPS 1024, MAX_RECV_BUFFER_SIZE 262144, MAX_URI_SIZE 1024, BUF_SIZE 65535, DEFAULT PKG_SIZE 8MB
poll method support: poll, epoll_lt, epoll_et, sigio_rt, select.
id: a19fe1
compiled on 20:40:47 Nov 22 2021 with gcc 4.8.5
[root@pbxphone-0866-fe-1 kamailio_crash]# uname -a
Linux pbxphone-0866-fe-1.company.domain 3.10.0-1160.15.2.el7.x86_64 #1 SMP Wed Feb 3 15:06:38 UTC 2021 x86_64 x86_64 x86_64 GNU/Linux
crashed with following backtraces (3 core files)
[root@pbxphone-0866-fe-1 kamailio_crash]# gdb kamailio core.kamailio.222447.pbxphone-0866-fe-1.company.name.1646903896
GNU gdb (GDB) Red Hat Enterprise Linux 7.6.1-120.el7
Copyright (C) 2013 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law. Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-redhat-linux-gnu".
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>...
Reading symbols from /root/kamailio_crash/kamailio...done.
warning: exec file is newer than core file.
[New LWP 222447]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib64/libthread_db.so.1".
Core was generated by `/usr/local/sbin/kamailio -P /run/kamailio/kamailio.pid -f /usr/local/etc/kamail'.
Program terminated with signal 6, Aborted.
#0 0x00007f04d6e16387 in raise () from /lib64/libc.so.6
Missing separate debuginfos, use: debuginfo-install audit-libs-2.8.5-4.el7.x86_64 bzip2-libs-1.0.6-13.el7.x86_64 elfutils-libelf-0.176-5.el7.x86_64 glibc-2.17-325.el7_9.x86_64 keyutils-libs-1.5.8-3.el7.x86_64 krb5-libs-1.15.1-51.el7_9.x86_64 libacl-2.2.51-15.el7.x86_64 libattr-2.4.46-13.el7.x86_64 libcap-2.22-11.el7.x86_64 libcap-ng-0.7.5-4.el7.x86_64 libcom_err-1.42.9-19.el7.x86_64 libdb-5.3.21-25.el7.x86_64 libffi-3.0.13-19.el7.x86_64 libgcc-4.8.5-44.el7.x86_64 libselinux-2.5-15.el7.x86_64 libstdc++-4.8.5-44.el7.x86_64 libunistring-0.9.3-9.el7.x86_64 libuuid-2.23.2-65.el7_9.1.x86_64 libxml2-2.9.1-6.el7_9.6.x86_64 lm_sensors-libs-3.4.0-8.20160601gitf9185e5.el7.x86_64 lua-5.1.4-15.el7.x86_64 net-snmp-agent-libs-5.7.2-49.el7_9.1.x86_64 net-snmp-libs-5.7.2-49.el7_9.1.x86_64 nspr-4.32.0-1.el7_9.x86_64 nss-3.67.0-4.el7_9.x86_64 nss-softokn-freebl-3.67.0-3.el7_9.x86_64 nss-util-3.67.0-1.el7_9.x86_64 openssl-libs-1.0.2k-24.el7_9.x86_64 pcre-8.32-17.el7.x86_64 perl-libs-5.16.3-299.el7_9.x86_64 popt-1.13-16.el7.x86_64 python-libs-2.7.5-90.el7.x86_64 rpm-libs-4.11.3-48.el7_9.x86_64 tcp_wrappers-libs-7.6-77.el7.x86_64 xz-libs-5.2.2-1.el7.x86_64 zlib-1.2.7-19.el7_9.x86_64
(gdb) bt full
#0 0x00007f04d6e16387 in raise () from /lib64/libc.so.6
No symbol table info available.
#1 0x00007f04d6e17a78 in abort () from /lib64/libc.so.6
No symbol table info available.
#2 0x000000000041e330 in sig_alarm_abort (signo=14) at main.c:693
__FUNCTION__ = "sig_alarm_abort"
#3 <signal handler called>
No symbol table info available.
#4 0x00007f04d6ed8e27 in syscall () from /lib64/libc.so.6
No symbol table info available.
#5 0x00000000004fa848 in futex_get (lock=0x7f042fe11ca0) at core/mem/../futexlock.h:121
v = 1
i = 0
#6 0x0000000000520dda in tcpconn_send_put (c=0x7f042fe11c98,
buf=0x7ffd8be90c90 "OPTIONS sip:73054@82.66.66.105:46050;transport=tls SIP/2.0\r\nVia: SIP/2.0/TLS tone.company.name:5061;branch=z9hG4bKx.14.1.0\r\nFrom: <sip:pbxphone@tone.company.name>;tag=uloc-6228658a-43655-2c-31dd201c-6229c21c-c"..., len=302, send_flags=...) at core/tcp_main.c:2422
tmp = 0x0
fd = 0
response = {140726950757744, 0}
n = 32516
do_close_fd = 1
rest_buf = 0x12fe11e10 <Address 0x12fe11e10 out of bounds>
t_buf = 0x0
rest_len = 32516
t_len = 794570504
resp = 139658225205256
t_send_flags = {f = 1296, blst_imask = 35817}
fd_cache_e = 0x0
use_fd_cache = 1
__FUNCTION__ = "tcpconn_send_put"
#7 0x00000000005203e7 in tcp_send (dst=0x7ffd8be90a20, from=0x0,
buf=0x7ffd8be90c90 "OPTIONS sip:73054@82.66.66.105:46050;transport=tls SIP/2.0\r\nVia: SIP/2.0/TLS tone.company.name:5061;branch=z9hG4bKx.14.1.0\r\nFrom: <sip:pbxphone@tone.company.name>;tag=uloc-6228658a-43655-2c-31dd201c-6229c21c-c"..., len=302) at core/tcp_main.c:2323
c = 0x7f042fe11c98
ip = {af = 2, len = 4, u = {addrl = {1765950034, 4294967295}, addr32 = {1765950034, 0, 4294967295, 0}, addr16 = {16978, 26946, 0, 0, 65535, 65535, 0, 0},
addr = "RBBi\000\000\000\000\377\377\377\377\000\000\000"}}
port = 46050
fd = 4
response = {139655957499112, 0}
n = 0
con_lifetime = 9600
try_local_port = 5061
rest_buf = 0x69d0c8 <dns_a_get_he+86> "\353CH\213Eؾ\001"
t_buf = 0x7ffd8be906e0 "\020\a\351\213\375\177"
rest_len = 0
t_len = 0
resp = 139655957577728
t_send_flags = {f = 19200, blst_imask = 180}
__FUNCTION__ = "tcp_send"
#8 0x00007f04b2aa0ceb in ul_ka_send (kamsg=0x7ffd8be90c80, kadst=0x7ffd8be90a20) at ul_keepalive.c:271
__FUNCTION__ = "ul_ka_send"
#9 0x00007f04b2aa0b58 in ul_ka_urecord (ur=0x7f042fca3340) at ul_keepalive.c:239
uc = 0x7f042fca3458
kabuf = "OPTIONS sip:73054@82.66.66.105:46050;transport=tls SIP/2.0\r\nVia: SIP/2.0/TLS tone.company.name:5061;branch=z9hG4bKx.14.1.0\r\nFrom: <sip:pbxphone@tone.company.name>;tag=uloc-6228658a-43655-2c-31dd201c-6229c21c-c"...
---Type <return> to continue, or q <return> to quit---
kabuf_len = 302
kamsg = {
s = 0x7ffd8be90c90 "OPTIONS sip:73054@82.66.66.105:46050;transport=tls SIP/2.0\r\nVia: SIP/2.0/TLS tone.company.name:5061;branch=z9hG4bKx.14.1.0\r\nFrom: <sip:pbxphone@tone.company.name>;tag=uloc-6228658a-43655-2c-31dd201c-6229c21c-c"..., len = 302}
vaddr = {s = 0x7f04b6e74008 "tone.company.name", len = 12}
vport = {s = 0x7f04b6e740f8 "5061", len = 4}
sdst = {s = 0x7f042fca3770 "sip:82.66.66.105:46050;transport=tls", len = 36}
sproto = {s = 0x81bb4b "TLS", len = 3}
duri = {user = {s = 0x0, len = 0}, passwd = {s = 0x0, len = 0}, host = {s = 0x7f042fca3774 "82.66.66.105:46050;transport=tls", len = 12}, port = {s = 0x7f042fca3781 "46050;transport=tls", len = 5},
params = {s = 0x7f042fca3787 "transport=tls", len = 13}, sip_params = {s = 0x7f042fca3787 "transport=tls", len = 13}, headers = {s = 0x0, len = 0}, port_no = 46050, proto = 3, type = SIP_URI_T,
flags = (unknown: 0), transport = {s = 0x7f042fca3787 "transport=tls", len = 13}, ttl = {s = 0x0, len = 0}, user_param = {s = 0x0, len = 0}, maddr = {s = 0x0, len = 0}, method = {s = 0x0, len = 0},
lr = {s = 0x0, len = 0}, r2 = {s = 0x0, len = 0}, gr = {s = 0x0, len = 0}, transport_val = {s = 0x7f042fca3791 "tls", len = 3}, ttl_val = {s = 0x0, len = 0}, user_param_val = {s = 0x0, len = 0},
maddr_val = {s = 0x0, len = 0}, method_val = {s = 0x0, len = 0}, lr_val = {s = 0x0, len = 0}, r2_val = {s = 0x0, len = 0}, gr_val = {s = 0x0, len = 0}}
dproto = 3 '\003'
he = 0xb88fa0 <he.5288>
ssock = 0x7f04b6e73d98
idst = {send_sock = 0x7f04b6e73d98, to = {s = {sa_family = 2, sa_data = "\263\342RBBi\000\000\000\000\000\000\000"}, sin = {sin_family = 2, sin_port = 58035, sin_addr = {s_addr = 1765950034},
sin_zero = "\000\000\000\000\000\000\000"}, sin6 = {sin6_family = 2, sin6_port = 58035, sin6_flowinfo = 1765950034, sin6_addr = {__in6_u = {__u6_addr8 = '\000' <repeats 15 times>, __u6_addr16 = {
0, 0, 0, 0, 0, 0, 0, 0}, __u6_addr32 = {0, 0, 0, 0}}}, sin6_scope_id = 0}, sas = {ss_family = 2, __ss_padding = "\263\342RBBi", '\000' <repeats 111 times>, __ss_align = 0}}, id = 0,
send_flags = {f = 0, blst_imask = 0}, proto = 3 '\003', proto_pad0 = 0 '\000', proto_pad1 = 0}
bcnt = 1
aortype = 0
i = 5
tv = {tv_sec = 1646903836, tv_usec = 846405}
tnow = 1646903836
__FUNCTION__ = "ul_ka_urecord"
#10 0x00007f04b2a6bb99 in mem_timer_udomain (_d=0x7f042f9c5388, istart=0, istep=1) at udomain.c:1234
ptr = 0x7f042fca3340
t = 0x1ea00000000
i = 28
#11 0x00007f04b2a9dd0e in synchronize_all_udomains (istart=0, istep=1) at dlist.c:966
res = 0
ptr = 0x7f042f9c5288
#12 0x00007f04b2a7ac5a in destroy () at usrloc_mod.c:498
__FUNCTION__ = "destroy"
#13 0x00000000005934ee in destroy_modules () at core/sr_module.c:772
t = 0x7f04b6e83828
foo = 0x7f04b6e83338
__FUNCTION__ = "destroy_modules"
#14 0x000000000041cda7 in cleanup (show_status=1) at main.c:569
memlog = 0
__FUNCTION__ = "cleanup"
#15 0x000000000041e682 in shutdown_children (sig=15, show_status=1) at main.c:712
__FUNCTION__ = "shutdown_children"
#16 0x000000000042172a in handle_sigs () at main.c:812
chld = 0
chld_status = 139
any_chld_stopped = 1
memlog = 798824048
__FUNCTION__ = "handle_sigs"
#17 0x000000000042babb in main_loop () at main.c:1845
i = 4
pid = 222489
si = 0x0
si_desc = "udp receiver child=3 sock=128.141.18.117:5060 (tone.company.name:5060)\000\000\000\000\000\000\000\000\200S/\004\177\000\000\000\000\000\000\000\000\000\000\220Q\234/\004\177\000\000\200\034\351\2---Type <return> to continue, or q <return> to quit---
13\375\177\000\000\000R\234/\004\177\000\000B\227Z\257\004\177\000\000\300\023\006\267\004\177\000"
nrprocs = 4
woneinit = 1
__FUNCTION__ = "main_loop"
#18 0x000000000043423e in main (argc=9, argv=0x7ffd8be921f8) at main.c:2942
cfg_stream = 0xf6a040
c = -1
r = 0
tmp = 0x7ffd8be935cc ""
tmp_len = 1472
port = 960
proto = 32516
ahost = 0x0
aport = 0
options = 0x7d7ef8 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:x:X:Y:"
ret = -1
seed = 1413124693
rfd = 4
debug_save = 0
debug_flag = 0
dont_fork_cnt = 0
n_lst = 0x7ffd8be920b0
p = 0xf0b5ff <Address 0xf0b5ff out of bounds>
st = {st_dev = 20, st_ino = 16553497, st_nlink = 2, st_mode = 16832, st_uid = 0, st_gid = 0, __pad0 = 0, st_rdev = 0, st_size = 40, st_blksize = 4096, st_blocks = 0, st_atim = {tv_sec = 1646819986,
tv_nsec = 228520360}, st_mtim = {tv_sec = 1646903531, tv_nsec = 836398702}, st_ctim = {tv_sec = 1646903531, tv_nsec = 836398702}, __unused = {0, 0, 0}}
tbuf = '\000' <repeats 392 times>...
option_index = 0
long_options = {{name = 0x7da22f "help", has_arg = 0, flag = 0x0, val = 104}, {name = 0x7d56d4 "version", has_arg = 0, flag = 0x0, val = 118}, {name = 0x7da234 "alias", has_arg = 1, flag = 0x0,
val = 1024}, {name = 0x7da23a "subst", has_arg = 1, flag = 0x0, val = 1025}, {name = 0x7da240 "substdef", has_arg = 1, flag = 0x0, val = 1026}, {name = 0x7da249 "substdefs", has_arg = 1,
flag = 0x0, val = 1027}, {name = 0x7da253 "server-id", has_arg = 1, flag = 0x0, val = 1028}, {name = 0x7da25d "loadmodule", has_arg = 1, flag = 0x0, val = 1029}, {name = 0x7da268 "modparam",
has_arg = 1, flag = 0x0, val = 1030}, {name = 0x7da271 "log-engine", has_arg = 1, flag = 0x0, val = 1031}, {name = 0x7da27c "debug", has_arg = 1, flag = 0x0, val = 1032}, {
name = 0x7da282 "atexit", has_arg = 1, flag = 0x0, val = 1034}, {name = 0x0, has_arg = 0, flag = 0x0, val = 0}}
__FUNCTION__ = "main"
2nd CORE backtrace
[root@pbxphone-0866-fe-1 kamailio_crash]# gdb kamailio core.kamailio.222460.pbxphone-0866-fe-1.company.domain.1646903834
GNU gdb (GDB) Red Hat Enterprise Linux 7.6.1-120.el7
Copyright (C) 2013 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law. Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-redhat-linux-gnu".
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>...
Reading symbols from /root/kamailio_crash/kamailio...done.
warning: exec file is newer than core file.
[New LWP 222460]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib64/libthread_db.so.1".
Core was generated by `/usr/local/sbin/kamailio -P /run/kamailio/kamailio.pid -f /usr/local/etc/kamail'.
Program terminated with signal 11, Segmentation fault.
#0 0x00007f04af578716 in tls_complete_init (c=0x7f042fe11c98) at tls_server.c:236
236 data->ssl = SSL_new(dom->ctx[process_no]);
Missing separate debuginfos, use: debuginfo-install audit-libs-2.8.5-4.el7.x86_64 bzip2-libs-1.0.6-13.el7.x86_64 elfutils-libelf-0.176-5.el7.x86_64 glibc-2.17-325.el7_9.x86_64 keyutils-libs-1.5.8-3.el7.x86_64 krb5-libs-1.15.1-51.el7_9.x86_64 libacl-2.2.51-15.el7.x86_64 libattr-2.4.46-13.el7.x86_64 libcap-2.22-11.el7.x86_64 libcap-ng-0.7.5-4.el7.x86_64 libcom_err-1.42.9-19.el7.x86_64 libdb-5.3.21-25.el7.x86_64 libffi-3.0.13-19.el7.x86_64 libgcc-4.8.5-44.el7.x86_64 libselinux-2.5-15.el7.x86_64 libstdc++-4.8.5-44.el7.x86_64 libunistring-0.9.3-9.el7.x86_64 libuuid-2.23.2-65.el7_9.1.x86_64 libxml2-2.9.1-6.el7_9.6.x86_64 lm_sensors-libs-3.4.0-8.20160601gitf9185e5.el7.x86_64 lua-5.1.4-15.el7.x86_64 net-snmp-agent-libs-5.7.2-49.el7_9.1.x86_64 net-snmp-libs-5.7.2-49.el7_9.1.x86_64 nspr-4.32.0-1.el7_9.x86_64 nss-3.67.0-4.el7_9.x86_64 nss-softokn-freebl-3.67.0-3.el7_9.x86_64 nss-util-3.67.0-1.el7_9.x86_64 openssl-libs-1.0.2k-24.el7_9.x86_64 pcre-8.32-17.el7.x86_64 perl-libs-5.16.3-299.el7_9.x86_64 popt-1.13-16.el7.x86_64 python-libs-2.7.5-90.el7.x86_64 rpm-libs-4.11.3-48.el7_9.x86_64 tcp_wrappers-libs-7.6-77.el7.x86_64 xz-libs-5.2.2-1.el7.x86_64 zlib-1.2.7-19.el7_9.x86_64
(gdb) bt full
#0 0x00007f04af578716 in tls_complete_init (c=0x7f042fe11c98) at tls_server.c:236
dom = 0x7f042fbcccf8
data = 0x7f042f96dec0
cfg = 0x7f042f96dec0
state = S_TLS_CONNECTING
sname = 0x0
srvid = 0x0
__FUNCTION__ = "tls_complete_init"
#1 0x00007f04af578d07 in tls_fix_connection_unsafe (c=0x7f042fe11c98) at tls_server.c:297
__FUNCTION__ = "tls_fix_connection_unsafe"
#2 0x00007f04af580784 in tls_h_encode_f (c=0x7f042fe11c98, pbuf=0x7ffd8be909e0, plen=0x7ffd8be909d8, rest_buf=0x7ffd8be909e8, rest_len=0x7ffd8be909dc, send_flags=0x7ffd8be909c0) at tls_server.c:772
n = 0
offs = 0
ssl = 0x1
tls_c = 0x7f042fe11ca0
wr_buf = "\026\003\001\002\000\001\000\001\374\003\003\276,'P^\252j\027\021S\351\252\021\342(~\347\216\351\332Hh!\vb\024\361\324+\003\177\a\000\000\254\300\060\300,\300(\300$\300\024\300\n\000\245\000\243\000\241\000\237\000k\000j\000i\000h\000\071\000\070\000\067\000\066\000\210\000\207\000\206\000\205\300\062\300.\300*\300&\300\017\300\005\000\235\000=\000\065\000\204\300/\300+\300'\300#\300\023\300\t\000\244\000\242\000\240\000\236\000g\000@\000?\000>\000\063\000\062\000\061\000\060\000\232\000\231\000\230\000\227\000E\000D\000C\000B\300\061\300-\300)\300%\300\016\300\004\000\234\000<\000/\000\226\000A\300\022\300\b\000\026\000\023\000\020\000\r\300\r\300\003"...
rd = {buf = 0x18 <Address 0x18 out of bounds>, pos = 22930096, used = 1771061403, size = -1947662592}
wr = {buf = 0x4b6e74a40 <Address 0x4b6e74a40 out of bounds>, pos = -1226359400, used = 32516, size = -1562640382}
ssl_error = 0
err_src = 0x7f04af5afc09 "TLS write:"
ip_buf = "\000\000\000\000\000\000\000\000\230\034\341/\004\177\000\000\000\000\000\000\000\000\000\000\370@\347\266\001\000\000\000\340\b\351\213\375\177\000\000f\211Q\000\000\000\000\000\000\000\000\000\001\000\000\000\060\000\000\000\060\000\000"
buf = 0x7ffd8be90fd0 "OPTIONS sip:73054@82.66.66.105:46050;transport=tls SIP/2.0\r\nVia: SIP/2.0/TLS tone.company.domain:5061;branch=z9hG4bKx.2854.1.0\r\nFrom: <sip:pbxphone@tone.company.domain>;tag=uloc-6228658a-43655-2c-31dd201c-6229c21a"...
len = 308
x = 4
__FUNCTION__ = "tls_h_encode_f"
#3 0x000000000051e729 in tcp_send (dst=0x7ffd8be90d60, from=0x0,
buf=0x7ffd8be90fd0 "OPTIONS sip:73054@82.66.66.105:46050;transport=tls SIP/2.0\r\nVia: SIP/2.0/TLS tone.company.domain:5061;branch=z9hG4bKx.2854.1.0\r\nFrom: <sip:pbxphone@tone.company.domain>;tag=uloc-6228658a-43655-2c-31dd201c-6229c21a"..., len=308) at core/tcp_main.c:2093
c = 0x7f042fe11c98
ip = {af = 2, len = 4, u = {addrl = {1765950034, 4294967295}, addr32 = {1765950034, 0, 4294967295, 0}, addr16 = {16978, 26946, 0, 0, 65535, 65535, 0, 0},
addr = "RBBi\000\000\000\000\377\377\377\377\000\000\000"}}
port = 46050
fd = 22
response = {139655966192064, 0}
n = 0
con_lifetime = 9600
try_local_port = 5061
rest_buf = 0x0
t_buf = 0x7ffd8be90fd0 "OPTIONS sip:73054@82.66.66.105:46050;transport=tls SIP/2.0\r\nVia: SIP/2.0/TLS tone.company.domain:5061;branch=z9hG4bKx.2854.1.0\r\nFrom: <sip:pbxphone@tone.company.domain>;tag=uloc-6228658a-43655-2c-31dd201c-6229c21a"...
rest_len = 0
t_len = 308
resp = 140726950756352
t_send_flags = {f = 0, blst_imask = 0}
__FUNCTION__ = "tcp_send"
#4 0x00007f04b2aa0ceb in ul_ka_send (kamsg=0x7ffd8be90fc0, kadst=0x7ffd8be90d60) at ul_keepalive.c:271
__FUNCTION__ = "ul_ka_send"
#5 0x00007f04b2aa0b58 in ul_ka_urecord (ur=0x7f042fca3340) at ul_keepalive.c:239
uc = 0x7f042fca3458
kabuf = "OPTIONS sip:73054@82.66.66.105:46050;transport=tls SIP/2.0\r\nVia: SIP/2.0/TLS tone.company.domain:5061;branch=z9hG4bKx.2854.1.0\r\nFrom: <sip:pbxphone@tone.company.domain>;tag=uloc-6228658a-43655-2c-31dd20---Type <return> to continue, or q <return> to quit---
1c-6229c21a"...
kabuf_len = 308
kamsg = {
s = 0x7ffd8be90fd0 "OPTIONS sip:73054@82.66.66.105:46050;transport=tls SIP/2.0\r\nVia: SIP/2.0/TLS tone.company.domain:5061;branch=z9hG4bKx.2854.1.0\r\nFrom: <sip:pbxphone@tone.company.domain>;tag=uloc-6228658a-43655-2c-31dd201c-6229c21a"..., len = 308}
vaddr = {s = 0x7f04b6e74008 "tone.company.domain", len = 12}
vport = {s = 0x7f04b6e740f8 "5061", len = 4}
sdst = {s = 0x7f042fca3770 "sip:82.66.66.105:46050;transport=tls", len = 36}
sproto = {s = 0x81bb4b "TLS", len = 3}
duri = {user = {s = 0x0, len = 0}, passwd = {s = 0x0, len = 0}, host = {s = 0x7f042fca3774 "82.66.66.105:46050;transport=tls", len = 12}, port = {s = 0x7f042fca3781 "46050;transport=tls", len = 5},
params = {s = 0x7f042fca3787 "transport=tls", len = 13}, sip_params = {s = 0x7f042fca3787 "transport=tls", len = 13}, headers = {s = 0x0, len = 0}, port_no = 46050, proto = 3, type = SIP_URI_T,
flags = (unknown: 0), transport = {s = 0x7f042fca3787 "transport=tls", len = 13}, ttl = {s = 0x0, len = 0}, user_param = {s = 0x0, len = 0}, maddr = {s = 0x0, len = 0}, method = {s = 0x0, len = 0},
lr = {s = 0x0, len = 0}, r2 = {s = 0x0, len = 0}, gr = {s = 0x0, len = 0}, transport_val = {s = 0x7f042fca3791 "tls", len = 3}, ttl_val = {s = 0x0, len = 0}, user_param_val = {s = 0x0, len = 0},
maddr_val = {s = 0x0, len = 0}, method_val = {s = 0x0, len = 0}, lr_val = {s = 0x0, len = 0}, r2_val = {s = 0x0, len = 0}, gr_val = {s = 0x0, len = 0}}
dproto = 3 '\003'
he = 0xb88fa0 <he.5288>
ssock = 0x7f04b6e73d98
idst = {send_sock = 0x7f04b6e73d98, to = {s = {sa_family = 2, sa_data = "\263\342RBBi\000\000\000\000\000\000\000"}, sin = {sin_family = 2, sin_port = 58035, sin_addr = {s_addr = 1765950034},
sin_zero = "\000\000\000\000\000\000\000"}, sin6 = {sin6_family = 2, sin6_port = 58035, sin6_flowinfo = 1765950034, sin6_addr = {__in6_u = {__u6_addr8 = '\000' <repeats 15 times>, __u6_addr16 = {
0, 0, 0, 0, 0, 0, 0, 0}, __u6_addr32 = {0, 0, 0, 0}}}, sin6_scope_id = 0}, sas = {ss_family = 2, __ss_padding = "\263\342RBBi", '\000' <repeats 111 times>, __ss_align = 0}}, id = 0,
send_flags = {f = 0, blst_imask = 0}, proto = 3 '\003', proto_pad0 = 0 '\000', proto_pad1 = 0}
bcnt = 1
aortype = 0
i = 5
tv = {tv_sec = 1646903834, tv_usec = 680904}
tnow = 1646903834
__FUNCTION__ = "ul_ka_urecord"
#6 0x00007f04b2a6bb99 in mem_timer_udomain (_d=0x7f042f9c5388, istart=0, istep=1) at udomain.c:1234
ptr = 0x7f042fca3340
t = 0x7f0429d60470 <str_record_route_col>
i = 28
#7 0x00007f04b2a9dd0e in synchronize_all_udomains (istart=0, istep=1) at dlist.c:966
res = 0
ptr = 0x7f042f9c5288
#8 0x00007f04b2a7b019 in ul_core_timer (ticks=1791845, param=0x0) at usrloc_mod.c:527
__FUNCTION__ = "ul_core_timer"
#9 0x00000000004f1e69 in compat_old_handler (ti=28669522, tl=0x7f042f82e060, data=0x7f042f82e060) at core/timer.c:980
t = 0x7f042f82e060
#10 0x00000000004f29a5 in slow_timer_main () at core/timer.c:1105
n = 12
ret = 4294967295
tl = 0x7f042f82e060
i = 308
__FUNCTION__ = "slow_timer_main"
#11 0x000000000042a707 in main_loop () at main.c:1765
i = 4
pid = 0
si = 0x0
si_desc = "udp receiver child=3 sock=128.141.18.117:5060 (tone.company.domain:5060)\000\000\000\000\000\000\000\000\200S/\004\177\000\000\000\000\000\000\000\000\000\000\220Q\234/\004\177\000\000\200\034\351\213\375\177\000\000\000R\234/\004\177\000\000B\227Z\257\004\177\000\000\300\023\006\267\004\177\000"
nrprocs = 4
woneinit = 1
__FUNCTION__ = "main_loop"
#12 0x000000000043423e in main (argc=9, argv=0x7ffd8be921f8) at main.c:2942
cfg_stream = 0xf6a040
---Type <return> to continue, or q <return> to quit---
c = -1
r = 0
tmp = 0x7ffd8be935cc ""
tmp_len = 1472
port = 960
proto = 32516
ahost = 0x0
aport = 0
options = 0x7d7ef8 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:x:X:Y:"
ret = -1
seed = 1413124693
rfd = 4
debug_save = 0
debug_flag = 0
dont_fork_cnt = 0
n_lst = 0x7ffd8be920b0
p = 0xf0b5ff <Address 0xf0b5ff out of bounds>
st = {st_dev = 20, st_ino = 16553497, st_nlink = 2, st_mode = 16832, st_uid = 0, st_gid = 0, __pad0 = 0, st_rdev = 0, st_size = 40, st_blksize = 4096, st_blocks = 0, st_atim = {tv_sec = 1646819986,
tv_nsec = 228520360}, st_mtim = {tv_sec = 1646903531, tv_nsec = 836398702}, st_ctim = {tv_sec = 1646903531, tv_nsec = 836398702}, __unused = {0, 0, 0}}
tbuf = '\000' <repeats 392 times>...
option_index = 0
long_options = {{name = 0x7da22f "help", has_arg = 0, flag = 0x0, val = 104}, {name = 0x7d56d4 "version", has_arg = 0, flag = 0x0, val = 118}, {name = 0x7da234 "alias", has_arg = 1, flag = 0x0,
val = 1024}, {name = 0x7da23a "subst", has_arg = 1, flag = 0x0, val = 1025}, {name = 0x7da240 "substdef", has_arg = 1, flag = 0x0, val = 1026}, {name = 0x7da249 "substdefs", has_arg = 1,
flag = 0x0, val = 1027}, {name = 0x7da253 "server-id", has_arg = 1, flag = 0x0, val = 1028}, {name = 0x7da25d "loadmodule", has_arg = 1, flag = 0x0, val = 1029}, {name = 0x7da268 "modparam",
has_arg = 1, flag = 0x0, val = 1030}, {name = 0x7da271 "log-engine", has_arg = 1, flag = 0x0, val = 1031}, {name = 0x7da27c "debug", has_arg = 1, flag = 0x0, val = 1032}, {
name = 0x7da282 "atexit", has_arg = 1, flag = 0x0, val = 1034}, {name = 0x0, has_arg = 0, flag = 0x0, val = 0}}
__FUNCTION__ = "main"
3rd CORE:
[root@pbxphone-0866-fe-1 kamailio_crash]# gdb kamailio core.kamailio.276013.pbxphone-0866-fe-1.company.domain.1646903518
GNU gdb (GDB) Red Hat Enterprise Linux 7.6.1-120.el7
Copyright (C) 2013 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law. Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-redhat-linux-gnu".
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>...
Reading symbols from /root/kamailio_crash/kamailio...done.
warning: exec file is newer than core file.
[New LWP 276013]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib64/libthread_db.so.1".
Core was generated by `/usr/local/sbin/kamailio -P /run/kamailio/kamailio.pid -f /usr/local/etc/kamail'.
Program terminated with signal 11, Segmentation fault.
#0 0x00007f7cfc309478 in atomic_cmpxchg_int (var=0x0, old=0, new_v=1) at ../../core/atomic/atomic_x86.h:224
224 ATOMIC_FUNC_CMPXCHG(cmpxchg, "cmpxchgl %2, %1", int , int)
Missing separate debuginfos, use: debuginfo-install audit-libs-2.8.5-4.el7.x86_64 bzip2-libs-1.0.6-13.el7.x86_64 elfutils-libelf-0.176-5.el7.x86_64 glibc-2.17-325.el7_9.x86_64 keyutils-libs-1.5.8-3.el7.x86_64 krb5-libs-1.15.1-51.el7_9.x86_64 libacl-2.2.51-15.el7.x86_64 libattr-2.4.46-13.el7.x86_64 libcap-2.22-11.el7.x86_64 libcap-ng-0.7.5-4.el7.x86_64 libcom_err-1.42.9-19.el7.x86_64 libdb-5.3.21-25.el7.x86_64 libffi-3.0.13-19.el7.x86_64 libgcc-4.8.5-44.el7.x86_64 libselinux-2.5-15.el7.x86_64 libstdc++-4.8.5-44.el7.x86_64 libunistring-0.9.3-9.el7.x86_64 libuuid-2.23.2-65.el7_9.1.x86_64 libxml2-2.9.1-6.el7_9.6.x86_64 lm_sensors-libs-3.4.0-8.20160601gitf9185e5.el7.x86_64 lua-5.1.4-15.el7.x86_64 net-snmp-agent-libs-5.7.2-49.el7_9.1.x86_64 net-snmp-libs-5.7.2-49.el7_9.1.x86_64 nspr-4.32.0-1.el7_9.x86_64 nss-3.67.0-4.el7_9.x86_64 nss-softokn-freebl-3.67.0-3.el7_9.x86_64 nss-util-3.67.0-1.el7_9.x86_64 openssl-libs-1.0.2k-24.el7_9.x86_64 pcre-8.32-17.el7.x86_64 perl-libs-5.16.3-299.el7_9.x86_64 popt-1.13-16.el7.x86_64 python-libs-2.7.5-90.el7.x86_64 rpm-libs-4.11.3-48.el7_9.x86_64 tcp_wrappers-libs-7.6-77.el7.x86_64 xz-libs-5.2.2-1.el7.x86_64 zlib-1.2.7-19.el7_9.x86_64
(gdb) bt full
#0 0x00007f7cfc309478 in atomic_cmpxchg_int (var=0x0, old=0, new_v=1) at ../../core/atomic/atomic_x86.h:224
ret = 0
#1 0x00007f7cfc3094c4 in futex_get (lock=0x0) at ../../core/futexlock.h:99
v = 32636
i = 1024
#2 0x00007f7cfc310461 in wsconn_get (id=992) at ws_conn.c:452
id_hash = 992
wsc = 0x100000001
__FUNCTION__ = "wsconn_get"
#3 0x00007f7cfc324d71 in ws_frame_transmit (evp=0x7ffcf5f31830) at ws_frame.c:728
wsev = 0x7ffcf5f31870
frame = {fin = 1, rsv1 = 0, rsv2 = 0, rsv3 = 0, opcode = 1, mask = 0, payload_len = 301, masking_key = "\000\000\000",
payload_data = 0x7ffcf5f31ea0 "OPTIONS sip:73904@7nprmvqiutl9.invalid;transport=wss SIP/2.0\r\nVia: SIP/2.0/WS tone.company.domain:8089;branch=z9hG4bKx.1.1.0\r\nFrom: <sip:pbxphone@tone.company.domain>;tag=uloc-6228658a-43655-44-31dd2800-6229c0de-6"..., wsc = 0x0}
__FUNCTION__ = "ws_frame_transmit"
#4 0x00000000004f3762 in sr_event_exec (type=11, evp=0x7ffcf5f31830) at core/events.c:296
ret = 0
i = 50304
#5 0x0000000000544edf in wss_send (dst=0x7ffcf5f31c30,
buf=0x7ffcf5f31ea0 "OPTIONS sip:73904@7nprmvqiutl9.invalid;transport=wss SIP/2.0\r\nVia: SIP/2.0/WS tone.company.domain:8089;branch=z9hG4bKx.1.1.0\r\nFrom: <sip:pbxphone@tone.company.domain>;tag=uloc-6228658a-43655-44-31dd2800-6229c0de-6"..., len=301) at core/tcp_main.c:5209
port = 50304
ip = {af = 2, len = 4, u = {addrl = {3232828610, 140724434836000}, addr32 = {3232828610, 0, 4126349856, 32764}, addr16 = {3266, 49329, 0, 0, 6688, 62963, 32764, 0},
addr = "\302\f\261\300\000\000\000\000 \032\363\365\374\177\000"}}
from = 0x0
local_addr = {s = {sa_family = 19200, sa_data = "\264\000\000\000\000\000\360\034\363\365\374\177\000"}, sin = {sin_family = 19200, sin_port = 180, sin_addr = {s_addr = 0},
sin_zero = "\360\034\363\365\374\177\000"}, sin6 = {sin6_family = 19200, sin6_port = 180, sin6_flowinfo = 0, sin6_addr = {__in6_u = {
__u6_addr8 = "\360\034\363\365\374\177\000\000\000\000\272\001}\177\000", __u6_addr16 = {7408, 62963, 32764, 0, 0, 442, 32637, 0}, __u6_addr32 = {4126350576, 32764, 28966912, 32637}}},
sin6_scope_id = 0}, sas = {ss_family = 19200,
__ss_padding = "\264\000\000\000\000\000\360\034\363\365\374\177\000\000\000\000\272\001}\177\000\000\000\000\000\000\000\000\000\000\360\030\363\365\374\177\000\000\310\320i\000\000\000\000\000\300}\271\001}\177\000\000\000\000\000\000\000\000\000\000\340\271A\000\004\000\000\000\360\362\200\000\000\000\000\000\377\377\377\377", '\000' <repeats 12 times>, " \031\363\365\374\177\000\000\200\200\025\251}\177\000\000\000\000\000\000\000\000\000", __ss_align = 140724434836720}}
con = 0x7f7d03e4c500
wsev = {type = 11,
buf = 0x7ffcf5f31ea0 "OPTIONS sip:73904@7nprmvqiutl9.invalid;transport=wss SIP/2.0\r\nVia: SIP/2.0/WS tone.company.domain:8089;branch=z9hG4bKx.1.1.0\r\nFrom: <sip:pbxphone@tone.company.domain>;tag=uloc-6228658a-43655-44-31dd2800-6229c0de-6"..., len = 301, id = 992}
evp = {data = 0x7ffcf5f31870, obuf = {s = 0x0, len = 0}, rcv = 0x0, dst = 0x0, req = 0x0, rpl = 0x0, rplcode = 0, mode = 0}
ret = 0
__FUNCTION__ = "wss_send"
#6 0x00007f7d84a54c47 in ul_ka_send (kamsg=0x7ffcf5f31e90, kadst=0x7ffcf5f31c30) at ul_keepalive.c:259
__FUNCTION__ = "ul_ka_send"
#7 0x00007f7d84a54b58 in ul_ka_urecord (ur=0x7f7d04fcb8f8) at ul_keepalive.c:239
uc = 0x7f7d04fcba10
kabuf = "OPTIONS sip:73904@7nprmvqiutl9.invalid;transport=wss SIP/2.0\r\nVia: SIP/2.0/WS tone.company.domain:8089;branch=z9hG4bKx.1.1.0\r\nFrom: <sip:pbxphone@tone.company.domain>;tag=uloc-6228658a-43655-44-31dd2800-6229c0de-6"...
kabuf_len = 301
kamsg = {
s = 0x7ffcf5f31ea0 "OPTIONS sip:73904@7nprmvqiutl9.invalid;transport=wss SIP/2.0\r\nVia: SIP/2.0/WS tone.company.domain:8089;branch=z9hG4bKx.1.1.0\r\nFrom: <sip:pbxphone@tone.company.domain>;tag=uloc-6228658a-43655-44-31dd2800-6229c0de-6"..., len = 301}
vaddr = {s = 0x7f7d88e28a40 "tone.company.domain", len = 12}
vport = {s = 0x7f7d88e28b30 "8089", len = 4}
sdst = {s = 0x7f7d051d1318 "sip:194.12.177.192:50304;transport=ws", len = 37}
sproto = {s = 0x81bb5d "WS", len = 2}
duri = {user = {s = 0x0, len = 0}, passwd = {s = 0x0, len = 0}, host = {s = 0x7f7d051d131c "194.12.177.192:50304;transport=ws", len = 14}, port = {s = 0x7f7d051d132b "50304;transport=ws", len = 5},
params = {s = 0x7f7d051d1331 "transport=ws", len = 12}, sip_params = {s = 0x7f7d051d1331 "transport=ws", len = 12}, headers = {s = 0x0, len = 0}, port_no = 50304, proto = 5, type = SIP_URI_T,
---Type <return> to continue, or q <return> to quit---
flags = (unknown: 0), transport = {s = 0x7f7d051d1331 "transport=ws", len = 12}, ttl = {s = 0x0, len = 0}, user_param = {s = 0x0, len = 0}, maddr = {s = 0x0, len = 0}, method = {s = 0x0, len = 0},
lr = {s = 0x0, len = 0}, r2 = {s = 0x0, len = 0}, gr = {s = 0x0, len = 0}, transport_val = {s = 0x7f7d051d133b "ws", len = 2}, ttl_val = {s = 0x0, len = 0}, user_param_val = {s = 0x0, len = 0},
maddr_val = {s = 0x0, len = 0}, method_val = {s = 0x0, len = 0}, lr_val = {s = 0x0, len = 0}, r2_val = {s = 0x0, len = 0}, gr_val = {s = 0x0, len = 0}}
dproto = 5 '\005'
he = 0xb88fa0 <he.5288>
ssock = 0x7f7d88e287d0
idst = {send_sock = 0x7f7d88e287d0, to = {s = {sa_family = 2, sa_data = "Ā\302\f\261\300\000\000\000\000\000\000\000"}, sin = {sin_family = 2, sin_port = 32964, sin_addr = {s_addr = 3232828610},
sin_zero = "\000\000\000\000\000\000\000"}, sin6 = {sin6_family = 2, sin6_port = 32964, sin6_flowinfo = 3232828610, sin6_addr = {__in6_u = {__u6_addr8 = '\000' <repeats 15 times>, __u6_addr16 = {
0, 0, 0, 0, 0, 0, 0, 0}, __u6_addr32 = {0, 0, 0, 0}}}, sin6_scope_id = 0}, sas = {ss_family = 2, __ss_padding = "Ā\302\f\261\300", '\000' <repeats 111 times>, __ss_align = 0}}, id = 0,
send_flags = {f = 0, blst_imask = 0}, proto = 5 '\005', proto_pad0 = 0 '\000', proto_pad1 = 0}
bcnt = 1
aortype = 0
i = 5
tv = {tv_sec = 1646903518, tv_usec = 435121}
tnow = 1646903518
__FUNCTION__ = "ul_ka_urecord"
#8 0x00007f7d84a1fb99 in mem_timer_udomain (_d=0x7f7d01990460, istart=0, istep=1) at udomain.c:1234
ptr = 0x7f7d04fcb8f8
t = 0x1ea00000000
i = 0
#9 0x00007f7d84a51d0e in synchronize_all_udomains (istart=0, istep=1) at dlist.c:966
res = 0
ptr = 0x7f7d01990360
#10 0x00007f7d84a2ec5a in destroy () at usrloc_mod.c:498
__FUNCTION__ = "destroy"
#11 0x00000000005934ee in destroy_modules () at core/sr_module.c:772
t = 0x7f7d88e37828
foo = 0x7f7d88e37338
__FUNCTION__ = "destroy_modules"
#12 0x000000000041cda7 in cleanup (show_status=1) at main.c:569
memlog = 0
__FUNCTION__ = "cleanup"
#13 0x000000000041e682 in shutdown_children (sig=15, show_status=1) at main.c:712
__FUNCTION__ = "shutdown_children"
#14 0x000000000041f1f7 in handle_sigs () at main.c:743
chld = 0
chld_status = 0
any_chld_stopped = 0
memlog = 26855240
__FUNCTION__ = "handle_sigs"
#15 0x000000000042babb in main_loop () at main.c:1845
i = 4
pid = 276058
si = 0x0
si_desc = "udp receiver child=3 sock=128.141.18.117:5060 (tone.company.domain:5060)\000\000\000\000\000\000\000\000\300N\001}\177\000\000\000\000\000\000\000\000\000\000h\002\231\001}\177\000\000\220.\363\365\374\177\000\000\330\002\231\001}\177\000\000B\327U\201}\177\000\000\300S\001\211}\177\000"
nrprocs = 4
woneinit = 1
__FUNCTION__ = "main_loop"
#16 0x000000000043423e in main (argc=9, argv=0x7ffcf5f33408) at main.c:2942
cfg_stream = 0xf06040
c = -1
r = 0
tmp = 0x7ffcf5f34eff ""
tmp_len = 1472
---Type <return> to continue, or q <return> to quit---
port = 960
proto = 32637
ahost = 0x0
aport = 0
options = 0x7d7ef8 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:x:X:Y:"
ret = -1
seed = 626051482
rfd = 4
debug_save = 0
debug_flag = 0
dont_fork_cnt = 0
n_lst = 0x7ffcf5f332c0
p = 0xf0b5ff ""
st = {st_dev = 20, st_ino = 16553497, st_nlink = 2, st_mode = 16832, st_uid = 0, st_gid = 0, __pad0 = 0, st_rdev = 0, st_size = 40, st_blksize = 4096, st_blocks = 0, st_atim = {tv_sec = 1614937626,
tv_nsec = 968330983}, st_mtim = {tv_sec = 1646814601, tv_nsec = 999994878}, st_ctim = {tv_sec = 1646814601, tv_nsec = 999994878}, __unused = {0, 0, 0}}
tbuf = '\000' <repeats 392 times>...
option_index = 0
long_options = {{name = 0x7da22f "help", has_arg = 0, flag = 0x0, val = 104}, {name = 0x7d56d4 "version", has_arg = 0, flag = 0x0, val = 118}, {name = 0x7da234 "alias", has_arg = 1, flag = 0x0,
val = 1024}, {name = 0x7da23a "subst", has_arg = 1, flag = 0x0, val = 1025}, {name = 0x7da240 "substdef", has_arg = 1, flag = 0x0, val = 1026}, {name = 0x7da249 "substdefs", has_arg = 1,
flag = 0x0, val = 1027}, {name = 0x7da253 "server-id", has_arg = 1, flag = 0x0, val = 1028}, {name = 0x7da25d "loadmodule", has_arg = 1, flag = 0x0, val = 1029}, {name = 0x7da268 "modparam",
has_arg = 1, flag = 0x0, val = 1030}, {name = 0x7da271 "log-engine", has_arg = 1, flag = 0x0, val = 1031}, {name = 0x7da27c "debug", has_arg = 1, flag = 0x0, val = 1032}, {
name = 0x7da282 "atexit", has_arg = 1, flag = 0x0, val = 1034}, {name = 0x0, has_arg = 0, flag = 0x0, val = 0}}
__FUNCTION__ = "main"
Is it a right place to report or better to make an issue on github?
Thanks in advance!