Kamailio 5.5 crashed when sending reply (bt full below). Before the
crash there had been several mysql related error messages ("Lost
connection to MySQL server during query" and "Too many connections") in
syslog when async worker tried to insert accounting data. So perhaps
this core dump has something to do with that.
-- Juha
Reading symbols from /usr/bin/sip-proxy...done.
[New LWP 21324]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
Core was generated by `/usr/bin/sip-proxy -f /etc/sip-proxy/sip-proxy.cfg -P /run/sip-proxy/sip-proxy.'.
Program terminated with signal SIGSEGV, Segmentation fault.
#0 0x00007fd274ce3b6e in run_trans_callbacks_internal (cb_lst=0x7fd2532503a8, type=512, trans=0x7fd253250330, params=0x7fff9b8eb2b0) at t_hooks.c:254
254 t_hooks.c: No such file or directory.
(gdb) bt full
#0 0x00007fd274ce3b6e in run_trans_callbacks_internal (cb_lst=0x7fd2532503a8, type=512, trans=0x7fd253250330, params=0x7fff9b8eb2b0) at t_hooks.c:254
cbp = 0x10
backup_from = 0x5613ccfb37f0 <def_list+16>
backup_to = 0x5613ccfb37f8 <def_list+24>
backup_dom_from = 0x5613ccfb3800 <def_list+32>
backup_dom_to = 0x5613ccfb3808 <def_list+40>
backup_uri_from = 0x5613ccfb37e0 <def_list>
backup_uri_to = 0x5613ccfb37e8 <def_list+8>
backup_xavps = 0x5613ccf56270 <_xavp_list_head>
backup_xavus = 0x5613ccf56278 <_xavu_list_head>
backup_xavis = 0x5613ccf56280 <_xavi_list_head>
__func__ = "run_trans_callbacks_internal"
#1 0x00007fd274ce41f7 in run_trans_callbacks_with_buf (type=512, rbuf=0x7fd253250400, req=0x7fd252d6dd20, repl=0x7fd2758bf0b0, flags=0) at t_hooks.c:303
params = {req = 0x7fd252d6dd20, rpl = 0x7fd2758bf0b0, param = 0x7fd254760f60, code = 486, flags = 0, branch = 0, t_rbuf = 0x7fd253250400, dst = 0x7fd253250450, send_buf = {
s = 0x7fd25325b9f0 "@", len = 579}}
trans = 0x7fd253250330
#2 0x00007fd274c88c15 in relay_reply (t=0x7fd253250330, p_msg=0x7fd2758bf0b0, branch=0, msg_status=486, cancel_data=0x7fff9b8eb580, do_put_on_wait=1) at t_reply.c:2133
relay = 0
save_clone = 0
buf = 0x7fd275ca0ff0 "SIP/2.0 486 Busy Here\r\nTo: <sip:XXXXX@XXXXX.com;user=phone>;tag=h7g4Esbg_11002529899813\r\nFrom: <sip:XXXXX@XXXXX>;tag=as6964f29d\r\nCal"...
res_len = 579
relayed_code = 486
relayed_msg = 0x7fd2758bf0b0
reply_bak = 0x7fd2510f1000
bm = {to_tag_val = {s = 0x7fd25216e4c0 "@\003", len = 1377234752}}
totag_retr = 0
reply_status = RPS_COMPLETED
uas_rb = 0x7fd253250400
to_tag = 0x7fd25216e820
reason = {s = 0x7fd25216e830 "0", len = 1394935248}
onsend_params = {req = 0x9b8eb420, rpl = 0x7fd274c6b10d <futex_get+40>, param = 0x1, code = 1359984720, flags = 32722, branch = 0, t_rbuf = 0x7fff9b8eb460, dst = 0xa0679800, send_buf = {
s = 0x19b8eb450 <error: Cannot access memory at address 0x19b8eb450>, len = 1359984720}}
ip = {af = 2609820752, len = 32767, u = {addrl = {94643040895493, 0}, addr32 = {3436528133, 22035, 0, 0}, addr16 = {16901, 52437, 22035, 0, 0, 0, 0, 0},
addr = "\005B\325\314\023V\000\000\000\000\000\000\000\000\000"}}
__func__ = "relay_reply"
#3 0x00007fd274c8de6b in reply_received (p_msg=0x7fd2758bf0b0) at t_reply.c:2680
msg_status = 486
last_uac_status = 180
ack = 0x7fd25216e830 "0"
ack_len = 459
branch = 0
reply_status = 32722
onreply_route = 2
cancel_data = {cancel_bitmap = 0, reason = {cause = 0, u = {text = {s = 0x0, len = 1977055608}, e2e_cancel = 0x0, packed_hdrs = {s = 0x0, len = 1977055608}}}}
uac = 0x7fd2532505c0
t = 0x7fd253250330
lack_dst = {send_sock = 0x40, to = {s = {sa_family = 49168, sa_data = "\207u\322\177\000\000\220\266\216\233\377\177\000"}, sin = {sin_family = 49168, sin_port = 30087, sin_addr = {
s_addr = 32722}, sin_zero = "\220\266\216\233\377\177\000"}, sin6 = {sin6_family = 49168, sin6_port = 30087, sin6_flowinfo = 32722, sin6_addr = {__in6_u = {
__u6_addr8 = "\220\266\216\233\377\177\000\000L=\215s\322\177\000", __u6_addr16 = {46736, 39822, 32767, 0, 15692, 29581, 32722, 0}, __u6_addr32 = {2609821328, 32767, 1938636108,
32722}}}, sin6_scope_id = 1971831024}, sas = {ss_family = 49168,
__ss_padding = "\207u\322\177\000\000\220\266\216\233\377\177\000\000L=\215s\322\177\000\000\360\300\207u\322\177", '\000' <repeats 13 times>, "\b\000\000\000\000\060Z\264u\322\177\000\000\213\t\342\316\023V\000\000;\b\342\316\023V\000\000\020\313\320\000\000\000\000\000\020\000\000\000\000\000\000\000I\001\000\000\000\000\000\000\000\230g\240\210\347A\357\300\266\216\233\377\177\000\000\000\230g\240\210\347", <incomplete sequence \357>, __ss_align = 0}}, id = 0, send_flags = {f = 0, blst_imask = 0}, proto = 0 '\000', proto_pad0 = 0 '\000', proto_pad1 = 0}
backup_user_from = 0x5613ccfb37f0 <def_list+16>
backup_user_to = 0x5613ccfb37f8 <def_list+24>
backup_domain_from = 0x5613ccfb3800 <def_list+32>
backup_domain_to = 0x5613ccfb3808 <def_list+40>
backup_uri_from = 0x5613ccfb37e0 <def_list>
backup_uri_to = 0x5613ccfb37e8 <def_list+8>
backup_xavps = 0x5613ccf56270 <_xavp_list_head>
backup_xavus = 0x5613ccf56278 <_xavu_list_head>
backup_xavis = 0x5613ccf56280 <_xavi_list_head>
replies_locked = 1
branch_ret = 0
prev_branch = 1971830800
blst_503_timeout = 32722
hf = 0xb5c750
onsend_params = {req = 0x7fff9b8eb5e0, rpl = 0xef41e788a0679800, param = 0x30, code = 0, flags = 0, branch = 0, t_rbuf = 0x0, dst = 0x7fd27d972cb0 <__syslog>, send_buf = {
s = 0x5613ccec69b3 "INFO", len = -1603823616}}
ctx = {rec_lev = 0, run_flags = 2, last_retcode = -1, jmp_env = {{__jmpbuf = {0, -5133089567146157739, 0, 140542026919088, 94643042412979, 536870912, -5133089567232140971, -1505551737941960363},
__mask_was_saved = 0, __saved_mask = {__val = {140735803210560, 13683472, 7765887371, 140735803209568, 94643041089583, 0, 17240315422543681536, 0, 0, 0, 0, 94643042412979,
17240315422543681536, 140735803209872, 94643041372964, 2866715017336}}}}}
bctx = 0x7fd27587c010
keng = 0x0
ret = 0
evname = {s = 0x7fd274d3696f "on_sl_reply", len = 11}
__func__ = "reply_received"
#4 0x00005613ccbba0d4 in do_forward_reply (msg=0x7fd2758bf0b0, mode=0) at core/forward.c:764
new_buf = 0x0
dst = {send_sock = 0x0, to = {s = {sa_family = 0, sa_data = '\000' <repeats 13 times>}, sin = {sin_family = 0, sin_port = 0, sin_addr = {s_addr = 0}, sin_zero = "\000\000\000\000\000\000\000"},
sin6 = {sin6_family = 0, sin6_port = 0, sin6_flowinfo = 0, sin6_addr = {__in6_u = {__u6_addr8 = '\000' <repeats 15 times>, __u6_addr16 = {0, 0, 0, 0, 0, 0, 0, 0}, __u6_addr32 = {0, 0, 0,
0}}}, sin6_scope_id = 0}, sas = {ss_family = 0, __ss_padding = '\000' <repeats 117 times>, __ss_align = 0}}, id = 0, send_flags = {f = 0, blst_imask = 0}, proto = 0 '\000',
proto_pad0 = 0 '\000', proto_pad1 = 0}
new_len = 0
r = 2
ip = {af = 1, len = 6356993, u = {addrl = {140735803210560, 140541891965104}, addr32 = {2609822528, 32767, 1972105392, 32722}, addr16 = {47936, 39822, 32767, 0, 61616, 30091, 32722, 0},
addr = "@\273\216\233\377\177\000\000\260\360\213u\322\177\000"}}
s = 0xef41e788a0679800 <error: Cannot access memory at address 0xef41e788a0679800>
len = -1685145520
__func__ = "do_forward_reply"
#5 0x00005613ccbbc23c in forward_reply (msg=0x7fd2758bf0b0) at core/forward.c:865
No locals.
#6 0x00005613ccc5eea9 in receive_msg (
buf=0x5613cee206f0 "SIP/2.0 486 Busy Here\r\nTo: <sip:XXXXX@XXXXX;user=phone>;tag=h7g4Esbg_11002529899813\r\nFrom: <sip:XXXXX@XXXXX>;tag=as6964f29d\r\nCal"..., len=667, rcv_info=0x7fd253b7cad8) at core/receive.c:587
msg = 0x7fd2758bf0b0
ctx = {rec_lev = 0, run_flags = 0, last_retcode = 1, jmp_env = {{__jmpbuf = {0, -5133089566915471019, 0, 140542026919088, 94643042412979, 536870912, -5133089567001454251, -1505551737941960363},
__mask_was_saved = 0, __saved_mask = {__val = {94643073835016, 65535, 17240315422543681536, 18446744073709551536, 2, 0, 140542026919088, 94643042412979, 536870912, 140735803211024,
94643040635195, 140735803211696, 65535, 65536, 4095, 140735803211580}}}}}
bctx = 0x0
ret = 1
tvb = {tv_sec = 0, tv_usec = 0}
tve = {tv_sec = 0, tv_usec = 0}
diff = 0
inb = {s = 0x5613cee206f0 "SIP/2.0 486 Busy Here\r\nTo: <sip:XXXXX@XXXXX;user=phone>;tag=h7g4Esbg_11002529899813\r\nFrom: <sip:XXXXX@XXXXX>;tag=as6964f29d\r\nCal"..., len = 667}
netinfo = {data = {s = 0x5613cee206f0 "SIP/2.0 486 Busy Here\r\nTo: <sip:XXXXX@XXXXX;user=phone>;tag=h7g4Esbg_11002529899813\r\nFrom: <sip:XXXXX@XXXXX;tag=as6964f29d\r\nCal"..., len = 667}, rcv = 0x7fd253b7cad8, dst = 0x0}
keng = 0x0
evp = {data = 0x7fff9b8eba90, obuf = {s = 0x0, len = 0}, rcv = 0x7fd253b7cad8, dst = 0x0, req = 0x0, rpl = 0x0, rplcode = 0, mode = 0}
cidlockidx = 0
cidlockset = 0
errsipmsg = 0
exectime = 0
__func__ = "receive_msg"
#7 0x00005613ccd1cf15 in receive_tcp_msg (tcpbuf=0x7fd253b7ce60 "SIP/2.0 486 Busy Here\r\nTo: <sip:XXXXX@XXXXX;user=phone>;tag=h7g4Esbg_11002529899813\r\nFrom: <sip:XXXXX@XXXXX>;tag=as6964f29d\r\nCal"..., len=667, rcv_info=0x7fd253b7cad8, con=0x7fd253b7cac0) at core/tcp_read.c:1424
buf = 0x5613cee206f0 "SIP/2.0 486 Busy Here\r\nTo: <sip:XXXXX@XXXXX;user=phone>;tag=h7g4Esbg_11002529899813\r\nFrom: <sip:XXXXX@XXXXX>;tag=as6964f29d\r\nCal"...
bsize = 65535
blen = 65535
__func__ = "receive_tcp_msg"
#8 0x00005613ccd1f554 in tcp_read_req (con=0x7fd253b7cac0, bytes_read=0x7fff9b8ebf34, read_flags=0x7fff9b8ebf3c) at core/tcp_read.c:1607
bytes = 667
total_bytes = 667
resp = 1
size = 0
req = 0x7fd253b7cbe8
dst = {send_sock = 0x7fff9b8ebec0, to = {s = {sa_family = 32062, sa_data = "\303\314\023V\000\000p\276\216\233\377\177\000"}, sin = {sin_family = 32062, sin_port = 52419, sin_addr = {s_addr = 22035}, sin_zero = "p\276\216\233\377\177\000"}, sin6 = {sin6_family = 32062, sin6_port = 52419, sin6_flowinfo = 22035, sin6_addr = {__in6_u = {__u6_addr8 = "p\276\216\233\377\177\000\000\030\000\000\000\000\000\000", __u6_addr16 = {48752, 39822, 32767, 0, 24, 0, 0, 0}, __u6_addr32 = {2609823344, 32767, 24, 0}}}, sin6_scope_id = 0}, sas = {ss_family = 32062, __ss_padding = "\303\314\023V\000\000p\276\216\233\377\177\000\000\030", '\000' <repeats 15 times>, "\225q\\\037P\000\000\000\070,\214u\322\177\000\000\000\244\374\314\023V\000\000\000\000\000\000\001 \000\000\070,\214u\322\177\000\000\001\000\000\000\000\000\000\000\300ʷS\322\177\000\000\200\276\216\233\377\177\000\000\070,\214u\322\177\000\000\200\277\216\233\377\177\000\000\b\000\000\000\000\000\000", __ss_align = 20}}, id = 1, send_flags = {f = 1, blst_imask = 0}, proto = 31 '\037', proto_pad0 = 0 '\000', proto_pad1 = 0}
c = 49 '1'
ret = 0
__func__ = "tcp_read_req"
#9 0x00005613ccd23eb4 in handle_io (fm=0x7fd2758c2c38, events=1, idx=-1) at core/tcp_read.c:1857
ret = 0
n = 1404553920
read_flags = RD_CONN_SHORT_READ
con = 0x7fd253b7cac0
s = 1404554328
resp = 1
t = 526152085
ee = 0x0
__func__ = "handle_io"
#10 0x00005613ccd0dd1e in io_wait_loop_epoll (h=0x5613ccfca240 <io_w>, t=2, repeat=0) at core/io_wait.h:1070
n = 1
r = 0
fm = 0x7fd2758c2c38
revents = 1
__func__ = "io_wait_loop_epoll"
#11 0x00005613ccd25981 in tcp_receive_loop (unix_sock=39) at core/tcp_read.c:1978
__func__ = "tcp_receive_loop"
#12 0x00005613ccb8561b in tcp_init_children (woneinit=0x7fff9b8ec338) at core/tcp_main.c:5139
r = 2
i = 7
reader_fd_1 = 39
pid = 0
si_desc = "tcp receiver (generic)\000\000\205\220\274\314\023V\000\000\020Î\233\377\177\000\000\301\200\313\314\000\000\000\000AS\000\000\000\000\000\000۸\352\314\023V\000\000\246\000\216\233\377\177\000\000\034\352\340\314\023V\000\000\340\302\216\233\001\000\000\000\360\311\017Q\322\177\000\000\340\302\216\233\377\177\000\000\340\227:S\001\000\000\000\000Î\233\377\177\000\000\274\353\340\314\023V\000"
si = 0x0
__func__ = "tcp_init_children"
#13 0x00005613cca62bba in main_loop () at main.c:1857
i = 8
pid = 21313
si = 0x0
si_desc = "udp receiver child=7 sock=XXX.XXX.XXX.XXX:5060\000\000\000\003\000\000\000 \000\000\000\000\230g\240\210\347A\357\000\000\000\000\000\000\000\000\210P\347\314\023V\000\000\000\000\000\000\000\000\000\000\260,\227}\322\177\000\000\263i\354\314\023V\000\000\000\000\000 \000\000\000\000\260Ď\233\377\177\000\000\t\205\317\314\023V\000"
nrprocs = 8
woneinit = 1
__func__ = "main_loop"
#14 0x00005613cca6d4aa in main (argc=17, argv=0x7fff9b8ec9f8) at main.c:3053
cfg_stream = 0x5613cecb6290
c = -1
r = 0
tmp = 0x7fff9b8ecf3f ""
tmp_len = 2109797928
port = 32722
proto = 2109962736
ahost = 0x0
aport = 0
options = 0x5613cce77ce0 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:x:X:Y:"
ret = -1
seed = 3769828628
rfd = 4
debug_save = 0
debug_flag = 0
dont_fork_cnt = 0
n_lst = 0x0
p = 0x0
st = {st_dev = 21, st_ino = 17087, st_nlink = 2, st_mode = 16832, st_uid = 109, st_gid = 115, __pad0 = 0, st_rdev = 0, st_size = 40, st_blksize = 4096, st_blocks = 0, st_atim = {tv_sec = 1620845793, tv_nsec = 887985330}, st_mtim = {tv_sec = 1625090397, tv_nsec = 157181302}, st_ctim = {tv_sec = 1625090401, tv_nsec = 197141517}, __glibc_reserved = {0, 0, 0}}
tbuf = "\000\000\000\000\000\000\000\000\b\000\000\000\000\000\000\000\060\207\303}\322\177\000\000\000\000\000\000\000\000\000\000\240q\237\233\377\177\000\000\364\212\301}\322\177\000\000\b\000\000\000\000\000\000\000\230r\237\233\377\177\000\000\060\207\303}\322\177\000\000\230ǎ\233\377\177\000\000\224ǎ\233\377\177", '\000' <repeats 18 times>, "\230r\237\233\377\177\000\000\250q\237\233\377\177\000\000'\376\237}\322\177\000\000&\260be\000\000\000\000\000Ɏ\233\377\177\000\000\300\212\225\001\000\000\000\000`Ȏ\233\377\177\000\000PȎ\233\377\177\000\000\230ǎ\233\377\177\000\000X\207\303}\322\177\000\000\001", '\000' <repeats 31 times>...
option_index = 0
long_options = {{name = 0x5613cce79f3f "help", has_arg = 0, flag = 0x0, val = 104}, {name = 0x5613cce754e0 "version", has_arg = 0, flag = 0x0, val = 118}, {name = 0x5613cce79f44 "alias", has_arg = 1, flag = 0x0, val = 1024}, {name = 0x5613cce79f4a "subst", has_arg = 1, flag = 0x0, val = 1025}, {name = 0x5613cce79f50 "substdef", has_arg = 1, flag = 0x0, val = 1026}, {name = 0x5613cce79f59 "substdefs", has_arg = 1, flag = 0x0, val = 1027}, {name = 0x5613cce79f63 "server-id", has_arg = 1, flag = 0x0, val = 1028}, {name = 0x5613cce79f6d "loadmodule", has_arg = 1, flag = 0x0, val = 1029}, {name = 0x5613cce79f78 "modparam", has_arg = 1, flag = 0x0, val = 1030}, {name = 0x5613cce79f81 "log-engine", has_arg = 1, flag = 0x0, val = 1031}, {name = 0x5613cce79f8c "debug", has_arg = 1, flag = 0x0, val = 1032}, {name = 0x5613cce79f92 "cfg-print", has_arg = 0, flag = 0x0, val = 1033}, {name = 0x5613cce79f9c "atexit", has_arg = 1, flag = 0x0, val = 1034}, {name = 0x0, has_arg = 0, flag = 0x0, val = 0}}
__func__ = "main"
