10 Sep
2015
10 Sep
'15
10:16 a.m.
I have two kamailio servers which is in staging area, so there are few of users on it. say
2 or 3 making some testing calls.
The only difference between staging area and production area is that staging area enabled
"ASYNC PUSH WAKE", similar concept and logic introduced by Danail at Kamailio
World 2014. [Aynchronous Processing in Kamailio Configuration
File](http://www.slideshare.net/miconda/26-danielconstantinmierlakamailiocf…
These days, I experienced some crash at staging area and I can't reproduce it.
But productions which has tens of thousands of users did not have this issue.
Since I've tested the ASYNC PUSH WAKE and all it works well, I believed (guess) the
problem may caused by two factors:
1. There are some bugs hidden in PUSH WAKE algorithm that suspend a transaction by tm
module, stores it to htable, then resume when matched some conditions.
2. sip client ( compromised Linphone ) used by my client.
( Note: I've tested all official CSipSimple / Bria / Linphone and all works well
)
But totally no ideas till now.
I've turn on debug mode with:
```
debug=3
memlog=4
memdbg=4
corelog=3
```
kamailio version is:
```
root@sip1:/home/pkg/kamailio/etc/kamailio# kamcmd core.info
{
version: kamailio 4.3.1
id: f38e67
compiler: gcc 4.7.2
compiled: 13:18:05 Aug 11 2015
flags: STATS: Off, USE_TCP, USE_TLS, USE_SCTP, TLS_HOOKS, USE_RAW_SOCKS, DISABLE_NAGLE,
USE_MCAST, DNS_IP_HACK, SHM_MEM, SHM_MMAP, PKG_MALLOC, F_MALLOC, DBG_F_MALLOC, USE_FUTEX,
FAST_LOCK-ADAPTIVE_WAIT, USE_DNS_CACHE, USE_DNS_FAILOVER, USE_NAPTR, USE_DST_BLACKLIST,
HAVE_RESOLV_RES
}
```
Before I catch the crash till it happens again, below is what I can give right now:
4 back traces and syslog indicate crash address
all corefiles are kept, I will feedback if I get another clues.
#### BT 1
```
===syslog===
Sep 4 14:24:57 sip1 kernel: [41174906.447276] kamailio[50056]: segfault at 7f97b600676e
ip 00007f97b600676e sp 00007fff6e141a28 error 14 in mi_rpc.so[7f97b5f3b000+200000]
Sep 4 14:25:17 sip1 media-dispatcher[76950]: debug: Connection to OpenSIPS lost:
Connection was closed cleanly.
Sep 4 14:25:17 sip1 /home/pkg/kamailio/sbin/kamailio[49990]: ALERT: <core>
[main.c:728]: handle_sigs(): child process 50056 exited by a signal 11
Sep 4 14:25:17 sip1 /home/pkg/kamailio/sbin/kamailio[49990]: ALERT: <core>
[main.c:731]: handle_sigs(): core was generated
===backtrace===
#0 0x00007f97b600676e in ?? ()
No symbol table info available.
#1 0x00007f97b803db5e in run_trans_callbacks_internal (cb_lst=0x7f97b98e5900, type=32,
trans=0x7f97b98e5890, params=0x7fff6e141b10) at t_hooks.c:268
cbp = 0x7f97b98e7380
backup_from = 0xa82430
backup_to = 0xa82438
backup_dom_from = 0xa82440
backup_dom_to = 0xa82448
backup_uri_from = 0xa82420
backup_uri_to = 0xa82428
backup_xavps = 0xa82560
__FUNCTION__ = "run_trans_callbacks_internal"
#2 0x00007f97b803dc6f in run_trans_callbacks (type=32, trans=0x7f97b98e5890,
req=0x7f97b98b38f8, rpl=0x7f98fb64e2b0, code=180) at t_hooks.c:295
params = {req = 0x7f97b98b38f8, rpl = 0x7f98fb64e2b0, param = 0x7f97b98e7390, code
= 180, flags = 0, branch = 0, t_rbuf = 0x0, dst = 0x0, send_buf = {s = 0x0, len = 0}}
#3 0x00007f97b809382e in relay_reply (t=0x7f97b98e5890, p_msg=0x7f98fb64e2b0, branch=1,
msg_status=180, cancel_data=0x7fff6e141e70, do_put_on_wait=1) at t_reply.c:1767
relay = 1
save_clone = 0
buf = 0x0
res_len = 0
relayed_code = 0
relayed_msg = 0x0
reply_bak = 0x7fff05051cb0
bm = {to_tag_val = {s = 0xffffffff01b46f02 <Address 0xffffffff01b46f02 out of
bounds>, len = -77247288}}
totag_retr = 0
reply_status = RPS_PROVISIONAL
uas_rb = 0x7f97b98e5950
to_tag = 0x7f97b80837bf
reason = {s = 0x7fff6e141c60 "\200\034\024n\005", len = -1207424906}
onsend_params = {req = 0x7fff6e141c80, rpl = 0x7f97b804db23, param =
0x7fff6e141c10, code = 1, flags = 0, branch = 0, t_rbuf = 0x7f97b98e5a50, dst =
0x50075b821, send_buf = {
s = 0xfab1b348 <Address 0xfab1b348 out of bounds>, len = 1024}}
__FUNCTION__ = "relay_reply"
#4 0x00007f97b8097959 in reply_received (p_msg=0x7f98fb64e2b0) at t_reply.c:2429
msg_status = 180
last_uac_status = 100
ack = 0x7fff6e141f60 "` \024n\377\177"
ack_len = 32664
branch = 1
reply_status = 6440375
onreply_route = 3
cancel_data = {cancel_bitmap = 0, reason = {cause = 0, u = {text = {s = 0x0, len =
6682541}, e2e_cancel = 0x0, packed_hdrs = {s = 0x0, len = 6682541}}}}
uac = 0x7f97b98e5c48
t = 0x7f97b98e5890
lack_dst = {send_sock = 0x7fff6e141e70, to = {s = {sa_family = 17847, sa_data =
"b\000\000\000\000\000\200\036\024n\377\177\000"}, sin = {sin_family = 17847,
sin_port = 98, sin_addr = {
s_addr = 0}, sin_zero = "\200\036\024n\377\177\000"}, sin6 =
{sin6_family = 17847, sin6_port = 98, sin6_flowinfo = 0, sin6_addr = {__in6_u = {
__u6_addr8 =
"\200\036\024n\377\177\000\000\020\260\261\372\230\177\000", __u6_addr16 =
{7808, 28180, 32767, 0, 45072, 64177, 32664, 0}, __u6_addr32 = {1846812288, 32767,
4205948944,
32664}}}, sin6_scope_id = 778}}, id = 10, proto = 84 'T',
send_flags = {f = 109 'm', blst_imask = 180 '\264'}}
backup_user_from = 0xa82430
backup_user_to = 0xa82438
backup_domain_from = 0xa82440
backup_domain_to = 0xa82448
backup_uri_from = 0xa82420
backup_uri_to = 0xa82428
backup_xavps = 0xa82560
replies_locked = 1
branch_ret = 1846812496
prev_branch = 32664
blst_503_timeout = -77274448
hf = 0xa96c10
onsend_params = {req = 0x4004188b0, rpl = 0x75fc90, param = 0x0, code = 0, flags =
3, branch = 0, t_rbuf = 0x7fff6e141ee0, dst = 0x631363, send_buf = {s = 0x7fff6e141ee0
"`\037\024n\377\177",
len = 6422143}}
ctx = {rec_lev = 0, run_flags = 0, last_retcode = 1, jmp_env = {{__jmpbuf = {797,
-5029273663833343261, 4294832, 140735040203504, 0, 0, -5029273663808177437,
5029028274090965731},
__mask_was_saved = 0, __saved_mask = {__val = {6422143, 140735040200416,
6475430, 140735040200128, 140295017705488, 140735040200192, 140289551658488,
140735040200160, 0, 1073741824,
1401221318984, 140295029470864, 7733692, 64, 28601913, 174}}}}}
__FUNCTION__ = "reply_received"
#5 0x00000000004901cc in do_forward_reply (msg=0x7f98fb64e2b0, mode=0) at forward.c:747
new_buf = 0x0
dst = {send_sock = 0x0, to = {s = {sa_family = 0, sa_data = '\000'
<repeats 13 times>}, sin = {sin_family = 0, sin_port = 0, sin_addr = {s_addr = 0},
sin_zero = "\000\000\000\000\000\000\000"},
sin6 = {sin6_family = 0, sin6_port = 0, sin6_flowinfo = 0, sin6_addr =
{__in6_u = {__u6_addr8 = '\000' <repeats 15 times>, __u6_addr16 = {0, 0, 0,
0, 0, 0, 0, 0}, __u6_addr32 = {0, 0, 0,
0}}}, sin6_scope_id = 0}}, id = 0, proto = 0 '\000',
send_flags = {f = 0 '\000', blst_imask = 0 '\000'}}
new_len = 0
r = 2
ip = {af = 0, len = 0, u = {addrl = {21482551329, 140295017709352}, addr32 =
{7714849, 5, 4205952808, 32664}, addr16 = {47137, 117, 5, 0, 48936, 64177, 32664, 0},
addr = "!\270u\000\005\000\000\000(\277\261\372\230\177\000"}}
s = 0x7f98fb64e2b8 "99\351U"
len = 32663
__FUNCTION__ = "do_forward_reply"
#6 0x0000000000491856 in forward_reply (msg=0x7f98fb64e2b0) at forward.c:849
No locals.
#7 0x000000000050e062 in receive_msg (
buf=0x1b46d40 "SIP/2.0 180 Ringing\r\nVia: SIP/2.0/TCP
211.78.19.1:5003;branch=z9hG4bK60b8.17e8d16cb9ef6d39be60d4cc68762171.1;i=9e02;received=211.78.19.1;rport=57074\r\nVia:
SIP/2.0/TCP 192.168.1.112:64044;received=218."..., len=797, rcv_info=0x7f97b98f1610)
at receive.c:255
msg = 0x7f98fb64e2b0
ctx = {rec_lev = -1181805064, run_flags = 32663, last_retcode = 16383, jmp_env =
{{__jmpbuf = {17179869186, 140289629072096, 4294832, 3424935750384, 0, 0, 4294832,
140735040203504},
__mask_was_saved = 1846813104, __saved_mask = {__val = {6234849, 28601648,
140296091965321, 0, 140296091950726, 140735040201912, 140289629951480, 65552, 797,
42949672963, 140289629953026,
0, 140289629953029, 140735040201360, 6235333, 4294832}}}}}
ret = 0
inb = {
s = 0x1b46d40 "SIP/2.0 180 Ringing\r\nVia: SIP/2.0/TCP
211.78.19.1:5003;branch=z9hG4bK60b8.17e8d16cb9ef6d39be60d4cc68762171.1;i=9e02;received=211.78.19.1;rport=57074\r\nVia:
SIP/2.0/TCP 192.168.1.112:64044;received=218."..., len = 797}
__FUNCTION__ = "receive_msg"
#8 0x00000000005f65fc in receive_tcp_msg (
tcpbuf=0x7f97b98f18e8 "SIP/2.0 180 Ringing\r\nVia: SIP/2.0/TCP
211.78.19.1:5003;branch=z9hG4bK60b8.17e8d16cb9ef6d39be60d4cc68762171.1;i=9e02;received=211.78.19.1;rport=57074\r\nVia:
SIP/2.0/TCP 192.168.1.112:64044;received=218."..., len=797, rcv_info=0x7f97b98f1610,
con=0x7f97b98f15f8) at tcp_read.c:1247
buf = 0x1b46d40 "SIP/2.0 180 Ringing\r\nVia: SIP/2.0/TCP
211.78.19.1:5003;branch=z9hG4bK60b8.17e8d16cb9ef6d39be60d4cc68762171.1;i=9e02;received=211.78.19.1;rport=57074\r\nVia:
SIP/2.0/TCP 192.168.1.112:64044;received=218."...
bsize = 65535
blen = 65535
__FUNCTION__ = "receive_tcp_msg"
#9 0x00000000005f7d9b in tcp_read_req (con=0x7f97b98f15f8, bytes_read=0x7fff6e1424c0,
read_flags=0x7fff6e1424b8) at tcp_read.c:1401
bytes = 797
total_bytes = 797
resp = 1
size = 140735040201552
req = 0x7f97b98f1678
dst = {send_sock = 0x14, to = {s = {sa_family = 1, sa_data =
"\000\000P\000\000\000\001 \000\200\370\320\366", <incomplete sequence
\372>}, sin = {sin_family = 1, sin_port = 0, sin_addr = {
s_addr = 80}, sin_zero = "\001 \000\200\370\320\366",
<incomplete sequence \372>}, sin6 = {sin6_family = 1, sin6_port = 0, sin6_flowinfo =
80, sin6_addr = {__in6_u = {
__u6_addr8 = "\001
\000\200\370\320\366\372\230\177\000\000\000\000\000", __u6_addr16 = {8193, 32768,
53496, 64246, 32664, 0, 0, 0}, __u6_addr32 = {2147491841, 4210479352, 32664, 0}}},
sin6_scope_id = 1846813872}}, id = 32767, proto = 8 '\b', send_flags
= {f = 0 '\000', blst_imask = 0 '\000'}}
c = 0 '\000'
ret = -1
__FUNCTION__ = "tcp_read_req"
#10 0x00000000005fadab in handle_io (fm=0x7f98faf6d0f8, events=1, idx=-1) at
tcp_read.c:1624
ret = 0
n = -1
read_flags = 1
con = 0x7f97b98f15f8
s = 11
resp = 1
t = 381376958
__FUNCTION__ = "handle_io"
#11 0x00000000005ee8c0 in io_wait_loop_epoll (h=0xa2dfe0, t=2, repeat=1) at
io_wait.h:1061
n = 1
r = 0
fm = 0x7f98faf6d0f8
revents = 1
__FUNCTION__ = "io_wait_loop_epoll"
#12 0x00000000005fbe1a in tcp_receive_loop (unix_sock=97) at tcp_read.c:1739
__FUNCTION__ = "tcp_receive_loop"
#13 0x00000000005e54e1 in tcp_init_children () at tcp_main.c:4787
r = 15
i = 23
reader_fd_1 = 97
pid = 0
si_desc = "tcp receiver
(generic)\000\000ygY\000\000\000\000\000P'\024n\377\177\000\000R\342B\000\000\000\000\000\060'\024n\377\177\000\000\260ò:\231\177\000\000\220\023\364\372\230\177\000\000_\301\211T\000\000\000\000\217sp\000\000\000\000\000\020\021\265\372\230\177\000\000(\000\000\000)\000\000\000UZ-;a\303\000\000m\303\000\000Tb\214K_\301\211T\000\000\000"
si = 0x0
__FUNCTION__ = "tcp_init_children"
#14 0x00000000004ab258 in main_loop () at main.c:1658
i = 24
pid = 50017
si = 0x0
si_desc = "udp receiver child=23
sock=211.78.19.1:5003\000n\000\000\000~/q\000\001\000\000\000\260䕸\227\177\000\000\060(\024n\377\177\000\000\032\264N\000\000\000\000\000\200(\024n\377\177\000\000\260\000\364\372\230\177\000\000\350\327⸗\177\000\000\260䕸\227\177\000\000_\301\211T\000\000\000\000\260\210A\000\001\000\000"
nrprocs = 24
__FUNCTION__ = "main_loop"
#15 0x00000000004b0b5b in main (argc=13, argv=0x7fff6e142af8) at main.c:2533
cfg_stream = 0x1a9a010
c = -1
r = 0
tmp = 0x7fff6e144d59 ""
tmp_len = 32665
port = 994981744
proto = 0
options = 0x707db8
":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
ret = -1
seed = 1686944683
rfd = 4
debug_save = 0
debug_flag = 0
dont_fork_cnt = 0
n_lst = 0x76
p = 0x7fff6e1429fe ""
__FUNCTION__ = "main"
```
#### BT 2
```
===syslog===
Sep 8 09:59:08 sip2 kernel: [58418949.036449] kamailio[145444]: segfault at 7fa6a900676e
ip 00007fa6a900676e sp 00007fff6fed90a8 error 14 in textops.so[7fa6a8ec6000+200000]
Sep 8 09:59:38 sip2 media-dispatcher[98511]: debug: Connection to OpenSIPS lost:
Connection was closed cleanly.
Sep 8 09:59:38 sip2 /home/pkg/kamailio/sbin/kamailio[145378]: ALERT: <core>
[main.c:728]: handle_sigs(): child process 145444 exited by a signal 11
Sep 8 09:59:38 sip2 /home/pkg/kamailio/sbin/kamailio[145378]: ALERT: <core>
[main.c:731]: handle_sigs(): core was generated
===backtrace===
#0 0x00007fa6a900676e in ?? ()
No symbol table info available.
#1 0x00007fa6ab3d9b5e in run_trans_callbacks_internal (cb_lst=0x7fa6acd75a88, type=32,
trans=0x7fa6acd75a18, params=0x7fff6fed9190) at t_hooks.c:268
cbp = 0x7fa6acd680f0
backup_from = 0xa82430
backup_to = 0xa82438
backup_dom_from = 0xa82440
backup_dom_to = 0xa82448
backup_uri_from = 0xa82420
backup_uri_to = 0xa82428
backup_xavps = 0xa82560
__FUNCTION__ = "run_trans_callbacks_internal"
#2 0x00007fa6ab3d9c6f in run_trans_callbacks (type=32, trans=0x7fa6acd75a18,
req=0x7fa6acd6c828, rpl=0x7fa7ee9ea2b0, code=180) at t_hooks.c:295
params = {req = 0x7fa6acd6c828, rpl = 0x7fa7ee9ea2b0, param = 0x7fa6acd68100, code
= 180, flags = 0, branch = 0, t_rbuf = 0x0, dst = 0x0, send_buf = {s = 0x0, len = 0}}
#3 0x00007fa6ab42f82e in relay_reply (t=0x7fa6acd75a18, p_msg=0x7fa7ee9ea2b0, branch=1,
msg_status=180, cancel_data=0x7fff6fed94f0, do_put_on_wait=1) at t_reply.c:1767
relay = 1
save_clone = 0
buf = 0x0
res_len = 0
relayed_code = 0
relayed_msg = 0x0
reply_bak = 0x7fff05059330
bm = {to_tag_val = {s = 0xffffffff00c75ead <Address 0xffffffff00c75ead out of
bounds>, len = -291574016}}
totag_retr = 0
reply_status = RPS_PROVISIONAL
uas_rb = 0x7fa6acd75ad8
to_tag = 0x7fa6ab41f7bf
reason = {s = 0x7fff6fed92e0 "", len = -1421744010}
onsend_params = {req = 0x7fff6fed9300, rpl = 0x7fa6ab3e9b23, param =
0x7fff6fed9290, code = 1, flags = 0, branch = 0, t_rbuf = 0x7fa6acd75bd8, dst =
0x50075b821, send_buf = {
s = 0xedeb7328 <Address 0xedeb7328 out of bounds>, len = 1024}}
__FUNCTION__ = "relay_reply"
#4 0x00007fa6ab433959 in reply_received (p_msg=0x7fa7ee9ea2b0) at t_reply.c:2429
msg_status = 180
last_uac_status = 100
ack = 0x7fff6fed95e0 "\340\226\355o\377\177"
ack_len = 32679
branch = 1
reply_status = 6440375
onreply_route = 3
cancel_data = {cancel_bitmap = 0, reason = {cause = 0, u = {text = {s = 0x0, len =
6682541}, e2e_cancel = 0x0, packed_hdrs = {s = 0x0, len = 6682541}}}}
uac = 0x7fa6acd75dd0
t = 0x7fa6acd75a18
lack_dst = {send_sock = 0x7fff6fed94f0, to = {s = {sa_family = 17847, sa_data =
"b\000\000\000\000\000\000\225\355o\377\177\000"}, sin = {sin_family = 17847,
sin_port = 98, sin_addr = {
s_addr = 0}, sin_zero = "\000\225\355o\377\177\000"}, sin6 =
{sin6_family = 17847, sin6_port = 98, sin6_flowinfo = 0, sin6_addr = {__in6_u = {
__u6_addr8 =
"\000\225\355o\377\177\000\000\020p\353\355\247\177\000", __u6_addr16 = {38144,
28653, 32767, 0, 28688, 60907, 32679, 0}, __u6_addr32 = {1877841152, 32767, 3991629840,
32679}}}, sin6_scope_id = 778}}, id = 10, proto = 84 'T',
send_flags = {f = 93 ']', blst_imask = 199 '\307'}}
backup_user_from = 0xa82430
backup_user_to = 0xa82438
backup_domain_from = 0xa82440
backup_domain_to = 0xa82448
backup_uri_from = 0xa82420
backup_uri_to = 0xa82428
backup_xavps = 0xa82560
replies_locked = 1
branch_ret = 1877841360
prev_branch = 32679
blst_503_timeout = -291593552
hf = 0xa96580
onsend_params = {req = 0x4004188b0, rpl = 0x75fc90, param = 0x0, code = 0, flags =
3, branch = 0, t_rbuf = 0x7fff6fed9560, dst = 0x631363, send_buf = {s = 0x7fff6fed9560
"\340\225\355o\377\177",
len = 6422143}}
ctx = {rec_lev = 0, run_flags = 0, last_retcode = 1, jmp_env = {{__jmpbuf = {797,
-4883994847374486000, 4294832, 140735071232368, 0, 0, -4883994847399651824,
4883678595199653392},
__mask_was_saved = 0, __saved_mask = {__val = {6422143, 140735071229280,
6475430, 140735071228992, 140359227895824, 140735071229056, 140353761848824,
140735071229024, 0, 1073741824,
1401221321056, 140359239663288, 7733692, 64, 13065785, 175}}}}}
__FUNCTION__ = "reply_received"
#5 0x00000000004901cc in do_forward_reply (msg=0x7fa7ee9ea2b0, mode=0) at forward.c:747
new_buf = 0x0
dst = {send_sock = 0x0, to = {s = {sa_family = 0, sa_data = '\000'
<repeats 13 times>}, sin = {sin_family = 0, sin_port = 0, sin_addr = {s_addr = 0},
sin_zero = "\000\000\000\000\000\000\000"},
sin6 = {sin6_family = 0, sin6_port = 0, sin6_flowinfo = 0, sin6_addr =
{__in6_u = {__u6_addr8 = '\000' <repeats 15 times>, __u6_addr16 = {0, 0, 0,
0, 0, 0, 0, 0}, __u6_addr32 = {0, 0, 0,
0}}}, sin6_scope_id = 0}}, id = 0, proto = 0 '\000',
send_flags = {f = 0 '\000', blst_imask = 0 '\000'}}
new_len = 0
r = 2
ip = {af = 0, len = 0, u = {addrl = {21482551329, 140359227899688}, addr32 =
{7714849, 5, 3991633704, 32679}, addr16 = {47137, 117, 5, 0, 32552, 60907, 32679, 0},
addr = "!\270u\000\005\000\000\000(\177\353\355\247\177\000"}}
s = 0x7fa7ee9ea2b8 "\354@\356U"
len = 32678
__FUNCTION__ = "do_forward_reply"
#6 0x0000000000491856 in forward_reply (msg=0x7fa7ee9ea2b0) at forward.c:849
No locals.
#7 0x000000000050e062 in receive_msg (
buf=0xc75d40 "SIP/2.0 180 Ringing\r\nVia: SIP/2.0/TCP
211.78.19.2:5003;branch=z9hG4bKd59d.0efeaa2488813a7c1ff6cbb4fa19f351.1;i=6592;received=211.78.19.2;rport=22430\r\nVia:
SIP/2.0/TCP 192.168.1.112:58885;received=218."..., len=797, rcv_info=0x7fa6acd7b088)
at receive.c:255
msg = 0x7fa7ee9ea2b0
ctx = {rec_lev = -1395150736, run_flags = 32678, last_retcode = 16383, jmp_env =
{{__jmpbuf = {17179869186, 140353840147104, 4294832, 3424966779248, 0, 0, 4294832,
140735071232368},
__mask_was_saved = 1877841968, __saved_mask = {__val = {6234849, 13065520,
140360302155657, 0, 140360302141062, 140735071230776, 140353841115248, 65552, 797,
42949672963, 140353841116794,
0, 140353841116797, 140735071230224, 6235333, 4294832}}}}}
ret = 0
inb = {
s = 0xc75d40 "SIP/2.0 180 Ringing\r\nVia: SIP/2.0/TCP
211.78.19.2:5003;branch=z9hG4bKd59d.0efeaa2488813a7c1ff6cbb4fa19f351.1;i=6592;received=211.78.19.2;rport=22430\r\nVia:
SIP/2.0/TCP 192.168.1.112:58885;received=218."..., len = 797}
__FUNCTION__ = "receive_msg"
#8 0x00000000005f65fc in receive_tcp_msg (
tcpbuf=0x7fa6acd7b360 "SIP/2.0 180 Ringing\r\nVia: SIP/2.0/TCP
211.78.19.2:5003;branch=z9hG4bKd59d.0efeaa2488813a7c1ff6cbb4fa19f351.1;i=6592;received=211.78.19.2;rport=22430\r\nVia:
SIP/2.0/TCP 192.168.1.112:58885;received=218."..., len=797, rcv_info=0x7fa6acd7b088,
con=0x7fa6acd7b070) at tcp_read.c:1247
buf = 0xc75d40 "SIP/2.0 180 Ringing\r\nVia: SIP/2.0/TCP
211.78.19.2:5003;branch=z9hG4bKd59d.0efeaa2488813a7c1ff6cbb4fa19f351.1;i=6592;received=211.78.19.2;rport=22430\r\nVia:
SIP/2.0/TCP 192.168.1.112:58885;received=218."...
bsize = 65535
blen = 65535
__FUNCTION__ = "receive_tcp_msg"
#9 0x00000000005f7d9b in tcp_read_req (con=0x7fa6acd7b070, bytes_read=0x7fff6fed9b40,
read_flags=0x7fff6fed9b38) at tcp_read.c:1401
bytes = 797
total_bytes = 797
resp = 1
size = 140735071230416
req = 0x7fa6acd7b0f0
dst = {send_sock = 0x14, to = {s = {sa_family = 1, sa_data =
"\000\000P\000\000\000\001 \000\200\370\220", <incomplete sequence \356>},
sin = {sin_family = 1, sin_port = 0, sin_addr = {
s_addr = 80}, sin_zero = "\001 \000\200\370\220", <incomplete
sequence \356>}, sin6 = {sin6_family = 1, sin6_port = 0, sin6_flowinfo = 80, sin6_addr
= {__in6_u = {
__u6_addr8 = "\001
\000\200\370\220\060\356\247\177\000\000\000\000\000", __u6_addr16 = {8193, 32768,
37112, 60976, 32679, 0, 0, 0}, __u6_addr32 = {2147491841, 3996160248, 32679, 0}}},
sin6_scope_id = 1877842736}}, id = 32767, proto = 8 '\b', send_flags
= {f = 0 '\000', blst_imask = 0 '\000'}}
c = 0 '\000'
ret = -1
__FUNCTION__ = "tcp_read_req"
#10 0x00000000005fadab in handle_io (fm=0x7fa7ee3090f8, events=1, idx=-1) at
tcp_read.c:1624
ret = 0
n = -1
read_flags = 1
con = 0x7fa6acd7b070
s = 11
resp = 1
t = 1530467254
__FUNCTION__ = "handle_io"
#11 0x00000000005ee8c0 in io_wait_loop_epoll (h=0xa2dfe0, t=2, repeat=1) at
io_wait.h:1061
n = 1
r = 0
fm = 0x7fa7ee3090f8
revents = 1
__FUNCTION__ = "io_wait_loop_epoll"
#12 0x00000000005fbe1a in tcp_receive_loop (unix_sock=97) at tcp_read.c:1739
__FUNCTION__ = "tcp_receive_loop"
#13 0x00000000005e54e1 in tcp_init_children () at tcp_main.c:4787
r = 15
i = 23
reader_fd_1 = 97
pid = 0
si_desc = "tcp receiver
(generic)\000\000ygY\000\000\000\000\000Н\355o\377\177\000\000R\342B\000\000\000\000\000\260\235\355o\377\177\000\000\260\203\354-\250\177\000\000\220\323-\356\247\177\000\000\023\255\r\020\000\000\000\000\217sp\000\000\000\000\000\020\321\356\355\247\177\000\000(\000\000\000)\000\000\000U\032g.\375\067\002\000\t8\002\000\267̑`\023\255\r\020\000\000\000"
si = 0x0
__FUNCTION__ = "tcp_init_children"
#14 0x00000000004ab258 in main_loop () at main.c:1658
i = 24
pid = 145405
si = 0x0
si_desc = "udp receiver child=23
sock=211.78.19.2:5003\000n\000\000\000~/q\000\001\000\000\000\260\244ϫ\246\177\000\000\260\236\355o\377\177\000\000\032\264N\000\000\000\000\000\000\237\355o\377\177\000\000\260\300-\356\247\177\000\000\350\227\034\254\246\177\000\000\260\244ϫ\246\177\000\000\023\255\r\020\000\000\000\000\260\210A\000\001\000\000"
nrprocs = 24
__FUNCTION__ = "main_loop"
#15 0x00000000004b0b5b in main (argc=13, argv=0x7fff6feda178) at main.c:2533
cfg_stream = 0xbc9010
c = -1
r = 0
tmp = 0x7fff6fedad59 ""
tmp_len = 32680
port = 780662640
proto = 0
options = 0x707db8
":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
ret = -1
seed = 503368684
rfd = 4
debug_save = 0
debug_flag = 0
dont_fork_cnt = 0
n_lst = 0x76
p = 0x7fff6feda07e ""
__FUNCTION__ = "main"
```
#### BT 3
it generates two core files 3 and 4 in 20 secs
```
===syslog===
Sep 10 14:39:29 sip1 kernel: [41693172.748753] kamailio[245950] general protection
ip:602d43 sp:7fff185e8500 error:0 in kamailio[400000+3cb000]
Sep 10 14:39:47 sip1 /home/pkg/kamailio/sbin/kamailio[245999]: CRITICAL: <core>
[pass_fd.c:275]: receive_fd(): EOF on 39
Sep 10 14:39:47 sip1 /home/pkg/kamailio/sbin/kamailio[245924]: ALERT: <core>
[main.c:728]: handle_sigs(): child process 245950 exited by a signal 11
Sep 10 14:39:47 sip1 /home/pkg/kamailio/sbin/kamailio[245924]: ALERT: <core>
[main.c:731]: handle_sigs(): core was generated
Sep 10 14:39:47 sip1 kernel: [41693191.570265] kamailio[245924] general protection
ip:7f1f2ba23800 sp:7fff185e7ff0 error:0 in tm.so[7f1f2b9c8000+114000]
===backtrace===
#0 0x0000000000602d43 in slow_timer_main () at timer.c:1130
n = 12
ret = 4294967295
tl = 0x7f1f2cca64f0
i = 957
__FUNCTION__ = "slow_timer_main"
#1 0x00000000004aacf9 in main_loop () at main.c:1628
i = 24
pid = 0
si = 0x0
si_desc = "udp receiver child=23
sock=211.78.19.1:5003\000n\000\000\000~/q\000\001\000\000\000\260D4,\037\177\000\000\060\206^\030\377\177\000\000\032\264N\000\000\000\000\000\200\206^\030\377\177\000\000\260`\222n
\177\000\000\350\067\201,\037\177\000\000\260D4,\037\177\000\000\224\321\306[\000\000\000\000\260\210A\000\001\000\000"
nrprocs = 24
__FUNCTION__ = "main_loop"
#2 0x00000000004b0b5b in main (argc=13, argv=0x7fff185e88f8) at main.c:2533
cfg_stream = 0x27a6010
c = -1
r = 0
tmp = 0x7fff185e8f32 ""
tmp_len = 32544
port = -1360226448
proto = 0
options = 0x707db8
":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
ret = -1
seed = 1739382561
rfd = 4
debug_save = 0
debug_flag = 0
dont_fork_cnt = 0
n_lst = 0x76
p = 0x7fff185e87fe ""
__FUNCTION__ = "main"
```
#### BT 4
it generates two core files 3 and 4 in 20 secs
```
===syslog===
Sep 10 14:39:29 sip1 kernel: [41693172.748753] kamailio[245950] general protection
ip:602d43 sp:7fff185e8500 error:0 in kamailio[400000+3cb000]
Sep 10 14:39:47 sip1 /home/pkg/kamailio/sbin/kamailio[245999]: CRITICAL: <core>
[pass_fd.c:275]: receive_fd(): EOF on 39
Sep 10 14:39:47 sip1 /home/pkg/kamailio/sbin/kamailio[245924]: ALERT: <core>
[main.c:728]: handle_sigs(): child process 245950 exited by a signal 11
Sep 10 14:39:47 sip1 /home/pkg/kamailio/sbin/kamailio[245924]: ALERT: <core>
[main.c:731]: handle_sigs(): core was generated
Sep 10 14:39:47 sip1 kernel: [41693191.570265] kamailio[245924] general protection
ip:7f1f2ba23800 sp:7fff185e7ff0 error:0 in tm.so[7f1f2b9c8000+114000]
===backtrace===
#0 0x00007f1f2ba23800 in run_trans_callbacks_internal (cb_lst=0x7f1f2c9ba700,
type=131072, trans=0x7f1f2c9ba690, params=0x7fff185e80d0) at t_hooks.c:264
cbp = 0x3c223d65636e6174
backup_from = 0xa82430
backup_to = 0xa82438
backup_dom_from = 0xa82440
backup_dom_to = 0xa82448
backup_uri_from = 0xa82420
backup_uri_to = 0xa82428
backup_xavps = 0xa82560
__FUNCTION__ = "run_trans_callbacks_internal"
#1 0x00007f1f2ba23c6f in run_trans_callbacks (type=131072, trans=0x7f1f2c9ba690, req=0x0,
rpl=0x0, code=0) at t_hooks.c:295
params = {req = 0x0, rpl = 0x0, param = 0x0, code = 0, flags = 0, branch = 0,
t_rbuf = 0x0, dst = 0x0, send_buf = {s = 0x0, len = 0}}
#2 0x00007f1f2b9e2aa1 in free_cell (dead_cell=0x7f1f2c9ba690) at h_table.c:128
b = 0x621325 "\353\a\220\353\004\220\353\001\220H\201Đ"
i = 0
rpl = 0x7fff185e8270
tt = 0x7fff185e8290
foo = 0x7f1f2c33c1c8
cbs = 0x621325
cbs_tmp = 0x7fff185e8280
__FUNCTION__ = "free_cell"
#3 0x00007f1f2b9e4779 in free_hash_table () at h_table.c:432
p_cell = 0x7f1f2c9ba690
tmp_cell = 0x3a2772657375277b
i = 53971
__FUNCTION__ = "free_hash_table"
#4 0x00007f1f2ba09733 in tm_shutdown () at t_funcs.c:90
__FUNCTION__ = "tm_shutdown"
#5 0x000000000059669b in destroy_modules () at sr_module.c:788
t = 0x7f206e524660
foo = 0x7f206e5241f8
__FUNCTION__ = "destroy_modules"
#6 0x00000000004a011a in cleanup (show_status=1) at main.c:513
memlog = 32544
__FUNCTION__ = "cleanup"
#7 0x00000000004a170e in shutdown_children (sig=15, show_status=1) at main.c:655
__FUNCTION__ = "shutdown_children"
#8 0x00000000004a3c8d in handle_sigs () at main.c:746
chld = 0
chld_status = 139
memlog = 0
__FUNCTION__ = "handle_sigs"
#9 0x00000000004abb64 in main_loop () at main.c:1701
i = 24
pid = 245999
si = 0x0
si_desc = "udp receiver child=23
sock=211.78.19.1:5003\000n\000\000\000~/q\000\001\000\000\000\260D4,\037\177\000\000\060\206^\030\377\177\000\000\032\264N\000\000\000\000\000\200\206^\030\377\177\000\000\260`\222n
\177\000\000\350\067\201,\037\177\000\000\260D4,\037\177\000\000\224\321\306[\000\000\000\000\260\210A\000\001\000\000"
nrprocs = 24
__FUNCTION__ = "main_loop"
#10 0x00000000004b0b5b in main (argc=13, argv=0x7fff185e88f8) at main.c:2533
cfg_stream = 0x27a6010
c = -1
r = 0
tmp = 0x7fff185e8f32 ""
tmp_len = 32544
port = -1360226448
proto = 0
options = 0x707db8
":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
ret = -1
seed = 1739382561
rfd = 4
debug_save = 0
debug_flag = 0
dont_fork_cnt = 0
n_lst = 0x76
p = 0x7fff185e87fe ""
__FUNCTION__ = "main"
```
---
Reply to this email directly or view it on GitHub:
https://github.com/kamailio/kamailio/issues/322