New subject: [kamailio] kamailio 4.3 crash at run_trans_callbacks_internal (#322)

10 Sep 2015


      I have two kamailio servers which is in staging area, so there are few of users on it. say 2 or 3 making some testing calls.
The only difference between staging area and production area is that staging area enabled "ASYNC PUSH WAKE", similar concept and logic introduced by Danail at Kamailio World 2014. [Aynchronous Processing in Kamailio Configuration File](http://www.slideshare.net/miconda/26-danielconstantinmierlakamailiocfgasync)
These days, I experienced some crash at staging area and I can't reproduce it.
But productions which has tens of thousands of users did not have this issue.
Since I've tested the ASYNC PUSH WAKE and all it works well, I believed (guess) the problem may caused by two factors:
1. There are some bugs hidden in PUSH WAKE algorithm that suspend a transaction by tm module, stores it to htable, then resume when matched some conditions.
2. sip client ( compromised Linphone ) used by my client.
    ( Note: I've tested all official CSipSimple / Bria / Linphone and all works well )
But totally no ideas till now.
I've turn on debug mode with:
```
debug=3
memlog=4
memdbg=4
corelog=3
```
kamailio version is:
```
root@sip1:/home/pkg/kamailio/etc/kamailio# kamcmd core.info
{
    version: kamailio 4.3.1
    id: f38e67 
    compiler: gcc 4.7.2
    compiled: 13:18:05 Aug 11 2015
    flags: STATS: Off, USE_TCP, USE_TLS, USE_SCTP, TLS_HOOKS, USE_RAW_SOCKS, DISABLE_NAGLE, USE_MCAST, DNS_IP_HACK, SHM_MEM, SHM_MMAP, PKG_MALLOC, F_MALLOC, DBG_F_MALLOC, USE_FUTEX, FAST_LOCK-ADAPTIVE_WAIT, USE_DNS_CACHE, USE_DNS_FAILOVER, USE_NAPTR, USE_DST_BLACKLIST, HAVE_RESOLV_RES
}
```
Before I catch the crash till it happens again, below is what I can give right now:
4 back traces and syslog indicate crash address
all corefiles are kept, I will feedback if I get another clues.
#### BT 1
```
===syslog===
Sep  4 14:24:57 sip1 kernel: [41174906.447276] kamailio[50056]: segfault at 7f97b600676e ip 00007f97b600676e sp 00007fff6e141a28 error 14 in mi_rpc.so[7f97b5f3b000+200000]
Sep  4 14:25:17 sip1 media-dispatcher[76950]: debug: Connection to OpenSIPS lost: Connection was closed cleanly.
Sep  4 14:25:17 sip1 /home/pkg/kamailio/sbin/kamailio[49990]: ALERT: <core> [main.c:728]: handle_sigs(): child process 50056 exited by a signal 11
Sep  4 14:25:17 sip1 /home/pkg/kamailio/sbin/kamailio[49990]: ALERT: <core> [main.c:731]: handle_sigs(): core was generated
===backtrace===
#0  0x00007f97b600676e in ?? ()
No symbol table info available.
#1  0x00007f97b803db5e in run_trans_callbacks_internal (cb_lst=0x7f97b98e5900, type=32, trans=0x7f97b98e5890, params=0x7fff6e141b10) at t_hooks.c:268
        cbp = 0x7f97b98e7380
        backup_from = 0xa82430
        backup_to = 0xa82438
        backup_dom_from = 0xa82440
        backup_dom_to = 0xa82448
        backup_uri_from = 0xa82420
        backup_uri_to = 0xa82428
        backup_xavps = 0xa82560
        __FUNCTION__ = "run_trans_callbacks_internal"
#2  0x00007f97b803dc6f in run_trans_callbacks (type=32, trans=0x7f97b98e5890, req=0x7f97b98b38f8, rpl=0x7f98fb64e2b0, code=180) at t_hooks.c:295
        params = {req = 0x7f97b98b38f8, rpl = 0x7f98fb64e2b0, param = 0x7f97b98e7390, code = 180, flags = 0, branch = 0, t_rbuf = 0x0, dst = 0x0, send_buf = {s = 0x0, len = 0}}
#3  0x00007f97b809382e in relay_reply (t=0x7f97b98e5890, p_msg=0x7f98fb64e2b0, branch=1, msg_status=180, cancel_data=0x7fff6e141e70, do_put_on_wait=1) at t_reply.c:1767
        relay = 1
        save_clone = 0
        buf = 0x0
        res_len = 0
        relayed_code = 0
        relayed_msg = 0x0
        reply_bak = 0x7fff05051cb0
        bm = {to_tag_val = {s = 0xffffffff01b46f02 <Address 0xffffffff01b46f02 out of bounds>, len = -77247288}}
        totag_retr = 0
        reply_status = RPS_PROVISIONAL
        uas_rb = 0x7f97b98e5950
        to_tag = 0x7f97b80837bf
        reason = {s = 0x7fff6e141c60 "\200\034\024n\005", len = -1207424906}
        onsend_params = {req = 0x7fff6e141c80, rpl = 0x7f97b804db23, param = 0x7fff6e141c10, code = 1, flags = 0, branch = 0, t_rbuf = 0x7f97b98e5a50, dst = 0x50075b821, send_buf = {
            s = 0xfab1b348 <Address 0xfab1b348 out of bounds>, len = 1024}}
        __FUNCTION__ = "relay_reply"
#4  0x00007f97b8097959 in reply_received (p_msg=0x7f98fb64e2b0) at t_reply.c:2429
        msg_status = 180
        last_uac_status = 100
        ack = 0x7fff6e141f60 "` \024n\377\177"
        ack_len = 32664
        branch = 1
        reply_status = 6440375
        onreply_route = 3
        cancel_data = {cancel_bitmap = 0, reason = {cause = 0, u = {text = {s = 0x0, len = 6682541}, e2e_cancel = 0x0, packed_hdrs = {s = 0x0, len = 6682541}}}}
        uac = 0x7f97b98e5c48
        t = 0x7f97b98e5890
        lack_dst = {send_sock = 0x7fff6e141e70, to = {s = {sa_family = 17847, sa_data = "b\000\000\000\000\000\200\036\024n\377\177\000"}, sin = {sin_family = 17847, sin_port = 98, sin_addr = {
                s_addr = 0}, sin_zero = "\200\036\024n\377\177\000"}, sin6 = {sin6_family = 17847, sin6_port = 98, sin6_flowinfo = 0, sin6_addr = {__in6_u = {
                  __u6_addr8 = "\200\036\024n\377\177\000\000\020\260\261\372\230\177\000", __u6_addr16 = {7808, 28180, 32767, 0, 45072, 64177, 32664, 0}, __u6_addr32 = {1846812288, 32767, 4205948944,
                    32664}}}, sin6_scope_id = 778}}, id = 10, proto = 84 'T', send_flags = {f = 109 'm', blst_imask = 180 '\264'}}
        backup_user_from = 0xa82430
        backup_user_to = 0xa82438
        backup_domain_from = 0xa82440
        backup_domain_to = 0xa82448
        backup_uri_from = 0xa82420
        backup_uri_to = 0xa82428
        backup_xavps = 0xa82560
        replies_locked = 1
        branch_ret = 1846812496
        prev_branch = 32664
        blst_503_timeout = -77274448
        hf = 0xa96c10
        onsend_params = {req = 0x4004188b0, rpl = 0x75fc90, param = 0x0, code = 0, flags = 3, branch = 0, t_rbuf = 0x7fff6e141ee0, dst = 0x631363, send_buf = {s = 0x7fff6e141ee0 "`\037\024n\377\177",
            len = 6422143}}
        ctx = {rec_lev = 0, run_flags = 0, last_retcode = 1, jmp_env = {{__jmpbuf = {797, -5029273663833343261, 4294832, 140735040203504, 0, 0, -5029273663808177437, 5029028274090965731},
              __mask_was_saved = 0, __saved_mask = {__val = {6422143, 140735040200416, 6475430, 140735040200128, 140295017705488, 140735040200192, 140289551658488, 140735040200160, 0, 1073741824,
                  1401221318984, 140295029470864, 7733692, 64, 28601913, 174}}}}}
        __FUNCTION__ = "reply_received"
#5  0x00000000004901cc in do_forward_reply (msg=0x7f98fb64e2b0, mode=0) at forward.c:747
        new_buf = 0x0
        dst = {send_sock = 0x0, to = {s = {sa_family = 0, sa_data = '\000' <repeats 13 times>}, sin = {sin_family = 0, sin_port = 0, sin_addr = {s_addr = 0}, sin_zero = "\000\000\000\000\000\000\000"},
            sin6 = {sin6_family = 0, sin6_port = 0, sin6_flowinfo = 0, sin6_addr = {__in6_u = {__u6_addr8 = '\000' <repeats 15 times>, __u6_addr16 = {0, 0, 0, 0, 0, 0, 0, 0}, __u6_addr32 = {0, 0, 0,
                    0}}}, sin6_scope_id = 0}}, id = 0, proto = 0 '\000', send_flags = {f = 0 '\000', blst_imask = 0 '\000'}}
        new_len = 0
        r = 2
        ip = {af = 0, len = 0, u = {addrl = {21482551329, 140295017709352}, addr32 = {7714849, 5, 4205952808, 32664}, addr16 = {47137, 117, 5, 0, 48936, 64177, 32664, 0},
            addr = "!\270u\000\005\000\000\000(\277\261\372\230\177\000"}}
        s = 0x7f98fb64e2b8 "99\351U"
        len = 32663
        __FUNCTION__ = "do_forward_reply"
#6  0x0000000000491856 in forward_reply (msg=0x7f98fb64e2b0) at forward.c:849
No locals.
#7  0x000000000050e062 in receive_msg (
    buf=0x1b46d40 "SIP/2.0 180 Ringing\r\nVia: SIP/2.0/TCP 211.78.19.1:5003;branch=z9hG4bK60b8.17e8d16cb9ef6d39be60d4cc68762171.1;i=9e02;received=211.78.19.1;rport=57074\r\nVia: SIP/2.0/TCP 192.168.1.112:64044;received=218."..., len=797, rcv_info=0x7f97b98f1610) at receive.c:255
        msg = 0x7f98fb64e2b0
        ctx = {rec_lev = -1181805064, run_flags = 32663, last_retcode = 16383, jmp_env = {{__jmpbuf = {17179869186, 140289629072096, 4294832, 3424935750384, 0, 0, 4294832, 140735040203504},
              __mask_was_saved = 1846813104, __saved_mask = {__val = {6234849, 28601648, 140296091965321, 0, 140296091950726, 140735040201912, 140289629951480, 65552, 797, 42949672963, 140289629953026,
                  0, 140289629953029, 140735040201360, 6235333, 4294832}}}}}
        ret = 0
        inb = {
          s = 0x1b46d40 "SIP/2.0 180 Ringing\r\nVia: SIP/2.0/TCP 211.78.19.1:5003;branch=z9hG4bK60b8.17e8d16cb9ef6d39be60d4cc68762171.1;i=9e02;received=211.78.19.1;rport=57074\r\nVia: SIP/2.0/TCP 192.168.1.112:64044;received=218."..., len = 797}
        __FUNCTION__ = "receive_msg"
#8  0x00000000005f65fc in receive_tcp_msg (
    tcpbuf=0x7f97b98f18e8 "SIP/2.0 180 Ringing\r\nVia: SIP/2.0/TCP 211.78.19.1:5003;branch=z9hG4bK60b8.17e8d16cb9ef6d39be60d4cc68762171.1;i=9e02;received=211.78.19.1;rport=57074\r\nVia: SIP/2.0/TCP 192.168.1.112:64044;received=218."..., len=797, rcv_info=0x7f97b98f1610, con=0x7f97b98f15f8) at tcp_read.c:1247
        buf = 0x1b46d40 "SIP/2.0 180 Ringing\r\nVia: SIP/2.0/TCP 211.78.19.1:5003;branch=z9hG4bK60b8.17e8d16cb9ef6d39be60d4cc68762171.1;i=9e02;received=211.78.19.1;rport=57074\r\nVia: SIP/2.0/TCP 192.168.1.112:64044;received=218."...
        bsize = 65535
        blen = 65535
        __FUNCTION__ = "receive_tcp_msg"
#9  0x00000000005f7d9b in tcp_read_req (con=0x7f97b98f15f8, bytes_read=0x7fff6e1424c0, read_flags=0x7fff6e1424b8) at tcp_read.c:1401
        bytes = 797
        total_bytes = 797
        resp = 1
        size = 140735040201552
        req = 0x7f97b98f1678
        dst = {send_sock = 0x14, to = {s = {sa_family = 1, sa_data = "\000\000P\000\000\000\001 \000\200\370\320\366", <incomplete sequence \372>}, sin = {sin_family = 1, sin_port = 0, sin_addr = {
                s_addr = 80}, sin_zero = "\001 \000\200\370\320\366", <incomplete sequence \372>}, sin6 = {sin6_family = 1, sin6_port = 0, sin6_flowinfo = 80, sin6_addr = {__in6_u = {
                  __u6_addr8 = "\001 \000\200\370\320\366\372\230\177\000\000\000\000\000", __u6_addr16 = {8193, 32768, 53496, 64246, 32664, 0, 0, 0}, __u6_addr32 = {2147491841, 4210479352, 32664, 0}}},
              sin6_scope_id = 1846813872}}, id = 32767, proto = 8 '\b', send_flags = {f = 0 '\000', blst_imask = 0 '\000'}}
        c = 0 '\000'
        ret = -1
        __FUNCTION__ = "tcp_read_req"
#10 0x00000000005fadab in handle_io (fm=0x7f98faf6d0f8, events=1, idx=-1) at tcp_read.c:1624
        ret = 0
        n = -1
        read_flags = 1
        con = 0x7f97b98f15f8
        s = 11
        resp = 1
        t = 381376958
        __FUNCTION__ = "handle_io"
#11 0x00000000005ee8c0 in io_wait_loop_epoll (h=0xa2dfe0, t=2, repeat=1) at io_wait.h:1061
        n = 1
        r = 0
        fm = 0x7f98faf6d0f8
        revents = 1
        __FUNCTION__ = "io_wait_loop_epoll"
#12 0x00000000005fbe1a in tcp_receive_loop (unix_sock=97) at tcp_read.c:1739
        __FUNCTION__ = "tcp_receive_loop"
#13 0x00000000005e54e1 in tcp_init_children () at tcp_main.c:4787
        r = 15
        i = 23
        reader_fd_1 = 97
        pid = 0
        si_desc = "tcp receiver (generic)\000\000ygY\000\000\000\000\000P'\024n\377\177\000\000R\342B\000\000\000\000\000\060'\024n\377\177\000\000\260ò:\231\177\000\000\220\023\364\372\230\177\000\000_\301\211T\000\000\000\000\217sp\000\000\000\000\000\020\021\265\372\230\177\000\000(\000\000\000)\000\000\000UZ-;a\303\000\000m\303\000\000Tb\214K_\301\211T\000\000\000"
        si = 0x0
        __FUNCTION__ = "tcp_init_children"
#14 0x00000000004ab258 in main_loop () at main.c:1658
        i = 24
        pid = 50017
        si = 0x0
        si_desc = "udp receiver child=23 sock=211.78.19.1:5003\000n\000\000\000~/q\000\001\000\000\000\260䕸\227\177\000\000\060(\024n\377\177\000\000\032\264N\000\000\000\000\000\200(\024n\377\177\000\000\260\000\364\372\230\177\000\000\350\327⸗\177\000\000\260䕸\227\177\000\000_\301\211T\000\000\000\000\260\210A\000\001\000\000"
        nrprocs = 24
        __FUNCTION__ = "main_loop"
#15 0x00000000004b0b5b in main (argc=13, argv=0x7fff6e142af8) at main.c:2533
        cfg_stream = 0x1a9a010
        c = -1
        r = 0
        tmp = 0x7fff6e144d59 ""
        tmp_len = 32665
        port = 994981744
        proto = 0
        options = 0x707db8 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
        ret = -1
        seed = 1686944683
        rfd = 4
        debug_save = 0
        debug_flag = 0
        dont_fork_cnt = 0
        n_lst = 0x76
        p = 0x7fff6e1429fe ""
        __FUNCTION__ = "main"
```
#### BT 2
```
===syslog===
Sep  8 09:59:08 sip2 kernel: [58418949.036449] kamailio[145444]: segfault at 7fa6a900676e ip 00007fa6a900676e sp 00007fff6fed90a8 error 14 in textops.so[7fa6a8ec6000+200000]
Sep  8 09:59:38 sip2 media-dispatcher[98511]: debug: Connection to OpenSIPS lost: Connection was closed cleanly.
Sep  8 09:59:38 sip2 /home/pkg/kamailio/sbin/kamailio[145378]: ALERT: <core> [main.c:728]: handle_sigs(): child process 145444 exited by a signal 11
Sep  8 09:59:38 sip2 /home/pkg/kamailio/sbin/kamailio[145378]: ALERT: <core> [main.c:731]: handle_sigs(): core was generated
===backtrace===
#0  0x00007fa6a900676e in ?? ()
No symbol table info available.
#1  0x00007fa6ab3d9b5e in run_trans_callbacks_internal (cb_lst=0x7fa6acd75a88, type=32, trans=0x7fa6acd75a18, params=0x7fff6fed9190) at t_hooks.c:268
        cbp = 0x7fa6acd680f0
        backup_from = 0xa82430
        backup_to = 0xa82438
        backup_dom_from = 0xa82440
        backup_dom_to = 0xa82448
        backup_uri_from = 0xa82420
        backup_uri_to = 0xa82428
        backup_xavps = 0xa82560
        __FUNCTION__ = "run_trans_callbacks_internal"
#2  0x00007fa6ab3d9c6f in run_trans_callbacks (type=32, trans=0x7fa6acd75a18, req=0x7fa6acd6c828, rpl=0x7fa7ee9ea2b0, code=180) at t_hooks.c:295
        params = {req = 0x7fa6acd6c828, rpl = 0x7fa7ee9ea2b0, param = 0x7fa6acd68100, code = 180, flags = 0, branch = 0, t_rbuf = 0x0, dst = 0x0, send_buf = {s = 0x0, len = 0}}
#3  0x00007fa6ab42f82e in relay_reply (t=0x7fa6acd75a18, p_msg=0x7fa7ee9ea2b0, branch=1, msg_status=180, cancel_data=0x7fff6fed94f0, do_put_on_wait=1) at t_reply.c:1767
        relay = 1
        save_clone = 0
        buf = 0x0
        res_len = 0
        relayed_code = 0
        relayed_msg = 0x0
        reply_bak = 0x7fff05059330
        bm = {to_tag_val = {s = 0xffffffff00c75ead <Address 0xffffffff00c75ead out of bounds>, len = -291574016}}
        totag_retr = 0
        reply_status = RPS_PROVISIONAL
        uas_rb = 0x7fa6acd75ad8
        to_tag = 0x7fa6ab41f7bf
        reason = {s = 0x7fff6fed92e0 "", len = -1421744010}
        onsend_params = {req = 0x7fff6fed9300, rpl = 0x7fa6ab3e9b23, param = 0x7fff6fed9290, code = 1, flags = 0, branch = 0, t_rbuf = 0x7fa6acd75bd8, dst = 0x50075b821, send_buf = {
            s = 0xedeb7328 <Address 0xedeb7328 out of bounds>, len = 1024}}
        __FUNCTION__ = "relay_reply"
#4  0x00007fa6ab433959 in reply_received (p_msg=0x7fa7ee9ea2b0) at t_reply.c:2429
        msg_status = 180
        last_uac_status = 100
        ack = 0x7fff6fed95e0 "\340\226\355o\377\177"
        ack_len = 32679
        branch = 1
        reply_status = 6440375
        onreply_route = 3
        cancel_data = {cancel_bitmap = 0, reason = {cause = 0, u = {text = {s = 0x0, len = 6682541}, e2e_cancel = 0x0, packed_hdrs = {s = 0x0, len = 6682541}}}}
        uac = 0x7fa6acd75dd0
        t = 0x7fa6acd75a18
        lack_dst = {send_sock = 0x7fff6fed94f0, to = {s = {sa_family = 17847, sa_data = "b\000\000\000\000\000\000\225\355o\377\177\000"}, sin = {sin_family = 17847, sin_port = 98, sin_addr = {
                s_addr = 0}, sin_zero = "\000\225\355o\377\177\000"}, sin6 = {sin6_family = 17847, sin6_port = 98, sin6_flowinfo = 0, sin6_addr = {__in6_u = {
                  __u6_addr8 = "\000\225\355o\377\177\000\000\020p\353\355\247\177\000", __u6_addr16 = {38144, 28653, 32767, 0, 28688, 60907, 32679, 0}, __u6_addr32 = {1877841152, 32767, 3991629840,
                    32679}}}, sin6_scope_id = 778}}, id = 10, proto = 84 'T', send_flags = {f = 93 ']', blst_imask = 199 '\307'}}
        backup_user_from = 0xa82430
        backup_user_to = 0xa82438
        backup_domain_from = 0xa82440
        backup_domain_to = 0xa82448
        backup_uri_from = 0xa82420
        backup_uri_to = 0xa82428
        backup_xavps = 0xa82560
        replies_locked = 1
        branch_ret = 1877841360
        prev_branch = 32679
        blst_503_timeout = -291593552
        hf = 0xa96580
        onsend_params = {req = 0x4004188b0, rpl = 0x75fc90, param = 0x0, code = 0, flags = 3, branch = 0, t_rbuf = 0x7fff6fed9560, dst = 0x631363, send_buf = {s = 0x7fff6fed9560 "\340\225\355o\377\177",
            len = 6422143}}
        ctx = {rec_lev = 0, run_flags = 0, last_retcode = 1, jmp_env = {{__jmpbuf = {797, -4883994847374486000, 4294832, 140735071232368, 0, 0, -4883994847399651824, 4883678595199653392},
              __mask_was_saved = 0, __saved_mask = {__val = {6422143, 140735071229280, 6475430, 140735071228992, 140359227895824, 140735071229056, 140353761848824, 140735071229024, 0, 1073741824,
                  1401221321056, 140359239663288, 7733692, 64, 13065785, 175}}}}}
        __FUNCTION__ = "reply_received"
#5  0x00000000004901cc in do_forward_reply (msg=0x7fa7ee9ea2b0, mode=0) at forward.c:747
        new_buf = 0x0
        dst = {send_sock = 0x0, to = {s = {sa_family = 0, sa_data = '\000' <repeats 13 times>}, sin = {sin_family = 0, sin_port = 0, sin_addr = {s_addr = 0}, sin_zero = "\000\000\000\000\000\000\000"},
            sin6 = {sin6_family = 0, sin6_port = 0, sin6_flowinfo = 0, sin6_addr = {__in6_u = {__u6_addr8 = '\000' <repeats 15 times>, __u6_addr16 = {0, 0, 0, 0, 0, 0, 0, 0}, __u6_addr32 = {0, 0, 0,
                    0}}}, sin6_scope_id = 0}}, id = 0, proto = 0 '\000', send_flags = {f = 0 '\000', blst_imask = 0 '\000'}}
        new_len = 0
        r = 2
        ip = {af = 0, len = 0, u = {addrl = {21482551329, 140359227899688}, addr32 = {7714849, 5, 3991633704, 32679}, addr16 = {47137, 117, 5, 0, 32552, 60907, 32679, 0},
            addr = "!\270u\000\005\000\000\000(\177\353\355\247\177\000"}}
        s = 0x7fa7ee9ea2b8 "\354@\356U"
        len = 32678
        __FUNCTION__ = "do_forward_reply"
#6  0x0000000000491856 in forward_reply (msg=0x7fa7ee9ea2b0) at forward.c:849
No locals.
#7  0x000000000050e062 in receive_msg (
    buf=0xc75d40 "SIP/2.0 180 Ringing\r\nVia: SIP/2.0/TCP 211.78.19.2:5003;branch=z9hG4bKd59d.0efeaa2488813a7c1ff6cbb4fa19f351.1;i=6592;received=211.78.19.2;rport=22430\r\nVia: SIP/2.0/TCP 192.168.1.112:58885;received=218."..., len=797, rcv_info=0x7fa6acd7b088) at receive.c:255
        msg = 0x7fa7ee9ea2b0
        ctx = {rec_lev = -1395150736, run_flags = 32678, last_retcode = 16383, jmp_env = {{__jmpbuf = {17179869186, 140353840147104, 4294832, 3424966779248, 0, 0, 4294832, 140735071232368},
              __mask_was_saved = 1877841968, __saved_mask = {__val = {6234849, 13065520, 140360302155657, 0, 140360302141062, 140735071230776, 140353841115248, 65552, 797, 42949672963, 140353841116794,
                  0, 140353841116797, 140735071230224, 6235333, 4294832}}}}}
        ret = 0
        inb = {
          s = 0xc75d40 "SIP/2.0 180 Ringing\r\nVia: SIP/2.0/TCP 211.78.19.2:5003;branch=z9hG4bKd59d.0efeaa2488813a7c1ff6cbb4fa19f351.1;i=6592;received=211.78.19.2;rport=22430\r\nVia: SIP/2.0/TCP 192.168.1.112:58885;received=218."..., len = 797}
        __FUNCTION__ = "receive_msg"
#8  0x00000000005f65fc in receive_tcp_msg (
    tcpbuf=0x7fa6acd7b360 "SIP/2.0 180 Ringing\r\nVia: SIP/2.0/TCP 211.78.19.2:5003;branch=z9hG4bKd59d.0efeaa2488813a7c1ff6cbb4fa19f351.1;i=6592;received=211.78.19.2;rport=22430\r\nVia: SIP/2.0/TCP 192.168.1.112:58885;received=218."..., len=797, rcv_info=0x7fa6acd7b088, con=0x7fa6acd7b070) at tcp_read.c:1247
        buf = 0xc75d40 "SIP/2.0 180 Ringing\r\nVia: SIP/2.0/TCP 211.78.19.2:5003;branch=z9hG4bKd59d.0efeaa2488813a7c1ff6cbb4fa19f351.1;i=6592;received=211.78.19.2;rport=22430\r\nVia: SIP/2.0/TCP 192.168.1.112:58885;received=218."...
        bsize = 65535
        blen = 65535
        __FUNCTION__ = "receive_tcp_msg"
#9  0x00000000005f7d9b in tcp_read_req (con=0x7fa6acd7b070, bytes_read=0x7fff6fed9b40, read_flags=0x7fff6fed9b38) at tcp_read.c:1401
        bytes = 797
        total_bytes = 797
        resp = 1
        size = 140735071230416
        req = 0x7fa6acd7b0f0
        dst = {send_sock = 0x14, to = {s = {sa_family = 1, sa_data = "\000\000P\000\000\000\001 \000\200\370\220", <incomplete sequence \356>}, sin = {sin_family = 1, sin_port = 0, sin_addr = {
                s_addr = 80}, sin_zero = "\001 \000\200\370\220", <incomplete sequence \356>}, sin6 = {sin6_family = 1, sin6_port = 0, sin6_flowinfo = 80, sin6_addr = {__in6_u = {
                  __u6_addr8 = "\001 \000\200\370\220\060\356\247\177\000\000\000\000\000", __u6_addr16 = {8193, 32768, 37112, 60976, 32679, 0, 0, 0}, __u6_addr32 = {2147491841, 3996160248, 32679, 0}}},
              sin6_scope_id = 1877842736}}, id = 32767, proto = 8 '\b', send_flags = {f = 0 '\000', blst_imask = 0 '\000'}}
        c = 0 '\000'
        ret = -1
        __FUNCTION__ = "tcp_read_req"
#10 0x00000000005fadab in handle_io (fm=0x7fa7ee3090f8, events=1, idx=-1) at tcp_read.c:1624
        ret = 0
        n = -1
        read_flags = 1
        con = 0x7fa6acd7b070
        s = 11
        resp = 1
        t = 1530467254
        __FUNCTION__ = "handle_io"
#11 0x00000000005ee8c0 in io_wait_loop_epoll (h=0xa2dfe0, t=2, repeat=1) at io_wait.h:1061
        n = 1
        r = 0
        fm = 0x7fa7ee3090f8
        revents = 1
        __FUNCTION__ = "io_wait_loop_epoll"
#12 0x00000000005fbe1a in tcp_receive_loop (unix_sock=97) at tcp_read.c:1739
        __FUNCTION__ = "tcp_receive_loop"
#13 0x00000000005e54e1 in tcp_init_children () at tcp_main.c:4787
        r = 15
        i = 23
        reader_fd_1 = 97
        pid = 0
        si_desc = "tcp receiver (generic)\000\000ygY\000\000\000\000\000Н\355o\377\177\000\000R\342B\000\000\000\000\000\260\235\355o\377\177\000\000\260\203\354-\250\177\000\000\220\323-\356\247\177\000\000\023\255\r\020\000\000\000\000\217sp\000\000\000\000\000\020\321\356\355\247\177\000\000(\000\000\000)\000\000\000U\032g.\375\067\002\000\t8\002\000\267̑`\023\255\r\020\000\000\000"
        si = 0x0
        __FUNCTION__ = "tcp_init_children"
#14 0x00000000004ab258 in main_loop () at main.c:1658
        i = 24
        pid = 145405
        si = 0x0
        si_desc = "udp receiver child=23 sock=211.78.19.2:5003\000n\000\000\000~/q\000\001\000\000\000\260\244ϫ\246\177\000\000\260\236\355o\377\177\000\000\032\264N\000\000\000\000\000\000\237\355o\377\177\000\000\260\300-\356\247\177\000\000\350\227\034\254\246\177\000\000\260\244ϫ\246\177\000\000\023\255\r\020\000\000\000\000\260\210A\000\001\000\000"
        nrprocs = 24
        __FUNCTION__ = "main_loop"
#15 0x00000000004b0b5b in main (argc=13, argv=0x7fff6feda178) at main.c:2533
        cfg_stream = 0xbc9010
        c = -1
        r = 0
        tmp = 0x7fff6fedad59 ""
        tmp_len = 32680
        port = 780662640
        proto = 0
        options = 0x707db8 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
        ret = -1
        seed = 503368684
        rfd = 4
        debug_save = 0
        debug_flag = 0
        dont_fork_cnt = 0
        n_lst = 0x76
        p = 0x7fff6feda07e ""
        __FUNCTION__ = "main"
```
#### BT 3
it generates two core files  3 and 4 in 20 secs
```
===syslog===
Sep 10 14:39:29 sip1 kernel: [41693172.748753] kamailio[245950] general protection ip:602d43 sp:7fff185e8500 error:0 in kamailio[400000+3cb000]
Sep 10 14:39:47 sip1 /home/pkg/kamailio/sbin/kamailio[245999]: CRITICAL: <core> [pass_fd.c:275]: receive_fd(): EOF on 39
Sep 10 14:39:47 sip1 /home/pkg/kamailio/sbin/kamailio[245924]: ALERT: <core> [main.c:728]: handle_sigs(): child process 245950 exited by a signal 11
Sep 10 14:39:47 sip1 /home/pkg/kamailio/sbin/kamailio[245924]: ALERT: <core> [main.c:731]: handle_sigs(): core was generated
Sep 10 14:39:47 sip1 kernel: [41693191.570265] kamailio[245924] general protection ip:7f1f2ba23800 sp:7fff185e7ff0 error:0 in tm.so[7f1f2b9c8000+114000]
===backtrace===
#0  0x0000000000602d43 in slow_timer_main () at timer.c:1130
        n = 12
        ret = 4294967295
        tl = 0x7f1f2cca64f0
        i = 957
        __FUNCTION__ = "slow_timer_main"
#1  0x00000000004aacf9 in main_loop () at main.c:1628
        i = 24
        pid = 0
        si = 0x0
        si_desc = "udp receiver child=23 sock=211.78.19.1:5003\000n\000\000\000~/q\000\001\000\000\000\260D4,\037\177\000\000\060\206^\030\377\177\000\000\032\264N\000\000\000\000\000\200\206^\030\377\177\000\000\260`\222n \177\000\000\350\067\201,\037\177\000\000\260D4,\037\177\000\000\224\321\306[\000\000\000\000\260\210A\000\001\000\000"
        nrprocs = 24
        __FUNCTION__ = "main_loop"
#2  0x00000000004b0b5b in main (argc=13, argv=0x7fff185e88f8) at main.c:2533
        cfg_stream = 0x27a6010
        c = -1
        r = 0
        tmp = 0x7fff185e8f32 ""
        tmp_len = 32544
        port = -1360226448
        proto = 0
        options = 0x707db8 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
        ret = -1
        seed = 1739382561
        rfd = 4
        debug_save = 0
        debug_flag = 0
        dont_fork_cnt = 0
        n_lst = 0x76
        p = 0x7fff185e87fe ""
        __FUNCTION__ = "main"
```
#### BT 4
it generates two core files  3 and 4 in 20 secs
```
===syslog===
Sep 10 14:39:29 sip1 kernel: [41693172.748753] kamailio[245950] general protection ip:602d43 sp:7fff185e8500 error:0 in kamailio[400000+3cb000]
Sep 10 14:39:47 sip1 /home/pkg/kamailio/sbin/kamailio[245999]: CRITICAL: <core> [pass_fd.c:275]: receive_fd(): EOF on 39
Sep 10 14:39:47 sip1 /home/pkg/kamailio/sbin/kamailio[245924]: ALERT: <core> [main.c:728]: handle_sigs(): child process 245950 exited by a signal 11
Sep 10 14:39:47 sip1 /home/pkg/kamailio/sbin/kamailio[245924]: ALERT: <core> [main.c:731]: handle_sigs(): core was generated
Sep 10 14:39:47 sip1 kernel: [41693191.570265] kamailio[245924] general protection ip:7f1f2ba23800 sp:7fff185e7ff0 error:0 in tm.so[7f1f2b9c8000+114000]
===backtrace===
#0  0x00007f1f2ba23800 in run_trans_callbacks_internal (cb_lst=0x7f1f2c9ba700, type=131072, trans=0x7f1f2c9ba690, params=0x7fff185e80d0) at t_hooks.c:264
        cbp = 0x3c223d65636e6174
        backup_from = 0xa82430
        backup_to = 0xa82438
        backup_dom_from = 0xa82440
        backup_dom_to = 0xa82448
        backup_uri_from = 0xa82420
        backup_uri_to = 0xa82428
        backup_xavps = 0xa82560
        __FUNCTION__ = "run_trans_callbacks_internal"
#1  0x00007f1f2ba23c6f in run_trans_callbacks (type=131072, trans=0x7f1f2c9ba690, req=0x0, rpl=0x0, code=0) at t_hooks.c:295
        params = {req = 0x0, rpl = 0x0, param = 0x0, code = 0, flags = 0, branch = 0, t_rbuf = 0x0, dst = 0x0, send_buf = {s = 0x0, len = 0}}
#2  0x00007f1f2b9e2aa1 in free_cell (dead_cell=0x7f1f2c9ba690) at h_table.c:128
        b = 0x621325 "\353\a\220\353\004\220\353\001\220H\201Đ"
        i = 0
        rpl = 0x7fff185e8270
        tt = 0x7fff185e8290
        foo = 0x7f1f2c33c1c8
        cbs = 0x621325
        cbs_tmp = 0x7fff185e8280
        __FUNCTION__ = "free_cell"
#3  0x00007f1f2b9e4779 in free_hash_table () at h_table.c:432
        p_cell = 0x7f1f2c9ba690
        tmp_cell = 0x3a2772657375277b
        i = 53971
        __FUNCTION__ = "free_hash_table"
#4  0x00007f1f2ba09733 in tm_shutdown () at t_funcs.c:90
        __FUNCTION__ = "tm_shutdown"
#5  0x000000000059669b in destroy_modules () at sr_module.c:788
        t = 0x7f206e524660
        foo = 0x7f206e5241f8
        __FUNCTION__ = "destroy_modules"
#6  0x00000000004a011a in cleanup (show_status=1) at main.c:513
        memlog = 32544
        __FUNCTION__ = "cleanup"
#7  0x00000000004a170e in shutdown_children (sig=15, show_status=1) at main.c:655
        __FUNCTION__ = "shutdown_children"
#8  0x00000000004a3c8d in handle_sigs () at main.c:746
        chld = 0
        chld_status = 139
        memlog = 0
        __FUNCTION__ = "handle_sigs"
#9  0x00000000004abb64 in main_loop () at main.c:1701
        i = 24
        pid = 245999
        si = 0x0
        si_desc = "udp receiver child=23 sock=211.78.19.1:5003\000n\000\000\000~/q\000\001\000\000\000\260D4,\037\177\000\000\060\206^\030\377\177\000\000\032\264N\000\000\000\000\000\200\206^\030\377\177\000\000\260`\222n \177\000\000\350\067\201,\037\177\000\000\260D4,\037\177\000\000\224\321\306[\000\000\000\000\260\210A\000\001\000\000"
        nrprocs = 24
        __FUNCTION__ = "main_loop"
#10 0x00000000004b0b5b in main (argc=13, argv=0x7fff185e88f8) at main.c:2533
        cfg_stream = 0x27a6010
        c = -1
        r = 0
        tmp = 0x7fff185e8f32 ""
        tmp_len = 32544
        port = -1360226448
        proto = 0
        options = 0x707db8 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
        ret = -1
        seed = 1739382561
        rfd = 4
        debug_save = 0
        debug_flag = 0
        dont_fork_cnt = 0
        n_lst = 0x76
        p = 0x7fff185e87fe ""
        __FUNCTION__ = "main"
```
---
Reply to this email directly or view it on GitHub:
https://github.com/kamailio/kamailio/issues/322