Segmentaion fault in Kamailio 4.0.3 - sr-users

21 Oct 2013


      Hi,
Our Kamailio has stopped with a segmentation fault 4 times the last week.
There has been no changes to the configuration file the last 15 days, so I
suspect a SIP phone is sending a SIP packet that Kamailio does not like.
We have a core dump file but I cannot read anything usefull from the
backtrace. Can you see what is wrong from the backtrace?
Regards
Morten
The output from gdb (bt full):
Core was generated by `/usr/local/sbin/kamailio -P
/var/run/kamailio/kamailio.pid -m 256 -M 8 -u kamai'.
Program terminated with signal 11, Segmentation fault.
#0  0x00007f0e10de17b2 in cancel_branch (t=0x7f0dfbf38e10, branch=0,
reason=<value optimized out>, flags=4) at t_cancel.c:284
284             if (cfg_get(tm, tm_cfg, reparse_invite) ||
Missing separate debuginfos, use: debuginfo-install
glibc-2.12-1.107.el6.x86_64 hiredis-0.10.1-3.el6.x86_64
keyutils-libs-1.4-4.el6.x86_64 krb5-libs-1.10.3-10.el6.x86_64
libcom_err-1.41.12-14.el6.x86_64 libselinux-2.0.94-5.3.el6.x86_64
libxml2-2.7.6-12.el6_4.1.x86_64 mysql-libs-5.1.67-1.el6_3.x86_64
nss-softokn-freebl-3.12.9-11.el6.x86_64 openssl-1.0.0-27.el6_4.2.x86_64
zlib-1.2.3-29.el6.x86_64
(gdb) bt full
#0  0x00007f0e10de17b2 in cancel_branch (t=0x7f0dfbf38e10, branch=0,
reason=<value optimized out>, flags=4) at t_cancel.c:284
        cancel = <value optimized out>
        len = <value optimized out>
        crb = 0x7f0dfbf39008
        irb = 0x7f0dfbf38f80
        ret = 1
        tmp_cd = {cancel_bitmap = 0, reason = {cause = 0, u = {text = {s =
0x0, len = 0}, e2e_cancel = 0x0, packed_hdrs = {s = 0x0, len = 0}}}}
        pcbuf = <value optimized out>
        __FUNCTION__ = "cancel_branch"
#1  0x00007f0e10e298ab in reply_received (p_msg=0x7f0e124ce760) at
t_reply.c:2194
        msg_status = <value optimized out>
        last_uac_status = 408
        ack = 0x7f0dfbf38e10 "Ð
                               \267\373\r\177"
        ack_len = <value optimized out>
        branch = 0
        reply_status = <value optimized out>
        onreply_route = <value optimized out>
        cancel_data = {cancel_bitmap = 0, reason = {cause = 408, u = {text
= {s = 0x0, len = 307468800}, e2e_cancel = 0x0, packed_hdrs = {s = 0x0, len
= 307468800}}}}
        uac = <value optimized out>
        t = 0x7f0dfbf38e10
        lack_dst = {send_sock = 0x0, to = {s = {sa_family = 6704, sa_data =
"\#\377\177\000\000\000\000\000\000\000\000\000"}, sin = {sin_family =
6704, sin_port = 9052, sin_addr = {s_addr = 32767},
              sin_zero = "\000\000\000\000\000\000\000"}, sin6 =
{sin6_family = 6704, sin6_port = 9052, sin6_flowinfo = 32767, sin6_addr =
{__in6_u = {
                  __u6_addr8 =
"\000\000\000\000\000\000\000\000p{1\022\016\177\000", __u6_addr16 = {0, 0,
0, 0, 31600, 4657, 32526, 0}, __u6_addr32 = {0, 0, 305232752, 32526}}},
              sin6_scope_id = 307029856}}, id = 32526, proto = 28 '\034',
send_flags = {f = 0 '\000', blst_imask = 0 '\000'}}
        backup_user_from = <value optimized out>
        backup_user_to = <value optimized out>
        backup_domain_from = <value optimized out>
        backup_domain_to = <value optimized out>
        backup_uri_from = <value optimized out>
        backup_uri_to = <value optimized out>
        backup_xavps = <value optimized out>
        replies_locked = 0
        branch_ret = <value optimized out>
        prev_branch = <value optimized out>
        blst_503_timeout = <value optimized out>
        hf = <value optimized out>
        onsend_params = {req = 0x8d8a39, rpl = 0x541db4, param =
0x7f0e1253e528, code = 307029856, flags = 32526, branch = 0, t_rbuf =
0x7fff235c1a30, dst = 0x7f0e12317b70, send_buf = {
            s = 0x375311000000000 <Address 0x375311000000000 out of
bounds>, len = 0}}
        ctx = {rec_lev = 307491008, run_flags = 32526, last_retcode =
5674412, jmp_env = {{__jmpbuf = {140733786626256, 63331951475841423,
139698413054576, 139698413299552, 9276465, 139698039855608,
                -63254168797292145, 63332490682325391}, __mask_was_saved =
0, __saved_mask = {__val = {139698413734928, 0, 139698411522821, 1,
140733786626608, 6185835, 5972697, 8586176, 9275699,
                  69026945952, 3, 9276465, 9275673, 139698413738496,
9275961, 139698413760704}}}}}
        __FUNCTION__ = "reply_received"
#2  0x0000000000456444 in do_forward_reply (msg=0x7f0e124ce760, mode=<value
optimized out>) at forward.c:799
        new_buf = 0x0
        dst = {send_sock = 0x0, to = {s = {sa_family = 0, sa_data = '\000'
<repeats 13 times>}, sin = {sin_family = 0, sin_port = 0, sin_addr =
{s_addr = 0}, sin_zero = "\000\000\000\000\000\000\000"},
            sin6 = {sin6_family = 0, sin6_port = 0, sin6_flowinfo = 0,
sin6_addr = {__in6_u = {__u6_addr8 = '\000' <repeats 15 times>, __u6_addr16
= {0, 0, 0, 0, 0, 0, 0, 0}, __u6_addr32 = {0, 0, 0, 0}}},
              sin6_scope_id = 0}}, id = 0, proto = 0 '\000', send_flags =
{f = 0 '\000', blst_imask = 0 '\000'}}
        new_len = <value optimized out>
        r = <value optimized out>
        s = <value optimized out>
        len = <value optimized out>
        __FUNCTION__ = "do_forward_reply"
#3  0x000000000049e15e in receive_msg (buf=<value optimized out>, len=313,
rcv_info=0x7fff235c1cd0) at receive.c:270
        msg = 0x7f0e124ce760
        ctx = {rec_lev = 11, run_flags = 0, last_retcode = 206110737,
jmp_env = {{__jmpbuf = {139698036884436, 11, 219309716216, 139698419720192,
140733786627520, 4294967295, 140733786627647, 1},
              __mask_was_saved = 8576456, __saved_mask = {__val = {0, 28,
16, 0, 219305533392, 1, 0, 139698411461552, 219309716216, 139698036884436,
139698413732672, 139698419717800, 139698413732680,
                  140733786627416, 219305559701, 140733786627288}}}}}
        ret = <value optimized out>
        inb = {
          s = 0x8d8900 "SIP/2.0 100 Trying\r\nVia: SIP/2.0/UDP
178.21.249.20;branch=z9hG4bK8149.c6575a95.0\r\nTo:
sip:201@78799865.pbx.one-connect.dk;tag=07c44e68\r\nFrom:
sip:201@78799865.pbx.one-connect.dk;tag=a6a1c5f60faecf035a"..., len = 313}
        __FUNCTION__ = "receive_msg"
#4  0x0000000000530e46 in udp_rcv_loop () at udp_server.c:557
---Type <return> to continue, or q <return> to quit---
        len = 313
        buf = "SIP/2.0 100 Trying\r\nVia: SIP/2.0/UDP
178.21.249.20;branch=z9hG4bK8149.c6575a95.0\r\nTo:
sip:201@78799865.pbx.one-connect.dk;tag=07c44e68\r\nFrom:
sip:201@78799865.pbx.one-connect.dk;tag=a6a1c5f60faecf035a"...
        from = 0x7f0e12538340
        fromlen = 16
        ri = {src_ip = {af = 2, len = 4, u = {addrl = {2993962576, 0},
addr32 = {2993962576, 0, 0, 0}, addr16 = {15952, 45684, 0, 0, 0, 0, 0, 0},
addr = "P>t\262", '\000' <repeats 11 times>}}, dst_ip = {
            af = 2, len = 4, u = {addrl = {351868338, 0}, addr32 =
{351868338, 0, 0, 0}, addr16 = {5554, 5369, 0, 0, 0, 0, 0, 0}, addr =
"\262\025\371\024", '\000' <repeats 11 times>}}, src_port = 35754,
          dst_port = 5060, proto_reserved1 = 0, proto_reserved2 = 0, src_su
= {s = {sa_family = 2, sa_data =
"\213\252P>t\262\000\000\000\000\000\000\000"}, sin = {sin_family = 2,
sin_port = 43659,
              sin_addr = {s_addr = 2993962576}, sin_zero =
"\000\000\000\000\000\000\000"}, sin6 = {sin6_family = 2, sin6_port =
43659, sin6_flowinfo = 2993962576, sin6_addr = {__in6_u = {
                  __u6_addr8 = '\000' <repeats 15 times>, __u6_addr16 = {0,
0, 0, 0, 0, 0, 0, 0}, __u6_addr32 = {0, 0, 0, 0}}}, sin6_scope_id = 0}},
bind_address = 0x7f0e124cfbd0, proto = 1 '\001'}
        __FUNCTION__ = "udp_rcv_loop"
#5  0x000000000046716a in main_loop () at main.c:1638
        i = <value optimized out>
        pid = <value optimized out>
        si = <value optimized out>
        si_desc = "udp receiver child=2 sock=178.21.249.20:5060
\000\000\000\000\200\303P\022\016\177\000\000\000\000\000\000\000\000\000\000\003\000\000\000\000\000\000\000\001\000\000\000\001\000\000\000@\350\216\000\000\000\000\000\001\000\000\000\000\000\000\000\200\350\216\000\000\000\000\000\000\000\200\020",
'\000' <repeats 12 times>, "\005\000\000\000\000\000\000"
        nrprocs = <value optimized out>
        __FUNCTION__ = "main_loop"
#6  0x000000000046a002 in main (argc=<value optimized out>, argv=<value
optimized out>) at main.c:2566
        cfg_stream = <value optimized out>
        c = <value optimized out>
        r = <value optimized out>
        tmp = 0x7fff235c377f ""
        tmp_len = 0
        options = 0x5c08c8
":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
        ret = -1
        seed = 1722854551
        rfd = <value optimized out>
        debug_save = <value optimized out>
        debug_flag = <value optimized out>
        dont_fork_cnt = <value optimized out>
        n_lst = <value optimized out>
        p = <value optimized out>
        __FUNCTION__ = "main"
(gdb)
-- 
Morten Isaksen