New subject: SEG FAULT dialog

20 Nov 2013

I can reproduce it consistently.
I am doing serial forking into a non-existent host, when the caller gets
impatient, it cancels the call and send another invite.
this happens after that.

full backtrace:

#0  qm_detach_free (frag=0x7fb2b552e2b0, qm=<optimized out>) at
mem/q_malloc.c:269
        prev = 0x7fb2b5192058
        next = 0x0
#1  qm_malloc (qm=0x7fb2b5192000, size=16) at mem/q_malloc.c:386
        f = 0x7fb2b552e2b0
        hash = -1256644520
#2  0x00007fb2b44f7e93 in set_dlg_variable_unsafe () from
/usr/local/lib64/kamailio/modules/dialog.so
No symbol table info available.
#3  0x00007fb2b44fa3aa in pv_set_dlg_variable () from
/usr/local/lib64/kamailio/modules/dialog.so
No symbol table info available.
#4  0x000000000047be59 in lval_pvar_assign (rv=0x7fb2bf8348f8,
lv=0x7fb2bf834750, msg=0x7fb2bf8c7a00,
    h=<optimized out>) at lvalue.c:353
        r_avp = <optimized out>
        ret = 1
        destroy_pval = 1
        pvar = 0x7fb2bf834758
        pval = {rs = {
            s = 0x92750b &quot;+1xx882xx111(a)2xx.xx1.3x.2xSIP/2.0\r\nRecord-RouteRoute:
<sip:6x.x1.4.195;lr=on;ftag=gK085a1dec>\r\nRecord-Route:
<sip:6X.2X1.8.8X;lr=on;ftag=gK085a1dec>\r\nAccept:
application/sdp\r\nAllow: INVITE,ACK,CANCEL,BYE"..., len = 12}, ri = 0,
flags = 4}
        avp_val = {n = 0, s = {s = 0x0, len = 0}, re = 0x0}
        v = <optimized out>
#5  lval_assign (h=<optimized out>, msg=0x7fb2bf8c7a00, lv=0x7fb2bf834750,
rve=0x7fb2bf8348f0) at lvalue.c:401
        rv = 0x7fb2bf8348f8
        ret = 0
        __FUNCTION__ = "lval_assign"
#6  0x000000000041cf14 in do_action (h=0x7fffe1b8f710, a=0x7fb2bf8332b0,
msg=0x7fb2bf8c7a00) at action.c:1453
        ret = -5
        v = <optimized out>
        dst = {send_sock = 0x7fb2bdb0f080, to = {s = {sa_family = 0,
              sa_data =
"\000\000\000\000\000\000\200-\210\000\000\000\000"}, sin = {sin_family =
0, sin_port = 0,
              sin_addr = {s_addr = 0}, sin_zero =
"\200-\210\000\000\000\000"}, sin6 = {sin6_family = 0,
              sin6_port = 0, sin6_flowinfo = 0, sin6_addr = {__in6_u = {
                  __u6_addr8 =
"\200-\210\000\000\000\000\000\016\000\000\000\000\000\000", __u6_addr16 =
{11648,
                    136, 0, 0, 14, 0, 0, 0}, __u6_addr32 = {8924544, 0, 14,
0}}}, sin6_scope_id = 13}}, id = 0,
          proto = 58 ':', send_flags = {f = 180 '\264', blst_imask = 138
'\212'}}
        tmp = <optimized out>
        new_uri = <optimized out>
        end = <optimized out>
        crt = <optimized out>
        cmd = <optimized out>
        len = <optimized out>
        user = <optimized out>
        uri = {user = {s = 0x7fffe1b8eef0 "/", len = 0}, passwd = {
            s = 0x3000000010 <Address 0x3000000010 out of bounds>, len =
-507973920}, host = {
            s = 0x7fffe1b8ee20 "", len = 17022112}, port = {s =
0x7fb2bffbd720 "", len = 8208}, params = {
            s = 0x2010 <Address 0x2010 out of bounds>, len = 17030320},
sip_params = {
            s = 0x2010 <Address 0x2010 out of bounds>, len = -1077387804},
headers = {
            s = 0x7fb2bffbdfb0 "0z\001\001", len = -507974144}, port_no =
55072, proto = 49147, type = 32690,
          flags = 17022128, transport = {s = 0x95 <Address 0x95 out of
bounds>, len = -1241200896}, ttl = {
            s = 0x3734000000a0 <Address 0x3734000000a0 out of bounds>, len
= -1081931184}, user_param = {s = 0x0,
            len = 0}, maddr = {s = 0x63371f "", len = -1}, method = {s =
0x7fb2b519cb30 "\220\240H\265\262\177",
            len = 4310469}, lr = {s = 0x3000000018 <Address 0x3000000018
out of bounds>, len = -507973440}, r2 = {
            s = 0x7fffe1b8f000 "\210u\223", len = 8187}, gr = {s = 0x2
<Address 0x2 out of bounds>, len = 1},
          transport_val = {s = 0x1 <Address 0x1 out of bounds>, len = 0},
ttl_val = {
            s = 0xffffffffffffffa0 <Address 0xffffffffffffffa0 out of
bounds>, len = 17022112}, user_param_val = {
            s = 0x7fb2bffbd720 "", len = -507974064}, maddr_val = {s =
0x7fb2bf8c7a00 "\016", len = -1081931184},
          method_val = {s = 0x2010 <Address 0x2010 out of bounds>, len =
-507971824}, lr_val = {s = 0x0,
            len = -507971824}, r2_val = {s = 0x7fffe1b8f580 "\001", len =
-1}, gr_val = {
            s = 0x7fb2b519cb30 "\220\240H\265\262\177", len = 4346308}}
        next_hop = {user = {s = 0x0, len = 1}, passwd = {s = 0x7fffe1b8f710
"\002", len = 1237187532}, host = {
            s = 0x7fffe1b8f710 "\002", len = -507972224}, port = {
            s = 0xffffffff <Address 0xffffffff out of bounds>, len =
-1256600784}, params = {
            s = 0x3542f9ae92fdfbcc <Address 0x3542f9ae92fdfbcc out of
bounds>, len = -589431860}, sip_params = {
            s = 0x0, len = 17022128}, headers = {s = 0x15 <Address 0x15 out
of bounds>, len = 1}, port_no = 0,
          proto = 0, type = ERROR_URI_T, flags = 4294967200, transport = {s
= 0x7fffe1b8efe8 "l\345N",
            len = -1076936721}, ttl = {s = 0x1 <Address 0x1 out of bounds>,
len = -1114716768}, user_param = {
            s = 0x1c0000002f <Address 0x1c0000002f out of bounds>, len =
7}, maddr = {
            s = 0x710000000a <Address 0x710000000a out of bounds>, len =
2}, method = {s = 0x0, len = -18000},
          lr = {s = 0x1017a30 "EST", len = 6323844}, r2 = {s =
0x7fb2bd8ed356 "tm [t_lookup.c:716]: ",
            len = -507972224}, gr = {s = 0x7fffe1b8f710 "\002", len =
5148368}, transport_val = {s = 0x0,
            len = 0}, ttl_val = {s = 0x7fb2bf8c7a00 "\016", len =
-507973348}, user_param_val = {
            s = 0x103bcb0 "<135>Nov 19 07:28:47
/usr/local/sbin/kamailio[15556]: DEBUG: dialog [dlg_hash.c:602]:
dlg_lookup(): dialog id=7094 found on entry 2049\n", len = 0}, maddr_val = {
            s = 0x103bcb0 "<135>Nov 19 07:28:47
/usr/local/sbin/kamailio[15556]: DEBUG: dialog [dlg_hash.c:602]:
dlg_lookup(): dialog id=7094 found on entry 2049\n", len = -1115163744},
method_val = {
            s = 0x99 <Address 0x99 out of bounds>, len = 1384864127},
lr_val = {s = 0x882d80 "",
            len = -1081312768}, r2_val = {s = 0x7fb2bdb0f080 "\016", len =
8924544}, gr_val = {
            s = 0xffffffff <Address 0xffffffff out of bounds>, len =
-1081930768}}
        u = <optimized out>
        port = <optimized out>
        dst_host = <optimized out>
        i = <optimized out>
        flags = <optimized out>
        avp = <optimized out>
        st = {flags = 3213654528, id = 32690, name = {n = -507971824, s =
{s = 0x7fffe1b8f710 "\002",
              len = -507972224}, re = 0x7fffe1b8f710}, avp = 0x4ee56c}
        sct = <optimized out>
        sjt = <optimized out>
        rve = <optimized out>
        mct = <optimized out>
        rv = <optimized out>
        rv1 = <optimized out>
        c1 = {cache_type = 3213020352, val_type = 32690, c = {avp_val = {n
= 0, s = {
                s = 0x7fb200000000 <Address 0x7fb200000000 out of bounds>,
len = -1081312768},
              re = 0x7fb200000000}, pval = {rs = {s = 0x7fb200000000
<Address 0x7fb200000000 out of bounds>,
                len = -1081312768}, ri = -1124131338, flags = 32690}},
          i2s = "\377\377\377\377\060", '\000' <repeats 16
times>}
        s = {s = 0x7fffe1b8f0d8 "\260\062\203\277\262\177", len =
-507973632}
        srevp = {0x937588, 0x7fb2bfc4f1fc}
        mod_f_params = {{type = NOSUBTYPE, u = {number = 0, string = 0x0,
str = {s = 0x0, len = 0}, data = 0x0,
              attr = 0x0, select = 0x0}}, {type = NOSUBTYPE, u = {number =
0, string = 0x0, str = {s = 0x0,
                len = 0}, data = 0x0, attr = 0x0, select = 0x0}}, {type =
NOSUBTYPE, u = {number = 0,
              string = 0x0, str = {s = 0x0, len = 0}, data = 0x0, attr =
0x0, select = 0x0}}, {type = NOSUBTYPE,
            u = {number = 0, string = 0x0, str = {s = 0x0, len = 0}, data =
0x0, attr = 0x0, select = 0x0}}, {
            type = NOSUBTYPE, u = {number = 0, string = 0x0, str = {s =
0x0, len = 0}, data = 0x0, attr = 0x0,
              select = 0x0}}, {type = NOSUBTYPE, u = {number = 0, string =
0x0, str = {s = 0x0, len = 0},
              data = 0x0, attr = 0x0, select = 0x0}}, {type = NOSUBTYPE, u
= {number = 0, string = 0x0, str = {
                s = 0x0, len = 0}, data = 0x0, attr = 0x0, select = 0x0}},
{type = NOSUBTYPE, u = {number = 0,
              string = 0x0, str = {s = 0x0, len = 0}, data = 0x0, attr =
0x0, select = 0x0}}}
        __FUNCTION__ = "do_action"
#7  0x000000000041c5c5 in run_actions (h=0x7fffe1b8f710, a=0x7fb2bf8313a0,
msg=0x7fb2bf8c7a00) at action.c:1573
        t = <optimized out>
        ret = <optimized out>
        mod = <optimized out>
        ms = 0
        __FUNCTION__ = "run_actions"
#8  0x000000000041d92e in do_action (h=0x7fffe1b8f710, a=0x7fb2bf8396d0,
msg=0x7fb2bf8c7a00) at action.c:1065
        ret = 1
        v = 1
        dst = {send_sock = 0x0, to = {s = {sa_family = 0,
              sa_data =
"\000\000\001\000\000\000\000\000\000\000\000\000\000"}, sin = {sin_family
= 0,
              sin_port = 0, sin_addr = {s_addr = 1}, sin_zero =
"\000\000\000\000\000\000\000"}, sin6 = {
              sin6_family = 0, sin6_port = 0, sin6_flowinfo = 1, sin6_addr
= {__in6_u = {
                  __u6_addr8 =
"\000\000\000\000\000\000\000\000\001\000\000\000\000\000\000", __u6_addr16
= {0,
                    0, 0, 0, 1, 0, 0, 0}, __u6_addr32 = {0, 0, 1, 0}}},
sin6_scope_id = 0}}, id = 0,
          proto = -80 '\260', send_flags = {f = 188 '\274', blst_imask =
3
'\003'}}
        tmp = <optimized out>
        new_uri = <optimized out>
        end = <optimized out>
        crt = <optimized out>
        cmd = <optimized out>
        len = <optimized out>
        user = <optimized out>
        uri = {user = {s = 0x3635353531000002 <Address 0x3635353531000002
out of bounds>, len = 1}, passwd = {
            s = 0x0, len = 17022128}, host = {s = 0x7fffe1b8f460 "/", len =
0}, port = {
            s = 0x3000000010 <Address 0x3000000010 out of bounds>, len =
-507972528}, params = {
            s = 0x7fffe1b8f390 "\260\337\373\277\262\177", len =
-1076936721}, sip_params = {
            s = 0xa2 <Address 0xa2 out of bounds>, len = -1114690816},
headers = {
            s = 0x1c0000002f <Address 0x1c0000002f out of bounds>, len =
7}, port_no = 10, proto = 0, type = 113,
          flags = URI_SIP_USER_PHONE, transport = {s = 0x0, len = -18000},
ttl = {s = 0x1017a30 "EST",
            len = -1076935806}, user_param = {s = 0x7fb2b5531d90
"P\213\065\265\262\177", len = 48}, maddr = {
            s = 0x3734e1b8f3b8 <Address 0x3734e1b8f3b8 out of bounds>, len
= -507972896}, method = {
            s = 0x3531320000000002 <Address 0x3531320000000002 out of
bounds>, len = -1076363582}, lr = {
            s = 0x103bcbc "07:28:47 /usr/local/sbin/kamailio[15556]: DEBUG:
dialog [dlg_hash.c:602]: dlg_lookup(): dialog id=7094 found on entry
2049\n", len = 17022140}, r2 = {s = 0x7fb2bffbb3c0 "", len = 7}, gr = {
            s = 0x3000000018 <Address 0x3000000018 out of bounds>, len =
-507972048}, transport_val = {
            s = 0x7fffe1b8f570 "\001", len = 8187}, ttl_val = {s = 0x2
<Address 0x2 out of bounds>, len = 1},
          user_param_val = {s = 0x1 <Address 0x1 out of bounds>, len = 0},
maddr_val = {
            s = 0xffffffffffffffa0 <Address 0xffffffffffffffa0 out of
bounds>, len = 17022112}, method_val = {
            s = 0x7fb2bffbd720 "", len = 8208}, lr_val = {s = 0x2010
<Address 0x2010 out of bounds>,
            len = 17030320}, r2_val = {s = 0x2010 <Address 0x2010 out of
bounds>, len = -1077387804}, gr_val = {
            s = 0x7fb2bffbdfb0 "0z\001\001", len = -1077164088}}
        next_hop = {user = {s = 0x7fb2bffbd720 "", len = 8180}, passwd = {
            s = 0x3635353531000002 <Address 0x3635353531000002 out of
bounds>, len = 17022112}, host = {s = 0x0,
            len = -1077381378}, port = {s = 0x7fffe1b8f5f0 "", len = 0},
params = {
            s = 0x3000000010 <Address 0x3000000010 out of bounds>, len =
-507972128}, sip_params = {
            s = 0x7fffe1b8f520 "\002", len = -1077164088}, headers = {s =
0x0, len = 8180}, port_no = 2,
          proto = 12544, type = 909456693, flags = 17022128, transport = {s
= 0x0, len = 1}, ttl = {
            s = 0x7fffe1b8f650 "0\313\031\265\262\177", len = 0},
user_param = {
            s = 0x3000000010 <Address 0x3000000010 out of bounds>, len =
-507972032}, maddr = {
            s = 0x7fffe1b8f580 "\001", len = 6526864}, method = {
            s = 0x37340000002f <Address 0x37340000002f out of bounds>, len
= 7}, lr = {
            s = 0x320000710000000a <Address 0x320000710000000a out of
bounds>, len = -1076363582}, r2 = {
            s = 0x103bcbc "07:28:47 /usr/local/sbin/kamailio[15556]: DEBUG:
dialog [dlg_hash.c:602]: dlg_lookup(): dialog id=7094 found on entry
2049\n", len = 17022140}, gr = {s = 0x7fb2bffbb3c0 "", len = 7},
transport_val = {
            s = 0x3000000018 <Address 0x3000000018 out of bounds>, len =
-507971648}, ttl_val = {
            s = 0x7fffe1b8f700 "0\313\031\265\262\177", len = 8187},
user_param_val = {
            s = 0x373400000002 <Address 0x373400000002 out of bounds>, len
= 1}, maddr_val = {
            s = 0x1 <Address 0x1 out of bounds>, len = -1076363582},
method_val = {
            s = 0x103bcbc "07:28:47 /usr/local/sbin/kamailio[15556]: DEBUG:
dialog [dlg_hash.c:602]: dlg_lookup(): dialog id=7094 found on entry
2049\n", len = 17022140}, lr_val = {s = 0x7fb2bffbb3c0 "", len = 7},
r2_val
= {
            s = 0x3000000018 <Address 0x3000000018 out of bounds>, len =
-507971552}, gr_val = {
            s = 0x7fffe1b8f760 "", len = 8187}}
        u = <optimized out>
        port = <optimized out>
        dst_host = <optimized out>
        i = <optimized out>
        flags = <optimized out>
        avp = <optimized out>
        st = {flags = 2, id = 0, name = {n = 1, s = {s = 0x1 <Address 0x1
out of bounds>, len = 1}, re = 0x1},
          avp = 0x0}
        sct = <optimized out>
        sjt = <optimized out>
        rve = <optimized out>
        mct = <optimized out>
        rv = <optimized out>
        rv1 = <optimized out>
        c1 = {cache_type = 137, val_type = RV_NONE, c = {avp_val = {n =
17022112, s = {s = 0x103bca0 "`\002",
                len = 160}, re = 0x103bca0}, pval = {rs = {s = 0x103bca0
"`\002", len = 160}, ri = -1077381378,
              flags = 32690}},
          i2s = "
\327\373\277\262\177\000\000\000\000\000\000\001\000\000\000\340\366\270\341\377\177"}
        s = {s = 0xffffffffffffffa0 <Address 0xffffffffffffffa0 out of
bounds>, len = 17022112}
        srevp = {0x7fb2bffbd720, 0x2010}
        mod_f_params = {{type = NOSUBTYPE, u = {number = 0, string = 0x0,
str = {s = 0x0, len = 0}, data = 0x0,
              attr = 0x0, select = 0x0}}, {type = NOSUBTYPE, u = {number =
0, string = 0x0, str = {s = 0x0,
                len = 0}, data = 0x0, attr = 0x0, select = 0x0}}, {type =
NOSUBTYPE, u = {number = 0,
              string = 0x0, str = {s = 0x0, len = 0}, data = 0x0, attr =
0x0, select = 0x0}}, {type = NOSUBTYPE,
            u = {number = 0, string = 0x0, str = {s = 0x0, len = 0}, data =
0x0, attr = 0x0, select = 0x0}}, {
            type = NOSUBTYPE, u = {number = 0, string = 0x0, str = {s =
0x0, len = 0}, data = 0x0, attr = 0x0,
              select = 0x0}}, {type = NOSUBTYPE, u = {number = 0, string =
0x0, str = {s = 0x0, len = 0},
              data = 0x0, attr = 0x0, select = 0x0}}, {type = NOSUBTYPE, u
= {number = 0, string = 0x0, str = {
                s = 0x0, len = 0}, data = 0x0, attr = 0x0, select = 0x0}},
{type = NOSUBTYPE, u = {number = 0,
              string = 0x0, str = {s = 0x0, len = 0}, data = 0x0, attr =
0x0, select = 0x0}}}
        __FUNCTION__ = "do_action"
#9  0x000000000041c5c5 in run_actions (h=0x7fffe1b8f710, a=0x7fb2bf82d4c0,
msg=0x7fb2bf8c7a00) at action.c:1573
        t = <optimized out>
        ret = <optimized out>
        mod = <optimized out>
        ms = 0#0  qm_detach_free (frag=0x7fb2b552e2b0, qm=<optimized out>)
at mem/q_malloc.c:269
        prev = 0x7fb2b5192058
        next = 0x0

... cut ....

Kelvin Chua

On Tue, Nov 19, 2013 at 9:40 PM, Daniel-Constantin Mierla &lt;miconda(a)gmail.com
...
  wrote: 
...
   Hello,

 can you get the output for 'bt full'? Probably you have to install the
 debug symbols (kamailio-dbg package) for getting something useful.

 Another question, can you reproduce it? Or it happens sporadically?

 Cheers,
 Daniel

 On 11/19/13 1:11 PM, Kelvin Chua wrote:

  kamailio 4.0.4
 has anybody experience this using dialog module?

  backtrace:
 #0  0x000000000057af45 in qm_malloc ()
 #1  0x00007f268ac0fe93 in set_dlg_variable_unsafe () from
 /usr/local/lib64/kamailio/modules/dialog.so
 #2  0x00007f268ac123aa in pv_set_dlg_variable () from
 /usr/local/lib64/kamailio/modules/dialog.so
 #3  0x000000000047be59 in lval_assign ()
 #4  0x000000000041cf14 in do_action ()
 #5  0x000000000041c5c5 in run_actions ()
 #6  0x000000000041d92e in do_action ()
 #7  0x000000000041c5c5 in run_actions ()
 #8  0x0000000000425250 in run_top_route ()
 #9  0x00000000004bfcce in receive_msg ()
 #10 0x000000000056a2df in udp_rcv_loop ()
 #11 0x0000000000481b43 in main_loop ()
 #12 0x000000000041c081 in main ()

  logfile:
  DEBUG: <core> [parser/msg_parser.c:623]: parse_msg(): SIP Request:
 DEBUG: <core> [parser/msg_parser.c:625]: parse_msg():  method:  <ACK>
 DEBUG: <core> [parser/msg_parser.c:627]: parse_msg():  uri:
 <sip:+1XX882XX111@2XX.XX1.X9.2X>
 DEBUG: <core> [parser/msg_parser.c:629]: parse_msg():  version: <SIP/2.0>
 DEBUG: <core> [parser/parse_via.c:1284]: parse_via_param(): Found param
 type 232, <branch> = <z9hG4bKe97b.5a4397f3.0>; state=16
 DEBUG: <core> [parser/parse_via.c:2672]: parse_via(): end of header
 reached, state=5
 DEBUG: <core> [parser/msg_parser.c:513]: parse_headers(): parse_headers:
 Via found, flags=2
 DEBUG: <core> [parser/msg_parser.c:515]: parse_headers(): parse_headers:
 this is the first via
  DEBUG: <core> [receive.c:151]: receive_msg(): After parse_msg...
 DEBUG: <core> [receive.c:192]: receive_msg(): preparing to run routing
 scripts...
 DEBUG: <core> [parser/parse_to.c:176]: parse_to_param(): DEBUG: add_param:
 tag=bbd932f8f9dbf9743f9b7cadcbf622ac.0dc7
 DEBUG: <core> [parser/parse_to.c:799]: parse_to(): end of header reached,
 state=29
 DEBUG: <core> [parser/msg_parser.c:190]: get_hdr_field(): DEBUG:
 get_hdr_field: <To> [74]; uri=[sip:+1XX882XX111@6X.X31.X.8X]
 DEBUG: <core> [parser/msg_parser.c:192]: get_hdr_field(): DEBUG: to body [
 <sip:+1XX882XX111@6X.X31.X.8X>]
 DEBUG: sl [sl_funcs.c:415]: sl_filter_ACK(): SL local ACK found ->
 dropping it!
 DEBUG: <core> [usr_avp.c:644]: destroy_avp_list(): DEBUG:destroy_avp_list:
 destroying list (nil)
 last message repeated 5 times
 DEBUG: <core> [xavp.c:447]: xavp_destroy_list(): destroying xavp list (nil)
 DEBUG: <core> [receive.c:295]: receive_msg(): receive_msg: cleaning up
 segfault at 0 ip 000000000057af45 sp 00007fff30e44ac0 error 4 in
 kamailio[400000+27c000]

 Kelvin Chua

 _______________________________________________
 SIP Express Router (SER) and Kamailio (OpenSER) - sr-users mailing
listsr-users@lists.sip-router.orghttp://lists.sip-router.org/cgi-bin/mailman/listinfo/sr-users

 --
 Daniel-Constantin Mierla - http://www.asipto.comhttp://twitter.com/#!/miconda -
http://www.linkedin.com/in/miconda
 Kamailio Advanced Trainings - Berlin, Nov 25-28
   - more details about Kamailio trainings at http://www.asipto.com -

 _______________________________________________
 SIP Express Router (SER) and Kamailio (OpenSER) - sr-users mailing list
 sr-users(a)lists.sip-router.org
 http://lists.sip-router.org/cgi-bin/mailman/listinfo/sr-users

Re: [SR-Users] SEG FAULT dialog