I can reproduce it consistently.
I am doing serial forking into a non-existent host, when the caller gets impatient, it cancels the call and send another invite.
this happens after that.

full backtrace:

#0  qm_detach_free (frag=0x7fb2b552e2b0, qm=<optimized out>) at mem/q_malloc.c:269
        prev = 0x7fb2b5192058
        next = 0x0
#1  qm_malloc (qm=0x7fb2b5192000, size=16) at mem/q_malloc.c:386
        f = 0x7fb2b552e2b0
        hash = -1256644520
#2  0x00007fb2b44f7e93 in set_dlg_variable_unsafe () from /usr/local/lib64/kamailio/modules/dialog.so
No symbol table info available.
#3  0x00007fb2b44fa3aa in pv_set_dlg_variable () from /usr/local/lib64/kamailio/modules/dialog.so
No symbol table info available.
#4  0x000000000047be59 in lval_pvar_assign (rv=0x7fb2bf8348f8, lv=0x7fb2bf834750, msg=0x7fb2bf8c7a00, 
    h=<optimized out>) at lvalue.c:353
        r_avp = <optimized out>
        ret = 1
        destroy_pval = 1
        pvar = 0x7fb2bf834758
        pval = {rs = {
            s = 0x92750b "+1xx882xx111@2xx.xx1.3x.2x SIP/2.0\r\nRecord-Route: <sip:6x.x1.4.195;lr=on;ftag=gK085a1dec>\r\nRecord-Route: <sip:6X.2X1.8.8X;lr=on;ftag=gK085a1dec>\r\nAccept: application/sdp\r\nAllow: INVITE,ACK,CANCEL,BYE"..., len = 12}, ri = 0, flags = 4}
        avp_val = {n = 0, s = {s = 0x0, len = 0}, re = 0x0}
        v = <optimized out>
#5  lval_assign (h=<optimized out>, msg=0x7fb2bf8c7a00, lv=0x7fb2bf834750, rve=0x7fb2bf8348f0) at lvalue.c:401
        rv = 0x7fb2bf8348f8
        ret = 0
        __FUNCTION__ = "lval_assign"
#6  0x000000000041cf14 in do_action (h=0x7fffe1b8f710, a=0x7fb2bf8332b0, msg=0x7fb2bf8c7a00) at action.c:1453
        ret = -5
        v = <optimized out>
        dst = {send_sock = 0x7fb2bdb0f080, to = {s = {sa_family = 0, 
              sa_data = "\000\000\000\000\000\000\200-\210\000\000\000\000"}, sin = {sin_family = 0, sin_port = 0, 
              sin_addr = {s_addr = 0}, sin_zero = "\200-\210\000\000\000\000"}, sin6 = {sin6_family = 0, 
              sin6_port = 0, sin6_flowinfo = 0, sin6_addr = {__in6_u = {
                  __u6_addr8 = "\200-\210\000\000\000\000\000\016\000\000\000\000\000\000", __u6_addr16 = {11648, 
                    136, 0, 0, 14, 0, 0, 0}, __u6_addr32 = {8924544, 0, 14, 0}}}, sin6_scope_id = 13}}, id = 0, 
          proto = 58 ':', send_flags = {f = 180 '\264', blst_imask = 138 '\212'}}
        tmp = <optimized out>
        new_uri = <optimized out>
        end = <optimized out>
        crt = <optimized out>
        cmd = <optimized out>
        len = <optimized out>
        user = <optimized out>
        uri = {user = {s = 0x7fffe1b8eef0 "/", len = 0}, passwd = {
            s = 0x3000000010 <Address 0x3000000010 out of bounds>, len = -507973920}, host = {
            s = 0x7fffe1b8ee20 "", len = 17022112}, port = {s = 0x7fb2bffbd720 "", len = 8208}, params = {
            s = 0x2010 <Address 0x2010 out of bounds>, len = 17030320}, sip_params = {
            s = 0x2010 <Address 0x2010 out of bounds>, len = -1077387804}, headers = {
            s = 0x7fb2bffbdfb0 "0z\001\001", len = -507974144}, port_no = 55072, proto = 49147, type = 32690, 
          flags = 17022128, transport = {s = 0x95 <Address 0x95 out of bounds>, len = -1241200896}, ttl = {
            s = 0x3734000000a0 <Address 0x3734000000a0 out of bounds>, len = -1081931184}, user_param = {s = 0x0, 
            len = 0}, maddr = {s = 0x63371f "", len = -1}, method = {s = 0x7fb2b519cb30 "\220\240H\265\262\177", 
            len = 4310469}, lr = {s = 0x3000000018 <Address 0x3000000018 out of bounds>, len = -507973440}, r2 = {
            s = 0x7fffe1b8f000 "\210u\223", len = 8187}, gr = {s = 0x2 <Address 0x2 out of bounds>, len = 1}, 
          transport_val = {s = 0x1 <Address 0x1 out of bounds>, len = 0}, ttl_val = {
            s = 0xffffffffffffffa0 <Address 0xffffffffffffffa0 out of bounds>, len = 17022112}, user_param_val = {
            s = 0x7fb2bffbd720 "", len = -507974064}, maddr_val = {s = 0x7fb2bf8c7a00 "\016", len = -1081931184}, 
          method_val = {s = 0x2010 <Address 0x2010 out of bounds>, len = -507971824}, lr_val = {s = 0x0, 
            len = -507971824}, r2_val = {s = 0x7fffe1b8f580 "\001", len = -1}, gr_val = {
            s = 0x7fb2b519cb30 "\220\240H\265\262\177", len = 4346308}}
        next_hop = {user = {s = 0x0, len = 1}, passwd = {s = 0x7fffe1b8f710 "\002", len = 1237187532}, host = {
            s = 0x7fffe1b8f710 "\002", len = -507972224}, port = {
            s = 0xffffffff <Address 0xffffffff out of bounds>, len = -1256600784}, params = {
            s = 0x3542f9ae92fdfbcc <Address 0x3542f9ae92fdfbcc out of bounds>, len = -589431860}, sip_params = {
            s = 0x0, len = 17022128}, headers = {s = 0x15 <Address 0x15 out of bounds>, len = 1}, port_no = 0, 
          proto = 0, type = ERROR_URI_T, flags = 4294967200, transport = {s = 0x7fffe1b8efe8 "l\345N", 
            len = -1076936721}, ttl = {s = 0x1 <Address 0x1 out of bounds>, len = -1114716768}, user_param = {
            s = 0x1c0000002f <Address 0x1c0000002f out of bounds>, len = 7}, maddr = {
            s = 0x710000000a <Address 0x710000000a out of bounds>, len = 2}, method = {s = 0x0, len = -18000}, 
          lr = {s = 0x1017a30 "EST", len = 6323844}, r2 = {s = 0x7fb2bd8ed356 "tm [t_lookup.c:716]: ", 
            len = -507972224}, gr = {s = 0x7fffe1b8f710 "\002", len = 5148368}, transport_val = {s = 0x0, 
            len = 0}, ttl_val = {s = 0x7fb2bf8c7a00 "\016", len = -507973348}, user_param_val = {
            s = 0x103bcb0 "<135>Nov 19 07:28:47 /usr/local/sbin/kamailio[15556]: DEBUG: dialog [dlg_hash.c:602]: dlg_lookup(): dialog id=7094 found on entry 2049\n", len = 0}, maddr_val = {
            s = 0x103bcb0 "<135>Nov 19 07:28:47 /usr/local/sbin/kamailio[15556]: DEBUG: dialog [dlg_hash.c:602]: dlg_lookup(): dialog id=7094 found on entry 2049\n", len = -1115163744}, method_val = {
            s = 0x99 <Address 0x99 out of bounds>, len = 1384864127}, lr_val = {s = 0x882d80 "", 
            len = -1081312768}, r2_val = {s = 0x7fb2bdb0f080 "\016", len = 8924544}, gr_val = {
            s = 0xffffffff <Address 0xffffffff out of bounds>, len = -1081930768}}
        u = <optimized out>
        port = <optimized out>
        dst_host = <optimized out>
        i = <optimized out>
        flags = <optimized out>
        avp = <optimized out>
        st = {flags = 3213654528, id = 32690, name = {n = -507971824, s = {s = 0x7fffe1b8f710 "\002", 
              len = -507972224}, re = 0x7fffe1b8f710}, avp = 0x4ee56c}
        sct = <optimized out>
        sjt = <optimized out>
        rve = <optimized out>
        mct = <optimized out>
        rv = <optimized out>
        rv1 = <optimized out>
        c1 = {cache_type = 3213020352, val_type = 32690, c = {avp_val = {n = 0, s = {
                s = 0x7fb200000000 <Address 0x7fb200000000 out of bounds>, len = -1081312768}, 
              re = 0x7fb200000000}, pval = {rs = {s = 0x7fb200000000 <Address 0x7fb200000000 out of bounds>, 
                len = -1081312768}, ri = -1124131338, flags = 32690}}, 
          i2s = "\377\377\377\377\060", '\000' <repeats 16 times>}
        s = {s = 0x7fffe1b8f0d8 "\260\062\203\277\262\177", len = -507973632}
        srevp = {0x937588, 0x7fb2bfc4f1fc}
        mod_f_params = {{type = NOSUBTYPE, u = {number = 0, string = 0x0, str = {s = 0x0, len = 0}, data = 0x0, 
              attr = 0x0, select = 0x0}}, {type = NOSUBTYPE, u = {number = 0, string = 0x0, str = {s = 0x0, 
                len = 0}, data = 0x0, attr = 0x0, select = 0x0}}, {type = NOSUBTYPE, u = {number = 0, 
              string = 0x0, str = {s = 0x0, len = 0}, data = 0x0, attr = 0x0, select = 0x0}}, {type = NOSUBTYPE, 
            u = {number = 0, string = 0x0, str = {s = 0x0, len = 0}, data = 0x0, attr = 0x0, select = 0x0}}, {
            type = NOSUBTYPE, u = {number = 0, string = 0x0, str = {s = 0x0, len = 0}, data = 0x0, attr = 0x0, 
              select = 0x0}}, {type = NOSUBTYPE, u = {number = 0, string = 0x0, str = {s = 0x0, len = 0}, 
              data = 0x0, attr = 0x0, select = 0x0}}, {type = NOSUBTYPE, u = {number = 0, string = 0x0, str = {
                s = 0x0, len = 0}, data = 0x0, attr = 0x0, select = 0x0}}, {type = NOSUBTYPE, u = {number = 0, 
              string = 0x0, str = {s = 0x0, len = 0}, data = 0x0, attr = 0x0, select = 0x0}}}
        __FUNCTION__ = "do_action"
#7  0x000000000041c5c5 in run_actions (h=0x7fffe1b8f710, a=0x7fb2bf8313a0, msg=0x7fb2bf8c7a00) at action.c:1573
        t = <optimized out>
        ret = <optimized out>
        mod = <optimized out>
        ms = 0
        __FUNCTION__ = "run_actions"
#8  0x000000000041d92e in do_action (h=0x7fffe1b8f710, a=0x7fb2bf8396d0, msg=0x7fb2bf8c7a00) at action.c:1065
        ret = 1
        v = 1
        dst = {send_sock = 0x0, to = {s = {sa_family = 0, 
              sa_data = "\000\000\001\000\000\000\000\000\000\000\000\000\000"}, sin = {sin_family = 0, 
              sin_port = 0, sin_addr = {s_addr = 1}, sin_zero = "\000\000\000\000\000\000\000"}, sin6 = {
              sin6_family = 0, sin6_port = 0, sin6_flowinfo = 1, sin6_addr = {__in6_u = {
                  __u6_addr8 = "\000\000\000\000\000\000\000\000\001\000\000\000\000\000\000", __u6_addr16 = {0, 
                    0, 0, 0, 1, 0, 0, 0}, __u6_addr32 = {0, 0, 1, 0}}}, sin6_scope_id = 0}}, id = 0, 
          proto = -80 '\260', send_flags = {f = 188 '\274', blst_imask = 3 '\003'}}
        tmp = <optimized out>
        new_uri = <optimized out>
        end = <optimized out>
        crt = <optimized out>
        cmd = <optimized out>
        len = <optimized out>
        user = <optimized out>
        uri = {user = {s = 0x3635353531000002 <Address 0x3635353531000002 out of bounds>, len = 1}, passwd = {
            s = 0x0, len = 17022128}, host = {s = 0x7fffe1b8f460 "/", len = 0}, port = {
            s = 0x3000000010 <Address 0x3000000010 out of bounds>, len = -507972528}, params = {
            s = 0x7fffe1b8f390 "\260\337\373\277\262\177", len = -1076936721}, sip_params = {
            s = 0xa2 <Address 0xa2 out of bounds>, len = -1114690816}, headers = {
            s = 0x1c0000002f <Address 0x1c0000002f out of bounds>, len = 7}, port_no = 10, proto = 0, type = 113, 
          flags = URI_SIP_USER_PHONE, transport = {s = 0x0, len = -18000}, ttl = {s = 0x1017a30 "EST", 
            len = -1076935806}, user_param = {s = 0x7fb2b5531d90 "P\213\065\265\262\177", len = 48}, maddr = {
            s = 0x3734e1b8f3b8 <Address 0x3734e1b8f3b8 out of bounds>, len = -507972896}, method = {
            s = 0x3531320000000002 <Address 0x3531320000000002 out of bounds>, len = -1076363582}, lr = {
            s = 0x103bcbc "07:28:47 /usr/local/sbin/kamailio[15556]: DEBUG: dialog [dlg_hash.c:602]: dlg_lookup(): dialog id=7094 found on entry 2049\n", len = 17022140}, r2 = {s = 0x7fb2bffbb3c0 "", len = 7}, gr = {
            s = 0x3000000018 <Address 0x3000000018 out of bounds>, len = -507972048}, transport_val = {
            s = 0x7fffe1b8f570 "\001", len = 8187}, ttl_val = {s = 0x2 <Address 0x2 out of bounds>, len = 1}, 
          user_param_val = {s = 0x1 <Address 0x1 out of bounds>, len = 0}, maddr_val = {
            s = 0xffffffffffffffa0 <Address 0xffffffffffffffa0 out of bounds>, len = 17022112}, method_val = {
            s = 0x7fb2bffbd720 "", len = 8208}, lr_val = {s = 0x2010 <Address 0x2010 out of bounds>, 
            len = 17030320}, r2_val = {s = 0x2010 <Address 0x2010 out of bounds>, len = -1077387804}, gr_val = {
            s = 0x7fb2bffbdfb0 "0z\001\001", len = -1077164088}}
        next_hop = {user = {s = 0x7fb2bffbd720 "", len = 8180}, passwd = {
            s = 0x3635353531000002 <Address 0x3635353531000002 out of bounds>, len = 17022112}, host = {s = 0x0, 
            len = -1077381378}, port = {s = 0x7fffe1b8f5f0 "", len = 0}, params = {
            s = 0x3000000010 <Address 0x3000000010 out of bounds>, len = -507972128}, sip_params = {
            s = 0x7fffe1b8f520 "\002", len = -1077164088}, headers = {s = 0x0, len = 8180}, port_no = 2, 
          proto = 12544, type = 909456693, flags = 17022128, transport = {s = 0x0, len = 1}, ttl = {
            s = 0x7fffe1b8f650 "0\313\031\265\262\177", len = 0}, user_param = {
            s = 0x3000000010 <Address 0x3000000010 out of bounds>, len = -507972032}, maddr = {
            s = 0x7fffe1b8f580 "\001", len = 6526864}, method = {
            s = 0x37340000002f <Address 0x37340000002f out of bounds>, len = 7}, lr = {
            s = 0x320000710000000a <Address 0x320000710000000a out of bounds>, len = -1076363582}, r2 = {
            s = 0x103bcbc "07:28:47 /usr/local/sbin/kamailio[15556]: DEBUG: dialog [dlg_hash.c:602]: dlg_lookup(): dialog id=7094 found on entry 2049\n", len = 17022140}, gr = {s = 0x7fb2bffbb3c0 "", len = 7}, transport_val = {
            s = 0x3000000018 <Address 0x3000000018 out of bounds>, len = -507971648}, ttl_val = {
            s = 0x7fffe1b8f700 "0\313\031\265\262\177", len = 8187}, user_param_val = {
            s = 0x373400000002 <Address 0x373400000002 out of bounds>, len = 1}, maddr_val = {
            s = 0x1 <Address 0x1 out of bounds>, len = -1076363582}, method_val = {
            s = 0x103bcbc "07:28:47 /usr/local/sbin/kamailio[15556]: DEBUG: dialog [dlg_hash.c:602]: dlg_lookup(): dialog id=7094 found on entry 2049\n", len = 17022140}, lr_val = {s = 0x7fb2bffbb3c0 "", len = 7}, r2_val = {
            s = 0x3000000018 <Address 0x3000000018 out of bounds>, len = -507971552}, gr_val = {
            s = 0x7fffe1b8f760 "", len = 8187}}
        u = <optimized out>
        port = <optimized out>
        dst_host = <optimized out>
        i = <optimized out>
        flags = <optimized out>
        avp = <optimized out>
        st = {flags = 2, id = 0, name = {n = 1, s = {s = 0x1 <Address 0x1 out of bounds>, len = 1}, re = 0x1}, 
          avp = 0x0}
        sct = <optimized out>
        sjt = <optimized out>
        rve = <optimized out>
        mct = <optimized out>
        rv = <optimized out>
        rv1 = <optimized out>
        c1 = {cache_type = 137, val_type = RV_NONE, c = {avp_val = {n = 17022112, s = {s = 0x103bca0 "`\002", 
                len = 160}, re = 0x103bca0}, pval = {rs = {s = 0x103bca0 "`\002", len = 160}, ri = -1077381378, 
              flags = 32690}}, 
          i2s = " \327\373\277\262\177\000\000\000\000\000\000\001\000\000\000\340\366\270\341\377\177"}
        s = {s = 0xffffffffffffffa0 <Address 0xffffffffffffffa0 out of bounds>, len = 17022112}
        srevp = {0x7fb2bffbd720, 0x2010}
        mod_f_params = {{type = NOSUBTYPE, u = {number = 0, string = 0x0, str = {s = 0x0, len = 0}, data = 0x0, 
              attr = 0x0, select = 0x0}}, {type = NOSUBTYPE, u = {number = 0, string = 0x0, str = {s = 0x0, 
                len = 0}, data = 0x0, attr = 0x0, select = 0x0}}, {type = NOSUBTYPE, u = {number = 0, 
              string = 0x0, str = {s = 0x0, len = 0}, data = 0x0, attr = 0x0, select = 0x0}}, {type = NOSUBTYPE, 
            u = {number = 0, string = 0x0, str = {s = 0x0, len = 0}, data = 0x0, attr = 0x0, select = 0x0}}, {
            type = NOSUBTYPE, u = {number = 0, string = 0x0, str = {s = 0x0, len = 0}, data = 0x0, attr = 0x0, 
              select = 0x0}}, {type = NOSUBTYPE, u = {number = 0, string = 0x0, str = {s = 0x0, len = 0}, 
              data = 0x0, attr = 0x0, select = 0x0}}, {type = NOSUBTYPE, u = {number = 0, string = 0x0, str = {
                s = 0x0, len = 0}, data = 0x0, attr = 0x0, select = 0x0}}, {type = NOSUBTYPE, u = {number = 0, 
              string = 0x0, str = {s = 0x0, len = 0}, data = 0x0, attr = 0x0, select = 0x0}}}
        __FUNCTION__ = "do_action"
#9  0x000000000041c5c5 in run_actions (h=0x7fffe1b8f710, a=0x7fb2bf82d4c0, msg=0x7fb2bf8c7a00) at action.c:1573
        t = <optimized out>
        ret = <optimized out>
        mod = <optimized out>
        ms = 0#0  qm_detach_free (frag=0x7fb2b552e2b0, qm=<optimized out>) at mem/q_malloc.c:269
        prev = 0x7fb2b5192058
        next = 0x0

... cut ....

Kelvin Chua


On Tue, Nov 19, 2013 at 9:40 PM, Daniel-Constantin Mierla <miconda@gmail.com> wrote:
Hello,

can you get the output for 'bt full'? Probably you have to install the debug symbols (kamailio-dbg package) for getting something useful.

Another question, can you reproduce it? Or it happens sporadically?

Cheers,
Daniel




On 11/19/13 1:11 PM, Kelvin Chua wrote:
kamailio 4.0.4
has anybody experience this using dialog module?

backtrace:
#0  0x000000000057af45 in qm_malloc ()
#1  0x00007f268ac0fe93 in set_dlg_variable_unsafe () from /usr/local/lib64/kamailio/modules/dialog.so
#2  0x00007f268ac123aa in pv_set_dlg_variable () from /usr/local/lib64/kamailio/modules/dialog.so
#3  0x000000000047be59 in lval_assign ()
#4  0x000000000041cf14 in do_action ()
#5  0x000000000041c5c5 in run_actions ()
#6  0x000000000041d92e in do_action ()
#7  0x000000000041c5c5 in run_actions ()
#8  0x0000000000425250 in run_top_route ()
#9  0x00000000004bfcce in receive_msg ()
#10 0x000000000056a2df in udp_rcv_loop ()
#11 0x0000000000481b43 in main_loop ()
#12 0x000000000041c081 in main ()

logfile:
DEBUG: <core> [parser/msg_parser.c:623]: parse_msg(): SIP Request:
DEBUG: <core> [parser/msg_parser.c:625]: parse_msg():  method:  <ACK>
DEBUG: <core> [parser/msg_parser.c:627]: parse_msg():  uri:     <sip:+1XX882XX111@2XX.XX1.X9.2X>
DEBUG: <core> [parser/msg_parser.c:629]: parse_msg():  version: <SIP/2.0>
DEBUG: <core> [parser/parse_via.c:1284]: parse_via_param(): Found param type 232, <branch> = <z9hG4bKe97b.5a4397f3.0>; state=16
DEBUG: <core> [parser/parse_via.c:2672]: parse_via(): end of header reached, state=5
DEBUG: <core> [parser/msg_parser.c:513]: parse_headers(): parse_headers: Via found, flags=2
DEBUG: <core> [parser/msg_parser.c:515]: parse_headers(): parse_headers: this is the first via
DEBUG: <core> [receive.c:151]: receive_msg(): After parse_msg...
DEBUG: <core> [receive.c:192]: receive_msg(): preparing to run routing scripts...
DEBUG: <core> [parser/parse_to.c:176]: parse_to_param(): DEBUG: add_param: tag=bbd932f8f9dbf9743f9b7cadcbf622ac.0dc7
DEBUG: <core> [parser/parse_to.c:799]: parse_to(): end of header reached, state=29
DEBUG: <core> [parser/msg_parser.c:190]: get_hdr_field(): DEBUG: get_hdr_field: <To> [74]; uri=[sip:+1XX882XX111@6X.X31.X.8X]
DEBUG: <core> [parser/msg_parser.c:192]: get_hdr_field(): DEBUG: to body [<sip:+1XX882XX111@6X.X31.X.8X>]
DEBUG: sl [sl_funcs.c:415]: sl_filter_ACK(): SL local ACK found -> dropping it!
DEBUG: <core> [usr_avp.c:644]: destroy_avp_list(): DEBUG:destroy_avp_list: destroying list (nil)
last message repeated 5 times
DEBUG: <core> [xavp.c:447]: xavp_destroy_list(): destroying xavp list (nil)
DEBUG: <core> [receive.c:295]: receive_msg(): receive_msg: cleaning up
segfault at 0 ip 000000000057af45 sp 00007fff30e44ac0 error 4 in kamailio[400000+27c000]

Kelvin Chua


_______________________________________________
SIP Express Router (SER) and Kamailio (OpenSER) - sr-users mailing list
sr-users@lists.sip-router.org
http://lists.sip-router.org/cgi-bin/mailman/listinfo/sr-users


-- 
Daniel-Constantin Mierla - http://www.asipto.com
http://twitter.com/#!/miconda - http://www.linkedin.com/in/miconda
Kamailio Advanced Trainings - Berlin, Nov 25-28
  - more details about Kamailio trainings at http://www.asipto.com -

_______________________________________________
SIP Express Router (SER) and Kamailio (OpenSER) - sr-users mailing list
sr-users@lists.sip-router.org
http://lists.sip-router.org/cgi-bin/mailman/listinfo/sr-users