Re: [SR-Users] Segmentaion fault in Kamailio 4.0.3

Hi, We upgraded to 4.0.4 and now we got a new core dump in q_malloc.c:269. It has been stable for about 5 days after the upgrade. /Morten The full backtrace is here: Loaded symbols for /lib64/libnss_files.so.2 Core was generated by `/usr/local/sbin/kamailio -P /var/run/kamailio/kamailio.pid -m 256 -M 8 -u kamai'. Program terminated with signal 11, Segmentation fault. #0 qm_detach_free (qm=0x7f5394de9000, size=0) at mem/q_malloc.c:269 269 FRAG_END(next)->prev_free=prev; Missing separate debuginfos, use: debuginfo-install glibc-2.12-1.107.el6.x86_64 hiredis-0.10.1-3.el6.x86_64 keyutils-libs-1.4-4.el6.x86_64 krb5-libs-1.10.3-10.el6.x86_64 libcom_err-1.41.12-14.el6.x86_64 libselinux-2.0.94-5.3.el6.x86_64 libxml2-2.7.6-12.el6_4.1.x86_64 mysql-libs-5.1.67-1.el6_3.x86_64 nss-softokn-freebl-3.12.9-11.el6.x86_64 openssl-1.0.0-27.el6_4.2.x86_64 zlib-1.2.3-29.el6.x86_64 (gdb) bt full #0 qm_detach_free (qm=0x7f5394de9000, size=0) at mem/q_malloc.c:269 prev = 0x7f539524ed60 next = 0x0 #1 qm_malloc (qm=0x7f5394de9000, size=0) at mem/q_malloc.c:386 f = 0x7f539524ed60 hash = 0 #2 0x00007f53a5662950 in shm_malloc (size=0) at ../../mem/shm_mem.h:262 p = <value optimized out> #3 0x00007f53a566b49b in dlg_set_leg_info (dlg=<value optimized out>, tag=0x7ffff680b9a0, rr=0x7ffff680b860, contact=0x7ffff680b870, cseq=0x7ffff680b880, leg=1) at dlg_hash.c:498 p = <value optimized out> __FUNCTION__ = "dlg_set_leg_info" #4 0x00007f53a5659e88 in populate_leg_info (dlg=0x7f53952312b0, msg=<value optimized out>, t=<value optimized out>, leg=1, tag=0x7ffff680b9a0) at dlg_handlers.c:276 skip_recs = 4 cseq = {s = 0x0, len = 0} contact = { s = 0x8da2c0 "sip:65993777@178.21.249.22:5060 <http://sip:65993777@178.21.249.22:5060>>\r\nContent-Type: application/sdp\r\nRequire: timer\r\nContent-Length: 252\r\n\r\nv=0\r\no=- 2042480911 2042480911 IN IP4 178.21.249.22\r\ns=Hosted PBX\r\nc=IN IP4 178.21.249.22\r\nt=0 0"..., len = 31} rr_set = {s = 0x7f53ab927c20 <incomplete sequence \361\237\215>, len = 0} __FUNCTION__ = "populate_leg_info" #5 0x00007f53a565b949 in dlg_onreply (t=0x7f5395245030, type=<value optimized out>, param=<value optimized out>) at dlg_handlers.c:500 dlg = 0x7f53952312b0 iuid = <value optimized out> new_state = 3 old_state = 1 unref = 0 event = 3 tag = { s = 0x8da1bc "as77c791c9\r\nCall-ID: 0dd71e51-ba5d-1231-c79f-00095200eb1e\r\nCSeq: 1206140 INVITE\r\nServer: one-connect\r\nAllow: INVITE, ACK, CANCEL, OPTIONS, BYE, REFER, SUBSCRIBE, NOTIFY, INFO, PUBLISH\r\nSupported: repl"..., len = 10} req = 0x7f5395640090 rpl = 0x7f53ab8cd7a0 __FUNCTION__ = "dlg_onreply" #6 0x00007f53aa1f58ad in run_trans_callbacks_internal (cb_lst=<value optimized out>, type=1048576, trans=0x7f5395245030, params=0x7ffff680baa0) at t_hooks.c:290 cbp = 0x7f5395311260 backup_from = 0x8e9ed0 backup_to = 0x8e9ed8 backup_dom_from = 0x8e9ee0 backup_dom_to = 0x8e9ee8 backup_uri_from = 0x8e9ec0 backup_uri_to = 0x8e9ec8 backup_xavps = 0x8e9fd0 __FUNCTION__ = "run_trans_callbacks_internal" #7 0x00007f53aa1f5a94 in run_trans_callbacks_with_buf (type=<value optimized out>, rbuf=0x7f53952450f0, req=<value optimized out>, repl=<value optimized out>, flags=<value optimized out>) at t_hooks.c:336 params = {req = 0x7f5395640090, rpl = 0x7f53ab8cd7a0, param = 0x7f5395311270, code = 200, flags = 200, branch = 0, t_rbuf = 0x7f53952450f0, dst = 0x7f5395245140, send_buf = { s = 0x7f53953ac460 "SIP/2.0 200 OK\r\nVia: SIP/2.0/UDP 178.21.248.30;rport=5060;branch=z9hG4bKd24f.32134635.0\r\nVia: SIP/2.0/UDP 178.21.248.8;rport=5060;branch=z9hG4bKd24f.b20f4bb3.0\r\nVia: SIP/2.0/UDP 178.21.248.20;branch=z"..., len = 1341}} trans = <value optimized out> #8 0x00007f53aa22728c in relay_reply (t=<value optimized out>, p_msg=<value optimized out>, branch=0, msg_status=200, cancel_data=0x7ffff680be40, do_put_on_wait=1) at t_reply.c:1867 relay = 0 save_clone = 0 buf = 0x7f53ab9361a0 "SIP/2.0 200 OK\r\nVia: SIP/2.0/UDP 178.21.248.30;rport=5060;branch=z9hG4bKd24f.32134635.0\r\nVia: SIP/2.0/UDP 178.21.248.8;rport=5060;branch=z9hG4bKd24f.b20f4bb3.0\r\nVia: SIP/2.0/UDP 178.21.248.20;branch=z"... res_len = 1341 relayed_code = 200 relayed_msg = 0x7f53ab8cd7a0 reply_bak = <value optimized out> bm = {to_tag_val = {s = 0x7f5395600c50 "P\233\365\224S\177", len = -1440728525}} totag_retr = 0 reply_status = RPS_COMPLETED uas_rb = <value optimized out> to_tag = <value optimized out> reason = {s = 0x3 <Address 0x3 out of bounds>, len = -159334788} onsend_params = {req = 0x7f53aa45e594, rpl = 0x7ffff680be78, param = 0x0, code = 0, flags = 0, branch = 0, t_rbuf = 0x10, dst = 0x7f53aa200a91, send_buf = {s = 0x7f5395601f38 "", len = -1440601726}} ---Type <return> to continue, or q <return> to quit--- __FUNCTION__ = "relay_reply" #9 0x00007f53aa227da6 in reply_received (p_msg=0x7f53ab8cd7a0) at t_reply.c:2374 msg_status = 200 last_uac_status = 100 ack = 0x1 <Address 0x1 out of bounds> ack_len = 0 branch = 0 reply_status = <value optimized out> onreply_route = <value optimized out> cancel_data = {cancel_bitmap = 0, reason = {cause = 200, u = {text = {s = 0x0, len = -1416571248}, e2e_cancel = 0x0, packed_hdrs = {s = 0x0, len = -1416571248}}}} uac = 0x7f5395245198 t = 0x7f5395245030 lack_dst = {send_sock = 0x0, to = {s = {sa_family = 48720, sa_data = "\200\366\377\177\000\000\000\000\000\000\000\000\000"}, sin = {sin_family = 48720, sin_port = 63104, sin_addr = { s_addr = 32767}, sin_zero = "\000\000\000\000\000\000\000"}, sin6 = {sin6_family = 48720, sin6_port = 63104, sin6_flowinfo = 32767, sin6_addr = {__in6_u = { __u6_addr8 = "\000\000\000\000\000\000\000\000\260kq\253S\177\000", __u6_addr16 = {0, 0, 0, 0, 27568, 43889, 32595, 0}, __u6_addr32 = {0, 0, 2876337072, 32595}}}, sin6_scope_id = 2878134176}}, id = 32595, proto = -76 '\264', send_flags = {f = 5 '\005', blst_imask = 0 '\000'}} backup_user_from = <value optimized out> backup_user_to = <value optimized out> backup_domain_from = <value optimized out> backup_domain_to = <value optimized out> backup_uri_from = <value optimized out> backup_uri_to = <value optimized out> backup_xavps = <value optimized out> replies_locked = <value optimized out> branch_ret = <value optimized out> prev_branch = <value optimized out> blst_503_timeout = <value optimized out> hf = <value optimized out> onsend_params = {req = 0x8da424, rpl = 0x543274, param = 0x10, code = -1416833120, flags = 32595, branch = 0, t_rbuf = 0x7ffff680be50, dst = 0x7f53a566690b, send_buf = { s = 0x4ff1e7b00000000 <Address 0x4ff1e7b00000000 out of bounds>, len = 0}} ctx = {rec_lev = 0, run_flags = 0, last_retcode = 1, jmp_env = {{__jmpbuf = {140737329020144, -5577311537021373234, 139997336902320, 139997337147296, 200, 139996961202968, 5577292203380126926, -5577312076254333746}, __mask_was_saved = 0, __saved_mask = {__val = {139997337420128, 9281488, 139997335445253, 1, 140737329020496, 6191275, 5978073, 8591616, 9281252, 68728758110, 5548121, 9281656, 9281173, 139997337409168, 9282596, 139997337519904}}}}} __FUNCTION__ = "reply_received" #10 0x0000000000456d54 in do_forward_reply (msg=0x7f53ab8cd7a0, mode=<value optimized out>) at forward.c:799 new_buf = 0x0 dst = {send_sock = 0x0, to = {s = {sa_family = 0, sa_data = '\000' <repeats 13 times>}, sin = {sin_family = 0, sin_port = 0, sin_addr = {s_addr = 0}, sin_zero = "\000\000\000\000\000\000\000"}, sin6 = {sin6_family = 0, sin6_port = 0, sin6_flowinfo = 0, sin6_addr = {__in6_u = {__u6_addr8 = '\000' <repeats 15 times>, __u6_addr16 = {0, 0, 0, 0, 0, 0, 0, 0}, __u6_addr32 = {0, 0, 0, 0}}}, sin6_scope_id = 0}}, id = 0, proto = 0 '\000', send_flags = {f = 0 '\000', blst_imask = 0 '\000'}} new_len = <value optimized out> r = <value optimized out> s = <value optimized out> len = <value optimized out> __FUNCTION__ = "do_forward_reply" #11 0x000000000049f4be in receive_msg ( buf=0x8d9e80 "SIP/2.0 200 OK\r\nVia: SIP/2.0/UDP 178.21.249.20;branch=z9hG4bKd24f.20eeb764.0;received=178.21.249.20;rport=5060\r\nVia: SIP/2.0/UDP 178.21.248.30;rport=5060;branch=z9hG4bKd24f.32134635.0\r\nVia: SIP/2.0/UD"..., len=<value optimized out>, rcv_info=0x7ffff680c100) at receive.c:272 msg = 0x7f53ab8cd7a0 ctx = {rec_lev = 12, run_flags = 0, last_retcode = -1517756399, jmp_env = {{__jmpbuf = {139996960369776, 12, 219309716216, 139997343567872, 140737329021424, 4294967295, 140737329021551, 1}, __mask_was_saved = 8581896, __saved_mask = {__val = {0, 28, 16, 0, 219305533392, 1, 0, 139997335309296, 219309716216, 139996960369776, 139997337398864, 139997343565480, 139997337398872, 140737329021320, 219305559701, 140737329021192}}}}} ret = <value optimized out> inb = { s = 0x8d9e80 "SIP/2.0 200 OK\r\nVia: SIP/2.0/UDP 178.21.249.20;branch=z9hG4bKd24f.20eeb764.0;received=178.21.249.20;rport=5060\r\nVia: SIP/2.0/UDP 178.21.248.30;rport=5060;branch=z9hG4bKd24f.32134635.0\r\nVia: SIP/2.0/UD"..., len = 1444} __FUNCTION__ = "receive_msg" #12 0x0000000000532306 in udp_rcv_loop () at udp_server.c:557 len = 1444 buf = "SIP/2.0 200 OK\r\nVia: SIP/2.0/UDP 178.21.249.20;branch=z9hG4bKd24f.20eeb764.0;received=178.21.249.20;rport=5060\r\nVia: SIP/2.0/UDP 178.21.248.30;rport=5060;branch=z9hG4bKd24f.32134635.0\r\nVia: SIP/2.0/UD"... from = 0x7f53ab90ae50 fromlen = 16 ri = {src_ip = {af = 2, len = 4, u = {addrl = {385422770, 0}, addr32 = {385422770, 0, 0, 0}, addr16 = {5554, 5881, 0, 0, 0, 0, 0, 0}, addr = "\262\025\371\026", '\000' <repeats 11 times>}}, ---Type <return> to continue, or q <return> to quit--- dst_ip = {af = 2, len = 4, u = {addrl = {351868338, 0}, addr32 = {351868338, 0, 0, 0}, addr16 = {5554, 5369, 0, 0, 0, 0, 0, 0}, addr = "\262\025\371\024", '\000' <repeats 11 times>}}, src_port = 5060, dst_port = 5060, proto_reserved1 = 0, proto_reserved2 = 0, src_su = {s = {sa_family = 2, sa_data = "\023IJ\025\371\026\000\000\000\000\000\000\000"}, sin = {sin_family = 2, sin_port = 50195, sin_addr = {s_addr = 385422770}, sin_zero = "\000\000\000\000\000\000\000"}, sin6 = {sin6_family = 2, sin6_port = 50195, sin6_flowinfo = 385422770, sin6_addr = {__in6_u = { __u6_addr8 = '\000' <repeats 15 times>, __u6_addr16 = {0, 0, 0, 0, 0, 0, 0, 0}, __u6_addr32 = {0, 0, 0, 0}}}, sin6_scope_id = 0}}, bind_address = 0x7f53ab8cec10, proto = 1 '\001'} __FUNCTION__ = "udp_rcv_loop" #13 0x0000000000467a7a in main_loop () at main.c:1638 i = <value optimized out> pid = <value optimized out> si = <value optimized out> si_desc = "udp receiver child=3 sock=178.21.249.20:5060 <http://178.21.249.20:5060>\000\000\000\000@\\\220\253S\177\000\000\000\000\000\000\000\000\000\000\003\000\000\000\000\000\000\000\001\000\000\000\001\000\000\000\300\375\216\000\000\000\000\000\001\000\000\000\000\000\000\000\000\376\216\000\000\000\000\000\000\000\200\020", '\000' <repeats 12 times>, "\005\000\000\000\000\000\000" nrprocs = <value optimized out> __FUNCTION__ = "main_loop" #14 0x000000000046a912 in main (argc=<value optimized out>, argv=<value optimized out>) at main.c:2566 cfg_stream = <value optimized out> c = <value optimized out> r = <value optimized out> tmp = 0x7ffff680d89a "" tmp_len = 0 options = 0x5c1de8 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:" ret = -1 seed = 1043769636 rfd = <value optimized out> debug_save = <value optimized out> debug_flag = <value optimized out> dont_fork_cnt = <value optimized out> n_lst = <value optimized out> p = <value optimized out> __FUNCTION__ = "main" On Mon, Oct 21, 2013 at 8:15 PM, Daniel-Constantin Mierla &lt;miconda(a)gmail.com <mailto:miconda@gmail.com>> wrote: Hello, can you upgrade to 4.0.4? there were some fixes related to new fields in sip message structure that resulted in some out bound memory inside transactions. The config file and database is the same, so just re-install over the previous installation. Cheers, Daniel On 10/21/13 11:10 AM, Morten Isaksen wrote: -- Daniel-Constantin Mierla -http://www.asipto.com http://twitter.com/#!/miconda <http://twitter.com/#%21/miconda> -http://www.linkedin.com/in/miconda Kamailio Advanced Trainings - Berlin, Nov 25-28; Miami, Nov 18-20, 2013 - more details about Kamailio trainings athttp://www.asipto.com - _______________________________________________ SIP Express Router (SER) and Kamailio (OpenSER) - sr-users mailing list sr-users(a)lists.sip-router.org <mailto:sr-users@lists.sip-router.org> http://lists.sip-router.org/cgi-bin/mailman/listinfo/sr-users -- Morten Isaksen _______________________________________________ SIP Express Router (SER) and Kamailio (OpenSER) - sr-users mailing list sr-users(a)lists.sip-router.org http://lists.sip-router.org/cgi-bin/mailman/listinfo/sr-users