[sr-dev] [kamailio/kamailio] Crashing Kamailio: store_reply. replacing stored reply (#1744)

29 Nov 2018


      ### Description
Did a parallel fork (added two destinations using `append_branch` with the same qvalue). Looks like something is wrong with "store_reply" function:
```
"store_reply(): ERROR: replacing stored reply; aborting"
```
### Troubleshooting
#### Reproduction
Randomly happening when executing the SIP flow attached.
#### Debugging Data
```
(gdb) bt
#0  0x00007f700eb4c4f5 in raise (sig=6) at ../nptl/sysdeps/unix/sysv/linux/raise.c:64
#1  0x00007f700eb4dcd5 in abort () at abort.c:92
#2  0x00007f6ffad16087 in store_reply (trans=0x7f6fb44b3d38, branch=0, rpl=0x7f6ffeca4118) at t_reply.c:1601
#3  0x00007f6ffad17275 in relay_reply (t=0x7f6fb44b3d38, p_msg=0x7f6ffeca4118, branch=0, msg_status=500, cancel_data=0x7ffde2759230, do_put_on_wait=1) at t_reply.c:1757
#4  0x00007f6ffad1bec4 in reply_received (p_msg=0x7f6ffeca4118) at t_reply.c:2466
#5  0x000000000049828f in do_forward_reply (msg=0x7f6ffeca4118, mode=0) at forward.c:747
#6  0x00000000004999fe in forward_reply (msg=0x7f6ffeca4118) at forward.c:849
#7  0x0000000000515d8a in receive_msg (
    buf=0xab0080 "SIP/2.0 500 I'm terribly sorry, server error occurred (6/SL)\r\nVia: SIP/2.0/UDP 192.168.178.35;branch=z9hG4bKecc7.4b5f187ecb5db7bceee392c8c43c0509.0\r\nVia: SIP/2.0/UDP 192.168.178.36:5060;received=192.1"..., len=518, rcv_info=0x7ffde27596f0) at receive.c:299
#8  0x0000000000628f95 in udp_rcv_loop () at udp_server.c:495
#9  0x00000000004b2dd6 in main_loop () at main.c:1614
#10 0x00000000004ba1eb in main (argc=14, argv=0x7ffde2759c68) at main.c:2631
```
```
(gdb) bt full
#0  0x00007f700eb4c4f5 in raise (sig=6) at ../nptl/sysdeps/unix/sysv/linux/raise.c:64
        resultvar = 0
        pid = <value optimized out>
        selftid = <value optimized out>
#1  0x00007f700eb4dcd5 in abort () at abort.c:92
        save_stage = 2
        act = {__sigaction_handler = {sa_handler = 0x8, sa_sigaction = 0x8}, sa_mask = {__val = {0, 7582136, 140118926563523, 140118926577440, 180, 206158430248, 140728402808320,
              140728402808112, 522715890832, 140118815048117, 17092387201, 137438953472000, 5000, 7582110, 140118926567681, 140118926577360}}, sa_flags = 1030059617, sa_restorer = 0x1f4}
        sigs = {__val = {32, 0 <repeats 15 times>}}
#2  0x00007f6ffad16087 in store_reply (trans=0x7f6fb44b3d38, branch=0, rpl=0x7f6ffeca4118) at t_reply.c:1601
        __FUNCTION__ = "store_reply"
#3  0x00007f6ffad17275 in relay_reply (t=0x7f6fb44b3d38, p_msg=0x7f6ffeca4118, branch=0, msg_status=500, cancel_data=0x7ffde2759230, do_put_on_wait=1) at t_reply.c:1757
        relay = -1
        save_clone = 1
        buf = 0x0
        res_len = 0
        relayed_code = 0
        relayed_msg = 0x0
        reply_bak = 0xb411e4c0
        bm = {to_tag_val = {s = 0x7ffde2758ff0 "\020\220u\342\375\177", len = -87011592}}
        totag_retr = 0
        reply_status = RPS_STORE
        uas_rb = 0x7f6fb4116000
        to_tag = 0x7ffde2759010
        reason = {s = 0xe2759040 <Address 0xe2759040 out of bounds>, len = -1270137096}
        onsend_params = {req = 0x7f6f00000003, rpl = 0x7ffde2759070, param = 0x7f6ffad7e6f0, code = -1270136928, flags = 2216, branch = 0, t_rbuf = 0x7f6ffad84de2, dst = 0x1, send_buf = {
            s = 0x7f6fb44b3ef8 "\002", len = -1273929728}}
        ip = {af = 0, len = 0, u = {addrl = {140728402809056, 140118992765208}, addr32 = {3799355616, 32765, 4274667800, 32623}, addr16 = {37088, 57973, 32765, 0, 16664, 65226, 32623, 0},
            addr = "\340\220u\342\375\177\000\000\030A\312\376o\177\000"}}
        __FUNCTION__ = "relay_reply"
#4  0x00007f6ffad1bec4 in reply_received (p_msg=0x7f6ffeca4118) at t_reply.c:2466
        msg_status = 500
        last_uac_status = 0
        ack = 0x7f6fb44b7980 "SIP/2.0 180 Ringing\r\nRecord-Route: sip:192.168.178.53:8080;transport=ws;r2=on;lr=on;ftag=56e.2c8\r\nRecord-Route: sip:192.168.178.53;r2=on;lr=on;ftag=56e.2c8\r\nRecord-Route: <sip:192.168.178.35;lr=on"...
---Type <return> to continue, or q <return> to quit---
        ack_len = 363
        branch = 0
        reply_status = -198019613
        onreply_route = 1
        cancel_data = {cancel_bitmap = 0, reason = {cause = 0, u = {text = {s = 0x0, len = 257199189}, e2e_cancel = 0x0, packed_hdrs = {s = 0x0, len = 257199189}}}}
        uac = 0x7f6fb44b3f40
        t = 0x7f6fb44b3d38
        lack_dst = {send_sock = 0xe00000001, to = {s = {sa_family = 0, sa_data = "\000\000\000\000\000\000\000\000\000\000\006\000\000"}, sin = {sin_family = 0, sin_port = 0, sin_addr = {
                s_addr = 0}, sin_zero = "\000\000\000\000\006\000\000"}, sin6 = {sin6_family = 0, sin6_port = 0, sin6_flowinfo = 0, sin6_addr = {__in6_u = {
                  __u6_addr8 = "\000\000\000\000\006\000\000\000\000\066\262\016p\177\000", __u6_addr16 = {0, 0, 6, 0, 13824, 3762, 32624, 0}, __u6_addr32 = {0, 6, 246560256, 32624}}},
              sin6_scope_id = 7582136}}, id = 0, proto = -29 '\343', send_flags = {f = 117 'u', blst_imask = 50 '2'}}
        backup_user_from = 0xac00b0
        backup_user_to = 0xac00b8
        backup_domain_from = 0xac00c0
        backup_domain_to = 0xac00c8
        backup_uri_from = 0xac00a0
        backup_uri_to = 0xac00a8
        backup_xavps = 0xac01e0
        replies_locked = 1
        branch_ret = 3
        prev_branch = 7582136
        blst_503_timeout = 0
        hf = 0x7004885c2
        onsend_params = {req = 0x77cdae, rpl = 0x1, param = 0xa17cb0, code = 0, flags = 0, branch = 0, t_rbuf = 0x7, dst = 0x0, send_buf = {s = 0x0, len = 257172512}}
        ctx = {rec_lev = 0, run_flags = 0, last_retcode = 128, jmp_env = {{__jmpbuf = {140117739134976, 2350401339451012324, 7582136, 0, 0, 0, 2350401339480372452, -2349319903025662748},
              __mask_was_saved = 0, __saved_mask = {__val = {4251813, 140118992648344, 140114718097411, 11206855, 3, 11206859, 140114718097422, 11206863, 140114718097408, 8035905, 459561500672,
                  140114718097427, 33089526208, 140119259624960, 140119272375504, 7582136}}}}}
        __FUNCTION__ = "reply_received"
#5  0x000000000049828f in do_forward_reply (msg=0x7f6ffeca4118, mode=0) at forward.c:747
        new_buf = 0x0
        dst = {send_sock = 0x0, to = {s = {sa_family = 0, sa_data = '\000' <repeats 13 times>}, sin = {sin_family = 0, sin_port = 0, sin_addr = {s_addr = 0},
              sin_zero = "\000\000\000\000\000\000\000"}, sin6 = {sin6_family = 0, sin6_port = 0, sin6_flowinfo = 0, sin6_addr = {__in6_u = {__u6_addr8 = '\000' <repeats 15 times>,
                  __u6_addr16 = {0, 0, 0, 0, 0, 0, 0, 0}, __u6_addr32 = {0, 0, 0, 0}}}, sin6_scope_id = 0}}, id = 0, proto = 0 '\000', send_flags = {f = 0 '\000', blst_imask = 0 '\000'}}
        new_len = 0
        r = 1
---Type <return> to continue, or q <return> to quit---
        ip = {af = 3799356400, len = 32765, u = {addrl = {140118935378591, 8391916088}, addr32 = {4217281183, 32623, 4096948792, 1}, addr16 = {39583, 64350, 32623, 0, 31288, 62514, 1, 0},
            addr = "\237\232^\373o\177\000\000\070z2\364\001\000\000"}}
        s = 0x7ffde27594a0 "\200\226u\342\375\177"
        len = 0
        __FUNCTION__ = "do_forward_reply"
#6  0x00000000004999fe in forward_reply (msg=0x7f6ffeca4118) at forward.c:849
No locals.
#7  0x0000000000515d8a in receive_msg (
    buf=0xab0080 "SIP/2.0 500 I'm terribly sorry, server error occurred (6/SL)\r\nVia: SIP/2.0/UDP 192.168.178.35;branch=z9hG4bKecc7.4b5f187ecb5db7bceee392c8c43c0509.0\r\nVia: SIP/2.0/UDP 192.168.178.36:5060;received=192.1"..., len=518, rcv_info=0x7ffde27596f0) at receive.c:299
        msg = 0x7f6ffeca4118
        ctx = {rec_lev = 0, run_flags = 0, last_retcode = 1, jmp_env = {{__jmpbuf = {140118815045091, 2350401338767340772, 7582136, 7, 7, 0, 2350401338796700900, -2349319903025662748},
              __mask_was_saved = 0, __saved_mask = {__val = {140118991147024, 140728402810496, 7582136, 20, 1, 140117739169952, 917678272, 0, 1024, 8094324272, 140117739169952, 140728402810416,
                  6429261, 85, 140117739169952, 32}}}}}
        ret = 1
        inb = {
          s = 0xab0080 "SIP/2.0 500 I'm terribly sorry, server error occurred (6/SL)\r\nVia: SIP/2.0/UDP 192.168.178.35;branch=z9hG4bKecc7.4b5f187ecb5db7bceee392c8c43c0509.0\r\nVia: SIP/2.0/UDP 192.168.178.36:5060;received=192.1"..., len = 518}
        netinfo = {data = {s = 0x7f700eb2a2f0 "\<", len = 259310800}, dst = 0x0, rcv = 0xffffffff}
        __FUNCTION__ = "receive_msg"
#8  0x0000000000628f95 in udp_rcv_loop () at udp_server.c:495
        len = 518
        buf = "SIP/2.0 500 I'm terribly sorry, server error occurred (6/SL)\r\nVia: SIP/2.0/UDP 192.168.178.35;branch=z9hG4bKecc7.4b5f187ecb5db7bceee392c8c43c0509.0\r\nVia: SIP/2.0/UDP 192.168.178.36:5060;received=192.1"...
        tmp = 0x600000000 <Address 0x600000000 out of bounds>
        from = 0x7f6ffec584f8
        fromlen = 16
        ri = {src_ip = {af = 2, len = 4, u = {addrl = {140725521131712, 140728402810640}, addr32 = {917678272, 32765, 3799357200, 32765}, addr16 = {43200, 14002, 32765, 0, 38672, 57973, 32765,
                0}, addr = "\300\250\262\066\375\177\000\000\020\227u\342\375\177\000"}}, dst_ip = {af = 2, len = 4, u = {addrl = {598911168, 0}, addr32 = {598911168, 0, 0, 0}, addr16 = {43200,
                9138, 0, 0, 0, 0, 0, 0}, addr = "\300\250\262#", '\000' <repeats 11 times>}}, src_port = 5060, dst_port = 5060, proto_reserved1 = 0, proto_reserved2 = 0, src_su = {s = {
              sa_family = 2, sa_data = "\023\304\300\250\262\066\000\000\000\000\000\000\000"}, sin = {sin_family = 2, sin_port = 50195, sin_addr = {s_addr = 917678272},
              sin_zero = "\000\000\000\000\000\000\000"}, sin6 = {sin6_family = 2, sin6_port = 50195, sin6_flowinfo = 917678272, sin6_addr = {__in6_u = {__u6_addr8 = '\000' <repeats 15 times>,
                  __u6_addr16 = {0, 0, 0, 0, 0, 0, 0, 0}, __u6_addr32 = {0, 0, 0, 0}}}, sin6_scope_id = 0}}, bind_address = 0x7f6ffeb3fe80, proto = 1 '\001'}
        __FUNCTION__ = "udp_rcv_loop"
#9  0x00000000004b2dd6 in main_loop () at main.c:1614
---Type <return> to continue, or q <return> to quit---
        i = 5
        pid = 0
        si = 0x7f6ffeb3fe80
        si_desc = "udp receiver child=5 sock=192.168.178.35:5060\000\000\000\024\000\000\000\000\000\000\000\a", '\000' <repeats 15 times>, "\003\000\000\000p\177\000\000\060\000\000\000\060\000\000\000P\231u\342\375\177\000\000\220\230u\342\375\177\000\000E;u\000\000\000\000\000\004\000\000\000\000\000\000\000\000`\021\264o\177\000"
        r = 0
        nrprocs = 8
        woneinit = 1
        __FUNCTION__ = "main_loop"
#10 0x00000000004ba1eb in main (argc=14, argv=0x7ffde2759c68) at main.c:2631
        cfg_stream = 0x198a010
        c = -1
        r = 0
        tmp = 0x7ffde275be25 ""
        tmp_len = 32624
        port = 259350928
        proto = 0
        options = 0x7493c8 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:x:X:"
        ret = -1
        seed = 2637875816
        rfd = 4
        debug_save = 0
        debug_flag = 0
        dont_fork_cnt = 2
        n_lst = 0x7f700ebaa457
        p = 0x7ffde2759b68 "`\234u\342\375\177"
        st = {st_dev = 64768, st_ino = 263284, st_nlink = 2, st_mode = 16832, st_uid = 0, st_gid = 0, __pad0 = 0, st_rdev = 0, st_size = 4096, st_blksize = 4096, st_blocks = 8, st_atim = {
            tv_sec = 1543247591, tv_nsec = 708943540}, st_mtim = {tv_sec = 1539696832, tv_nsec = 509518950}, st_ctim = {tv_sec = 1539696832, tv_nsec = 509518950}, __unused = {0, 0, 0}}
        __FUNCTION__ = "main"
```
```
(gdb) list
59	    if (__builtin_expect (pid <= 0, 0))
60	      pid = (pid & INT_MAX) == 0 ? selftid : -pid;
61	#endif
62
63	#if __ASSUME_TGKILL
64	  return INLINE_SYSCALL (tgkill, 3, pid, selftid, sig);
65	#else
66	# ifdef __NR_tgkill
67	  int res = INLINE_SYSCALL (tgkill, 3, pid, selftid, sig);
68	  if (res != -1 || errno != ENOSYS)
```
#### Log Messages
```
2018-11-28T21:03:44Z DEBUG DEBUG: tm [t_reply.c:1233]: t_should_relay_response(): ->>>>>>>>> T_code=0, new_code=180
2018-11-28T21:03:44Z DEBUG DEBUG: tm [t_reply.c:1752]: relay_reply(): DEBUG: relay_reply: branch=1, save=0, relay=1 icode=0
2018-11-28T21:03:44Z DEBUG DEBUG: tm [t_hooks.c:266]: run_trans_callbacks_internal(): DBG: trans=0x7f6fb44b3d38, callback type 32, id 0 entered
2018-11-28T21:03:44Z DEBUG DEBUG: dialog [dlg_hash.c:690]: dlg_lookup(): ref dlg 0x7f6fb44b2f58 with 1 -> 2
2018-11-28T21:03:44Z DEBUG DEBUG: dialog [dlg_hash.c:692]: dlg_lookup(): dialog id=6583 found on entry 3685
2018-11-28T21:03:44Z DEBUG DEBUG: dialog [dlg_hash.c:922]: dlg_unref_helper(): unref op on 0x7f6fb44b2f58 with 1 from dlg_hash.c:940
2018-11-28T21:03:44Z DEBUG DEBUG: dialog [dlg_hash.c:926]: dlg_unref_helper(): unref dlg 0x7f6fb44b2f58 with 1 -> 1
2018-11-28T21:03:44Z DEBUG DEBUG: <core> [msg_translator.c:2261]: generate_res_buf_from_sip_res(): old size: 813, new size: 725
2018-11-28T21:03:44Z DEBUG DEBUG: <core> [msg_translator.c:2279]: generate_res_buf_from_sip_res(): copied size: orig:811, new: 723, rest: 2 msg=
kamailio.log-SIP/2.0 180 Ringing
kamailio.log-Record-Route: sip:192.168.178.53:8080;transport=ws;r2=on;lr=on;ftag=56e.2c8
kamailio.log-Record-Route: sip:192.168.178.53;r2=on;lr=on;ftag=56e.2c8
kamailio.log-Record-Route: sip:192.168.178.35;lr=on;ftag=56e.7b91
kamailio.log-Via: SIP/2.0/UDP 192.168.178.36:5060;received=192.168.178.36;branch=z9hG4bKd237e465abb1c1bee01727d790654f65;rport=5060
kamailio.log-To: sip:911@192.168.178.35;tag=njbalhm9t3
kamailio.log-From: sip:2609184641@192.168.178.36;tag=b6344cba503381b36bdc2b52f4ae77e2
kamailio.log-Call-ID: ed3095um65kdm8h2rlle
kamailio.log-CSeq: 200 INVITE
kamailio.log-Contact: sip:kfp5hkj9@icb3p4n1vfqk.in;alias=192.168.178.33~50982~5;transport=ws;reguser=118-085-5011;regrealm=indigitaldev.net
kamailio.log-User-Agent: MyID SIP/MSRP v1.0
kamailio.log-Content-Length: 0
kamailio.log-Server: MyID-kamailio-
kamailio.log-
kamailio.log-
2018-11-28T21:03:44Z DEBUG DEBUG: tm [t_hooks.c:266]: run_trans_callbacks_internal(): DBG: trans=0x7f6fb44b3d38, callback type 1048576, id 0 entered
2018-11-28T21:03:44Z DEBUG DEBUG: dialog [dlg_hash.c:690]: dlg_lookup(): ref dlg 0x7f6fb44b2f58 with 1 -> 2
2018-11-28T21:03:44Z DEBUG DEBUG: dialog [dlg_hash.c:692]: dlg_lookup(): dialog id=6583 found on entry 3685
2018-11-28T21:03:44Z DEBUG DEBUG: dialog [dlg_hash.c:1122]: next_state_dlg(): dialog 0x7f6fb44b2f58 changed from state 1 to state 2, due event 2 (ref 2)
2018-11-28T21:03:44Z DEBUG DEBUG: dialog [dlg_hash.c:922]: dlg_unref_helper(): unref op on 0x7f6fb44b2f58 with 1 from dlg_hash.c:940
2018-11-28T21:03:44Z DEBUG DEBUG: dialog [dlg_hash.c:926]: dlg_unref_helper(): unref dlg 0x7f6fb44b2f58 with 1 -> 1
2018-11-28T21:03:44Z DEBUG DEBUG: <core> [pvapi.c:321]: pv_cache_lookup(): pvar [$mt] found in cache
2018-11-28T21:03:44Z DEBUG DEBUG: <core> [pvapi.c:321]: pv_cache_lookup(): pvar [$cs] found in cache
2018-11-28T21:03:44Z DEBUG DEBUG: <core> [pvapi.c:321]: pv_cache_lookup(): pvar [$si] found in cache
2018-11-28T21:03:44Z DEBUG DEBUG: <core> [pvapi.c:321]: pv_cache_lookup(): pvar [$rm] found in cache
2018-11-28T21:03:44Z DEBUG DEBUG: <core> [pvapi.c:321]: pv_cache_lookup(): pvar [$ci] found in cache
2018-11-28T21:03:44Z DEBUG DEBUG: <core> [pvapi.c:1393]: pv_printf(): final buffer length 54
2018-11-28T21:03:44Z DEBUG DEBUG: <core> [pvapi.c:1393]: pv_printf(): final buffer length 39
kamailio.log:kamailio  2018-11-28T21:03:44Z INFO {2 200 192.168.178.54 INVITE ed3095um65kdm8h2rlle} In MANAGE_REPLY, incoming reply 500 [0]
2018-11-28T21:03:44Z DEBUG DEBUG: tm [t_reply.c:1233]: t_should_relay_response(): ->>>>>>>>> T_code=180, new_code=500
2018-11-28T21:03:44Z DEBUG DEBUG: tm [t_reply.c:1752]: relay_reply(): DEBUG: relay_reply: branch=0, save=1, relay=-1 icode=0
2018-11-28T21:03:44Z DEBUG DEBUG: dialog [dlg_var.c:86]: cb_dlg_locals_reset(): resetting the local dialog shortcuts on script callback: 2147483652
2018-11-28T21:03:44Z DEBUG DEBUG: <core> [usr_avp.c:631]: destroy_avp_list(): destroying list (nil)
2018-11-28T21:03:44Z DEBUG DEBUG: <core> [usr_avp.c:631]: destroy_avp_list(): destroying list (nil)
2018-11-28T21:03:44Z DEBUG DEBUG: <core> [usr_avp.c:631]: destroy_avp_list(): destroying list (nil)
2018-11-28T21:03:44Z DEBUG DEBUG: <core> [usr_avp.c:631]: destroy_avp_list(): destroying list (nil)
2018-11-28T21:03:44Z DEBUG DEBUG: <core> [usr_avp.c:631]: destroy_avp_list(): destroying list (nil)
2018-11-28T21:03:44Z DEBUG DEBUG: <core> [usr_avp.c:631]: destroy_avp_list(): destroying list (nil)
2018-11-28T21:03:44Z DEBUG DEBUG: <core> [xavp.c:446]: xavp_destroy_list(): destroying xavp list (nil)
2018-11-28T21:03:44Z DEBUG DEBUG: <core> [receive.c:322]: receive_msg(): cleaning up
2018-11-28T21:03:44Z DEBUG DEBUG: <core> [pvapi.c:321]: pv_cache_lookup(): pvar [$mt] found in cache
2018-11-28T21:03:44Z DEBUG DEBUG: <core> [pvapi.c:321]: pv_cache_lookup(): pvar [$cs] found in cache
2018-11-28T21:03:44Z DEBUG DEBUG: <core> [pvapi.c:321]: pv_cache_lookup(): pvar [$si] found in cache
2018-11-28T21:03:44Z DEBUG DEBUG: <core> [pvapi.c:321]: pv_cache_lookup(): pvar [$rm] found in cache
2018-11-28T21:03:44Z DEBUG DEBUG: <core> [pvapi.c:321]: pv_cache_lookup(): pvar [$ci] found in cache
2018-11-28T21:03:44Z DEBUG DEBUG: <core> [pvapi.c:1393]: pv_printf(): final buffer length 54
2018-11-28T21:03:44Z DEBUG DEBUG: <core> [pvapi.c:1393]: pv_printf(): final buffer length 39
--
2018-11-28T21:03:44Z DEBUG DEBUG: tm [t_reply.c:1233]: t_should_relay_response(): ->>>>>>>>> T_code=180, new_code=500
2018-11-28T21:03:44Z DEBUG DEBUG: tm [t_reply.c:1752]: relay_reply(): DEBUG: relay_reply: branch=0, save=1, relay=-1 icode=0
2018-11-28T21:03:44Z DEBUG DEBUG: dialog [dlg_var.c:86]: cb_dlg_locals_reset(): resetting the local dialog shortcuts on script callback: 2147483652
2018-11-28T21:03:44Z DEBUG DEBUG: <core> [usr_avp.c:631]: destroy_avp_list(): destroying list (nil)
2018-11-28T21:03:44Z DEBUG DEBUG: <core> [usr_avp.c:631]: destroy_avp_list(): destroying list (nil)
2018-11-28T21:03:44Z DEBUG DEBUG: <core> [usr_avp.c:631]: destroy_avp_list(): destroying list (nil)
2018-11-28T21:03:44Z DEBUG DEBUG: <core> [usr_avp.c:631]: destroy_avp_list(): destroying list (nil)
2018-11-28T21:03:44Z DEBUG DEBUG: <core> [usr_avp.c:631]: destroy_avp_list(): destroying list (nil)
2018-11-28T21:03:44Z DEBUG DEBUG: <core> [usr_avp.c:631]: destroy_avp_list(): destroying list (nil)
2018-11-28T21:03:44Z DEBUG DEBUG: <core> [xavp.c:446]: xavp_destroy_list(): destroying xavp list (nil)
2018-11-28T21:03:44Z DEBUG DEBUG: <core> [receive.c:322]: receive_msg(): cleaning up
2018-11-28T21:03:44Z DEBUG DEBUG: <core> [pvapi.c:321]: pv_cache_lookup(): pvar [$mt] found in cache
2018-11-28T21:03:44Z DEBUG DEBUG: <core> [pvapi.c:321]: pv_cache_lookup(): pvar [$cs] found in cache
2018-11-28T21:03:44Z DEBUG DEBUG: <core> [pvapi.c:321]: pv_cache_lookup(): pvar [$si] found in cache
2018-11-28T21:03:44Z DEBUG DEBUG: <core> [pvapi.c:321]: pv_cache_lookup(): pvar [$rm] found in cache
2018-11-28T21:03:44Z DEBUG DEBUG: <core> [pvapi.c:321]: pv_cache_lookup(): pvar [$ci] found in cache
2018-11-28T21:03:44Z DEBUG DEBUG: <core> [pvapi.c:1393]: pv_printf(): final buffer length 54
2018-11-28T21:03:44Z DEBUG DEBUG: <core> [pvapi.c:1393]: pv_printf(): final buffer length 39
kamailio.log:kamailio  2018-11-28T21:03:44Z INFO {2 200 192.168.178.54 INVITE ed3095um65kdm8h2rlle} In MANAGE_REPLY, incoming reply 500 [0]
2018-11-28T21:03:44Z DEBUG DEBUG: tm [t_reply.c:1233]: t_should_relay_response(): ->>>>>>>>> T_code=180, new_code=500
2018-11-28T21:03:44Z DEBUG DEBUG: tm [t_reply.c:1752]: relay_reply(): DEBUG: relay_reply: branch=0, save=1, relay=-1 icode=0
2018-11-28T21:03:44Z ERROR ERROR: tm [t_reply.c:1600]: store_reply(): ERROR: replacing stored reply; aborting
2018-11-28T21:03:47Z ALERT ALERT: <core> [main.c:740]: handle_sigs(): child process 7554 exited by a signal 6
2018-11-28T21:03:47Z ALERT ALERT: <core> [main.c:743]: handle_sigs(): core was generated
2018-11-28T21:03:47Z INFO INFO: <core> [main.c:755]: handle_sigs(): terminating due to SIGCHLD
2018-11-28T21:03:47Z DEBUG DEBUG: <core> [main.c:757]: handle_sigs(): terminating due to SIGCHLD
2018-11-28T21:03:47Z INFO INFO: <core> [main.c:810]: sig_usr(): signal 15 received
2018-11-28T21:03:47Z INFO INFO: <core> [main.c:810]: sig_usr(): signal 15 received
2018-11-28T21:03:47Z INFO INFO: <core> [main.c:810]: sig_usr(): signal 15 received
2018-11-28T21:03:47Z INFO INFO: <core> [main.c:810]: sig_usr(): signal 15 received
2018-11-28T21:03:47Z INFO INFO: <core> [main.c:810]: sig_usr(): signal 15 received
2018-11-28T21:03:47Z INFO INFO: <core> [main.c:810]: sig_usr(): signal 15 received
2018-11-28T21:03:47Z INFO INFO: <core> [main.c:810]: sig_usr(): signal 15 received
2018-11-28T21:03:47Z INFO INFO: <core> [main.c:810]: sig_usr(): signal 15 received
2018-11-28T21:03:47Z INFO INFO: <core> [main.c:810]: sig_usr(): signal 15 received
2018-11-28T21:03:47Z INFO INFO: <core> [main.c:810]: sig_usr(): signal 15 received
2018-11-28T21:03:47Z INFO INFO: <core> [main.c:810]: sig_usr(): signal 15 received
2018-11-28T21:03:47Z INFO INFO: <core> [main.c:810]: sig_usr(): signal 15 received
2018-11-28T21:03:47Z INFO INFO: <core> [main.c:810]: sig_usr(): signal 15 received
2018-11-28T21:03:47Z INFO INFO: <core> [main.c:810]: sig_usr(): signal 15 received
2018-11-28T21:03:47Z INFO INFO: <core> [main.c:810]: sig_usr(): signal 15 received
2018-11-28T21:03:47Z INFO INFO: <core> [main.c:810]: sig_usr(): signal 15 received
2018-11-28T21:03:47Z INFO INFO: <core> [main.c:810]: sig_usr(): signal 15 received
2018-11-28T21:03:48Z DEBUG DEBUG: dialog [dlg_hash.c:327]: destroy_dlg(): destroying dialog 0x7f6fb44b2f58 (ref 1)
2018-11-28T21:03:48Z DEBUG DEBUG: dialog [dlg_hash.c:343]: destroy_dlg(): removed timer for dlg 0x7f6fb44b2f58 [3685:6583] with clid 'ed3095um65kdm8h2rlle' and tags 'b6344cba503381b36bdc2b52f4ae77e2' ''
2018-11-28T21:03:48Z DEBUG DEBUG: <core> [db_pool.c:100]: pool_remove(): removing connection from the pool
2018-11-28T21:03:48Z DEBUG DEBUG: tm [t_funcs.c:86]: tm_shutdown(): DEBUG: tm_shutdown : start
2018-11-28T21:03:48Z DEBUG DEBUG: tm [t_funcs.c:89]: tm_shutdown(): DEBUG: tm_shutdown : emptying hash table
2018-11-28T21:03:48Z DEBUG DEBUG: tm [h_table.c:127]: free_cell_helper(): freeing transaction 0x7f6fb44b3d38 from h_table.c:449
2018-11-28T21:03:48Z DEBUG DEBUG: dialog [dlg_handlers.c:303]: dlg_iuid_sfree(): freeing dlg iuid [3685:6583] (0x7f6fb44b3950)
2018-11-28T21:03:48Z DEBUG DEBUG: tm [t_funcs.c:91]: tm_shutdown(): DEBUG: tm_shutdown : removing semaphores
2018-11-28T21:03:48Z DEBUG DEBUG: tm [t_funcs.c:93]: tm_shutdown(): DEBUG: tm_shutdown : destroying tmcb lists
2018-11-28T21:03:48Z DEBUG DEBUG: tm [t_funcs.c:96]: tm_shutdown(): DEBUG: tm_shutdown : done
2018-11-28T21:03:48Z INFO INFO: <core> [sctp_core.c:53]: sctp_core_destroy(): SCTP API not initialized
2018-11-28T21:03:48Z DEBUG DEBUG: <core> [mem/shm.c:174]: shm_core_lock_destroy(): destroying the shared memory lock
2018-11-28T21:03:48Z DEBUG DEBUG: <core> [mem/shm.c:266]: shm_destroy_manager(): destroying memory manager: f_malloc
2018-11-28T21:03:48Z DEBUG DEBUG: <core> [mem/pkg.c:89]: pkg_destroy_manager(): destroying memory manager: f_malloc
```
#### SIP Traffic
![kamailio-store_reply-crash](https://user-images.githubusercontent.com/1906859/49220087-79b09780-f3d5-11e...)
### Additional Information
* **Kamailio Version** - output of `kamailio -v`
```
kamailio-4.4.5
```
* **Operating System**:
```
CentOS release 6.10
Linux mybox 2.6.32-754.6.3.el6.x86_64 #1 SMP Tue Oct 9 17:27:49 UTC 2018 x86_64 x86_64 x86_64 GNU/Linux
```
-- 
You are receiving this because you are subscribed to this thread.
Reply to this email directly or view it on GitHub:
https://github.com/kamailio/kamailio/issues/1744

2025

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

2008

[sr-dev] [kamailio/kamailio] Crashing Kamailio: store_reply. replacing stored reply (#1744)