First core file:
``` GNU gdb (GDB) 7.4.1-debian Copyright (C) 2012 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later http://gnu.org/licenses/gpl.html This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "x86_64-linux-gnu". For bug reporting instructions, please see: http://www.gnu.org/software/gdb/bugs/... Reading symbols from /usr/sbin/kamailio...Reading symbols from /usr/lib/debug/.build-id/bf/0f481eacd5661e7adec0c358237730d825ea2a.debug...done. done. [New LWP 31359]
warning: Can't read pathname for load map: Input/output error. [Thread debugging using libthread_db enabled] Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1". Core was generated by `/usr/sbin/kamailio -f /etc/kamailio/kamailio_sip_proxy.cfg -P /var/run/kamailio'. Program terminated with signal 11, Segmentation fault. #0 0x0000000000663091 in destroy_avp_list_unsafe (list=0x7efe49050bd0) at core/usr_avp.c:625 625 core/usr_avp.c: No such file or directory. (gdb) frame 1 #1 0x00007efe6b4daadc in free_cell_helper (dead_cell=0x7efe49050a48, silent=0, fname=0x7efe6b5db100 "timer.c", fline=654) at h_table.c:242 242 h_table.c: No such file or directory. (gdb) p *dead_cell $1 = {next_c = 0x0, prev_c = 0x0, hash_index = 30681, label = 705812179, flags = 32, nr_of_outgoings = 1, ref_count = {val = 0}, from = { s = 0x7efe48f3796d "From: sip:1234567@sipgate.de;tag=7fcea877-80e3-4069-b\r\nCall-ID: DF246541ADA4EDA2@84.149.15.231\r\nEvent: message-summary\r\nSubscription-State: Active\r\nRoute: sip:217.10.79.9;lr;ftag=3951995175\r\nConte"..., len = 57}, callid = { s = 0x7efe48f379a6 "Call-ID: DF246541ADA4EDA2@84.149.15.231\r\nEvent: message-summary\r\nSubscription-State: Active\r\nRoute: sip:217.10.79.9;lr;ftag=3951995175\r\nContent-Type: application/simple-message-summary\r\nVia: SIP/2.0"..., len = 41}, cseq_n = { s = 0x7efe48f3792d "CSeq: 7950 NOTIFY\r\nTo: sip:1234567@sipgate.de;tag=3951995175\r\nFrom: sip:1234567@sipgate.de;tag=7fcea877-80e3-4069-b\r\nCall-ID: DF246541ADA4EDA2@84.149.15.231\r\nEvent: message-summary\r\nSubscription-S"..., len = 10}, to = { s = 0x7efe48f37940 "To: sip:1234567@sipgate.de;tag=3951995175\r\nFrom: sip:1234567@sipgate.de;tag=7fcea877-80e3-4069-b\r\nCall-ID: DF246541ADA4EDA2@84.149.15.231\r\nEvent: message-summary\r\nSubscription-State: Active\r\nRoute"..., len = 45}, method = { s = 0x7efe48f378e0 "NOTIFY sip:1442863@84.149.15.231;uniq=29A7ECC65F8884BD635517C89AFD8 SIP/2.0\r\nCSeq: 7950 NOTIFY\r\nTo: sip:1442863@sipgate.de;tag=3951995175\r\nFrom: sip:1442863@sipgate.de;tag=7fcea877-80e3-4069-b\r\nCa"..., len = 6}, tmcb_hl = {first = 0x0, reg_types = 0}, wait_timer = {next = 0x0, prev = 0x0, expire = 1289862248, initial_timeout = 320, data = 0x7efe49050a48, f = 0x7efe6b5875fa <wait_handler>, flags = 513, slow_idx = 0}, uas = {request = 0x7efe48f371c8, end_request = 0x7efe48f38070 "\210", response = {activ_type = 481, flags = 128, t_active = 0 '\000', branch = 0, buffer_len = 323, buffer = 0x7efe49077660 "SIP/2.0 481 Call Leg/Transaction Does Not Exist\r\nVia: SIP/2.0/UDP 217.10.76.144:5060\r\nFrom: sip:1442863@sipgate.de;tag=7fcea877-80e3-4069-b\r\nTo: sip:1442863@sipgate.de;tag=3951995175\r\nCall-ID: DF2"..., my_T = 0x7efe49050a48, timer = {next = 0x0, prev = 0x0, expire = 0, initial_timeout = 0, data = 0x0, f = 0x7efe6b5870b2 <retr_buf_handler>, flags = 0, slow_idx = 0}, dst = { send_sock = 0x7efe6dff77e0, to = {s = {sa_family = 2, sa_data = "\023\304\331\nL\220\000\000\000\000\000\000\000"}, sin = {sin_family = 2, sin_port = 50195, sin_addr = {s_addr = 2420902617}, sin_zero = "\000\000\000\000\000\000\000"}, sin6 = {sin6_family = 2, sin6_port = 50195, sin6_flowinfo = 2420902617, sin6_addr = {__in6_u = {__u6_addr8 = '\000' <repeats 15 times>, __u6_addr16 = {0, 0, 0, 0, 0, 0, 0, 0}, __u6_addr32 = {0, 0, 0, 0}}}, sin6_scope_id = 0}}, id = 0, proto = 1 '\001', send_flags = {f = 0, blst_imask = 0}}, retr_expire = 0, fr_expire = 0}, local_totag = { s = 0x0, len = 0}, cancel_reas = 0x0, status = 481}, uac = 0x7efe49050c50, async_backup = {backup_route = 0, backup_branch = 0, blind_uac = 0, ruri_new = 0}, fwded_totags = 0x0, uri_avps_from = 0x7efe49030900, uri_avps_to = 0x0, user_avps_from = 0x0, user_avps_to = 0x0, domain_avps_from = 0x0, domain_avps_to = 0x0, xavps_list = 0x0, reply_mutex = {val = 0}, reply_locker_pid = { val = 0}, reply_rec_lock_level = 0, fr_timeout = 320, fr_inv_timeout = 2000, rt_t1_timeout_ms = 500, rt_t2_timeout_ms = 4000, end_of_life = 1289862439, relayed_reply_branch = 0, on_failure = 0, on_branch_failure = 0, on_reply = 0, on_branch = 0, on_branch_delayed = 0, md5 = 0x7efe49050c30 "f17e9566ecae744152b7aed3c5ba0273"} ```
The other core: ``` GNU gdb (GDB) 7.4.1-debian Copyright (C) 2012 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later http://gnu.org/licenses/gpl.html This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "x86_64-linux-gnu". For bug reporting instructions, please see: http://www.gnu.org/software/gdb/bugs/... Reading symbols from /usr/sbin/kamailio...Reading symbols from /usr/lib/debug/.build-id/bf/0f481eacd5661e7adec0c358237730d825ea2a.debug...done. done. [New LWP 34381]
warning: Can't read pathname for load map: Input/output error. [Thread debugging using libthread_db enabled] Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1". Core was generated by `/usr/sbin/kamailio -f /etc/kamailio/kamailio_sip_proxy.cfg -P /var/run/kamailio'. Program terminated with signal 11, Segmentation fault. #0 0x00007f2dea9d1998 in t_forward_nonack (t=0x7f2dc7310660, p_msg=0x7f2ded9f17a0, proxy=0x0, proto=0) at t_fwd.c:1721 1721 t_fwd.c: No such file or directory. (gdb) list 1716 in t_fwd.c (gdb) p *t $1 = {next_c = 0x7f2dc212e0c0, prev_c = 0x7f2dc212e0c0, hash_index = 3986, label = 1504273493, flags = 2, nr_of_outgoings = 1, ref_count = {val = 1}, from = { s = 0x7f2dc71e8dbb "From: sip:dispatcher@localhost;tag=3d02c524d32baf26b7c31e3ac94b01dc-7e61\r\nCSeq: 10 OPTIONS\r\nCall-ID: 1afe16d6638c333a-34402@172.20.40.6\r\nMax-Forwards: 70\r\nContent-Length: 0\r\n\r\n", len = 76}, callid = {s = 0x7f2dc71e8e19 "Call-ID: 1afe16d6638c333a-34402@172.20.40.6\r\nMax-Forwards: 70\r\nContent-Length: 0\r\n\r\n", len = 45}, cseq_n = { s = 0x7f2dc71e8e07 "CSeq: 10 OPTIONS\r\nCall-ID: 1afe16d6638c333a-34402@172.20.40.6\r\nMax-Forwards: 70\r\nContent-Length: 0\r\n\r\n", len = 8}, to = { s = 0x7f2dc71e8da4 "To: sip:172.20.40.5\r\nFrom: sip:dispatcher@localhost;tag=3d02c524d32baf26b7c31e3ac94b01dc-7e61\r\nCSeq: 10 OPTIONS\r\nCall-ID: 1afe16d6638c333a-34402@172.20.40.6\r\nMax-Forwards: 70\r\nContent-Length: 0\r\n\r"..., len = 23}, method = { s = 0x7f2dc71e8d30 "OPTIONS sip:172.20.40.5 SIP/2.0\r\nVia: SIP/2.0/UDP 172.20.40.6;branch=z9hG4bK29f.8a425ca5", '0' <repeats 24 times>, ".0\r\nTo: sip:172.20.40.5\r\nFrom: sip:dispatcher@localhost;tag=3d02c524d32baf26b7c31e3a"..., len = 7}, tmcb_hl = {first = 0x7f2dc72fe838, reg_types = 1024}, wait_timer = {next = 0x0, prev = 0x0, expire = 0, initial_timeout = 0, data = 0x7f2dc7310660, f = 0x7f2deaa385fa <wait_handler>, flags = 1, slow_idx = 0}, uas = {request = 0x0, end_request = 0x0, response = {activ_type = 0, flags = 0, t_active = 0 '\000', branch = 0, buffer_len = 0, buffer = 0x0, my_T = 0x7f2dc7310660, timer = {next = 0x0, prev = 0x0, expire = 0, initial_timeout = 0, data = 0x0, f = 0x7f2deaa380b2 <retr_buf_handler>, flags = 0, slow_idx = 0}, dst = {send_sock = 0x0, to = {s = { sa_family = 0, sa_data = '\000' <repeats 13 times>}, sin = {sin_family = 0, sin_port = 0, sin_addr = {s_addr = 0}, sin_zero = "\000\000\000\000\000\000\000"}, sin6 = {sin6_family = 0, sin6_port = 0, sin6_flowinfo = 0, sin6_addr = {__in6_u = {__u6_addr8 = '\000' <repeats 15 times>, __u6_addr16 = {0, 0, 0, 0, 0, 0, 0, 0}, __u6_addr32 = {0, 0, 0, 0}}}, sin6_scope_id = 0}}, id = 0, proto = 0 '\000', send_flags = {f = 0, blst_imask = 0}}, retr_expire = 0, fr_expire = 0}, local_totag = {s = 0x0, len = 0}, cancel_reas = 0x0, status = 0}, uac = 0x7f2dc7310868, async_backup = { backup_route = 0, backup_branch = 0, blind_uac = 0, ruri_new = 0}, fwded_totags = 0x0, uri_avps_from = 0x0, uri_avps_to = 0x0, user_avps_from = 0x0, user_avps_to = 0x0, domain_avps_from = 0x0, domain_avps_to = 0x0, xavps_list = 0x0, reply_mutex = {val = 0}, reply_locker_pid = {val = 0}, reply_rec_lock_level = 0, fr_timeout = 320, fr_inv_timeout = 2000, rt_t1_timeout_ms = 500, rt_t2_timeout_ms = 4000, end_of_life = 1323165532, relayed_reply_branch = -1, on_failure = 1, on_branch_failure = 0, on_reply = 2, on_branch = 0, on_branch_delayed = 0, md5 = 0x7f2dc7310848 "8a425ca5", '0' <repeats 24 times>} ```