Hello,

Here is the bt of all cores. This time, the scenario is different, when kamailio receives an ACK and try to forward the message

Log file:
Feb 13 11:09:19 kamailio23 /usr/local/sbin/kamailio[21086]: : <core> [pass_fd.c:293]: receive_fd(): ERROR: receive_fd: EOF on 31
Feb 13 11:09:19 kamailio23 /usr/local/sbin/kamailio[21038]: ALERT: <core> [main.c:788]: handle_sigs(): child process 21063 exited by a signal 11
Feb 13 11:09:19 kamailio23 /usr/local/sbin/kamailio[21038]: ALERT: <core> [main.c:791]: handle_sigs(): core was generated
Feb 13 11:09:19 kamailio23 /usr/local/sbin/kamailio[21038]: INFO: <core> [main.c:803]: handle_sigs(): INFO: terminating due to SIGCHLD

root@kamailio23:/var/log/kamailio# gdb /usr/local/sbin/kamailio cores/core.kamailio.21063
(gdb) bt
#0 0x000000000052971e in timer_list_expire (t=275552247, h=0x7f5ea9f30aa0, slow_l=0x7f5ea9f31a50, slow_mark=59628) at timer.c:883
#1 0x0000000000529a80 in timer_handler () at timer.c:959
#2 0x0000000000529d0f in timer_main () at timer.c:998
#3 0x0000000000469eda in main_loop () at main.c:1709
#4 0x000000000046ca85 in main (argc=5, argv=0x7fffcd0f5828) at main.c:2566
(gdb) bt full
#0 0x000000000052971e in timer_list_expire (t=275552247, h=0x7f5ea9f30aa0, slow_l=0x7f5ea9f31a50, slow_mark=59628) at timer.c:883
        tl = 0x7f5eaa1bf5f0
        ret = 32767
#1 0x0000000000529a80 in timer_handler () at timer.c:959
        saved_ticks = 275552247
        run_slow_timer = 0
        i = 236
        __FUNCTION__ = "timer_handler"
#2 0x0000000000529d0f in timer_main () at timer.c:998
No locals.
#3 0x0000000000469eda in main_loop () at main.c:1709
        i = 8
        pid = 0
        si = 0x0
        si_desc = "udp receiver child=7 sock=91.213.79.31:5060\000^\177", '\000' <repeats 18 times>, "0V\017\315\377\177\000\000u\234\021O\000\000\000\000\240mA\000\000\000\000\000 X\017\315\377\177", '\000' <repeats 18 times>, "`V\017\315\377\177\000\000\030>K\000\000\000\000"
        nrprocs = 8
        __FUNCTION__ = "main_loop"
#4 0x000000000046ca85 in main (argc=5, argv=0x7fffcd0f5828) at main.c:2566
        cfg_stream = 0x1bd2010
        c = -1
        r = 0
        tmp = 0x7fffcd0f5858 "\233n\017\315\377\177"
        tmp_len = 0
        port = 5
        proto = 0
        options = 0x5c7c18 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
        ret = -1
        seed = 1955773403
        rfd = 4
        debug_save = 0
        debug_flag = 0
        dont_fork_cnt = 0
        n_lst = 0xbf
        p = 0x4162c9 "H\203\304\b\303" <Address 0x4162ce out of bounds>
        __FUNCTION__ = "main"
(gdb)

root@kamailio23:/var/log/kamailio# gdb /usr/local/sbin/kamailio cores/core.kamailio.21038
(gdb) bt full
#0 0x000000000053a81d in qm_free (qm=0x7f5ea9ee2000, p=0x424f9a6b00000000) at mem/q_malloc.c:476
        f = 0x424f9a6afffffff0
        size = 4880
        next = 0x7f5eaa17eff0
        prev = 0x7f5ea9eec840
        __FUNCTION__ = "qm_free"
#1 0x00007f5eb150d81c in free_cell (dead_cell=0x7f5eaa047710) at h_table.c:162
        b = 0x0
        i = 1
        rpl = 0x0
        tt = 0x0
        foo = 0x0
        cbs = 0x0
        cbs_tmp = 0x7f5eaa186950
        __FUNCTION__ = "free_cell"
#2 0x00007f5eb150e783 in free_hash_table () at h_table.c:443
        p_cell = 0x7f5eaa047710
        tmp_cell = 0x7f5eaa047710
        i = 59334
#3 0x00007f5eb152181b in tm_shutdown () at t_funcs.c:122
        __FUNCTION__ = "tm_shutdown"
#4 0x00000000004ee6f0 in destroy_modules () at sr_module.c:790
        t = 0x7f5eb2aea2f0
        foo = 0x7f5eb2ae9cc0
#5 0x0000000000463baa in cleanup (show_status=1) at main.c:573
        memlog = 32606
        __FUNCTION__ = "cleanup"
#6 0x0000000000464c32 in shutdown_children (sig=15, show_status=1) at main.c:715
        __FUNCTION__ = "shutdown_children"
#7 0x0000000000466223 in handle_sigs () at main.c:806
        chld = 0
        chld_status = 139
        memlog = 0
        __FUNCTION__ = "handle_sigs"
#8 0x000000000046a459 in main_loop () at main.c:1767
        i = 8
        pid = 21086
        si = 0x0
        si_desc = "udp receiver child=7 sock=91.213.79.31:5060\000^\177", '\000' <repeats 18 times>, "0V\017\315\377\177\000\000u\234\021O\000\000\000\000\240mA\000\000\000\000\000 X\017\315\377\177", '\000' <repeats 18 times>, "`V\017\315\377\177\000\000\030>K\000\000\000\000"
        nrprocs = 8
        __FUNCTION__ = "main_loop"
#9 0x000000000046ca85 in main (argc=5, argv=0x7fffcd0f5828) at main.c:2566
        cfg_stream = 0x1bd2010
        c = -1
        r = 0
        tmp = 0x7fffcd0f5858 "\233n\017\315\377\177"
        tmp_len = 0
        port = 5
        proto = 0
        options = 0x5c7c18 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
        ret = -1
        seed = 1955773403
        rfd = 4
        debug_save = 0
        debug_flag = 0
        dont_fork_cnt = 0
        n_lst = 0xbf
        p = 0x4162c9 "H\203\304\b\303" <Address 0x4162ce out of bounds>
        __FUNCTION__ = "main"
(gdb)

Regards,

On Wed, Feb 12, 2014 at 12:24 PM, Daniel-Constantin Mierla <miconda@gmail.com> wrote:

Hello,

the backtrace you got is not relevant, because it is from the main process that is handling the singnal of the child process that crashed. So the corefile from crashed process has been overwritten.

Can you enable one core file per process and try to reproduce? Then get the backtrace from each core generated.

Cheers,
Daniel

On 11/02/14 12:38, Tuan Viet Nguyen wrote:
Hello,

Upon receipt an 502 Bad gateway, I have the following error then Kamailio crashed
Feb 11 11:49:57 kamailio23 /usr/local/sbin/kamailio[28895]: WARNING: tm [t_lookup.c:1564]: t_unref(): WARNING: script writer didn't release transaction
Feb 11 11:50:03 kamailio23 /usr/local/sbin/kamailio[28926]: : <core> [pass_fd.c:293]: receive_fd(): ERROR: receive_fd: EOF on 31
Feb 11 11:50:03 kamailio23 /usr/local/sbin/kamailio[28876]: ALERT: <core> [main.c:788]: handle_sigs(): child process 28903 exited by a signal 11
Feb 11 11:50:03 kamailio23 /usr/local/sbin/kamailio[28876]: ALERT: <core> [main.c:791]: handle_sigs(): core was generated

(gdb) bt
#0 0x000000000053a81d in qm_free (qm=0x7fd6f962a000, p=0x75d63ff100000000) at mem/q_malloc.c:476
#1 0x00007fd700c5581c in free_cell (dead_cell=0x7fd6f9777bf0) at h_table.c:162
#2 0x00007fd700c56783 in free_hash_table () at h_table.c:443
#3 0x00007fd700c6981b in tm_shutdown () at t_funcs.c:122
#4 0x00000000004ee6f0 in destroy_modules () at sr_module.c:790
#5 0x0000000000463baa in cleanup (show_status=1) at main.c:573
#6 0x0000000000464c32 in shutdown_children (sig=15, show_status=1) at main.c:715
#7 0x0000000000466223 in handle_sigs () at main.c:806
#8 0x000000000046a459 in main_loop () at main.c:1767
#9 0x000000000046ca85 in main (argc=5, argv=0x7fff2f13ec38) at main.c:2566
(gdb) bt full
#0 0x000000000053a81d in qm_free (qm=0x7fd6f962a000, p=0x75d63ff100000000) at mem/q_malloc.c:476
        f = 0x75d63ff0fffffff0
        size = 4880
        next = 0x7fd6f98c5380
        prev = 0x7fd6f9634840
        __FUNCTION__ = "qm_free"
#1 0x00007fd700c5581c in free_cell (dead_cell=0x7fd6f9777bf0) at h_table.c:162
        b = 0x0
        i = 1
        rpl = 0x0
        tt = 0x0
        foo = 0x59e92da4
        cbs = 0x0
        cbs_tmp = 0x7fd6f98c5370
        __FUNCTION__ = "free_cell"
#2 0x00007fd700c56783 in free_hash_table () at h_table.c:443
        p_cell = 0x7fd6f9777bf0
        tmp_cell = 0x7fd6f98fb9c0
        i = 3084
#3 0x00007fd700c6981b in tm_shutdown () at t_funcs.c:122
        __FUNCTION__ = "tm_shutdown"
#4 0x00000000004ee6f0 in destroy_modules () at sr_module.c:790
        t = 0x7fd7022322f0
        foo = 0x7fd702231cc0
#5 0x0000000000463baa in cleanup (show_status=1) at main.c:573
        memlog = 32727
        __FUNCTION__ = "cleanup"
#6 0x0000000000464c32 in shutdown_children (sig=15, show_status=1) at main.c:715
        __FUNCTION__ = "shutdown_children"
#7 0x0000000000466223 in handle_sigs () at main.c:806
        chld = 0
        chld_status = 139
        memlog = 0
        __FUNCTION__ = "handle_sigs"
#8 0x000000000046a459 in main_loop () at main.c:1767
        i = 8
        pid = 28926
        si = 0x0
        si_desc = "udp receiver child=7 sock=91.213.79.31:5060\000\326\177", '\000' <repeats 18 times>, "@\352\023/\377\177\000\000\244-\351Y\000\000\000\000\240mA\000\000\000\000\000\060\354\023/\377\177", '\000' <repeats 18 times>, "p\352\023/\377\177\000\000\030>K\000\000\000\000"
        nrprocs = 8
        __FUNCTION__ = "main_loop"
#9 0x000000000046ca85 in main (argc=5, argv=0x7fff2f13ec38) at main.c:2566
        cfg_stream = 0xc36010
        c = -1
        r = 0
        tmp = 0x7fff2f13ec68 "\233\356\023/\377\177"
        tmp_len = 0
        port = 5
        proto = 0
        options = 0x5c7c18 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
        ret = -1
        seed = 3628627122
        rfd = 4
        debug_save = 0
        debug_flag = 0
        dont_fork_cnt = 0
        n_lst = 0xbf
        p = 0x4162c9 "H\203\304\b\303" <Address 0x4162ce out of bounds>
        __FUNCTION__ = "main"
(gdb)

Regards,
_______________________________________________
SIP Express Router (SER) and Kamailio (OpenSER) - sr-users mailing list
sr-users@lists.sip-router.org
http://lists.sip-router.org/cgi-bin/mailman/listinfo/sr-users
-- Daniel-Constantin Mierla - http://www.asipto.com http://twitter.com/#!/miconda - http://www.linkedin.com/in/miconda
_______________________________________________
SIP Express Router (SER) and Kamailio (OpenSER) - sr-users mailing list
sr-users@lists.sip-router.org
http://lists.sip-router.org/cgi-bin/mailman/listinfo/sr-users