i noticed that my sip proxy running quite recent master had crashed. perhaps this has something to do with recent memory allocation changes?
-- juha
(gdb) where #0 fm_split_frag (size=104, frag=0x7fa1e3879fe8, qm=0x7fa1e187a000) at mem/f_malloc.c:285 #1 fm_malloc (qmp=0x7fa1e187a000, size=<optimized out>) at mem/f_malloc.c:497 #2 0x00000000005f94eb in fm_shm_malloc (qmp=0x7fa1e187a000, size=97) at mem/f_malloc.c:1059 #3 0x000000000055a67d in create_avp (flags=275, name=..., val=...) at usr_avp.c:175 #4 0x000000000055af4c in add_avp_list (list=0x9faa60 <def_list>, flags=<optimized out>, name=..., val=...) at usr_avp.c:232 #5 0x00007fa1dc898f3d in pv_set_avp (msg=0x7fa1e9ad0ef0, param=0x0, op=-128, val=0x7fff3dbcf7b0) at pv_core.c:1954 #6 0x00000000005444b1 in lval_pvar_assign (lv=<optimized out>, rv=<optimized out>, msg=<optimized out>, h=<optimized out>) at lvalue.c:351 #7 lval_assign (h=0x1, msg=0x7fa1e9ad0ef0, lv=0x7fa1e931fa70, rve=0x7fa1e93202e8) at lvalue.c:399 #8 0x0000000000459a53 in do_action (h=0x7fff3dbd17a0, a=0x7fa1e93209c8, msg=0x7fa1e9ad0ef0) at action.c:1429 #9 0x0000000000458d05 in run_actions (h=0x7fa1e387a068, a=0x68, msg=0xffffffffffffff80) at action.c:1548 #10 0x000000000045a7b6 in do_action (h=0x7fff3dbd17a0, a=0x7fa1e9320ac8, msg=0x7fa1e9ad0ef0) at action.c:1048 #11 0x0000000000458d05 in run_actions (h=0x7fa1e387a068, a=0x68, msg=0xffffffffffffff80) at action.c:1548 #12 0x000000000045a98d in do_action (h=0x7fff3dbd17a0, a=0x7fa1e99442b8, msg=0x7fa1e9ad0ef0) at action.c:677 #13 0x0000000000458d05 in run_actions (h=0x7fa1e387a068, a=0x68, msg=0xffffffffffffff80) at action.c:1548 #14 0x000000000045a7b6 in do_action (h=0x7fff3dbd17a0, a=0x7fa1e920d088, msg=0x7fa1e9ad0ef0) at action.c:1048 #15 0x0000000000458d05 in run_actions (h=0x7fa1e387a068, a=0x68, msg=0xffffffffffffff80) at action.c:1548 #16 0x000000000045a98d in do_action (h=0x7fff3dbd17a0, a=0x7fa1e99442b8, msg=0x7fa1e9ad0ef0) at action.c:677 #17 0x0000000000458d05 in run_actions (h=0x7fa1e387a068, a=0x68, msg=0xffffffffffffff80) at action.c:1548 #18 0x000000000045a7b6 in do_action (h=0x7fff3dbd17a0, a=0x7fa1e91d0b70, msg=0x7fa1e9ad0ef0) at action.c:1048 #19 0x0000000000458d05 in run_actions (h=0x7fa1e387a068, h@entry=0x7fff3dbd17a0, a=0x68, a@entry=0x7fa1e91bc958, msg=0xffffffffffffff80, msg@entry=0x7fa1e9ad0ef0) at action.c:1548 #20 0x0000000000465575 in run_top_route (a=0x7fa1e91bc958, msg=0x7fa1e9ad0ef0, c=<optimized out>) at action.c:1634 ---Type <return> to continue, or q <return> to quit--- #21 0x000000000054692e in receive_msg (buf=0x0, len=104, rcv_info=0x7fa1e1f09e10) at receive.c:195 #22 0x00000000005c28b5 in tcp_read_req (con=0x7fa1e1f09df8, bytes_read=0x7fff3dbd1ae0, read_flags=0x7fff3dbd1ae8) at tcp_read.c:1382 #23 0x00000000005c4d29 in handle_io (fm=0x0, events=104, idx=-1) at tcp_read.c:1624 #24 0x00000000005c994b in io_wait_loop_epoll (h=<optimized out>, t=<optimized out>, repeat=<optimized out>) at io_wait.h:1061 #25 tcp_receive_loop (unix_sock=-477650840) at tcp_read.c:1733 #26 0x00000000004e1f67 in tcp_init_children () at tcp_main.c:4787 #27 0x0000000000513bfb in main_loop () at main.c:1664 #28 0x000000000041c92b in main (argc=0, argv=0x6) at main.c:2566
Hello,
I wonder why parameters in frame 0 for fm_split_frag() are displayed in the reverse order.
Anyhow, can you give the output from gdb for:
frame 0 list info locals p *qm p *frag p *n
Cheers, Daniel
On 20/09/15 17:52, Juha Heinanen wrote:
i noticed that my sip proxy running quite recent master had crashed. perhaps this has something to do with recent memory allocation changes?
-- juha
(gdb) where #0 fm_split_frag (size=104, frag=0x7fa1e3879fe8, qm=0x7fa1e187a000) at mem/f_malloc.c:285 #1 fm_malloc (qmp=0x7fa1e187a000, size=<optimized out>) at mem/f_malloc.c:497 #2 0x00000000005f94eb in fm_shm_malloc (qmp=0x7fa1e187a000, size=97) at mem/f_malloc.c:1059 #3 0x000000000055a67d in create_avp (flags=275, name=..., val=...) at usr_avp.c:175 #4 0x000000000055af4c in add_avp_list (list=0x9faa60 <def_list>, flags=<optimized out>, name=..., val=...) at usr_avp.c:232 #5 0x00007fa1dc898f3d in pv_set_avp (msg=0x7fa1e9ad0ef0, param=0x0, op=-128, val=0x7fff3dbcf7b0) at pv_core.c:1954 #6 0x00000000005444b1 in lval_pvar_assign (lv=<optimized out>, rv=<optimized out>, msg=<optimized out>, h=<optimized out>) at lvalue.c:351 #7 lval_assign (h=0x1, msg=0x7fa1e9ad0ef0, lv=0x7fa1e931fa70, rve=0x7fa1e93202e8) at lvalue.c:399 #8 0x0000000000459a53 in do_action (h=0x7fff3dbd17a0, a=0x7fa1e93209c8, msg=0x7fa1e9ad0ef0) at action.c:1429 #9 0x0000000000458d05 in run_actions (h=0x7fa1e387a068, a=0x68, msg=0xffffffffffffff80) at action.c:1548 #10 0x000000000045a7b6 in do_action (h=0x7fff3dbd17a0, a=0x7fa1e9320ac8, msg=0x7fa1e9ad0ef0) at action.c:1048 #11 0x0000000000458d05 in run_actions (h=0x7fa1e387a068, a=0x68, msg=0xffffffffffffff80) at action.c:1548 #12 0x000000000045a98d in do_action (h=0x7fff3dbd17a0, a=0x7fa1e99442b8, msg=0x7fa1e9ad0ef0) at action.c:677 #13 0x0000000000458d05 in run_actions (h=0x7fa1e387a068, a=0x68, msg=0xffffffffffffff80) at action.c:1548 #14 0x000000000045a7b6 in do_action (h=0x7fff3dbd17a0, a=0x7fa1e920d088, msg=0x7fa1e9ad0ef0) at action.c:1048 #15 0x0000000000458d05 in run_actions (h=0x7fa1e387a068, a=0x68, msg=0xffffffffffffff80) at action.c:1548 #16 0x000000000045a98d in do_action (h=0x7fff3dbd17a0, a=0x7fa1e99442b8, msg=0x7fa1e9ad0ef0) at action.c:677 #17 0x0000000000458d05 in run_actions (h=0x7fa1e387a068, a=0x68, msg=0xffffffffffffff80) at action.c:1548 #18 0x000000000045a7b6 in do_action (h=0x7fff3dbd17a0, a=0x7fa1e91d0b70, msg=0x7fa1e9ad0ef0) at action.c:1048 #19 0x0000000000458d05 in run_actions (h=0x7fa1e387a068, h@entry=0x7fff3dbd17a0, a=0x68, a@entry=0x7fa1e91bc958, msg=0xffffffffffffff80, msg@entry=0x7fa1e9ad0ef0) at action.c:1548 #20 0x0000000000465575 in run_top_route (a=0x7fa1e91bc958, msg=0x7fa1e9ad0ef0, c=<optimized out>) at action.c:1634 ---Type <return> to continue, or q <return> to quit--- #21 0x000000000054692e in receive_msg (buf=0x0, len=104, rcv_info=0x7fa1e1f09e10) at receive.c:195 #22 0x00000000005c28b5 in tcp_read_req (con=0x7fa1e1f09df8, bytes_read=0x7fff3dbd1ae0, read_flags=0x7fff3dbd1ae8) at tcp_read.c:1382 #23 0x00000000005c4d29 in handle_io (fm=0x0, events=104, idx=-1) at tcp_read.c:1624 #24 0x00000000005c994b in io_wait_loop_epoll (h=<optimized out>, t=<optimized out>, repeat=<optimized out>) at io_wait.h:1061 #25 tcp_receive_loop (unix_sock=-477650840) at tcp_read.c:1733 #26 0x00000000004e1f67 in tcp_init_children () at tcp_main.c:4787 #27 0x0000000000513bfb in main_loop () at main.c:1664 #28 0x000000000041c92b in main (argc=0, argv=0x6) at main.c:2566
sr-dev mailing list sr-dev@lists.sip-router.org http://lists.sip-router.org/cgi-bin/mailman/listinfo/sr-dev
Daniel-Constantin Mierla writes:
Anyhow, can you give the output from gdb for:
frame 0 list info locals p *qm p *frag p *n
below, juha
(gdb) frame 0 #0 fm_split_frag (size=104, frag=0x7fa1e3879fe8, qm=0x7fa1e187a000) at mem/f_malloc.c:285 285 in mem/f_malloc.c (gdb) list 280 in mem/f_malloc.c (gdb) info locals rest = 18446744073709551512 n = 0x7fa1e387a068 (gdb) p *qm $1 = {type = 0, size = 33554432, used = 6945608, real_used = 7824440, max_real_used = 10067232, ffrags = 315, first_frag = 0x7fa1e1882478, last_frag = 0x7fa1e3879fe8, free_bitmap = {4100, 0, 17179869184, 0 <repeats 29 times>, 2054}, free_hash = {{first = 0x0, no = 18446744073709551615}, { first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e1eb63d0, no = 2}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, { first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e24cb560, no = 1}, { first = 0x7fa1e3879fe8, no = 0} <repeats 149 times>, {first = 0x7fa1e3879fe8, no = 18446744073709551613}, {first = 0x7fa1e3879fe8, no = 0} <repeats 785 times>, {first = 0x0, no = 0}, {first = 0x7fa1e3879fe8, no = 0} <repeats 56 times>, {first = 0x0, no = 0}, { first = 0x7fa1e3879fe8, no = 0} <repeats 38 times>, {first = 0x0, no = 0}, {first = 0x7fa1e3879fe8, no = 0} <repeats 102 times>, {first = 0x0, no = 0}, {first = 0x7fa1e3879fe8, no = 0} <repeats 17 times>, {first = 0x0, no = 0}, {first = 0x7fa1e3879fe8, no = 0} <repeats 291 times>, {first = 0x0, no = 0}, {first = 0x7fa1e3879fe8, no = 0} <repeats 29 times>, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x7fa1e3879fe8, no = 0} <repeats 40 times>, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, { first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x0, no = 0}, {first = 0x7fa1e3879fe8, no = 0} <repeats 114 times>, {first = 0x0, no = 0}, {first = 0x7fa1e3879fe8, no = 0} <repeats 13 times>, {first = 0x0, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, { first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, { first = 0x7fa1e3879fe8, no = 0}, {first = 0x0, no = 0}, {first = 0x7fa1e3879fe8, no = 0} <repeats 47 times>, {first = 0x0, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, { first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x0, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, { first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, { first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, { first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, ---Type <return> to continue, or q <return> to quit---p *frag {first = 0x0, no = 0}, {first = 0x7fa1e3879fe8, no = 0} <repeats 22 times>, {first = 0x0, no = 0}, { first = 0x7fa1e3879fe8, no = 0} <repeats 20 times>...}} (gdb) p *n $2 = {size = 504, u = {nxt_free = 0x8, reserved = 8}, prv_free = 0x400000003}
The 'list' command didn't print the source code around the line of crash?
Also, 'p *frag' is missing.
Daniel
On 20/09/15 19:19, Juha Heinanen wrote:
Daniel-Constantin Mierla writes:
Anyhow, can you give the output from gdb for:
frame 0 list info locals p *qm p *frag p *n
below, juha
(gdb) frame 0 #0 fm_split_frag (size=104, frag=0x7fa1e3879fe8, qm=0x7fa1e187a000) at mem/f_malloc.c:285 285 in mem/f_malloc.c (gdb) list 280 in mem/f_malloc.c (gdb) info locals rest = 18446744073709551512 n = 0x7fa1e387a068 (gdb) p *qm $1 = {type = 0, size = 33554432, used = 6945608, real_used = 7824440, max_real_used = 10067232, ffrags = 315, first_frag = 0x7fa1e1882478, last_frag = 0x7fa1e3879fe8, free_bitmap = {4100, 0, 17179869184, 0 <repeats 29 times>, 2054}, free_hash = {{first = 0x0, no = 18446744073709551615}, { first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e1eb63d0, no = 2}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, { first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e24cb560, no = 1}, { first = 0x7fa1e3879fe8, no = 0} <repeats 149 times>, {first = 0x7fa1e3879fe8, no = 18446744073709551613}, {first = 0x7fa1e3879fe8, no = 0} <repeats 785 times>, {first = 0x0, no = 0}, {first = 0x7fa1e3879fe8, no = 0} <repeats 56 times>, {first = 0x0, no = 0}, { first = 0x7fa1e3879fe8, no = 0} <repeats 38 times>, {first = 0x0, no = 0}, {first = 0x7fa1e3879fe8, no = 0} <repeats 102 times>, {first = 0x0, no = 0}, {first = 0x7fa1e3879fe8, no = 0} <repeats 17 times>, {first = 0x0, no = 0}, {first = 0x7fa1e3879fe8, no = 0} <repeats 291 times>, {first = 0x0, no = 0}, {first = 0x7fa1e3879fe8, no = 0} <repeats 29 times>, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x7fa1e3879fe8, no = 0} <repeats 40 times>, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, { first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x0, no = 0}, {first = 0x7fa1e3879fe8, no = 0} <repeats 114 times>, {first = 0x0, no = 0}, {first = 0x7fa1e3879fe8, no = 0} <repeats 13 times>, {first = 0x0, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, { first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, { first = 0x7fa1e3879fe8, no = 0}, {first = 0x0, no = 0}, {first = 0x7fa1e3879fe8, no = 0} <repeats 47 times>, {first = 0x0, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, { first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x0, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, { first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, { first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, { first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, ---Type <return> to continue, or q <return> to quit---p *frag {first = 0x0, no = 0}, {first = 0x7fa1e3879fe8, no = 0} <repeats 22 times>, {first = 0x0, no = 0}, { first = 0x7fa1e3879fe8, no = 0} <repeats 20 times>...}} (gdb) p *n $2 = {size = 504, u = {nxt_free = 0x8, reserved = 8}, prv_free = 0x400000003}
Daniel-Constantin Mierla writes:
The 'list' command didn't print the source code around the line of crash?
may be it is because of this:
Program terminated with signal SIGSEGV, Segmentation fault. #0 fm_split_frag (size=104, frag=0x7fa1e3879fe8, qm=0x7fa1e187a000) at mem/f_malloc.c:285 285 mem/f_malloc.c: No such file or directory.
another try:
Program terminated with signal SIGSEGV, Segmentation fault. #0 fm_split_frag (size=104, frag=0x7fa1e3879fe8, qm=0x7fa1e187a000) at mem/f_malloc.c:285 285 mem/f_malloc.c: No such file or directory. (gdb) frame 0 #0 fm_split_frag (size=104, frag=0x7fa1e3879fe8, qm=0x7fa1e187a000) at mem/f_malloc.c:285 285 in mem/f_malloc.c (gdb) list 280 in mem/f_malloc.c (gdb) info locals rest = 18446744073709551512 n = 0x7fa1e387a068 (gdb) p *qm $1 = {type = 0, size = 33554432, used = 6945608, real_used = 7824440, max_real_used = 10067232, ffrags = 315, first_frag = 0x7fa1e1882478, last_frag = 0x7fa1e3879fe8, free_bitmap = {4100, 0, 17179869184, 0 <repeats 29 times>, 2054}, free_hash = {{first = 0x0, no = 18446744073709551615}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e1eb63d0, no = 2}, {first = 0x7fa1e3879fe8, no = 0}, { first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, { first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, { first = 0x7fa1e24cb560, no = 1}, {first = 0x7fa1e3879fe8, no = 0} <repeats 149 times>, {first = 0x7fa1e3879fe8, no = 18446744073709551613}, {first = 0x7fa1e3879fe8, no = 0} <repeats 785 times>, {first = 0x0, no = 0}, {first = 0x7fa1e3879fe8, no = 0} <repeats 56 times>, { first = 0x0, no = 0}, {first = 0x7fa1e3879fe8, no = 0} <repeats 38 times>, {first = 0x0, no = 0}, {first = 0x7fa1e3879fe8, no = 0} <repeats 102 times>, {first = 0x0, no = 0}, {first = 0x7fa1e3879fe8, no = 0} <repeats 17 times>, {first = 0x0, no = 0}, { first = 0x7fa1e3879fe8, no = 0} <repeats 291 times>, {first = 0x0, no = 0}, {first = 0x7fa1e3879fe8, no = 0} <repeats 29 times>, { first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x7fa1e3879fe8, no = 0} <repeats 40 times>, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x0, no = 0}, { first = 0x7fa1e3879fe8, no = 0} <repeats 114 times>, {first = 0x0, no = 0}, {first = 0x7fa1e3879fe8, no = 0} <repeats 13 times>, { first = 0x0, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, { first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, { first = 0x7fa1e3879fe8, no = 0}, {first = 0x0, no = 0}, {first = 0x7fa1e3879fe8, no = 0} <repeats 47 times>, {first = 0x0, no = 0}, { first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, { first = 0x7fa1e3879fe8, no = 0}, {first = 0x0, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, { first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x7fa1e3879fe8, no = 0}, {first = 0x0, no = 0}, {first = 0x7fa1e3879fe8, no = 0} <repeats 22 times>, {first = 0x0, no = 0}, {first = 0x7fa1e3879fe8, no = 0} <repeats 20 times>...}} (gdb) p *frag $2 = {size = 104, u = {nxt_free = 0x0, reserved = 0}, prv_free = 0x0} (gdb) p *n $3 = {size = 504, u = {nxt_free = 0x8, reserved = 8}, prv_free = 0x400000003} (gdb)
-- juha
i built new kamailio from latest master and got again the crash. this time it produced two core files:
45852 -rw------- 1 root root 59764736 Sep 20 21:21 /var/cores/core.sip-proxy.sig11.32373 45856 -rw------- 1 root root 59805696 Sep 20 21:21 /var/cores/core.sip-proxy.sig11.32406
/var/cores/core.sip-proxy.sig11.32373 gives:
(gdb) frame 0 #0 fm_free (qmp=0xa42d00 <_shm_root>, p=0x2d64726f6365520a) at mem/f_malloc.c:589 589 in mem/f_malloc.c (gdb) list 584 in mem/f_malloc.c (gdb) info locals f = 0x7f78ef610780 __FUNCTION__ = "fm_free" (gdb) p *qm value has been optimized out (gdb) p *frag No symbol "frag" in current context. (gdb) p *n No symbol "n" in current context.
and /var/cores/core.sip-proxy.sig11.32406 gives:
(gdb) frame 0 #0 fm_extract_free (frag=<optimized out>, qm=<optimized out>) at mem/f_malloc.c:181 181 in mem/f_malloc.c (gdb) list 176 in mem/f_malloc.c (gdb) info locals hash = 2 (gdb) p *qm value has been optimized out (gdb) p *frag value has been optimized out (gdb) p *n No symbol "n" in current context.
-- juha
These two have different backtraces, so the previous commands won't work because of variables not being present in the frame. Send first 'bt' for each of the cores.
Daniel
On 20/09/15 20:31, Juha Heinanen wrote:
i built new kamailio from latest master and got again the crash. this time it produced two core files:
45852 -rw------- 1 root root 59764736 Sep 20 21:21 /var/cores/core.sip-proxy.sig11.32373 45856 -rw------- 1 root root 59805696 Sep 20 21:21 /var/cores/core.sip-proxy.sig11.32406
/var/cores/core.sip-proxy.sig11.32373 gives:
(gdb) frame 0 #0 fm_free (qmp=0xa42d00 <_shm_root>, p=0x2d64726f6365520a) at mem/f_malloc.c:589 589 in mem/f_malloc.c (gdb) list 584 in mem/f_malloc.c (gdb) info locals f = 0x7f78ef610780 __FUNCTION__ = "fm_free" (gdb) p *qm value has been optimized out (gdb) p *frag No symbol "frag" in current context. (gdb) p *n No symbol "n" in current context.
and /var/cores/core.sip-proxy.sig11.32406 gives:
(gdb) frame 0 #0 fm_extract_free (frag=<optimized out>, qm=<optimized out>) at mem/f_malloc.c:181 181 in mem/f_malloc.c (gdb) list 176 in mem/f_malloc.c (gdb) info locals hash = 2 (gdb) p *qm value has been optimized out (gdb) p *frag value has been optimized out (gdb) p *n No symbol "n" in current context.
-- juha
I just pushed a patch, no need to send the backtraces anymore, try with latest master to see if there is any issue.
Daniel
On 20/09/15 22:20, Daniel-Constantin Mierla wrote:
These two have different backtraces, so the previous commands won't work because of variables not being present in the frame. Send first 'bt' for each of the cores.
Daniel
On 20/09/15 20:31, Juha Heinanen wrote:
i built new kamailio from latest master and got again the crash. this time it produced two core files:
45852 -rw------- 1 root root 59764736 Sep 20 21:21 /var/cores/core.sip-proxy.sig11.32373 45856 -rw------- 1 root root 59805696 Sep 20 21:21 /var/cores/core.sip-proxy.sig11.32406
/var/cores/core.sip-proxy.sig11.32373 gives:
(gdb) frame 0 #0 fm_free (qmp=0xa42d00 <_shm_root>, p=0x2d64726f6365520a) at mem/f_malloc.c:589 589 in mem/f_malloc.c (gdb) list 584 in mem/f_malloc.c (gdb) info locals f = 0x7f78ef610780 __FUNCTION__ = "fm_free" (gdb) p *qm value has been optimized out (gdb) p *frag No symbol "frag" in current context. (gdb) p *n No symbol "n" in current context.
and /var/cores/core.sip-proxy.sig11.32406 gives:
(gdb) frame 0 #0 fm_extract_free (frag=<optimized out>, qm=<optimized out>) at mem/f_malloc.c:181 181 in mem/f_malloc.c (gdb) list 176 in mem/f_malloc.c (gdb) info locals hash = 2 (gdb) p *qm value has been optimized out (gdb) p *frag value has been optimized out (gdb) p *n No symbol "n" in current context.
-- juha
Daniel-Constantin Mierla writes:
I just pushed a patch, no need to send the backtraces anymore, try with latest master to see if there is any issue.
the crash has not anymore happened after a few tests. i'll keep watching.
was this also in 4.3, because the fix was backported? if so, for some reason i have not seen it there.
-- juha
On 21/09/15 08:09, Juha Heinanen wrote:
Daniel-Constantin Mierla writes:
I just pushed a patch, no need to send the backtraces anymore, try with latest master to see if there is any issue.
the crash has not anymore happened after a few tests. i'll keep watching.
was this also in 4.3, because the fix was backported? if so, for some reason i have not seen it there.
There was another backport in charge of the issue, done Friday, for the management of the free fragments when prev free fragment could point to an invalid value -- the regression was that next free ended up with invalid value in some cases. The patch was in master for quite long time, but probably surfaced by the last changes (not to be backported).
Daniel
i got the crash again with latest kamailio master.
-- juha
Program terminated with signal SIGSEGV, Segmentation fault. #0 fm_insert_free (frag=<optimized out>, qm=<optimized out>) at mem/f_malloc.c:221 221 mem/f_malloc.c: No such file or directory. (gdb) where #0 fm_insert_free (frag=<optimized out>, qm=<optimized out>) at mem/f_malloc.c:221 #1 fm_free (qmp=0x7f40f262f000, p=0x7f40f31d9d78) at mem/f_malloc.c:609 #2 0x00007f40f7dd578c in free_cell (dead_cell=0x7f40f31d8718) at h_table.c:133 #3 0x00007f40f7e409f0 in wait_handler (ti=<optimized out>, wait_tl=<optimized out>, data=0x7f40f31d8718) at timer.c:648 #4 0x000000000049947e in timer_list_expire (slow_mark=<optimized out>, slow_l=<optimized out>, h=<optimized out>, t=<optimized out>) at timer.c:873 #5 timer_handler () at timer.c:938 #6 timer_main () at timer.c:977 #7 0x0000000000513bdd in main_loop () at main.c:1650 #8 0x000000000041c92b in main (argc=0, argv=0x6) at main.c:2566
Can you get:
frame 0 info locals p *qm p *frag p *f
Cheers, Daniel
On 21/09/15 08:56, Juha Heinanen wrote:
i got the crash again with latest kamailio master.
-- juha
Program terminated with signal SIGSEGV, Segmentation fault. #0 fm_insert_free (frag=<optimized out>, qm=<optimized out>) at mem/f_malloc.c:221 221 mem/f_malloc.c: No such file or directory. (gdb) where #0 fm_insert_free (frag=<optimized out>, qm=<optimized out>) at mem/f_malloc.c:221 #1 fm_free (qmp=0x7f40f262f000, p=0x7f40f31d9d78) at mem/f_malloc.c:609 #2 0x00007f40f7dd578c in free_cell (dead_cell=0x7f40f31d8718) at h_table.c:133 #3 0x00007f40f7e409f0 in wait_handler (ti=<optimized out>, wait_tl=<optimized out>, data=0x7f40f31d8718) at timer.c:648 #4 0x000000000049947e in timer_list_expire (slow_mark=<optimized out>, slow_l=<optimized out>, h=<optimized out>, t=<optimized out>) at timer.c:873 #5 timer_handler () at timer.c:938 #6 timer_main () at timer.c:977 #7 0x0000000000513bdd in main_loop () at main.c:1650 #8 0x000000000041c92b in main (argc=0, argv=0x6) at main.c:2566
Daniel-Constantin Mierla writes:
Can you get:
frame 0 info locals p *qm p *frag p *f
not much luck with those:
(gdb) frame 0 #0 fm_insert_free (frag=<optimized out>, qm=<optimized out>) at mem/f_malloc.c:221 221 in mem/f_malloc.c (gdb) info locals hash = 2049 (gdb) p *qm value has been optimized out (gdb) p *frag value has been optimized out (gdb) p *f value has been optimized out
-- juha
Let's try on frame 1:
frame 1 info locals p *qm p *f
Daniel
On 21/09/15 15:48, Juha Heinanen wrote:
Daniel-Constantin Mierla writes:
Can you get:
frame 0 info locals p *qm p *frag p *f
not much luck with those:
(gdb) frame 0 #0 fm_insert_free (frag=<optimized out>, qm=<optimized out>) at mem/f_malloc.c:221 221 in mem/f_malloc.c (gdb) info locals hash = 2049 (gdb) p *qm value has been optimized out (gdb) p *frag value has been optimized out (gdb) p *f value has been optimized out
-- juha
Daniel-Constantin Mierla writes:
Let's try on frame 1:
frame 1 info locals p *qm p *f
(gdb) frame 1 #1 fm_free (qmp=0x7f40f262f000, p=0x7f40f31d9d78) at mem/f_malloc.c:609 609 in mem/f_malloc.c (gdb) info locals f = 0x7f40f31d9d60 __FUNCTION__ = "fm_free" (gdb) p *qm value has been optimized out (gdb) p *f $1 = {size = 23896, u = {nxt_free = 0x0, reserved = 0}, prv_free = 0x0}
-- juha
What is the level of optimization you set for compiler? Lots of local vars can't be retrieved.
Let's see if we can get the qm via pointer:
frame 1 p *((struct fm_block*)0x7f40f262f000)
frame 0 p ((struct fm_block*)0x7f40f262f000)->free_hash[hash]
Daniel
On 21/09/15 15:56, Juha Heinanen wrote:
Daniel-Constantin Mierla writes:
Let's try on frame 1:
frame 1 info locals p *qm p *f
(gdb) frame 1 #1 fm_free (qmp=0x7f40f262f000, p=0x7f40f31d9d78) at mem/f_malloc.c:609 609 in mem/f_malloc.c (gdb) info locals f = 0x7f40f31d9d60 __FUNCTION__ = "fm_free" (gdb) p *qm value has been optimized out (gdb) p *f $1 = {size = 23896, u = {nxt_free = 0x0, reserved = 0}, prv_free = 0x0}
-- juha
Daniel-Constantin Mierla writes:
What is the level of optimization you set for compiler? Lots of local vars can't be retrieved.
i don't think i have set anything what is not in kamailio debian rules. it has:
export DEB_BUILD_MAINT_OPTIONS = hardening=+all DPKG_EXPORT_BUILDFLAGS = 1 include /usr/share/dpkg/buildflags.mk
Let's see if we can get the qm via pointer:
frame 1 p *((struct fm_block*)0x7f40f262f000)
frame 0 p ((struct fm_block*)0x7f40f262f000)->free_hash[hash]
(gdb) frame 1 #1 fm_free (qmp=0x7f40f262f000, p=0x7f40f31d9d78) at mem/f_malloc.c:609 609 in mem/f_malloc.c (gdb) p *((struct fm_block*)0x7f40f262f000) $2 = {type = 0, size = 33554432, used = 6130440, real_used = 6996048, max_real_used = 8823136, ffrags = 421, first_frag = 0x7f40f2637478, last_frag = 0x7f40f462efe8, free_bitmap = {171834903804, 723812902624044032, 2378468964866203673, 1152921576547560512, 65536, 142936511611136, 18014398509481984, 4398046511104, 35184372088832, 1024, 1168231104516, 140739635840512, 4323456466917785600, 2305843421532934272, 1729399849633316865, 1073741824, 268435464, 563018672898048, 0, 0, 140737488355328, 1100048498688, 0, 0, 67108864, 65536, 2199031644160, 81064930732703747, 2533825619951616, 3023660122144, 36028797622943744, 1155208505972506624, 2054}, free_hash = {{ first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x7f40f2c6aad8, no = 2}, {first = 0x7f40f3211a88, no = 5}, { first = 0x7f40f3254fb0, no = 1}, {first = 0x7f40f2c6a8b8, no = 2}, {first = 0x7f40f2c654b0, no = 2}, { first = 0x7f40f2c6ac68, no = 1}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x7f40f2d77070, no = 3}, { first = 0x7f40f2ca42d0, no = 3}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, { first = 0x7f40f2d76f38, no = 1}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, { first = 0x7f40f2c6e970, no = 1}, {first = 0x0, no = 0}, {first = 0x7f40f3251dd8, no = 2}, {first = 0x0, no = 0}, { first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x7f40f2f64688, no = 1}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x7f40f2f6b620, no = 1}, {first = 0x0, no = 0}, { first = 0x7f40f2f64768, no = 1}, {first = 0x0, no = 0} <repeats 36 times>, {first = 0x7f40f31720b0, no = 1}, { first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x7f40f3288b58, no = 5}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x7f40f317a068, no = 7}, {first = 0x7f40f2d85aa8, no = 1}, {first = 0x0, no = 0} <repeats 23 times>, {first = 0x7f40f2eb00b0, no = 2}, {first = 0x7f40f322ae80, no = 1}, { first = 0x7f40f30b9cf8, no = 1}, {first = 0x0, no = 0}, {first = 0x7f40f315b260, no = 1}, {first = 0x0, no = 0}, { first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x7f40f3047cf0, no = 1}, {first = 0x0, no = 0}, {first = 0x7f40f2c967d8, no = 1}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, { first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x7f40f2ec6110, no = 1}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x7f40f2c96bc8, no = 1}, {first = 0x7f40f309eed8, no = 1}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x7f40f30a45b8, no = 1}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, { first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x7f40f3295108, no = 6}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x7f40f2f8c410, no = 1}, {first = 0x0, no = 0} <repeats 12 times>, {first = 0x7f40f32955b0, no = 13}, {first = 0x7f40f317abc8, no = 5}, {first = 0x0, no = 0}, {first = 0x7f40f2cef330, no = 2}, { first = 0x7f40f32b5560, no = 1}, {first = 0x7f40f2c6de58, no = 1}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, { ---Type <return> to continue, or q <return> to quit--- first = 0x7f40f2fc7060, no = 1}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x7f40f31e2c88, no = 1}, {first = 0x0, no = 0}, { first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, { first = 0x7f40f308d4b0, no = 1}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x7f40f30a3cf0, no = 5}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, { first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, { first = 0x7f40f30f3f98, no = 1}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, { first = 0x7f40f2e8bf50, no = 1}, {first = 0x7f40f3163008, no = 9}, {first = 0x0, no = 0}, {first = 0x7f40f30dbd70, no = 1}, {first = 0x0, no = 0} <repeats 16 times>, {first = 0x7f40f3295ad8, no = 1}, {first = 0x7f40f315bdb8, no = 1}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, { first = 0x7f40f30cefc0, no = 1}, {first = 0x0, no = 0} <repeats 23 times>, {first = 0x7f40f2eaf8b8, no = 2}, { first = 0x0, no = 0} <repeats 19 times>...}} (gdb) frame 0 #0 fm_insert_free (frag=<optimized out>, qm=<optimized out>) at mem/f_malloc.c:221 221 in mem/f_malloc.c (gdb) p ((struct fm_block*)0x7f40f262f000)->free_hash[hash] $3 = {first = 0x7f40f2c9ebb0, no = 192} (gdb)
-- juha
Can you try with latest master again?
I think the situation was when the fragment size to be freed was greater than the one of last free fragment in the slot of big fragments.
Daniel
On 21/09/15 16:26, Juha Heinanen wrote:
Daniel-Constantin Mierla writes:
What is the level of optimization you set for compiler? Lots of local vars can't be retrieved.
i don't think i have set anything what is not in kamailio debian rules. it has:
export DEB_BUILD_MAINT_OPTIONS = hardening=+all DPKG_EXPORT_BUILDFLAGS = 1 include /usr/share/dpkg/buildflags.mk
Let's see if we can get the qm via pointer:
frame 1 p *((struct fm_block*)0x7f40f262f000)
frame 0 p ((struct fm_block*)0x7f40f262f000)->free_hash[hash]
(gdb) frame 1 #1 fm_free (qmp=0x7f40f262f000, p=0x7f40f31d9d78) at mem/f_malloc.c:609 609 in mem/f_malloc.c (gdb) p *((struct fm_block*)0x7f40f262f000) $2 = {type = 0, size = 33554432, used = 6130440, real_used = 6996048, max_real_used = 8823136, ffrags = 421, first_frag = 0x7f40f2637478, last_frag = 0x7f40f462efe8, free_bitmap = {171834903804, 723812902624044032, 2378468964866203673, 1152921576547560512, 65536, 142936511611136, 18014398509481984, 4398046511104, 35184372088832, 1024, 1168231104516, 140739635840512, 4323456466917785600, 2305843421532934272, 1729399849633316865, 1073741824, 268435464, 563018672898048, 0, 0, 140737488355328, 1100048498688, 0, 0, 67108864, 65536, 2199031644160, 81064930732703747, 2533825619951616, 3023660122144, 36028797622943744, 1155208505972506624, 2054}, free_hash = {{ first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x7f40f2c6aad8, no = 2}, {first = 0x7f40f3211a88, no = 5}, { first = 0x7f40f3254fb0, no = 1}, {first = 0x7f40f2c6a8b8, no = 2}, {first = 0x7f40f2c654b0, no = 2}, { first = 0x7f40f2c6ac68, no = 1}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x7f40f2d77070, no = 3}, { first = 0x7f40f2ca42d0, no = 3}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, { first = 0x7f40f2d76f38, no = 1}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, { first = 0x7f40f2c6e970, no = 1}, {first = 0x0, no = 0}, {first = 0x7f40f3251dd8, no = 2}, {first = 0x0, no = 0}, { first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x7f40f2f64688, no = 1}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x7f40f2f6b620, no = 1}, {first = 0x0, no = 0}, { first = 0x7f40f2f64768, no = 1}, {first = 0x0, no = 0} <repeats 36 times>, {first = 0x7f40f31720b0, no = 1}, { first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x7f40f3288b58, no = 5}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x7f40f317a068, no = 7}, {first = 0x7f40f2d85aa8, no = 1}, {first = 0x0, no = 0} <repeats 23 times>, {first = 0x7f40f2eb00b0, no = 2}, {first = 0x7f40f322ae80, no = 1}, { first = 0x7f40f30b9cf8, no = 1}, {first = 0x0, no = 0}, {first = 0x7f40f315b260, no = 1}, {first = 0x0, no = 0}, { first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x7f40f3047cf0, no = 1}, {first = 0x0, no = 0}, {first = 0x7f40f2c967d8, no = 1}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, { first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x7f40f2ec6110, no = 1}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x7f40f2c96bc8, no = 1}, {first = 0x7f40f309eed8, no = 1}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x7f40f30a45b8, no = 1}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, { first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x7f40f3295108, no = 6}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x7f40f2f8c410, no = 1}, {first = 0x0, no = 0} <repeats 12 times>, {first = 0x7f40f32955b0, no = 13}, {first = 0x7f40f317abc8, no = 5}, {first = 0x0, no = 0}, {first = 0x7f40f2cef330, no = 2}, { first = 0x7f40f32b5560, no = 1}, {first = 0x7f40f2c6de58, no = 1}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, { ---Type <return> to continue, or q <return> to quit--- first = 0x7f40f2fc7060, no = 1}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x7f40f31e2c88, no = 1}, {first = 0x0, no = 0}, { first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, { first = 0x7f40f308d4b0, no = 1}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x7f40f30a3cf0, no = 5}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, { first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, { first = 0x7f40f30f3f98, no = 1}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, { first = 0x7f40f2e8bf50, no = 1}, {first = 0x7f40f3163008, no = 9}, {first = 0x0, no = 0}, {first = 0x7f40f30dbd70, no = 1}, {first = 0x0, no = 0} <repeats 16 times>, {first = 0x7f40f3295ad8, no = 1}, {first = 0x7f40f315bdb8, no = 1}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, { first = 0x7f40f30cefc0, no = 1}, {first = 0x0, no = 0} <repeats 23 times>, {first = 0x7f40f2eaf8b8, no = 2}, { first = 0x0, no = 0} <repeats 19 times>...}} (gdb) frame 0 #0 fm_insert_free (frag=<optimized out>, qm=<optimized out>) at mem/f_malloc.c:221 221 in mem/f_malloc.c (gdb) p ((struct fm_block*)0x7f40f262f000)->free_hash[hash] $3 = {first = 0x7f40f2c9ebb0, no = 192} (gdb)
-- juha
Daniel-Constantin Mierla writes:
Can you try with latest master again?
still crashing.
-- juha
45836 -rw------- 1 root root 59764736 Sep 21 18:44 core.sip-proxy.sig11.22435 45852 -rw------- 1 root root 59805696 Sep 21 18:44 core.sip-proxy.sig11.22460
# gdb /usr/bin/sip-proxy /var/cores/core.sip-proxy.sig11.22435 (gdb) where #0 0x00000000005f7575 in fm_insert_free (frag=<optimized out>, qm=<optimized out>) at mem/f_malloc.c:242 #1 fm_free (qmp=0x7fc3c325f000, p=0x7fc3c365b438) at mem/f_malloc.c:624 #2 0x00000000005f7aeb in fm_shm_free (qmp=0x7fc3c325f000, p=0x7fc3c365b438) at mem/f_malloc.c:1101 #3 0x00007fc3bd79fb52 in free_udomain (_d=0x7fc3c3646308) at udomain.c:152 #4 0x00007fc3bd7c5b11 in free_all_udomains () at dlist.c:697 #5 0x00007fc3bd7b3272 in destroy () at ul_mod.c:493 #6 0x0000000000517022 in destroy_modules () at sr_module.c:805 #7 0x000000000050c5f3 in cleanup (show_status=1) at main.c:519 #8 0x000000000050d63a in shutdown_children (show_status=1, sig=<optimized out>) at main.c:661 #9 0x000000000050f3e8 in handle_sigs () at main.c:752 #10 0x0000000000513df5 in main_loop () at main.c:1707 #11 0x000000000041c92b in main (argc=0, argv=0x6) at main.c:2566
# gdb /usr/bin/sip-proxy /var/cores/core.sip-proxy.sig11.22460 (gdb) where #0 0x00000000005f7575 in fm_insert_free (frag=<optimized out>, qm=<optimized out>) at mem/f_malloc.c:242 #1 fm_free (qmp=0x7fc3c325f000, p=0x7fc3c3a758c8) at mem/f_malloc.c:624 #2 0x00007fc3c8a0577a in free_cell (dead_cell=0x7fc3c3ae68d8) at h_table.c:133 #3 0x00007fc3c8a709f0 in wait_handler (ti=<optimized out>, wait_tl=<optimized out>, data=0x7fc3c3ae68d8) at timer.c:648 #4 0x000000000049947e in timer_list_expire (slow_mark=<optimized out>, slow_l=<optimized out>, h=<optimized out>, t=<optimized out>) at timer.c:873 #5 timer_handler () at timer.c:938 #6 timer_main () at timer.c:977 #7 0x0000000000513bdd in main_loop () at main.c:1650 #8 0x000000000041c92b in main (argc=0, argv=0x6) at main.c:2566
From the second core, get:
frame 1 info locals p *((struct fm_block*)0x7fc3c325f000) p *f
frame 0 info locals p ((struct fm_block*)0x7fc3c325f000)->free_hash[hash] p *f
Daniel
On 21/09/15 18:28, Juha Heinanen wrote:
Daniel-Constantin Mierla writes:
Can you try with latest master again?
still crashing.
-- juha
45836 -rw------- 1 root root 59764736 Sep 21 18:44 core.sip-proxy.sig11.22435 45852 -rw------- 1 root root 59805696 Sep 21 18:44 core.sip-proxy.sig11.22460
# gdb /usr/bin/sip-proxy /var/cores/core.sip-proxy.sig11.22435 (gdb) where #0 0x00000000005f7575 in fm_insert_free (frag=<optimized out>, qm=<optimized out>) at mem/f_malloc.c:242 #1 fm_free (qmp=0x7fc3c325f000, p=0x7fc3c365b438) at mem/f_malloc.c:624 #2 0x00000000005f7aeb in fm_shm_free (qmp=0x7fc3c325f000, p=0x7fc3c365b438) at mem/f_malloc.c:1101 #3 0x00007fc3bd79fb52 in free_udomain (_d=0x7fc3c3646308) at udomain.c:152 #4 0x00007fc3bd7c5b11 in free_all_udomains () at dlist.c:697 #5 0x00007fc3bd7b3272 in destroy () at ul_mod.c:493 #6 0x0000000000517022 in destroy_modules () at sr_module.c:805 #7 0x000000000050c5f3 in cleanup (show_status=1) at main.c:519 #8 0x000000000050d63a in shutdown_children (show_status=1, sig=<optimized out>) at main.c:661 #9 0x000000000050f3e8 in handle_sigs () at main.c:752 #10 0x0000000000513df5 in main_loop () at main.c:1707 #11 0x000000000041c92b in main (argc=0, argv=0x6) at main.c:2566
# gdb /usr/bin/sip-proxy /var/cores/core.sip-proxy.sig11.22460 (gdb) where #0 0x00000000005f7575 in fm_insert_free (frag=<optimized out>, qm=<optimized out>) at mem/f_malloc.c:242 #1 fm_free (qmp=0x7fc3c325f000, p=0x7fc3c3a758c8) at mem/f_malloc.c:624 #2 0x00007fc3c8a0577a in free_cell (dead_cell=0x7fc3c3ae68d8) at h_table.c:133 #3 0x00007fc3c8a709f0 in wait_handler (ti=<optimized out>, wait_tl=<optimized out>, data=0x7fc3c3ae68d8) at timer.c:648 #4 0x000000000049947e in timer_list_expire (slow_mark=<optimized out>, slow_l=<optimized out>, h=<optimized out>, t=<optimized out>) at timer.c:873 #5 timer_handler () at timer.c:938 #6 timer_main () at timer.c:977 #7 0x0000000000513bdd in main_loop () at main.c:1650 #8 0x000000000041c92b in main (argc=0, argv=0x6) at main.c:2566
Daniel-Constantin Mierla writes:
From the second core, get:
frame 1 info locals p *((struct fm_block*)0x7fc3c325f000) p *f
frame 0 info locals p ((struct fm_block*)0x7fc3c325f000)->free_hash[hash] p *f
here, juha
(gdb) frame 1 #1 fm_free (qmp=0x7fc3c325f000, p=0x7fc3c3a758c8) at mem/f_malloc.c:624 624 in mem/f_malloc.c (gdb) info locals f = 0x7fc3c3a758b0 __FUNCTION__ = "fm_free" (gdb) p *((struct fm_block*)0x7fc3c325f000) $1 = {type = 0, size = 33554432, used = 6355208, real_used = 7214840, max_real_used = 8902688, ffrags = 110, first_frag = 0x7fc3c3267478, last_frag = 0x7fc3c525efe8, free_bitmap = {18014398509482118, 12599296, 4035225283304128528, 2048, 8594128896, 0, 0, 0, 0, 2048, 2199023255556, 140737488355840, 274877906944, 16384, 140737488355328, 134217728, 216172782115880964, 0, 0, 0, 2251799813687296, 0, 576460752304472064, 4503599627370496, 0, 288230376151711744, 1073741824, 1048576, 281474976710656, 0, 8796093022208, 0, 2054}, free_hash = {{first = 0x0, no = 0}, {first = 0x7fc3c389b918, no = 1}, {first = 0x7fc3c3898d40, no = 2}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x7fc3c3ae1458, no = 1}, {first = 0x0, no = 0} <repeats 46 times>, {first = 0x7fc3c3933d08, no = 1}, {first = 0x0, no = 0} <repeats 23 times>, { first = 0x7fc3c3a8f898, no = 1}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x7fc3c39d0880, no = 1}, { first = 0x7fc3c3984360, no = 2}, {first = 0x0, no = 0} <repeats 44 times>, {first = 0x7fc3c3a0eb48, no = 1}, { first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, { first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, { first = 0x7fc3c3a0e6b8, no = 1}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x7fc3c39f8278, no = 4}, { first = 0x0, no = 0} <repeats 15 times>, {first = 0x7fc3c3a103b8, no = 1}, {first = 0x0, no = 0} <repeats 24 times>, {first = 0x7fc3c39c2398, no = 1}, {first = 0x7fc3c39fe0a0, no = 1}, {first = 0x7fc3c3a44878, no = 2}, {first = 0x0, no = 0} <repeats 13 times>, {first = 0x7fc3c3ae4e28, no = 3}, {first = 0x0, no = 0} <repeats 74 times>, { first = 0x7fc3c3a93890, no = 5}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x7fc3c39be508, no = 1}, {first = 0x0, no = 0} <repeats 297 times>, { first = 0x7fc3c3afaa88, no = 1}, {first = 0x0, no = 0} <repeats 54 times>, {first = 0x7fc3c3ae5498, no = 2}, { first = 0x0, no = 0} <repeats 38 times>, {first = 0x7fc3c3aeff90, no = 2}, {first = 0x0, no = 0} <repeats 31 times>, {first = 0x7fc3c3a57370, no = 18446744073709551615}, {first = 0x0, no = 0} <repeats 37 times>, { first = 0x7fc3c3aff538, no = 3}, {first = 0x0, no = 0} <repeats 54 times>, {first = 0x7fc3c3a510a0, no = 4}, { first = 0x0, no = 0} <repeats 39 times>, {first = 0x7fc3c3a7e538, no = 2}, {first = 0x0, no = 0} <repeats 96 times>...}} (gdb) p *f $2 = {size = 35952, u = {nxt_free = 0x7fc3c39d1c3b, reserved = 140478777203771}, prv_free = 0x430a0d6f666e6977} (gdb) frame 0 #0 0x00000000005f7575 in fm_insert_free (frag=<optimized out>, qm=<optimized out>) at mem/f_malloc.c:242 242 in mem/f_malloc.c (gdb) info locals hash = 2050 (gdb) p ((struct fm_block*)0x7fc3c325f000)->free_hash[hash] $4 = {first = 0x7fc3c39d13e0, no = 25} (gdb) p *f value has been optimized out (gdb)
Strange that 'info locals' doesn't print all local variables, can you get:
frame 0 print after
Saniel
On 21/09/15 19:15, Juha Heinanen wrote:
Daniel-Constantin Mierla writes:
From the second core, get:
frame 1 info locals p *((struct fm_block*)0x7fc3c325f000) p *f
frame 0 info locals p ((struct fm_block*)0x7fc3c325f000)->free_hash[hash] p *f
here, juha
(gdb) frame 1 #1 fm_free (qmp=0x7fc3c325f000, p=0x7fc3c3a758c8) at mem/f_malloc.c:624 624 in mem/f_malloc.c (gdb) info locals f = 0x7fc3c3a758b0 __FUNCTION__ = "fm_free" (gdb) p *((struct fm_block*)0x7fc3c325f000) $1 = {type = 0, size = 33554432, used = 6355208, real_used = 7214840, max_real_used = 8902688, ffrags = 110, first_frag = 0x7fc3c3267478, last_frag = 0x7fc3c525efe8, free_bitmap = {18014398509482118, 12599296, 4035225283304128528, 2048, 8594128896, 0, 0, 0, 0, 2048, 2199023255556, 140737488355840, 274877906944, 16384, 140737488355328, 134217728, 216172782115880964, 0, 0, 0, 2251799813687296, 0, 576460752304472064, 4503599627370496, 0, 288230376151711744, 1073741824, 1048576, 281474976710656, 0, 8796093022208, 0, 2054}, free_hash = {{first = 0x0, no = 0}, {first = 0x7fc3c389b918, no = 1}, {first = 0x7fc3c3898d40, no = 2}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x7fc3c3ae1458, no = 1}, {first = 0x0, no = 0} <repeats 46 times>, {first = 0x7fc3c3933d08, no = 1}, {first = 0x0, no = 0} <repeats 23 times>, { first = 0x7fc3c3a8f898, no = 1}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x7fc3c39d0880, no = 1}, { first = 0x7fc3c3984360, no = 2}, {first = 0x0, no = 0} <repeats 44 times>, {first = 0x7fc3c3a0eb48, no = 1}, { first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, { first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, { first = 0x7fc3c3a0e6b8, no = 1}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x7fc3c39f8278, no = 4}, { first = 0x0, no = 0} <repeats 15 times>, {first = 0x7fc3c3a103b8, no = 1}, {first = 0x0, no = 0} <repeats 24 times>, {first = 0x7fc3c39c2398, no = 1}, {first = 0x7fc3c39fe0a0, no = 1}, {first = 0x7fc3c3a44878, no = 2}, {first = 0x0, no = 0} <repeats 13 times>, {first = 0x7fc3c3ae4e28, no = 3}, {first = 0x0, no = 0} <repeats 74 times>, { first = 0x7fc3c3a93890, no = 5}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x7fc3c39be508, no = 1}, {first = 0x0, no = 0} <repeats 297 times>, { first = 0x7fc3c3afaa88, no = 1}, {first = 0x0, no = 0} <repeats 54 times>, {first = 0x7fc3c3ae5498, no = 2}, { first = 0x0, no = 0} <repeats 38 times>, {first = 0x7fc3c3aeff90, no = 2}, {first = 0x0, no = 0} <repeats 31 times>, {first = 0x7fc3c3a57370, no = 18446744073709551615}, {first = 0x0, no = 0} <repeats 37 times>, { first = 0x7fc3c3aff538, no = 3}, {first = 0x0, no = 0} <repeats 54 times>, {first = 0x7fc3c3a510a0, no = 4}, { first = 0x0, no = 0} <repeats 39 times>, {first = 0x7fc3c3a7e538, no = 2}, {first = 0x0, no = 0} <repeats 96 times>...}} (gdb) p *f $2 = {size = 35952, u = {nxt_free = 0x7fc3c39d1c3b, reserved = 140478777203771}, prv_free = 0x430a0d6f666e6977} (gdb) frame 0 #0 0x00000000005f7575 in fm_insert_free (frag=<optimized out>, qm=<optimized out>) at mem/f_malloc.c:242 242 in mem/f_malloc.c (gdb) info locals hash = 2050 (gdb) p ((struct fm_block*)0x7fc3c325f000)->free_hash[hash] $4 = {first = 0x7fc3c39d13e0, no = 25} (gdb) p *f value has been optimized out (gdb)
Daniel-Constantin Mierla writes:
Strange that 'info locals' doesn't print all local variables, can you get:
frame 0 print after
no luck:
(gdb) frame 0 #0 0x00000000005f7575 in fm_insert_free (frag=<optimized out>, qm=<optimized out>) at mem/f_malloc.c:242 242 in mem/f_malloc.c (gdb) print after $2 = <optimized out>
-- juha
I wonder what makes the local var 'hash' to be printed but not 'after'. If you installed from deb, do you have the kamailio-dbg package installed?
Daniel
On 21/09/15 20:52, Juha Heinanen wrote:
Daniel-Constantin Mierla writes:
Strange that 'info locals' doesn't print all local variables, can you get:
frame 0 print after
no luck:
(gdb) frame 0 #0 0x00000000005f7575 in fm_insert_free (frag=<optimized out>, qm=<optimized out>) at mem/f_malloc.c:242 242 in mem/f_malloc.c (gdb) print after $2 = <optimized out>
-- juha
Daniel-Constantin Mierla writes:
I wonder what makes the local var 'hash' to be printed but not 'after'. If you installed from deb, do you have the kamailio-dbg package installed?
no, i don't. but i have commented out
# dh_strip --dbg-package=kamailio-dbg
so symbols should be included.
this is how i make kamailio:
/usr/bin/make FLAVOUR=kamailio cfg prefix=/usr cfg_prefix=/usr/src/opensipg/trunk/src/openxg-sip-proxy/debian/openxg-sip-proxy \ cfg_target=/etc/sip-proxy/ \ run_target=/var/run/sip-proxy \ BASEDIR=/usr/src/opensipg/trunk/src/openxg-sip-proxy/debian/openxg-sip-proxy \ FMSTATS=1 KMSTATS=1 \ MAIN_NAME=sip-proxy STUN=1 \ skip_modules=" " \ CC_EXTRA_OPTS="-D_FORTIFY_SOURCE=2 -g -O2 -fstack-protector-strong -Wformat -Werror=format-security -DVERSION_NODATE" \ LD_EXTRA_OPTS="-Wl,-z,relro" \ group_include="openxg"
-- juha
Can you remove -O3 from CC_EXTRA_OPS?
Daniel
On 21/09/15 21:27, Juha Heinanen wrote:
Daniel-Constantin Mierla writes:
I wonder what makes the local var 'hash' to be printed but not 'after'. If you installed from deb, do you have the kamailio-dbg package installed?
no, i don't. but i have commented out
# dh_strip --dbg-package=kamailio-dbg
so symbols should be included.
this is how i make kamailio:
/usr/bin/make FLAVOUR=kamailio cfg prefix=/usr cfg_prefix=/usr/src/opensipg/trunk/src/openxg-sip-proxy/debian/openxg-sip-proxy \ cfg_target=/etc/sip-proxy/ \ run_target=/var/run/sip-proxy \ BASEDIR=/usr/src/opensipg/trunk/src/openxg-sip-proxy/debian/openxg-sip-proxy \ FMSTATS=1 KMSTATS=1 \ MAIN_NAME=sip-proxy STUN=1 \ skip_modules=" " \ CC_EXTRA_OPTS="-D_FORTIFY_SOURCE=2 -g -O2 -fstack-protector-strong -Wformat -Werror=format-security -DVERSION_NODATE" \ LD_EXTRA_OPTS="-Wl,-z,relro" \ group_include="openxg"
-- juha
Daniel-Constantin Mierla writes:
Can you remove -O3 from CC_EXTRA_OPS?
done, and i got again two cores. juha
45828 -rw------- 1 root root 59711488 Sep 21 23:00 core.sip-proxy.sig11.27810 45828 -rw------- 1 root root 59752448 Sep 21 23:00 core.sip-proxy.sig11.27844
# gdb /usr/bin/sip-proxy core.sip-proxy.sig11.27810 (gdb) where #0 0x0000000000639a74 in fm_insert_free (qm=0x7fe71ab64000, frag=0x7fe71abe2de0) at mem/f_malloc.c:242 #1 0x000000000063afdc in fm_free (qmp=0x7fe71ab64000, p=0x7fe71abe2df8) at mem/f_malloc.c:626 #2 0x000000000063d59a in fm_shm_free (qmp=0x7fe71ab64000, p=0x7fe71abe2df8) at mem/f_malloc.c:1103 #3 0x00007fe71d3b3402 in ser_free (ptr=0x7fe71abe2df8) at tls_init.c:294 #4 0x00007fe71e13143d in CRYPTO_free () from /usr/lib/x86_64-linux-gnu/libcrypto.so.1.0.0 #5 0x00007fe71e1ba627 in lh_free () from /usr/lib/x86_64-linux-gnu/libcrypto.so.1.0.0 #6 0x00007fe71e1bc8a0 in ?? () from /usr/lib/x86_64-linux-gnu/libcrypto.so.1.0.0 #7 0x00007fe71a3b96a8 in ?? () from /usr/lib/x86_64-linux-gnu/libcurl.so.4 #8 0x00007fe71a3ba199 in ?? () from /usr/lib/x86_64-linux-gnu/libcurl.so.4 #9 0x00007fe71a38f605 in curl_global_cleanup () from /usr/lib/x86_64-linux-gnu/libcurl.so.4 #10 0x00007fe71a952029 in destroy () at utils.c:321 #11 0x000000000053d201 in destroy_modules () at sr_module.c:805 #12 0x0000000000523420 in cleanup (show_status=1) at main.c:519 #13 0x0000000000524a2a in shutdown_children (sig=15, show_status=1) at main.c:661 #14 0x00000000005270d2 in handle_sigs () at main.c:752 #15 0x000000000052f651 in main_loop () at main.c:1707 #16 0x00000000005348b1 in main (argc=17, argv=0x7ffdfd0f9c78) at main.c:2566
# gdb /usr/bin/sip-proxy core.sip-proxy.sig11.27844 (gdb) where #0 0x0000000000639a74 in fm_insert_free (qm=0x7fe71ab64000, frag=0x7fe71b259928) at mem/f_malloc.c:242 #1 0x000000000063afdc in fm_free (qmp=0x7fe71ab64000, p=0x7fe71b259940) at mem/f_malloc.c:626 #2 0x00007fe720350983 in free_cell (dead_cell=0x7fe71b2652b8) at h_table.c:133 #3 0x00007fe7203d2bfc in wait_handler (ti=1006656376, wait_tl=0x7fe71b265338, data=0x7fe71b2652b8) at timer.c:648 #4 0x00000000004aa755 in timer_list_expire (t=1006656376, h=0x7fe71abb0ea0, slow_l=0x7fe71abb25b8, slow_mark=359) at timer.c:873 #5 0x00000000004aabb2 in timer_handler () at timer.c:938 #6 0x00000000004ab020 in timer_main () at timer.c:977 #7 0x000000000052ea59 in main_loop () at main.c:1650 #8 0x00000000005348b1 in main (argc=17, argv=0x7ffdfd0f9c78) at main.c:2566 (gdb)
From second core, get:
frame 0 p *qm p *frag p *f info locals p qm->free_hash[hash] p *qm->free_hash[hash].first
Daniel
On 21/09/15 22:03, Juha Heinanen wrote:
Daniel-Constantin Mierla writes:
Can you remove -O3 from CC_EXTRA_OPS?
done, and i got again two cores. juha
45828 -rw------- 1 root root 59711488 Sep 21 23:00 core.sip-proxy.sig11.27810 45828 -rw------- 1 root root 59752448 Sep 21 23:00 core.sip-proxy.sig11.27844
# gdb /usr/bin/sip-proxy core.sip-proxy.sig11.27810 (gdb) where #0 0x0000000000639a74 in fm_insert_free (qm=0x7fe71ab64000, frag=0x7fe71abe2de0) at mem/f_malloc.c:242 #1 0x000000000063afdc in fm_free (qmp=0x7fe71ab64000, p=0x7fe71abe2df8) at mem/f_malloc.c:626 #2 0x000000000063d59a in fm_shm_free (qmp=0x7fe71ab64000, p=0x7fe71abe2df8) at mem/f_malloc.c:1103 #3 0x00007fe71d3b3402 in ser_free (ptr=0x7fe71abe2df8) at tls_init.c:294 #4 0x00007fe71e13143d in CRYPTO_free () from /usr/lib/x86_64-linux-gnu/libcrypto.so.1.0.0 #5 0x00007fe71e1ba627 in lh_free () from /usr/lib/x86_64-linux-gnu/libcrypto.so.1.0.0 #6 0x00007fe71e1bc8a0 in ?? () from /usr/lib/x86_64-linux-gnu/libcrypto.so.1.0.0 #7 0x00007fe71a3b96a8 in ?? () from /usr/lib/x86_64-linux-gnu/libcurl.so.4 #8 0x00007fe71a3ba199 in ?? () from /usr/lib/x86_64-linux-gnu/libcurl.so.4 #9 0x00007fe71a38f605 in curl_global_cleanup () from /usr/lib/x86_64-linux-gnu/libcurl.so.4 #10 0x00007fe71a952029 in destroy () at utils.c:321 #11 0x000000000053d201 in destroy_modules () at sr_module.c:805 #12 0x0000000000523420 in cleanup (show_status=1) at main.c:519 #13 0x0000000000524a2a in shutdown_children (sig=15, show_status=1) at main.c:661 #14 0x00000000005270d2 in handle_sigs () at main.c:752 #15 0x000000000052f651 in main_loop () at main.c:1707 #16 0x00000000005348b1 in main (argc=17, argv=0x7ffdfd0f9c78) at main.c:2566
# gdb /usr/bin/sip-proxy core.sip-proxy.sig11.27844 (gdb) where #0 0x0000000000639a74 in fm_insert_free (qm=0x7fe71ab64000, frag=0x7fe71b259928) at mem/f_malloc.c:242 #1 0x000000000063afdc in fm_free (qmp=0x7fe71ab64000, p=0x7fe71b259940) at mem/f_malloc.c:626 #2 0x00007fe720350983 in free_cell (dead_cell=0x7fe71b2652b8) at h_table.c:133 #3 0x00007fe7203d2bfc in wait_handler (ti=1006656376, wait_tl=0x7fe71b265338, data=0x7fe71b2652b8) at timer.c:648 #4 0x00000000004aa755 in timer_list_expire (t=1006656376, h=0x7fe71abb0ea0, slow_l=0x7fe71abb25b8, slow_mark=359) at timer.c:873 #5 0x00000000004aabb2 in timer_handler () at timer.c:938 #6 0x00000000004ab020 in timer_main () at timer.c:977 #7 0x000000000052ea59 in main_loop () at main.c:1650 #8 0x00000000005348b1 in main (argc=17, argv=0x7ffdfd0f9c78) at main.c:2566 (gdb)
Daniel-Constantin Mierla writes:
From second core, get:
frame 0 p *qm p *frag p *f info locals p qm->free_hash[hash] p *qm->free_hash[hash].first
done, juha
(gdb) where #0 0x0000000000639a74 in fm_insert_free (qm=0x7fe71ab64000, frag=0x7fe71b259928) at mem/f_malloc.c:242 #1 0x000000000063afdc in fm_free (qmp=0x7fe71ab64000, p=0x7fe71b259940) at mem/f_malloc.c:626 #2 0x00007fe720350983 in free_cell (dead_cell=0x7fe71b2652b8) at h_table.c:133 #3 0x00007fe7203d2bfc in wait_handler (ti=1006656376, wait_tl=0x7fe71b265338, data=0x7fe71b2652b8) at timer.c:648 #4 0x00000000004aa755 in timer_list_expire (t=1006656376, h=0x7fe71abb0ea0, slow_l=0x7fe71abb25b8, slow_mark=359) at timer.c:873 #5 0x00000000004aabb2 in timer_handler () at timer.c:938 #6 0x00000000004ab020 in timer_main () at timer.c:977 #7 0x000000000052ea59 in main_loop () at main.c:1650 #8 0x00000000005348b1 in main (argc=17, argv=0x7ffdfd0f9c78) at main.c:2566 (gdb) frame 0 #0 0x0000000000639a74 in fm_insert_free (qm=0x7fe71ab64000, frag=0x7fe71b259928) at mem/f_malloc.c:242 242 in mem/f_malloc.c (gdb) p *qm $1 = {type = 0, size = 33554432, used = 6722696, real_used = 7586240, max_real_used = 8279488, ffrags = 49, first_frag = 0x7fe71ab6c478, last_frag = 0x7fe71cb63fe8, free_bitmap = {730742, 0, 8192, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 268435456, 0, 0, 0, 2097152, 2097152, 0 <repeats 13 times>, 2054}, free_hash = {{first = 0x0, no = 0}, { first = 0x7fe71b1a0918, no = 1}, {first = 0x7fe71b19dd40, no = 2}, {first = 0x0, no = 0}, {first = 0x7fe71b1a0dd0, no = 4}, {first = 0x7fe71b19a340, no = 3}, {first = 0x7fe71b1c31e8, no = 1}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x7fe71b1c0448, no = 2}, {first = 0x7fe71b1e1728, no = 1}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x7fe71b1e1920, no = 1}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x7fe71b1dd988, no = 1}, {first = 0x7fe71b1e1688, no = 1}, {first = 0x0, no = 0}, {first = 0x7fe71b323940, no = 1}, {first = 0x0, no = 0} <repeats 121 times>, {first = 0x7fe71b1e8468, no = 1}, {first = 0x0, no = 0} <repeats 57 times>, { first = 0x7fe71b303d10, no = 1}, {first = 0x0, no = 0} <repeats 660 times>, {first = 0x7fe71b2bcee8, no = 1}, { first = 0x0, no = 0} <repeats 248 times>, {first = 0x7fe71b1fe5c0, no = 1}, {first = 0x0, no = 0} <repeats 63 times>, {first = 0x7fe71b250638, no = 1}, {first = 0x0, no = 0} <repeats 875 times>, { first = 0x7fe71b298548, no = 22}, {first = 0x0, no = 3}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x7fe71b349e28, no = 1}, {first = 0x0, no = 0} <repeats 39 times>}} (gdb) p *frag $2 = {size = 28888, u = {nxt_free = 0x7fe71b298981, reserved = 140630569879937}, prv_free = 0x392e3239313a7069} (gdb) p *f $3 = {size = 8245933083814097524, u = {nxt_free = 0x7300007063743d74, reserved = 8286623797066612084}, prv_free = 0x392e3239313a7069} (gdb) info locals f = 0x7fe71b298981 hash = 2049 after = 0 (gdb) p qm->free_hash[hash] $4 = {first = 0x7fe71b298548, no = 22} (gdb) p *qm->free_hash[hash].first $5 = {size = 5, u = {nxt_free = 0x7fe71b298981, reserved = 140630569879937}, prv_free = 0x0}
This looks like a buffer overflow somewhere else.
Have you changed the value of MEMDBG in Makefile.defs? It should be 1 and that enables memory debugging, but I don't see the extra fields in fm fragmed structure.
Can you try building with MEMDBG=1, then add '-x qm' to command line starting kamailio? Let's see if q_malloc gets more hints.
Daniel
On 21/09/15 22:10, Juha Heinanen wrote:
Daniel-Constantin Mierla writes:
From second core, get:
frame 0 p *qm p *frag p *f info locals p qm->free_hash[hash] p *qm->free_hash[hash].first
done, juha
(gdb) where #0 0x0000000000639a74 in fm_insert_free (qm=0x7fe71ab64000, frag=0x7fe71b259928) at mem/f_malloc.c:242 #1 0x000000000063afdc in fm_free (qmp=0x7fe71ab64000, p=0x7fe71b259940) at mem/f_malloc.c:626 #2 0x00007fe720350983 in free_cell (dead_cell=0x7fe71b2652b8) at h_table.c:133 #3 0x00007fe7203d2bfc in wait_handler (ti=1006656376, wait_tl=0x7fe71b265338, data=0x7fe71b2652b8) at timer.c:648 #4 0x00000000004aa755 in timer_list_expire (t=1006656376, h=0x7fe71abb0ea0, slow_l=0x7fe71abb25b8, slow_mark=359) at timer.c:873 #5 0x00000000004aabb2 in timer_handler () at timer.c:938 #6 0x00000000004ab020 in timer_main () at timer.c:977 #7 0x000000000052ea59 in main_loop () at main.c:1650 #8 0x00000000005348b1 in main (argc=17, argv=0x7ffdfd0f9c78) at main.c:2566 (gdb) frame 0 #0 0x0000000000639a74 in fm_insert_free (qm=0x7fe71ab64000, frag=0x7fe71b259928) at mem/f_malloc.c:242 242 in mem/f_malloc.c (gdb) p *qm $1 = {type = 0, size = 33554432, used = 6722696, real_used = 7586240, max_real_used = 8279488, ffrags = 49, first_frag = 0x7fe71ab6c478, last_frag = 0x7fe71cb63fe8, free_bitmap = {730742, 0, 8192, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 268435456, 0, 0, 0, 2097152, 2097152, 0 <repeats 13 times>, 2054}, free_hash = {{first = 0x0, no = 0}, { first = 0x7fe71b1a0918, no = 1}, {first = 0x7fe71b19dd40, no = 2}, {first = 0x0, no = 0}, {first = 0x7fe71b1a0dd0, no = 4}, {first = 0x7fe71b19a340, no = 3}, {first = 0x7fe71b1c31e8, no = 1}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x7fe71b1c0448, no = 2}, {first = 0x7fe71b1e1728, no = 1}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x7fe71b1e1920, no = 1}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x7fe71b1dd988, no = 1}, {first = 0x7fe71b1e1688, no = 1}, {first = 0x0, no = 0}, {first = 0x7fe71b323940, no = 1}, {first = 0x0, no = 0} <repeats 121 times>, {first = 0x7fe71b1e8468, no = 1}, {first = 0x0, no = 0} <repeats 57 times>, { first = 0x7fe71b303d10, no = 1}, {first = 0x0, no = 0} <repeats 660 times>, {first = 0x7fe71b2bcee8, no = 1}, { first = 0x0, no = 0} <repeats 248 times>, {first = 0x7fe71b1fe5c0, no = 1}, {first = 0x0, no = 0} <repeats 63 times>, {first = 0x7fe71b250638, no = 1}, {first = 0x0, no = 0} <repeats 875 times>, { first = 0x7fe71b298548, no = 22}, {first = 0x0, no = 3}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x7fe71b349e28, no = 1}, {first = 0x0, no = 0} <repeats 39 times>}} (gdb) p *frag $2 = {size = 28888, u = {nxt_free = 0x7fe71b298981, reserved = 140630569879937}, prv_free = 0x392e3239313a7069} (gdb) p *f $3 = {size = 8245933083814097524, u = {nxt_free = 0x7300007063743d74, reserved = 8286623797066612084}, prv_free = 0x392e3239313a7069} (gdb) info locals f = 0x7fe71b298981 hash = 2049 after = 0 (gdb) p qm->free_hash[hash] $4 = {first = 0x7fe71b298548, no = 22} (gdb) p *qm->free_hash[hash].first $5 = {size = 5, u = {nxt_free = 0x7fe71b298981, reserved = 140630569879937}, prv_free = 0x0}
Daniel-Constantin Mierla writes:
Have you changed the value of MEMDBG in Makefile.defs? It should be 1 and that enables memory debugging, but I don't see the extra fields in fm fragmed structure.
Yes, I have have these two Makefile.defs changes:
# Turn off memory debuging sed -i -e 's/MEMDBG ?= 1/MEMDBG ?= 0/' $SRC_DIR/Makefile.defs
# Enable joining of free memory chunks sed -i -e 's/-DWITH_XAVP/-DWITH_XAVP -DMEM_JOIN_FREE/' $SRC_DIR/Makefile.defs
Can you try building with MEMDBG=1, then add '-x qm' to command line starting kamailio? Let's see if q_malloc gets more hints.
Will try, Juha
Daniel-Constantin Mierla writes:
Can you try building with MEMDBG=1, then add '-x qm' to command line starting kamailio? Let's see if q_malloc gets more hints.
does not start with -x qm:
Sep 22 08:15:33 lohi /usr/bin/sip-proxy[19391]: : <core> [mem/q_malloc.c:446]: qm_free(): BUG: qm_free: bad pointer 0x7f6e2aa6ab31 (out of memory block!) called from domain: hash.c: hash_table_free(230) - aborting Sep 22 08:15:33 lohi kernel: [116650.296517] sip-proxy[19391]: segfault at f0f0f0f0 ip 00007f6e22b1eb46 sp 00007fff2f809b10 error 4 in domain.so[7f6e22b10000+17000]
-- juha
On 22/09/15 07:17, Juha Heinanen wrote:
Daniel-Constantin Mierla writes:
Can you try building with MEMDBG=1, then add '-x qm' to command line starting kamailio? Let's see if q_malloc gets more hints.
does not start with -x qm:
Sep 22 08:15:33 lohi /usr/bin/sip-proxy[19391]: : <core> [mem/q_malloc.c:446]: qm_free(): BUG: qm_free: bad pointer 0x7f6e2aa6ab31 (out of memory block!) called from domain: hash.c: hash_table_free(230) - aborting Sep 22 08:15:33 lohi kernel: [116650.296517] sip-proxy[19391]: segfault at f0f0f0f0 ip 00007f6e22b1eb46 sp 00007fff2f809b10 error 4 in domain.so[7f6e22b10000+17000]
Did you get a core for this one? There might be an issue inside domain module...
Daniel
On 22/09/15 08:40, Daniel-Constantin Mierla wrote:
On 22/09/15 07:17, Juha Heinanen wrote:
Daniel-Constantin Mierla writes:
Can you try building with MEMDBG=1, then add '-x qm' to command line starting kamailio? Let's see if q_malloc gets more hints.
does not start with -x qm:
Sep 22 08:15:33 lohi /usr/bin/sip-proxy[19391]: : <core> [mem/q_malloc.c:446]: qm_free(): BUG: qm_free: bad pointer 0x7f6e2aa6ab31 (out of memory block!) called from domain: hash.c: hash_table_free(230) - aborting Sep 22 08:15:33 lohi kernel: [116650.296517] sip-proxy[19391]: segfault at f0f0f0f0 ip 00007f6e22b1eb46 sp 00007fff2f809b10 error 4 in domain.so[7f6e22b10000+17000]
Did you get a core for this one? There might be an issue inside domain module...
I was able to start with qmalloc and domain module using the default config file:
./kamailio -f etc/kamailio.cfg -A WITH_SRCPATH -A WITH_MYSQL -A WITH_MULTIDOMAIN -a no -E -e -ddd -x qm
Domain table has two records:
+----+-----------+------+---------------------+ | id | domain | did | last_modified | +----+-----------+------+---------------------+ | 1 | test.com | abc | 2015-09-22 09:08:42 | | 2 | test2.com | xyz | 2015-09-22 09:08:55 | +----+-----------+------+---------------------+
If you get a coredump when using -x qm, send the backtrace.
Daniel
Daniel-Constantin Mierla writes:
If you get a coredump when using -x qm, send the backtrace.
below, juha
Sep 22 17:49:10 lohi /usr/bin/sip-proxy[10873]: : <core> [mem/q_malloc.c:446]: qm_free(): BUG: qm_free: bad pointer 0x7f179f2a9b31 (out of memory block!) called from domain: hash.c: hash_table_free(230) - aborting Sep 22 17:49:10 lohi kernel: [122098.794641] sip-proxy[10873]: segfault at f0f0f0f0 ip 00007f179735db46 sp 00007ffc6a5e0280 error 4 in domain.so[7f179734f000+17000]
(gdb) where #0 0x00007f179735db46 in hash_table_free (hash_table=0x7f179ce6cf68) at hash.c:233 #1 0x00007f179735633b in reload_tables () at domain.c:305 #2 0x00007f17973603d0 in mod_init () at domain_mod.c:231 #3 0x000000000053e860 in init_mod (m=0x7f17a440e308) at sr_module.c:960 #4 0x000000000053e5b7 in init_mod (m=0x7f17a440e7c8) at sr_module.c:957 #5 0x000000000053e5b7 in init_mod (m=0x7f17a440ecc0) at sr_module.c:957 #6 0x000000000053e5b7 in init_mod (m=0x7f17a440f070) at sr_module.c:957 #7 0x000000000053e5b7 in init_mod (m=0x7f17a440f888) at sr_module.c:957 #8 0x000000000053e5b7 in init_mod (m=0x7f17a440fc98) at sr_module.c:957 #9 0x000000000053e5b7 in init_mod (m=0x7f17a44101c8) at sr_module.c:957 #10 0x000000000053e5b7 in init_mod (m=0x7f17a4410bd0) at sr_module.c:957 #11 0x000000000053e5b7 in init_mod (m=0x7f17a4411220) at sr_module.c:957 #12 0x000000000053e5b7 in init_mod (m=0x7f17a4411680) at sr_module.c:957 #13 0x000000000053e5b7 in init_mod (m=0x7f17a4411e00) at sr_module.c:957 #14 0x000000000053e5b7 in init_mod (m=0x7f17a44121a8) at sr_module.c:957 #15 0x000000000053e5b7 in init_mod (m=0x7f17a4412630) at sr_module.c:957 #16 0x000000000053e5b7 in init_mod (m=0x7f17a4413248) at sr_module.c:957 #17 0x000000000053e5b7 in init_mod (m=0x7f17a4413a20) at sr_module.c:957 #18 0x000000000053e5b7 in init_mod (m=0x7f17a4413f68) at sr_module.c:957 #19 0x000000000053e5b7 in init_mod (m=0x7f17a4414d20) at sr_module.c:957 #20 0x000000000053e5b7 in init_mod (m=0x7f17a4414ff8) at sr_module.c:957 #21 0x000000000053e5b7 in init_mod (m=0x7f17a44152d0) at sr_module.c:957 #22 0x000000000053e5b7 in init_mod (m=0x7f17a44156e0) at sr_module.c:957 #23 0x000000000053e5b7 in init_mod (m=0x7f17a44159b8) at sr_module.c:957 #24 0x000000000053eb7e in init_modules () at sr_module.c:989 #25 0x0000000000534a1d in main (argc=19, argv=0x7ffc6a5e0ef8) at main.c:2507
this is getting very serious. i went back to latest stable release 4.3 and kamailio crashes at start even without -x qm at start.
-- juha
(gdb) where #0 0x00007fbed17576fc in hash_table_free ( hash_table=0x7fbed6dc2000) at hash.c:230 #1 0x00007fbed1750fe0 in reload_tables () at domain.c:305 #2 0x00007fbed1758f15 in mod_init () at domain_mod.c:231 #3 0x000000000051b532 in init_mod (m=0x7fbed6dc2000) at sr_module.c:943 #4 0x000000000051b47c in init_mod (m=0x7fbed6dc2000) at sr_module.c:940 #5 0x000000000051b47c in init_mod (m=0x7fbed6dc2000) at sr_module.c:940 #6 0x000000000051b47c in init_mod (m=0x7fbed6dc2000) at sr_module.c:940 #7 0x000000000051b47c in init_mod (m=0x7fbed6dc2000) at sr_module.c:940 #8 0x000000000051b47c in init_mod (m=0x7fbed6dc2000) at sr_module.c:940 #9 0x000000000051b47c in init_mod (m=0x7fbed6dc2000) at sr_module.c:940 #10 0x000000000051b47c in init_mod (m=0x7fbed6dc2000) at sr_module.c:940 #11 0x000000000051b47c in init_mod (m=0x7fbed6dc2000) at sr_module.c:940 #12 0x000000000051b47c in init_mod (m=0x7fbed6dc2000) at sr_module.c:940 #13 0x000000000051b47c in init_mod (m=0x7fbed6dc2000) at sr_module.c:940 #14 0x000000000051b47c in init_mod (m=0x7fbed6dc2000) at sr_module.c:940 #15 0x000000000051b47c in init_mod (m=0x7fbed6dc2000) at sr_module.c:940 #16 0x000000000051b47c in init_mod (m=0x7fbed6dc2000) at sr_module.c:940 ---Type <return> to continue, or q <return> to quit--- #17 0x000000000051b47c in init_mod (m=0x7fbed6dc2000) at sr_module.c:940 #18 0x000000000051b47c in init_mod (m=0x7fbed6dc2000) at sr_module.c:940 #19 0x000000000051b47c in init_mod (m=0x7fbed6dc2000) at sr_module.c:940 #20 0x000000000051b47c in init_mod (m=0x7fbed6dc2000) at sr_module.c:940 #21 0x000000000051b47c in init_mod (m=0x7fbed6dc2000) at sr_module.c:940 #22 0x000000000051b47c in init_mod (m=0x7fbed6dc2000) at sr_module.c:940 #23 0x000000000051b47c in init_mod (m=0x7fbed6dc2000) at sr_module.c:940 #24 0x000000000051c8ef in init_modules () at sr_module.c:972 #25 0x000000000041ab40 in main (argc=0, argv=0x7ffd75ede808) at main.c:2474 (gdb)
On 22/09/15 17:29, Juha Heinanen wrote:
Juha Heinanen writes:
this is getting very serious. i went back to latest stable release 4.3 and kamailio crashes at start even without -x qm at start.
would it be possible to revert 4.3 back asap to an old non crashing state while this is being investigated?
I changed the default memory manager for 4.3 to q_malloc in order to allow to look at f_malloc changes.
q_malloc was not changed for ages. So it might be an issue in domain module that we need to fix.
Daniel
Daniel-Constantin Mierla writes:
I changed the default memory manager for 4.3 to q_malloc in order to allow to look at f_malloc changes.
good.
q_malloc was not changed for ages. So it might be an issue in domain module that we need to fix.
i checked and there has not been changes to 4.3 domain module for three months. i'm sure that i my 4.3 sip proxy has been able to start and run without any issues until very recently.
-- juha
On 22/09/15 17:45, Juha Heinanen wrote:
Daniel-Constantin Mierla writes:
I changed the default memory manager for 4.3 to q_malloc in order to allow to look at f_malloc changes.
good.
q_malloc was not changed for ages. So it might be an issue in domain module that we need to fix.
i checked and there has not been changes to 4.3 domain module for three months. i'm sure that i my 4.3 sip proxy has been able to start and run without any issues until very recently.
Before was f_malloc, which doesn't do strict checking of memory operations. But because some reported crashes look like some buffer overflow, I wanted to get better reports on that, which can be done by q malloc.
Daniel
Can you get:
bt full
frame 0 p *np
Daniel
On 22/09/15 17:26, Juha Heinanen wrote:
this is getting very serious. i went back to latest stable release 4.3 and kamailio crashes at start even without -x qm at start.
-- juha
(gdb) where #0 0x00007fbed17576fc in hash_table_free ( hash_table=0x7fbed6dc2000) at hash.c:230 #1 0x00007fbed1750fe0 in reload_tables () at domain.c:305 #2 0x00007fbed1758f15 in mod_init () at domain_mod.c:231 #3 0x000000000051b532 in init_mod (m=0x7fbed6dc2000) at sr_module.c:943 #4 0x000000000051b47c in init_mod (m=0x7fbed6dc2000) at sr_module.c:940 #5 0x000000000051b47c in init_mod (m=0x7fbed6dc2000) at sr_module.c:940 #6 0x000000000051b47c in init_mod (m=0x7fbed6dc2000) at sr_module.c:940 #7 0x000000000051b47c in init_mod (m=0x7fbed6dc2000) at sr_module.c:940 #8 0x000000000051b47c in init_mod (m=0x7fbed6dc2000) at sr_module.c:940 #9 0x000000000051b47c in init_mod (m=0x7fbed6dc2000) at sr_module.c:940 #10 0x000000000051b47c in init_mod (m=0x7fbed6dc2000) at sr_module.c:940 #11 0x000000000051b47c in init_mod (m=0x7fbed6dc2000) at sr_module.c:940 #12 0x000000000051b47c in init_mod (m=0x7fbed6dc2000) at sr_module.c:940 #13 0x000000000051b47c in init_mod (m=0x7fbed6dc2000) at sr_module.c:940 #14 0x000000000051b47c in init_mod (m=0x7fbed6dc2000) at sr_module.c:940 #15 0x000000000051b47c in init_mod (m=0x7fbed6dc2000) at sr_module.c:940 #16 0x000000000051b47c in init_mod (m=0x7fbed6dc2000) at sr_module.c:940 ---Type <return> to continue, or q <return> to quit--- #17 0x000000000051b47c in init_mod (m=0x7fbed6dc2000) at sr_module.c:940 #18 0x000000000051b47c in init_mod (m=0x7fbed6dc2000) at sr_module.c:940 #19 0x000000000051b47c in init_mod (m=0x7fbed6dc2000) at sr_module.c:940 #20 0x000000000051b47c in init_mod (m=0x7fbed6dc2000) at sr_module.c:940 #21 0x000000000051b47c in init_mod (m=0x7fbed6dc2000) at sr_module.c:940 #22 0x000000000051b47c in init_mod (m=0x7fbed6dc2000) at sr_module.c:940 #23 0x000000000051b47c in init_mod (m=0x7fbed6dc2000) at sr_module.c:940 #24 0x000000000051c8ef in init_modules () at sr_module.c:972 #25 0x000000000041ab40 in main (argc=0, argv=0x7ffd75ede808) at main.c:2474 (gdb)
Daniel-Constantin Mierla writes:
Can you get:
bt full
frame 0 p *np
this is my production proxy and thus has debian default -O2.
-- juha
(gdb) bt full #0 0x00007fbed17576fc in hash_table_free (hash_table=0x7fbed6dc2000) at hash.c:230 No locals. #1 0x00007fbed1750fe0 in reload_tables () at domain.c:305 cols = {0x7fbee0b0b2a0, 0x7fbee0b0b470, 0x7ffd75edee68, 0x7fbee047d1a7 <db_do_close+263>} res = 0x0 new_hash_table = 0x7fbed1961760 <reload_lock> type = -11072 did = {s = 0x7fbee0b0b4c0 "", len = -525290384} domain = {s = 0x7ffd75edee68 "root", len = 0} name = {s = 0x7fbee0b0b470 "ರ\340\276\177", len = 10656912} val = {n = 1978527336, s = {s = 0x7ffd75edee68 "root", len = -536582867}, re = 0x7ffd75edee68} __FUNCTION__ = "reload_tables" #2 0x00007fbed1758f15 in mod_init () at domain_mod.c:231 __FUNCTION__ = "mod_init" #3 0x000000000051b532 in init_mod (m=0x7fbed6dc2000) at sr_module.c:943 __FUNCTION__ = "init_mod" #4 0x000000000051b47c in init_mod (m=0x7fbed6dc2000) at sr_module.c:940 __FUNCTION__ = "init_mod" #5 0x000000000051b47c in init_mod (m=0x7fbed6dc2000) at sr_module.c:940 __FUNCTION__ = "init_mod" #6 0x000000000051b47c in init_mod (m=0x7fbed6dc2000) at sr_module.c:940 __FUNCTION__ = "init_mod" #7 0x000000000051b47c in init_mod (m=0x7fbed6dc2000) at sr_module.c:940 __FUNCTION__ = "init_mod" #8 0x000000000051b47c in init_mod (m=0x7fbed6dc2000) at sr_module.c:940 __FUNCTION__ = "init_mod" #9 0x000000000051b47c in init_mod (m=0x7fbed6dc2000) at sr_module.c:940 __FUNCTION__ = "init_mod" #10 0x000000000051b47c in init_mod (m=0x7fbed6dc2000) at sr_module.c:940 __FUNCTION__ = "init_mod" #11 0x000000000051b47c in init_mod (m=0x7fbed6dc2000) at sr_module.c:940 __FUNCTION__ = "init_mod" ---Type <return> to continue, or q <return> to quit--- #12 0x000000000051b47c in init_mod (m=0x7fbed6dc2000) at sr_module.c:940 __FUNCTION__ = "init_mod" #13 0x000000000051b47c in init_mod (m=0x7fbed6dc2000) at sr_module.c:940 __FUNCTION__ = "init_mod" #14 0x000000000051b47c in init_mod (m=0x7fbed6dc2000) at sr_module.c:940 __FUNCTION__ = "init_mod" #15 0x000000000051b47c in init_mod (m=0x7fbed6dc2000) at sr_module.c:940 __FUNCTION__ = "init_mod" #16 0x000000000051b47c in init_mod (m=0x7fbed6dc2000) at sr_module.c:940 __FUNCTION__ = "init_mod" #17 0x000000000051b47c in init_mod (m=0x7fbed6dc2000) at sr_module.c:940 __FUNCTION__ = "init_mod" #18 0x000000000051b47c in init_mod (m=0x7fbed6dc2000) at sr_module.c:940 __FUNCTION__ = "init_mod" #19 0x000000000051b47c in init_mod (m=0x7fbed6dc2000) at sr_module.c:940 __FUNCTION__ = "init_mod" #20 0x000000000051b47c in init_mod (m=0x7fbed6dc2000) at sr_module.c:940 __FUNCTION__ = "init_mod" #21 0x000000000051b47c in init_mod (m=0x7fbed6dc2000) at sr_module.c:940 __FUNCTION__ = "init_mod" #22 0x000000000051b47c in init_mod (m=0x7fbed6dc2000) at sr_module.c:940 __FUNCTION__ = "init_mod" #23 0x000000000051b47c in init_mod (m=0x7fbed6dc2000) at sr_module.c:940 __FUNCTION__ = "init_mod" #24 0x000000000051c8ef in init_modules () at sr_module.c:972 i = 1978525704 __FUNCTION__ = "init_modules" #25 0x000000000041ab40 in main (argc=0, argv=0x7ffd75ede808) at main.c:2474 cfg_stream = 0x7ffd75edee8f r = 0 tmp = 0x7ffd75edee8f "" tmp_len = -502542336 ---Type <return> to continue, or q <return> to quit--- port = 32702 proto = -502447704 ret = 1978525704 seed = 378659289 debug_save = 0 debug_flag = 1978527336 dont_fork_cnt = 0 __FUNCTION__ = "main" (gdb) (gdb) (gdb) frame 0 #0 0x00007fbed17576fc in hash_table_free (hash_table=0x7fbed6dc2000) at hash.c:230 230 in hash.c (gdb) p *np value has been optimized out
i just tried with earlier 4.3 sip proxy that i built on sept 17 at about 9 am cet and it starts without problems. so whatever commits happened after that caused the crash at start.
-- juha
Daniel-Constantin Mierla writes:
bt full
frame 0 p *np
below without -O2,
-- juhabt full #0 0x00007f806f885870 in hash_table_free (hash_table=0x7f8075320e40) at hash.c:230 i = 128 np = 0x298bd800 next = 0x7f0100000001 ap = 0x0 next_ap = 0x94ba8dd1faa09900 #1 0x00007f806f87e168 in reload_tables () at domain.c:305 cols = {0x1, 0x7f8075321270, 0x415f40 <_start>, 0x74f35850} res = 0x0 row = 0x7ffc76d29bd0 new_hash_table = 0x7f8074f351f8 i = 32640 type = 31884 did = {s = 0x623c5f <split_frag+201> "\270", len = 16} domain = {s = 0x7f8075321260 "\020", len = 1962061824} name = {s = 0x7f8075321260 "\020", len = 29400432} value = {s = 0x7f8075321280 "\020", len = 1966215824} val = {n = 1993514000, s = {s = 0x176d29c10 <error: Cannot access memory at address 0x176d29c10>, len = 1962104912}, re = 0x176d29c10} __FUNCTION__ = "reload_tables" #2 0x00007f806f88806c in mod_init () at domain_mod.c:231 __FUNCTION__ = "mod_init" #3 0x000000000053570b in init_mod (m=0x7f807c8d8580) at sr_module.c:943 __FUNCTION__ = "init_mod" #4 0x0000000000535476 in init_mod (m=0x7f807c8d8910) at sr_module.c:940 __FUNCTION__ = "init_mod" #5 0x0000000000535476 in init_mod (m=0x7f807c8d8cd0) at sr_module.c:940 __FUNCTION__ = "init_mod" #6 0x0000000000535476 in init_mod (m=0x7f807c8d8f40) at sr_module.c:940 __FUNCTION__ = "init_mod" #7 0x0000000000535476 in init_mod (m=0x7f807c8d95b0) at sr_module.c:940 ---Type <return> to continue, or q <return> to quit--- __FUNCTION__ = "init_mod" #8 0x0000000000535476 in init_mod (m=0x7f807c8d9890) at sr_module.c:940 __FUNCTION__ = "init_mod" #9 0x0000000000535476 in init_mod (m=0x7f807c8d9c90) at sr_module.c:940 __FUNCTION__ = "init_mod" #10 0x0000000000535476 in init_mod (m=0x7f807c8da560) at sr_module.c:940 __FUNCTION__ = "init_mod" #11 0x0000000000535476 in init_mod (m=0x7f807c8daa70) at sr_module.c:940 __FUNCTION__ = "init_mod" #12 0x0000000000535476 in init_mod (m=0x7f807c8dada0) at sr_module.c:940 __FUNCTION__ = "init_mod" #13 0x0000000000535476 in init_mod (m=0x7f807c8db380) at sr_module.c:940 __FUNCTION__ = "init_mod" #14 0x0000000000535476 in init_mod (m=0x7f807c8db5f0) at sr_module.c:940 __FUNCTION__ = "init_mod" #15 0x0000000000535476 in init_mod (m=0x7f807c8db940) at sr_module.c:940 __FUNCTION__ = "init_mod" #16 0x0000000000535476 in init_mod (m=0x7f807c8dc230) at sr_module.c:940 __FUNCTION__ = "init_mod" #17 0x0000000000535476 in init_mod (m=0x7f807c8dc8a0) at sr_module.c:940 __FUNCTION__ = "init_mod" #18 0x0000000000535476 in init_mod (m=0x7f807c8dcc80) at sr_module.c:940 __FUNCTION__ = "init_mod" #19 0x0000000000535476 in init_mod (m=0x7f807c8dd900) at sr_module.c:940 __FUNCTION__ = "init_mod" #20 0x0000000000535476 in init_mod (m=0x7f807c8ddae0) at sr_module.c:940 __FUNCTION__ = "init_mod" #21 0x0000000000535476 in init_mod (m=0x7f807c8ddcc0) at sr_module.c:940 __FUNCTION__ = "init_mod" #22 0x0000000000535476 in init_mod (m=0x7f807c8ddfa0) at sr_module.c:940 __FUNCTION__ = "init_mod" ---Type <return> to continue, or q <return> to quit--- #23 0x0000000000535476 in init_mod (m=0x7f807c8de180) at sr_module.c:940 __FUNCTION__ = "init_mod" #24 0x0000000000535a0b in init_modules () at sr_module.c:972 t = 0x2cb3d66f i = 2089519184 __FUNCTION__ = "init_modules" #25 0x000000000052c1b0 in main (argc=17, argv=0x7ffc76d2a568) at main.c:2474 cfg_stream = 0x18c2010 c = -1 r = 0 tmp = 0x7ffc76d2bf69 "" tmp_len = 32640 port = 1 proto = 0 options = 0x7223c0 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:" ret = -1 seed = 2169338037 rfd = 4 debug_save = 0 debug_flag = 0 dont_fork_cnt = 0 n_lst = 0xf0b6ff p = 0x6e471d <__libc_csu_init+77> "H\203\303\001H9\353u\352H\203\304\b[]A\A]A^A_\303ff.\017\037\204" __FUNCTION__ = "main" (gdb) frame 0 #0 0x00007f806f885870 in hash_table_free (hash_table=0x7f8075320e40) at hash.c:230 230 in hash.c (gdb) p *np Cannot access memory at address 0x298bd800
I think I found the issue in domain module and pushed a fix for it -- the hash tables used by domain module were not properly filled with 0 affer allocation. Can you try with latest master of latest 4.3 branch?
Cheers, Daniel
On 22/09/15 18:20, Juha Heinanen wrote:
Daniel-Constantin Mierla writes:
bt full
frame 0 p *np
below without -O2,
-- juhabt full #0 0x00007f806f885870 in hash_table_free (hash_table=0x7f8075320e40) at hash.c:230 i = 128 np = 0x298bd800 next = 0x7f0100000001 ap = 0x0 next_ap = 0x94ba8dd1faa09900 #1 0x00007f806f87e168 in reload_tables () at domain.c:305 cols = {0x1, 0x7f8075321270, 0x415f40 <_start>, 0x74f35850} res = 0x0 row = 0x7ffc76d29bd0 new_hash_table = 0x7f8074f351f8 i = 32640 type = 31884 did = {s = 0x623c5f <split_frag+201> "\270", len = 16} domain = {s = 0x7f8075321260 "\020", len = 1962061824} name = {s = 0x7f8075321260 "\020", len = 29400432} value = {s = 0x7f8075321280 "\020", len = 1966215824} val = {n = 1993514000, s = {s = 0x176d29c10 <error: Cannot access memory at address 0x176d29c10>, len = 1962104912}, re = 0x176d29c10} __FUNCTION__ = "reload_tables" #2 0x00007f806f88806c in mod_init () at domain_mod.c:231 __FUNCTION__ = "mod_init" #3 0x000000000053570b in init_mod (m=0x7f807c8d8580) at sr_module.c:943 __FUNCTION__ = "init_mod" #4 0x0000000000535476 in init_mod (m=0x7f807c8d8910) at sr_module.c:940 __FUNCTION__ = "init_mod" #5 0x0000000000535476 in init_mod (m=0x7f807c8d8cd0) at sr_module.c:940 __FUNCTION__ = "init_mod" #6 0x0000000000535476 in init_mod (m=0x7f807c8d8f40) at sr_module.c:940 __FUNCTION__ = "init_mod" #7 0x0000000000535476 in init_mod (m=0x7f807c8d95b0) at sr_module.c:940 ---Type <return> to continue, or q <return> to quit--- __FUNCTION__ = "init_mod" #8 0x0000000000535476 in init_mod (m=0x7f807c8d9890) at sr_module.c:940 __FUNCTION__ = "init_mod" #9 0x0000000000535476 in init_mod (m=0x7f807c8d9c90) at sr_module.c:940 __FUNCTION__ = "init_mod" #10 0x0000000000535476 in init_mod (m=0x7f807c8da560) at sr_module.c:940 __FUNCTION__ = "init_mod" #11 0x0000000000535476 in init_mod (m=0x7f807c8daa70) at sr_module.c:940 __FUNCTION__ = "init_mod" #12 0x0000000000535476 in init_mod (m=0x7f807c8dada0) at sr_module.c:940 __FUNCTION__ = "init_mod" #13 0x0000000000535476 in init_mod (m=0x7f807c8db380) at sr_module.c:940 __FUNCTION__ = "init_mod" #14 0x0000000000535476 in init_mod (m=0x7f807c8db5f0) at sr_module.c:940 __FUNCTION__ = "init_mod" #15 0x0000000000535476 in init_mod (m=0x7f807c8db940) at sr_module.c:940 __FUNCTION__ = "init_mod" #16 0x0000000000535476 in init_mod (m=0x7f807c8dc230) at sr_module.c:940 __FUNCTION__ = "init_mod" #17 0x0000000000535476 in init_mod (m=0x7f807c8dc8a0) at sr_module.c:940 __FUNCTION__ = "init_mod" #18 0x0000000000535476 in init_mod (m=0x7f807c8dcc80) at sr_module.c:940 __FUNCTION__ = "init_mod" #19 0x0000000000535476 in init_mod (m=0x7f807c8dd900) at sr_module.c:940 __FUNCTION__ = "init_mod" #20 0x0000000000535476 in init_mod (m=0x7f807c8ddae0) at sr_module.c:940 __FUNCTION__ = "init_mod" #21 0x0000000000535476 in init_mod (m=0x7f807c8ddcc0) at sr_module.c:940 __FUNCTION__ = "init_mod" #22 0x0000000000535476 in init_mod (m=0x7f807c8ddfa0) at sr_module.c:940 __FUNCTION__ = "init_mod" ---Type <return> to continue, or q <return> to quit--- #23 0x0000000000535476 in init_mod (m=0x7f807c8de180) at sr_module.c:940 __FUNCTION__ = "init_mod" #24 0x0000000000535a0b in init_modules () at sr_module.c:972 t = 0x2cb3d66f i = 2089519184 __FUNCTION__ = "init_modules" #25 0x000000000052c1b0 in main (argc=17, argv=0x7ffc76d2a568) at main.c:2474 cfg_stream = 0x18c2010 c = -1 r = 0 tmp = 0x7ffc76d2bf69 "" tmp_len = 32640 port = 1 proto = 0 options = 0x7223c0 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:" ret = -1 seed = 2169338037 rfd = 4 debug_save = 0 debug_flag = 0 dont_fork_cnt = 0 n_lst = 0xf0b6ff p = 0x6e471d <__libc_csu_init+77> "H\203\303\001H9\353u\352H\203\304\b[]A\A]A^A_\303ff.\017\037\204" __FUNCTION__ = "main" (gdb) frame 0 #0 0x00007f806f885870 in hash_table_free (hash_table=0x7f8075320e40) at hash.c:230 230 in hash.c (gdb) p *np Cannot access memory at address 0x298bd800
Daniel-Constantin Mierla writes:
I think I found the issue in domain module and pushed a fix for it -- the hash tables used by domain module were not properly filled with 0 affer allocation. Can you try with latest master of latest 4.3 branch?
master now started ok. wil start 4.3 soon. how about 4.2? were some of the changes done after morning of sep 17 backported to 4.2 too that could cause 4.2 based kamailios crashing at start?
-- juha
Juha Heinanen writes:
master now started ok. wil start 4.3 soon. how about 4.2? were some of the changes done after morning of sep 17 backported to 4.2 too that could cause 4.2 based kamailios crashing at start?
i checked from git and lots of changes were backported to 4.2 on sept 17, which i think may cause also 4.2 kamailio failing to start. i thus just backported the domain module fix to 4.2 too.
-- juha
For some reason, I didn't find this message from sr-dev archives.
-- Juha
----------------------------------------------------------------------
Daniel-Constantin Mierla writes:
bt full
frame 0 p *np
below without -O2,
-- juhabt full #0 0x00007f806f885870 in hash_table_free (hash_table=0x7f8075320e40) at hash.c:230 i = 128 np = 0x298bd800 next = 0x7f0100000001 ap = 0x0 next_ap = 0x94ba8dd1faa09900 #1 0x00007f806f87e168 in reload_tables () at domain.c:305 cols = {0x1, 0x7f8075321270, 0x415f40 <_start>, 0x74f35850} res = 0x0 row = 0x7ffc76d29bd0 new_hash_table = 0x7f8074f351f8 i = 32640 type = 31884 did = {s = 0x623c5f <split_frag+201> "\270", len = 16} domain = {s = 0x7f8075321260 "\020", len = 1962061824} name = {s = 0x7f8075321260 "\020", len = 29400432} value = {s = 0x7f8075321280 "\020", len = 1966215824} val = {n = 1993514000, s = {s = 0x176d29c10 <error: Cannot access memory at address 0x176d29c10>, len = 1962104912}, re = 0x176d29c10} __FUNCTION__ = "reload_tables" #2 0x00007f806f88806c in mod_init () at domain_mod.c:231 __FUNCTION__ = "mod_init" #3 0x000000000053570b in init_mod (m=0x7f807c8d8580) at sr_module.c:943 __FUNCTION__ = "init_mod" #4 0x0000000000535476 in init_mod (m=0x7f807c8d8910) at sr_module.c:940 __FUNCTION__ = "init_mod" #5 0x0000000000535476 in init_mod (m=0x7f807c8d8cd0) at sr_module.c:940 __FUNCTION__ = "init_mod" #6 0x0000000000535476 in init_mod (m=0x7f807c8d8f40) at sr_module.c:940 __FUNCTION__ = "init_mod" #7 0x0000000000535476 in init_mod (m=0x7f807c8d95b0) at sr_module.c:940 ---Type <return> to continue, or q <return> to quit--- __FUNCTION__ = "init_mod" #8 0x0000000000535476 in init_mod (m=0x7f807c8d9890) at sr_module.c:940 __FUNCTION__ = "init_mod" #9 0x0000000000535476 in init_mod (m=0x7f807c8d9c90) at sr_module.c:940 __FUNCTION__ = "init_mod" #10 0x0000000000535476 in init_mod (m=0x7f807c8da560) at sr_module.c:940 __FUNCTION__ = "init_mod" #11 0x0000000000535476 in init_mod (m=0x7f807c8daa70) at sr_module.c:940 __FUNCTION__ = "init_mod" #12 0x0000000000535476 in init_mod (m=0x7f807c8dada0) at sr_module.c:940 __FUNCTION__ = "init_mod" #13 0x0000000000535476 in init_mod (m=0x7f807c8db380) at sr_module.c:940 __FUNCTION__ = "init_mod" #14 0x0000000000535476 in init_mod (m=0x7f807c8db5f0) at sr_module.c:940 __FUNCTION__ = "init_mod" #15 0x0000000000535476 in init_mod (m=0x7f807c8db940) at sr_module.c:940 __FUNCTION__ = "init_mod" #16 0x0000000000535476 in init_mod (m=0x7f807c8dc230) at sr_module.c:940 __FUNCTION__ = "init_mod" #17 0x0000000000535476 in init_mod (m=0x7f807c8dc8a0) at sr_module.c:940 __FUNCTION__ = "init_mod" #18 0x0000000000535476 in init_mod (m=0x7f807c8dcc80) at sr_module.c:940 __FUNCTION__ = "init_mod" #19 0x0000000000535476 in init_mod (m=0x7f807c8dd900) at sr_module.c:940 __FUNCTION__ = "init_mod" #20 0x0000000000535476 in init_mod (m=0x7f807c8ddae0) at sr_module.c:940 __FUNCTION__ = "init_mod" #21 0x0000000000535476 in init_mod (m=0x7f807c8ddcc0) at sr_module.c:940 __FUNCTION__ = "init_mod" #22 0x0000000000535476 in init_mod (m=0x7f807c8ddfa0) at sr_module.c:940 __FUNCTION__ = "init_mod" ---Type <return> to continue, or q <return> to quit--- #23 0x0000000000535476 in init_mod (m=0x7f807c8de180) at sr_module.c:940 __FUNCTION__ = "init_mod" #24 0x0000000000535a0b in init_modules () at sr_module.c:972 t = 0x2cb3d66f i = 2089519184 __FUNCTION__ = "init_modules" #25 0x000000000052c1b0 in main (argc=17, argv=0x7ffc76d2a568) at main.c:2474 cfg_stream = 0x18c2010 c = -1 r = 0 tmp = 0x7ffc76d2bf69 "" tmp_len = 32640 port = 1 proto = 0 options = 0x7223c0 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:" ret = -1 seed = 2169338037 rfd = 4 debug_save = 0 debug_flag = 0 dont_fork_cnt = 0 n_lst = 0xf0b6ff p = 0x6e471d <__libc_csu_init+77> "H\203\303\001H9\353u\352H\203\304\b[]A\A]A^A_\303ff.\017\037\204" __FUNCTION__ = "main" (gdb) frame 0 #0 0x00007f806f885870 in hash_table_free (hash_table=0x7f8075320e40) at hash.c:230 230 in hash.c (gdb) p *np Cannot access memory at address 0x298bd800
same with this message. is the archive system somehow broken? the message was sent at 18:18:31 cet and sr-dev archive claims;
Last message date: Tue Sep 22 19:01:36 CEST 2015 Archived on: Tue Sep 22 19:01:43 CEST 2015
still i can't find the message.
-------------------------------------------------------------------------
i just tried with earlier 4.3 sip proxy that i built on sept 17 at about 9 am cet and it starts without problems. so whatever commits happened after that caused the crash at start.
-- juha
Daniel-Constantin Mierla writes:
From second core, get:
frame 0 p *qm p *frag p *f info locals p qm->free_hash[hash] p *qm->free_hash[hash].first
below is the same with MEMDBG ?=1 from another crash.
-- juha
(gdb) where #0 0x0000000000641d6a in fm_insert_free (qm=0x7f1756261000, frag=0x7f1756a32ab8) at mem/f_malloc.c:242 #1 0x00000000006445a9 in fm_free (qmp=0x7f1756261000, p=0x7f1756a32af0, file=0x7f175bb14f2d "tm: h_table.c", func=0x7f175bb151cb <__FUNCTION__.9593> "free_cell", line=135) at mem/f_malloc.c:626 #2 0x00007f175ba4eaa5 in free_cell (dead_cell=0x7f1756a0b168) at h_table.c:135 #3 0x00007f175bad14b0 in wait_handler (ti=285224763, wait_tl=0x7f1756a0b1e8, data=0x7f1756a0b168) at timer.c:648 #4 0x00000000004aaf88 in timer_list_expire (t=285224763, h=0x7f17562ae360, slow_l=0x7f17562af4e8, slow_mark=260) at timer.c:873 #5 0x00000000004ab3e5 in timer_handler () at timer.c:938 #6 0x00000000004ab853 in timer_main () at timer.c:977 #7 0x000000000052f67a in main_loop () at main.c:1650 #8 0x00000000005354d2 in main (argc=17, argv=0x7ffc55e2d168) at main.c:2566 (gdb) frame 0 #0 0x0000000000641d6a in fm_insert_free (qm=0x7f1756261000, frag=0x7f1756a32ab8) at mem/f_malloc.c:242 242 in mem/f_malloc.c (gdb) p *qm $1 = {type = 0, size = 33554432, used = 6473272, real_used = 8437168, max_real_used = 8820360, ffrags = 114, first_frag = 0x7f1756269478, last_frag = 0x7f1758260fc8, free_bitmap = {580542139582204, 576460752311812096, 281492173619200, 288230930219337730, 1099645845632, 4611686018427420672, 0, 0, 0, 2048, 4, 9227875636482146816, 8796093030400, 2199023779840, 2, 0, 0, 0, 134217728, 0, 137438953472, 0, 73016541184, 576460752303423488, 34359738372, 137438986240, 576460752303685632, 262144, 0, 0, 0, 0, 2050}, free_hash = {{first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x7f17569af710, no = 2}, {first = 0x7f17569b6d00, no = 3}, {first = 0x7f17569f41d0, no = 3}, { first = 0x7f17569af7b0, no = 2}, {first = 0x7f1756a21dc8, no = 1}, {first = 0x7f1756a8a8d0, no = 2}, {first = 0x0, no = 0}, { first = 0x7f1756a87610, no = 1}, {first = 0x7f1756a87800, no = 5}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, { first = 0x7f1756a22008, no = 1}, {first = 0x7f1756a67828, no = 2}, {first = 0x7f1756a87748, no = 1}, {first = 0x0, no = 0} <repeats 27 times>, {first = 0x7f17569bc118, no = 1}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, { first = 0x0, no = 0}, {first = 0x7f1756a7cff0, no = 1}, {first = 0x0, no = 0} <repeats 37 times>, {first = 0x7f1756a327c8, no = 6}, { first = 0x0, no = 0} <repeats 35 times>, {first = 0x7f17569fe5a8, no = 1}, {first = 0x0, no = 0} <repeats 22 times>, { first = 0x7f1756a31a68, no = 7}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x7f17569f4228, no = 1}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, { first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x7f1756a31f30, no = 6}, {first = 0x0, no = 0} <repeats 13 times>, {first = 0x7f1756a6e5e8, no = 1}, {first = 0x0, no = 0} <repeats 16 times>, { first = 0x7f1756a2a290, no = 6}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x7f1756a2abc0, no = 6}, { first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x7f1756a2e3a0, no = 5}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x7f1756a604e8, no = 1}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, { first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x7f17569f4888, no = 1}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x0, no = 0}, {first = 0x7f17569f3a60, no = 1}, { first = 0x0, no = 0} <repeats 18 times>, {first = 0x7f17569bb4b8, no = 1}, {first = 0x0, no = 0} <repeats 12 times>, { first = 0x7f17569ede78, no = 1}, {first = 0x0, no = 0} <repeats 19 times>, {first = 0x7f1756a75c38, no = 1}, {first = 0x0, no = 0} <repeats 12 times>, {first = 0x7f1756a7fd48, no = 1}, {first = 0x0, no = 0} <repeats 38 times>, {first = 0x7f1756a8a9f8, no = 1}, { first = 0x0, no = 0} <repeats 46 times>...}} (gdb) p *frag $2 = {size = 17944, u = {nxt_free = 0x7f1756a2b288, reserved = 139738214478472}, prv_free = 0x62876e620000de4f, file = 0x7f175bb14f2d "tm: h_table.c", func = 0x7f175bb151cb <__FUNCTION__.9593> "free_cell", line = 135, check = 4042322160} (gdb) p *f $3 = {size = 139738209821792, u = {nxt_free = 0x7f17565ba460, reserved = 139738209821792}, prv_free = 0x62876e620000de4f, file = 0x10020 <error: Cannot access memory at address 0x10020>, func = 0x7f1756a2d0b0 "From: "Juha Heinanen" sip:foo@test.tutpro.com;tag=UPUUEy88QXYNOoJhqQWQIrSwoVM31O0y\r\nTo: sip:foo@test.tutpro.com\r\nContact: sip:42351086@192.98.102.10:40198;transport=tcp\r\nCall-ID: x7hS1hVpUOS2EGm"..., line = 86, check = 139738214486368} (gdb) info locals f = 0x7f1756a2b288 hash = 2049 after = 0 (gdb) p qm->free_hash[hash] $4 = {first = 0x7f1756a2c2b8, no = 11} (gdb) p *qm->free_hash[hash].first $5 = {size = 0, u = {nxt_free = 0x7f1756a2b288, reserved = 139738214478472}, prv_free = 0x0, file = 0x0, func = 0x0, line = 0, check = 139738299043583}
i just resumed crash testing and built latest master. unfortunately, kamailio is still crashing.
-- juha (gdb) where
#0 0x0000000000641d80 in fm_insert_free (qm=0x7f3e927df000, frag=0x7f3e930a1970) at mem/f_malloc.c:242 #1 0x00000000006445bf in fm_free (qmp=0x7f3e927df000, p=0x7f3e930a19a8, file=0x7f3e98092f1d "tm: h_table.c", func=0x7f3e980931bb <__FUNCTION__.9593> "free_cell", line=162) at mem/f_malloc.c:626 #2 0x00007f3e97fccc12 in free_cell (dead_cell=0x7f3e92f79e00) at h_table.c:162 #3 0x00007f3e9804f49e in wait_handler (ti=686713932, wait_tl=0x7f3e92f79e80, data=0x7f3e92f79e00) at timer.c:648 #4 0x00000000004aaf88 in timer_list_expire (t=686713932, h=0x7f3e9282c360, slow_l=0x7f3e9282d218, slow_mark=215) at timer.c:873 #5 0x00000000004ab3e5 in timer_handler () at timer.c:938 #6 0x00000000004ab853 in timer_main () at timer.c:977 #7 0x000000000052f690 in main_loop () at main.c:1650 #8 0x00000000005354e8 in main (argc=17, argv=0x7ffd5da92638) at main.c:2566
-
Daniel-Constantin Mierla -
Juha Heinanen