11 Feb
2014
11 Feb
'14
1:38 p.m.
Hello,
Upon receipt an 502 Bad gateway, I have the following error then Kamailio
crashed
Feb 11 11:49:57 kamailio23 /usr/local/sbin/kamailio[28895]: WARNING: tm
[t_lookup.c:1564]: t_unref(): WARNING: script writer didn't release
transaction
Feb 11 11:50:03 kamailio23 /usr/local/sbin/kamailio[28926]: : <core>
[pass_fd.c:293]: receive_fd(): ERROR: receive_fd: EOF on 31
Feb 11 11:50:03 kamailio23 /usr/local/sbin/kamailio[28876]: ALERT: <core>
[main.c:788]: handle_sigs(): child process 28903 exited by a signal 11
Feb 11 11:50:03 kamailio23 /usr/local/sbin/kamailio[28876]: ALERT: <core>
[main.c:791]: handle_sigs(): core was generated
(gdb) bt
#0 0x000000000053a81d in qm_free (qm=0x7fd6f962a000, p=0x75d63ff100000000)
at mem/q_malloc.c:476
#1 0x00007fd700c5581c in free_cell (dead_cell=0x7fd6f9777bf0) at
h_table.c:162
#2 0x00007fd700c56783 in free_hash_table () at h_table.c:443
#3 0x00007fd700c6981b in tm_shutdown () at t_funcs.c:122
#4 0x00000000004ee6f0 in destroy_modules () at sr_module.c:790
#5 0x0000000000463baa in cleanup (show_status=1) at main.c:573
#6 0x0000000000464c32 in shutdown_children (sig=15, show_status=1) at
main.c:715
#7 0x0000000000466223 in handle_sigs () at main.c:806
#8 0x000000000046a459 in main_loop () at main.c:1767
#9 0x000000000046ca85 in main (argc=5, argv=0x7fff2f13ec38) at main.c:2566
(gdb) bt full
#0 0x000000000053a81d in qm_free (qm=0x7fd6f962a000, p=0x75d63ff100000000)
at mem/q_malloc.c:476
f = 0x75d63ff0fffffff0
size = 4880
next = 0x7fd6f98c5380
prev = 0x7fd6f9634840
__FUNCTION__ = "qm_free"
#1 0x00007fd700c5581c in free_cell (dead_cell=0x7fd6f9777bf0) at
h_table.c:162
b = 0x0
i = 1
rpl = 0x0
tt = 0x0
foo = 0x59e92da4
cbs = 0x0
cbs_tmp = 0x7fd6f98c5370
__FUNCTION__ = "free_cell"
#2 0x00007fd700c56783 in free_hash_table () at h_table.c:443
p_cell = 0x7fd6f9777bf0
tmp_cell = 0x7fd6f98fb9c0
i = 3084
#3 0x00007fd700c6981b in tm_shutdown () at t_funcs.c:122
__FUNCTION__ = "tm_shutdown"
#4 0x00000000004ee6f0 in destroy_modules () at sr_module.c:790
t = 0x7fd7022322f0
foo = 0x7fd702231cc0
#5 0x0000000000463baa in cleanup (show_status=1) at main.c:573
memlog = 32727
__FUNCTION__ = "cleanup"
#6 0x0000000000464c32 in shutdown_children (sig=15, show_status=1) at
main.c:715
__FUNCTION__ = "shutdown_children"
#7 0x0000000000466223 in handle_sigs () at main.c:806
chld = 0
chld_status = 139
memlog = 0
__FUNCTION__ = "handle_sigs"
#8 0x000000000046a459 in main_loop () at main.c:1767
i = 8
pid = 28926
si = 0x0
si_desc = "udp receiver child=7 sock=91.213.79.31:5060\000\326\177",
'\000' <repeats 18 times>,
"@\352\023/\377\177\000\000\244-\351Y\000\000\000\000\240mA\000\000\000\000\000\060\354\023/\377\177",
'\000' <repeats 18 times>,
"p\352\023/\377\177\000\000\030>K\000\000\000\000"
nrprocs = 8
__FUNCTION__ = "main_loop"
#9 0x000000000046ca85 in main (argc=5, argv=0x7fff2f13ec38) at main.c:2566
cfg_stream = 0xc36010
c = -1
r = 0
tmp = 0x7fff2f13ec68 "\233\356\023/\377\177"
tmp_len = 0
port = 5
proto = 0
options = 0x5c7c18
":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
ret = -1
seed = 3628627122
rfd = 4
debug_save = 0
debug_flag = 0
dont_fork_cnt = 0
n_lst = 0xbf
p = 0x4162c9 "H\203\304\b\303" <Address 0x4162ce out of bounds>
__FUNCTION__ = "main"
(gdb)
Regards,
12 Feb
12 Feb
1:24 p.m.
Hello,
the backtrace you got is not relevant, because it is from the main
process that is handling the singnal of the child process that crashed.
So the corefile from crashed process has been overwritten.
Can you enable one core file per process and try to reproduce? Then get
the backtrace from each core generated.
Cheers,
Daniel
On 11/02/14 12:38, Tuan Viet Nguyen wrote:
Hello,
Upon receipt an 502 Bad gateway, I have the following error then
Kamailio crashed
Feb 11 11:49:57 kamailio23 /usr/local/sbin/kamailio[28895]: WARNING:
tm [t_lookup.c:1564]: t_unref(): WARNING: script writer didn't release
transaction
Feb 11 11:50:03 kamailio23 /usr/local/sbin/kamailio[28926]: : <core>
[pass_fd.c:293]: receive_fd(): ERROR: receive_fd: EOF on 31
Feb 11 11:50:03 kamailio23 /usr/local/sbin/kamailio[28876]: ALERT:
<core> [main.c:788]: handle_sigs(): child process 28903 exited by a
signal 11
Feb 11 11:50:03 kamailio23 /usr/local/sbin/kamailio[28876]: ALERT:
<core> [main.c:791]: handle_sigs(): core was generated
(gdb) bt
#0 0x000000000053a81d in qm_free (qm=0x7fd6f962a000,
p=0x75d63ff100000000) at mem/q_malloc.c:476
#1 0x00007fd700c5581c in free_cell (dead_cell=0x7fd6f9777bf0) at
h_table.c:162
#2 0x00007fd700c56783 in free_hash_table () at h_table.c:443
#3 0x00007fd700c6981b in tm_shutdown () at t_funcs.c:122
#4 0x00000000004ee6f0 in destroy_modules () at sr_module.c:790
#5 0x0000000000463baa in cleanup (show_status=1) at main.c:573
#6 0x0000000000464c32 in shutdown_children (sig=15, show_status=1) at
main.c:715
#7 0x0000000000466223 in handle_sigs () at main.c:806
#8 0x000000000046a459 in main_loop () at main.c:1767
#9 0x000000000046ca85 in main (argc=5, argv=0x7fff2f13ec38) at
main.c:2566
(gdb) bt full
#0 0x000000000053a81d in qm_free (qm=0x7fd6f962a000,
p=0x75d63ff100000000) at mem/q_malloc.c:476
f = 0x75d63ff0fffffff0
size = 4880
next = 0x7fd6f98c5380
prev = 0x7fd6f9634840
__FUNCTION__ = "qm_free"
#1 0x00007fd700c5581c in free_cell (dead_cell=0x7fd6f9777bf0) at
h_table.c:162
b = 0x0
i = 1
rpl = 0x0
tt = 0x0
foo = 0x59e92da4
cbs = 0x0
cbs_tmp = 0x7fd6f98c5370
__FUNCTION__ = "free_cell"
#2 0x00007fd700c56783 in free_hash_table () at h_table.c:443
p_cell = 0x7fd6f9777bf0
tmp_cell = 0x7fd6f98fb9c0
i = 3084
#3 0x00007fd700c6981b in tm_shutdown () at t_funcs.c:122
__FUNCTION__ = "tm_shutdown"
#4 0x00000000004ee6f0 in destroy_modules () at sr_module.c:790
t = 0x7fd7022322f0
foo = 0x7fd702231cc0
#5 0x0000000000463baa in cleanup (show_status=1) at main.c:573
memlog = 32727
__FUNCTION__ = "cleanup"
#6 0x0000000000464c32 in shutdown_children (sig=15, show_status=1) at
main.c:715
__FUNCTION__ = "shutdown_children"
#7 0x0000000000466223 in handle_sigs () at main.c:806
chld = 0
chld_status = 139
memlog = 0
__FUNCTION__ = "handle_sigs"
#8 0x000000000046a459 in main_loop () at main.c:1767
i = 8
pid = 28926
si = 0x0
si_desc = "udp receiver child=7 sock=91.213.79.31:5060
<http://91.213.79.31:5060>\000\326\177", '\000' <repeats 18
times>,
"@\352\023/\377\177\000\000\244-\351Y\000\000\000\000\240mA\000\000\000\000\000\060\354\023/\377\177",
'\000' <repeats 18 times>,
"p\352\023/\377\177\000\000\030>K\000\000\000\000"
nrprocs = 8
__FUNCTION__ = "main_loop"
#9 0x000000000046ca85 in main (argc=5, argv=0x7fff2f13ec38) at
main.c:2566
cfg_stream = 0xc36010
c = -1
r = 0
tmp = 0x7fff2f13ec68 "\233\356\023/\377\177"
tmp_len = 0
port = 5
proto = 0
options = 0x5c7c18
":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
ret = -1
seed = 3628627122
rfd = 4
debug_save = 0
debug_flag = 0
dont_fork_cnt = 0
n_lst = 0xbf
p = 0x4162c9 "H\203\304\b\303" <Address 0x4162ce out of bounds>
__FUNCTION__ = "main"
(gdb)
Regards,
_______________________________________________
SIP Express Router (SER) and Kamailio (OpenSER) - sr-users mailing list
sr-users(a)lists.sip-router.org
http://lists.sip-router.org/cgi-bin/mailman/listinfo/sr-users
--
Daniel-Constantin Mierla - http://www.asipto.com
http://twitter.com/#!/miconda - http://www.linkedin.com/in/miconda
13 Feb
13 Feb
12:28 p.m.
Hello,
Here is the bt of all cores. This time, the scenario is different, when
kamailio receives an ACK and try to forward the message
Log file:
*Feb 13 11:09:19 kamailio23 /usr/local/sbin/kamailio[21086]: : <core>
[pass_fd.c:293]: receive_fd(): ERROR: receive_fd: EOF on 31 *
Feb 13 11:09:19 kamailio23 /usr/local/sbin/kamailio[21038]: ALERT: <core>
[main.c:788]: handle_sigs(): child process 21063 exited by a signal 11
Feb 13 11:09:19 kamailio23 /usr/local/sbin/kamailio[21038]: ALERT: <core>
[main.c:791]: handle_sigs(): core was generated
Feb 13 11:09:19 kamailio23 /usr/local/sbin/kamailio[21038]: INFO: <core>
[main.c:803]: handle_sigs(): INFO: terminating due to SIGCHLD
*root@kamailio23:/var/log/kamailio# gdb /usr/local/sbin/kamailio
cores/core.kamailio.21063*
(gdb) bt
#0 0x000000000052971e in timer_list_expire (t=275552247, h=0x7f5ea9f30aa0,
slow_l=0x7f5ea9f31a50, slow_mark=59628) at timer.c:883
#1 0x0000000000529a80 in timer_handler () at timer.c:959
#2 0x0000000000529d0f in timer_main () at timer.c:998
#3 0x0000000000469eda in main_loop () at main.c:1709
#4 0x000000000046ca85 in main (argc=5, argv=0x7fffcd0f5828) at main.c:2566
(gdb) bt full
#0 0x000000000052971e in timer_list_expire (t=275552247, h=0x7f5ea9f30aa0,
slow_l=0x7f5ea9f31a50, slow_mark=59628) at timer.c:883
tl = 0x7f5eaa1bf5f0
ret = 32767
#1 0x0000000000529a80 in timer_handler () at timer.c:959
saved_ticks = 275552247
run_slow_timer = 0
i = 236
__FUNCTION__ = "timer_handler"
#2 0x0000000000529d0f in timer_main () at timer.c:998
No locals.
#3 0x0000000000469eda in main_loop () at main.c:1709
i = 8
pid = 0
si = 0x0
si_desc = "udp receiver child=7 sock=91.213.79.31:5060\000^\177",
'\000' <repeats 18 times>,
"0V\017\315\377\177\000\000u\234\021O\000\000\000\000\240mA\000\000\000\000\000
X\017\315\377\177", '\000' <repeats 18 times>,
"`V\017\315\377\177\000\000\030>K\000\000\000\000"
nrprocs = 8
__FUNCTION__ = "main_loop"
#4 0x000000000046ca85 in main (argc=5, argv=0x7fffcd0f5828) at main.c:2566
cfg_stream = 0x1bd2010
c = -1
r = 0
tmp = 0x7fffcd0f5858 "\233n\017\315\377\177"
tmp_len = 0
port = 5
proto = 0
options = 0x5c7c18
":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
ret = -1
seed = 1955773403
rfd = 4
debug_save = 0
debug_flag = 0
dont_fork_cnt = 0
n_lst = 0xbf
p = 0x4162c9 "H\203\304\b\303" <Address 0x4162ce out of bounds>
__FUNCTION__ = "main"
(gdb)
*root@kamailio23:/var/log/kamailio# gdb /usr/local/sbin/kamailio
cores/core.kamailio.21038*
(gdb) bt full
#0 0x000000000053a81d in qm_free (qm=0x7f5ea9ee2000, p=0x424f9a6b00000000)
at mem/q_malloc.c:476
f = 0x424f9a6afffffff0
size = 4880
next = 0x7f5eaa17eff0
prev = 0x7f5ea9eec840
__FUNCTION__ = "qm_free"
#1 0x00007f5eb150d81c in free_cell (dead_cell=0x7f5eaa047710) at
h_table.c:162
b = 0x0
i = 1
rpl = 0x0
tt = 0x0
foo = 0x0
cbs = 0x0
cbs_tmp = 0x7f5eaa186950
__FUNCTION__ = "free_cell"
#2 0x00007f5eb150e783 in free_hash_table () at h_table.c:443
p_cell = 0x7f5eaa047710
tmp_cell = 0x7f5eaa047710
i = 59334
#3 0x00007f5eb152181b in tm_shutdown () at t_funcs.c:122
__FUNCTION__ = "tm_shutdown"
#4 0x00000000004ee6f0 in destroy_modules () at sr_module.c:790
t = 0x7f5eb2aea2f0
foo = 0x7f5eb2ae9cc0
#5 0x0000000000463baa in cleanup (show_status=1) at main.c:573
memlog = 32606
__FUNCTION__ = "cleanup"
#6 0x0000000000464c32 in shutdown_children (sig=15, show_status=1) at
main.c:715
__FUNCTION__ = "shutdown_children"
#7 0x0000000000466223 in handle_sigs () at main.c:806
chld = 0
chld_status = 139
memlog = 0
__FUNCTION__ = "handle_sigs"
#8 0x000000000046a459 in main_loop () at main.c:1767
i = 8
pid = 21086
si = 0x0
si_desc = "udp receiver child=7 sock=91.213.79.31:5060\000^\177",
'\000' <repeats 18 times>,
"0V\017\315\377\177\000\000u\234\021O\000\000\000\000\240mA\000\000\000\000\000
X\017\315\377\177", '\000' <repeats 18 times>,
"`V\017\315\377\177\000\000\030>K\000\000\000\000"
nrprocs = 8
__FUNCTION__ = "main_loop"
#9 0x000000000046ca85 in main (argc=5, argv=0x7fffcd0f5828) at main.c:2566
cfg_stream = 0x1bd2010
c = -1
r = 0
tmp = 0x7fffcd0f5858 "\233n\017\315\377\177"
tmp_len = 0
port = 5
proto = 0
options = 0x5c7c18
":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
ret = -1
seed = 1955773403
rfd = 4
debug_save = 0
debug_flag = 0
dont_fork_cnt = 0
n_lst = 0xbf
p = 0x4162c9 "H\203\304\b\303" <Address 0x4162ce out of bounds>
__FUNCTION__ = "main"
(gdb)
Regards,
On Wed, Feb 12, 2014 at 12:24 PM, Daniel-Constantin Mierla <
miconda(a)gmail.com> wrote:
Hello,
the backtrace you got is not relevant, because it is from the main process
that is handling the singnal of the child process that crashed. So the
corefile from crashed process has been overwritten.
Can you enable one core file per process and try to reproduce? Then get
the backtrace from each core generated.
Cheers,
Daniel
On 11/02/14 12:38, Tuan Viet Nguyen wrote:
Hello,
Upon receipt an 502 Bad gateway, I have the following error then Kamailio
crashed
Feb 11 11:49:57 kamailio23 /usr/local/sbin/kamailio[28895]: WARNING: tm
[t_lookup.c:1564]: t_unref(): WARNING: script writer didn't release
transaction
Feb 11 11:50:03 kamailio23 /usr/local/sbin/kamailio[28926]: : <core>
[pass_fd.c:293]: receive_fd(): ERROR: receive_fd: EOF on 31
Feb 11 11:50:03 kamailio23 /usr/local/sbin/kamailio[28876]: ALERT: <core>
[main.c:788]: handle_sigs(): child process 28903 exited by a signal 11
Feb 11 11:50:03 kamailio23 /usr/local/sbin/kamailio[28876]: ALERT: <core>
[main.c:791]: handle_sigs(): core was generated
(gdb) bt
#0 0x000000000053a81d in qm_free (qm=0x7fd6f962a000,
p=0x75d63ff100000000) at mem/q_malloc.c:476
#1 0x00007fd700c5581c in free_cell (dead_cell=0x7fd6f9777bf0) at
h_table.c:162
#2 0x00007fd700c56783 in free_hash_table () at h_table.c:443
#3 0x00007fd700c6981b in tm_shutdown () at t_funcs.c:122
#4 0x00000000004ee6f0 in destroy_modules () at sr_module.c:790
#5 0x0000000000463baa in cleanup (show_status=1) at main.c:573
#6 0x0000000000464c32 in shutdown_children (sig=15, show_status=1) at
main.c:715
#7 0x0000000000466223 in handle_sigs () at main.c:806
#8 0x000000000046a459 in main_loop () at main.c:1767
#9 0x000000000046ca85 in main (argc=5, argv=0x7fff2f13ec38) at main.c:2566
(gdb) bt full
#0 0x000000000053a81d in qm_free (qm=0x7fd6f962a000,
p=0x75d63ff100000000) at mem/q_malloc.c:476
f = 0x75d63ff0fffffff0
size = 4880
next = 0x7fd6f98c5380
prev = 0x7fd6f9634840
__FUNCTION__ = "qm_free"
#1 0x00007fd700c5581c in free_cell (dead_cell=0x7fd6f9777bf0) at
h_table.c:162
b = 0x0
i = 1
rpl = 0x0
tt = 0x0
foo = 0x59e92da4
cbs = 0x0
cbs_tmp = 0x7fd6f98c5370
__FUNCTION__ = "free_cell"
#2 0x00007fd700c56783 in free_hash_table () at h_table.c:443
p_cell = 0x7fd6f9777bf0
tmp_cell = 0x7fd6f98fb9c0
i = 3084
#3 0x00007fd700c6981b in tm_shutdown () at t_funcs.c:122
__FUNCTION__ = "tm_shutdown"
#4 0x00000000004ee6f0 in destroy_modules () at sr_module.c:790
t = 0x7fd7022322f0
foo = 0x7fd702231cc0
#5 0x0000000000463baa in cleanup (show_status=1) at main.c:573
memlog = 32727
__FUNCTION__ = "cleanup"
#6 0x0000000000464c32 in shutdown_children (sig=15, show_status=1) at
main.c:715
__FUNCTION__ = "shutdown_children"
#7 0x0000000000466223 in handle_sigs () at main.c:806
chld = 0
chld_status = 139
memlog = 0
__FUNCTION__ = "handle_sigs"
#8 0x000000000046a459 in main_loop () at main.c:1767
i = 8
pid = 28926
si = 0x0
si_desc = "udp receiver child=7 sock=91.213.79.31:5060\000\326\177",
'\000' <repeats 18 times>,
"@\352\023/\377\177\000\000\244-\351Y\000\000\000\000\240mA\000\000\000\000\000\060\354\023/\377\177",
'\000' <repeats 18 times>,
"p\352\023/\377\177\000\000\030>K\000\000\000\000"
nrprocs = 8
__FUNCTION__ = "main_loop"
#9 0x000000000046ca85 in main (argc=5, argv=0x7fff2f13ec38) at main.c:2566
cfg_stream = 0xc36010
c = -1
r = 0
tmp = 0x7fff2f13ec68 "\233\356\023/\377\177"
tmp_len = 0
port = 5
proto = 0
options = 0x5c7c18
":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
ret = -1
seed = 3628627122
rfd = 4
debug_save = 0
debug_flag = 0
dont_fork_cnt = 0
n_lst = 0xbf
p = 0x4162c9 "H\203\304\b\303" <Address 0x4162ce out of bounds>
__FUNCTION__ = "main"
(gdb)
Regards,
_______________________________________________
SIP Express Router (SER) and Kamailio (OpenSER) - sr-users mailing
listsr-users@lists.sip-router.orghttp://lists.sip-router.org/cgi-bin/mailman/listinfo/sr-users
--
Daniel-Constantin Mierla - http://www.asipto.comhttp://twitter.com/#!/miconda -
http://www.linkedin.com/in/miconda
_______________________________________________
SIP Express Router (SER) and Kamailio (OpenSER) - sr-users mailing list
sr-users(a)lists.sip-router.org
http://lists.sip-router.org/cgi-bin/mailman/listinfo/sr-users
3935
days inactive
3937
days old
2 comments
2 participants
participants (2)
-
Daniel-Constantin Mierla -
Tuan Viet Nguyen