_______________________________________________Hello,
this seems similar to the issue reported in the tracker:
- https://github.com/kamailio/kamailio/issues/1895#issuecomment-500895303
I was troubleshooting on that system in the past, but recently I couldn't get more there. Not sure why the mysql query is not timing out, on the net there were mentions that is cannot be ensured, it's more like a best try.
You should install debug symbols for libmysqlclient library and see if you get when in the libmysqlclient code is getting stuck.
Do you have tls enabled in kamailio?
Cheers,
Daniel
On 05.08.19 17:03, David Villasmil wrote:
At least try setting db_mode for the dialog module to 0 (zero)--
On Mon, 5 Aug 2019 at 16:02, David Villasmil <david.villasmil.work@gmail.com> wrote:
Have you tried taking out the dB support and see if it keeps happening? The by talks a lot about MySQL... no bT expert here...--
On Mon, 5 Aug 2019 at 14:07, M. Salman <msalman212@gmail.com> wrote:
Hi Team,
Facing 100% Kamilio CPU issue, here are the details, any pointers/hints would help. Thanks
Scenario:Random Kamailio CPU shoot, restart fixes it. Happens once or twice in a day.It's a pre-production server with around 30 extensions, not much load. For now, it has 2 CPUs and 4 G memory.
[Info]
uname -a
Linux ip-X-aws #46-Ubuntu SMP Thu Jul 4 13:38:28 UTC 2019 x86_64 x86_64 x86_64 GNU/Linux
kamailio -V
version: kamailio 5.2.3 (x86_64/linux) c36229
flags: STATS: Off, USE_TCP, USE_TLS, USE_SCTP, TLS_HOOKS, USE_RAW_SOCKS, DISABLE_NAGLE, USE_MCAST, DNS_IP_HACK, SHM_MEM, SHM_MMAP, PKG_MALLOC, Q_MALLOC, F_MALLOC, TLSF_MALLOC, DBG_SR_MEMORY, USE_FUTEX, FAST_LOCK-ADAPTIVE_WAIT, USE_DNS_CACHE, USE_DNS_FAILOVER, USE_NAPTR, USE_DST_BLACKLIST, HAVE_RESOLV_RES
ADAPTIVE_WAIT_LOOPS=1024, MAX_RECV_BUFFER_SIZE 262144 MAX_URI_SIZE 1024, BUF_SIZE 65535, DEFAULT PKG_SIZE 8MB
poll method support: poll, epoll_lt, epoll_et, sigio_rt, select.
id: c36229
(gdb) bt full
#0 0x00007f0069d97aab in ?? () from /usr/lib/x86_64-linux-gnu/libmysqlclient.so.20
No symbol table info available.
#1 0x00007f0069d7d6bd in ?? () from /usr/lib/x86_64-linux-gnu/libmysqlclient.so.20
No symbol table info available.
#2 0x00007f0069d657c7 in ?? () from /usr/lib/x86_64-linux-gnu/libmysqlclient.so.20
No symbol table info available.
#3 0x00007f0069d3ebce in ?? () from /usr/lib/x86_64-linux-gnu/libmysqlclient.so.20
No symbol table info available.
#4 0x00007f0069d3fc4c in ?? () from /usr/lib/x86_64-linux-gnu/libmysqlclient.so.20
No symbol table info available.
#5 0x00007f0069d33edc in ?? () from /usr/lib/x86_64-linux-gnu/libmysqlclient.so.20
No symbol table info available.
#6 0x00007f0069d34fa8 in ?? () from /usr/lib/x86_64-linux-gnu/libmysqlclient.so.20
No symbol table info available.
#7 0x00007f0069d36d71 in mysql_real_query () from /usr/lib/x86_64-linux-gnu/libmysqlclient.so.20
No symbol table info available.
#8 0x00007f006a31e4b4 in db_mysql_submit_query (h=0x7f006a6cdb20, s=0x7f0069af92d0 <sql_str>) at km_dbase.c:116
t = 1564528172
i = 0
code = 0
_func_ = "db_mysql_submit_query"
#9 0x00007f00698e8fb3 in db_do_submit_query (h=0x7f006a6cdb20, query=0x7f0069af92d0 <sql_str>,
submit_query=0x7f006a31dd3c <db_mysql_submit_query>) at db_query.c:58
ret = 10
tvb = {tv_sec = 0, tv_usec = 0}
tve = {tv_sec = 0, tv_usec = 2077076196242373120}
tz = {tz_minuteswest = 0, tz_dsttime = 0}
tdiff = 0
_func_ = "db_do_submit_query"
#10 0x00007f00698ec1dc in db_do_delete (h=0x7f006a6cdb20, k=0x7ffcdb440fa0, o=0x0, v=0x7ffcdb440fb0, _n=2,
val2str=0x7f006a340429 <db_mysql_val2str>, submit_query=0x7f006a31dd3c <db_mysql_submit_query>) at db_query.c:299
off = 68
ret = 36
---Type <return> to continue, or q <return> to quit---
_func_ = "db_do_delete"
#11 0x00007f006a322f6a in db_mysql_delete (h=0x7f006a6cdb20, k=0x7ffcdb440fa0, o=0x0, v=0x7ffcdb440fb0, _n=2) at km_dbase.c:515
No locals.
#12 0x00007f0065b6da70 in remove_dialog_from_db (cell=0x7f005c5ed3d8) at dlg_db_handler.c:722
values = {{type = DB1_INT, nul = 0, free = 477963473, val = {int_val = 1460, ll_val = 140720308487604,
double_val = 6.9525070095907518e-310, time_val = 140720308487604,
string_val = 0x7ffc000005b4 <error: Cannot access memory at address 0x7ffc000005b4>, str_val = {
s = 0x7ffc000005b4 <error: Cannot access memory at address 0x7ffc000005b4>, len = 0}, blob_val = {
s = 0x7ffc000005b4 <error: Cannot access memory at address 0x7ffc000005b4>, len = 0}, bitmap_val = 1460, uint_val = 1460,
ull_val = 140720308487604}}, {type = DB1_INT, nul = 0, free = 0, val = {int_val = 8020, ll_val = 139637976735572,
double_val = 6.899032715982534e-310, time_val = 139637976735572,
string_val = 0x7f0000001f54 <error: Cannot access memory at address 0x7f0000001f54>, str_val = {
s = 0x7f0000001f54 <error: Cannot access memory at address 0x7f0000001f54>, len = 0}, blob_val = {
s = 0x7f0000001f54 <error: Cannot access memory at address 0x7f0000001f54>, len = 0}, bitmap_val = 8020, uint_val = 8020,
ull_val = 139637976735572}}}
match_keys = {0x7f0065dfc6c0 <h_entry_column>, 0x7f0065dfc6b0 <h_id_column>}
vars_match_keys = {0x7f0065dfc7f0 <vars_h_entry_column>, 0x7f0065dfc7e0 <vars_h_id_column>}
_func_ = "remove_dialog_from_db"
#13 0x00007f0065b49375 in destroy_dlg (dlg=0x7f005c5ed3d8) at dlg_hash.c:371
ret = 1
var = 0x7f0065b45442 <futex_get+40>
_func_ = "destroy_dlg"
#14 0x00007f0065b50ca0 in dlg_unref_helper (dlg=0x7f005c5ed3d8, cnt=2, fname=0x7f0065be1eff "dlg_handlers.c", fline=1182) at dlg_hash.c:961
d_entry = 0x7f005c45b4c0
_func_ = "dlg_unref_helper"
#15 0x00007f0065b7f708 in unref_dlg_from_cb (t=0x7f005c5b5dc0, type=131072, param=0x7ffcdb4412a0) at dlg_handlers.c:1182
dlg = 0x7f005c5ed3d8
iuid = 0x7f005c568c38
#16 0x00007f0068295b05 in run_trans_callbacks_internal (cb_lst=0x7f005c5b5e38, type=131072, trans=0x7f005c5b5dc0, params=0x7ffcdb4412a0)
at t_hooks.c:260
cbp = 0x7f005c606118
backup_from = 0x5569cc3da8b0 <def_list+16>
backup_to = 0x5569cc3da8b8 <def_list+24>
---Type <return> to continue, or q <return> to quit---
backup_dom_from = 0x5569cc3da8c0 <def_list+32>
backup_dom_to = 0x5569cc3da8c8 <def_list+40>
backup_uri_from = 0x5569cc3da8a0 <def_list>
backup_uri_to = 0x5569cc3da8a8 <def_list+8>
backup_xavps = 0x5569cc3da2c0 <_xavp_list_head>
_func_ = "run_trans_callbacks_internal"
#17 0x00007f0068295c2c in run_trans_callbacks (type=131072, trans=0x7f005c5b5dc0, req=0x0, rpl=0x0, code=0) at t_hooks.c:287
params = {req = 0x0, rpl = 0x0, param = 0x7f005c606128, code = 0, flags = 0, branch = 0, t_rbuf = 0x0, dst = 0x0, send_buf = {
s = 0x0, len = 0}}
#18 0x00007f00681cbf45 in free_cell_helper (dead_cell=0x7f005c5b5dc0, silent=0, fname=0x7f00682de882 "timer.c", fline=689) at h_table.c:166
b = 0x7f005c5b5dc0 ""
i = 1
rpl = 0x7ffcdb4413a0
tt = 0x0
foo = 0x7ffcdb441390
cbs = 0x7f005c378700
cbs_tmp = 0x1681ca80f
_func_ = "free_cell_helper"
#19 0x00007f0068283bf6 in wait_handler (ti=855479619, wait_tl=0x7f005c5b5e48, data=0x7f005c5b5dc0) at timer.c:689
p_cell = 0x7f005c5b5dc0
ret = 1
unlinked = 0
rcount = 1
_func_ = "wait_handler"
#20 0x00005569cbdb0661 in timer_list_expire (t=855479619, h=0x7f005c238af0, slow_l=0x7f005c23ae08, slow_mark=5646) at core/timer.c:874
tl = 0x7f005c5b5e48
ret = 0
#21 0x00005569cbdb0b09 in timer_handler () at core/timer.c:939
saved_ticks = 855479619
run_slow_timer = 0
i = 526
_func_ = "timer_handler"
#22 0x00005569cbdb0fd2 in timer_main () at core/timer.c:978
---Type <return> to continue, or q <return> to quit---
No locals.
#23 0x00005569cbcf1382 in main_loop () at main.c:1727
i = 8
pid = 0
si = 0x0
si_desc = "udp receiver child=7 sock=172.31.2.33:6000 (13.54.252.29:6000)\000\034\004\000\000\000\000\000\000\000\321$}\034\000\000\000\000P0\316\313iU\000\000 \031D\333\374\177", '\000' <repeats 18 times>, "`\026D\333\374\177\000\000\360S\361\313iU\000"
nrprocs = 8
woneinit = 1
_func_ = "main_loop"
#24 0x00005569cbcf8a7b in main (argc=13, argv=0x7ffcdb441928) at main.c:2696
cfg_stream = 0x5569cd7c7260
c = -1
r = 0
tmp = 0x7ffcdb441f39 ""
tmp_len = -616294480
port = 32764
proto = 3
options = 0x5569cc074190 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:x:X:Y:"
ret = -1
seed = 3114863458
rfd = 4
debug_save = 0
debug_flag = 0
dont_fork_cnt = 0
n_lst = 0x7f006af01787
p = 0x9c0 <error: Cannot access memory at address 0x9c0>
st = {st_dev = 23, st_ino = 456, st_nlink = 2, st_mode = 16877, st_uid = 0, st_gid = 0, __pad0 = 0, st_rdev = 0, st_size = 60,
st_blksize = 4096, st_blocks = 0, st_atim = {tv_sec = 1564229247, tv_nsec = 809557407}, st_mtim = {tv_sec = 1564229229,
tv_nsec = 217250917}, st_ctim = {tv_sec = 1564229279, tv_nsec = 666090748}, __glibc_reserved = {0, 0, 0}}
_func_ = "main"
(gdb)_______________________________________________
--
Regards
M. Salman
Kamailio (SER) - Users Mailing List
sr-users@lists.kamailio.org
https://lists.kamailio.org/cgi-bin/mailman/listinfo/sr-users
_______________________________________________ Kamailio (SER) - Users Mailing List sr-users@lists.kamailio.org https://lists.kamailio.org/cgi-bin/mailman/listinfo/sr-users-- Daniel-Constantin Mierla -- www.asipto.com www.twitter.com/miconda -- www.linkedin.com/in/miconda
Kamailio (SER) - Users Mailing List
sr-users@lists.kamailio.org
https://lists.kamailio.org/cgi-bin/mailman/listinfo/sr-users
M. Salman
VoIP Professional