Hello all,
I've been experiencing memory exhaustion (all 8GiB of it) on hosts running
kamailio for quite some time now while on the 5.4.x train. I can't really
say about previous versions, because the setup was quite different back
when I was using those.
After about 2 days of operation, RAM is exhausted and kamailio is killed by
the kernel OOM which frees up memory again. Restarting kamailio via systemd
will also free up most memory. I can't seem to find what is using it up, so
I'm guessing there might be a memory leak, but I'm not 100% sure so I
thought I'd ask the list for opinions.
I'm including the output of kamcmd pkg and shm stats and kamctl stats here.
My take on those is that there's nothing to show where all this RAM is
going. Kamailio version is 5.4.5+bpo10 on debian buster and is started with
'-P /run/kamailio/kamailio.pid -f /etc/kamailio/kamailio.cfg -m 2048 -M 16'
(51 processes total). Thanks!
kamctl stats
{
"jsonrpc": "2.0",
"result": [
"core:bad_URIs_rcvd = 6470677",
"core:bad_msg_hdr = 25370",
"core:drop_replies = 0",
"core:drop_requests = 342064",
"core:err_replies = 0",
"core:err_requests = 0",
"core:fwd_replies = 24367",
"core:fwd_requests = 375703",
"core:rcv_replies = 45021597",
"core:rcv_replies_18x = 834856",
"core:rcv_replies_1xx = 1505425",
"core:rcv_replies_1xx_bye = 17",
"core:rcv_replies_1xx_cancel = 7",
"core:rcv_replies_1xx_invite = 1497313",
"core:rcv_replies_1xx_message = 0",
"core:rcv_replies_1xx_prack = 0",
"core:rcv_replies_1xx_refer = 0",
"core:rcv_replies_1xx_reg = 0",
"core:rcv_replies_1xx_update = 0",
"core:rcv_replies_2xx = 41764997",
"core:rcv_replies_2xx_bye = 328464",
"core:rcv_replies_2xx_cancel = 223572",
"core:rcv_replies_2xx_invite = 371129",
"core:rcv_replies_2xx_message = 190",
"core:rcv_replies_2xx_prack = 0",
"core:rcv_replies_2xx_refer = 3205",
"core:rcv_replies_2xx_reg = 12783948",
"core:rcv_replies_2xx_update = 0",
"core:rcv_replies_3xx = 1052",
"core:rcv_replies_3xx_bye = 0",
"core:rcv_replies_3xx_cancel = 0",
"core:rcv_replies_3xx_invite = 1052",
"core:rcv_replies_3xx_message = 0",
"core:rcv_replies_3xx_prack = 0",
"core:rcv_replies_3xx_refer = 0",
"core:rcv_replies_3xx_reg = 0",
"core:rcv_replies_3xx_update = 0",
"core:rcv_replies_401 = 17120",
"core:rcv_replies_404 = 525917",
"core:rcv_replies_407 = 553",
"core:rcv_replies_480 = 23138",
"core:rcv_replies_486 = 74201",
"core:rcv_replies_4xx = 1715492",
"core:rcv_replies_4xx_bye = 2146",
"core:rcv_replies_4xx_cancel = 594",
"core:rcv_replies_4xx_invite = 317850",
"core:rcv_replies_4xx_message = 208",
"core:rcv_replies_4xx_prack = 0",
"core:rcv_replies_4xx_refer = 1",
"core:rcv_replies_4xx_reg = 2382",
"core:rcv_replies_4xx_update = 0",
"core:rcv_replies_5xx = 29462",
"core:rcv_replies_5xx_bye = 15",
"core:rcv_replies_5xx_cancel = 10",
"core:rcv_replies_5xx_invite = 3501",
"core:rcv_replies_5xx_message = 0",
"core:rcv_replies_5xx_prack = 0",
"core:rcv_replies_5xx_refer = 0",
"core:rcv_replies_5xx_reg = 25157",
"core:rcv_replies_5xx_update = 0",
"core:rcv_replies_6xx = 5169",
"core:rcv_replies_6xx_bye = 0",
"core:rcv_replies_6xx_cancel = 0",
"core:rcv_replies_6xx_invite = 5169",
"core:rcv_replies_6xx_message = 0",
"core:rcv_replies_6xx_prack = 0",
"core:rcv_replies_6xx_refer = 0",
"core:rcv_replies_6xx_reg = 0",
"core:rcv_replies_6xx_update = 0",
"core:rcv_requests = 67620661",
"core:rcv_requests_ack = 707431",
"core:rcv_requests_bye = 336874",
"core:rcv_requests_cancel = 226878",
"core:rcv_requests_info = 70",
"core:rcv_requests_invite = 1038320",
"core:rcv_requests_message = 796",
"core:rcv_requests_notify = 6027113",
"core:rcv_requests_options = 29079952",
"core:rcv_requests_prack = 0",
"core:rcv_requests_publish = 40344",
"core:rcv_requests_refer = 3217",
"core:rcv_requests_register = 20919429",
"core:rcv_requests_subscribe = 5296608",
"core:rcv_requests_update = 0",
"core:unsupported_methods = 0",
"dialog:active_dialogs = 458",
"dialog:early_dialogs = 123",
"dialog:expired_dialogs = 105",
"dialog:failed_dialogs = 321224",
"dialog:processed_dialogs = 645092",
"dns:failed_dns_request = 534",
"dns:slow_dns_request = 0",
"mysql:driver_errors = 18",
"shmem:fragments = 8836",
"shmem:free_size = 1955034032",
"shmem:max_used_size = 250791392",
"shmem:real_used_size = 192449616",
"shmem:total_size = 2147483648",
"shmem:used_size = 105997528",
"siptrace:traced_replies = 0",
"siptrace:traced_requests = 0",
"sl:1xx_replies = 0",
"sl:200_replies = 14330540",
"sl:202_replies = 0",
"sl:2xx_replies = 0",
"sl:300_replies = 0",
"sl:301_replies = 0",
"sl:302_replies = 0",
"sl:3xx_replies = 0",
"sl:400_replies = 3",
"sl:401_replies = 7979168",
"sl:403_replies = 14845",
"sl:404_replies = 1159237",
"sl:407_replies = 2445205",
"sl:408_replies = 0",
"sl:483_replies = 79336",
"sl:4xx_replies = 28",
"sl:500_replies = 0",
"sl:5xx_replies = 0",
"sl:6xx_replies = 0",
"sl:failures = 1",
"sl:received_ACKs = 342064",
"sl:sent_err_replies = 0",
"sl:sent_replies = 26008362",
"sl:xxx_replies = 0",
"sst:expired_sst = 0",
"tcp:con_reset = 0",
"tcp:con_timeout = 0",
"tcp:connect_failed = 0",
"tcp:connect_success = 0",
"tcp:current_opened_connections = 0",
"tcp:current_write_queue_size = 0",
"tcp:established = 24562",
"tcp:local_reject = 0",
"tcp:passive_open = 24562",
"tcp:send_timeout = 0",
"tcp:sendq_full = 0",
"tmx:2xx_transactions = 41122607",
"tmx:3xx_transactions = 1049",
"tmx:4xx_transactions = 1904640",
"tmx:5xx_transactions = 8025",
"tmx:6xx_transactions = 5165",
"tmx:UAC_transactions = 2602132",
"tmx:UAS_transactions = 43036902",
"tmx:active_transactions = 145",
"tmx:inuse_transactions = 888",
"tmx:rpl_absorbed = 1541413",
"tmx:rpl_generated = 1111996",
"tmx:rpl_received = 44996889",
"tmx:rpl_relayed = 43455476",
"tmx:rpl_sent = 44567472"
],
"id": 15546
}
kamcmd mod.stats all pkg
Module: core
{
init_io_wait(469): 8040
init_modules(911): 16
get_hdr_field(116): 464
parse_headers(325): 128
rpc_hash_add(151): 2048
cnt_hash_add(394): 2048
register_select_table(458): 48
sr_wtimer_add(373): 80
add_callback(59): 312
sr_wtimer_init(351): 136
cfg_declare(50): 3168
fix_sock_str(500): 480
fix_hostname(1466): 96
grp_hash_add(241): 728
init_dst_set(89): 32296
cfg_script_fixup(303): 8
cfg_script_fixup(247): 288
cfg_script_fixup(241): 120
new_db_id(318): 304
fix_expr(554): 192
parse_pvname_list(1599): 480
mk_switch_jmp_table(73): 448
fix_param(1140): 672
pv_parse_index(776): 1232
subst_parser(291): 144
subst_parser(274): 64
subst_parser(301): 8
route_new_list(199): 1152
init_io_wait(516): 7256
db_do_init2(299): 280
parse_select(204): 2496
fix_param(1079): 704
mk_match_cond_table(100): 944
mk_name_lst(3754): 72
mk_listen_id(3771): 96
yyparse(2853): 19728
fix_param(1045): 11792
fixup_regexp_null(213): 368
tr_new(1730): 12352
fix_match(490): 384
fixup_pvar_all(279): 14512
mk_rval_expr2(2703): 845408
mk_elem(90): 528
fix_match_rve(3059): 1464
pv_cache_add(349): 63872
mk_rval_expr1(2643): 522656
mk_case_stm(3879): 3432
mk_rval_expr_v(2562): 2513784
mk_action(118): 493240
parse_params2(593): 6496
set_mod_param_regex(145): 1248
set_mod_param_regex(134): 2520
tr_table_add(1959): 360
register_module(253): 39928
register_module(236): 4784
load_module(437): 2624
new_cfg_script_var(134): 88
dupl_string_name(74): 56
dupl_string(49): 184
new_cfg_script_var(126): 104
pv_parse_format(1173): 30744
cfg_new_group(79): 864
new_sock_info(304): 96
new_sock_info(299): 2400
yyparse(659): 352
yyparse(1879): 216
add_alias(103): 256
add_alias(99): 320
yyparse(2388): 408
yyparse(678): 368
sr_push_yy_state(1695): 208
sr_push_yy_state(1634): 584
addstr(1448): 572072
rpc_hash_add(105): 28128
str_hash_alloc(59): 512
pv_table_add(236): 19448
pv_init_buffer(2139): 327680
pv_init_buffer(2129): 320
init_nonsip_hooks(43): 24
rval_get_str(1281): 54920
init_rlist(146): 24
fix_socket_list(1675): 56
route_add(124): 5312
str_hash_alloc(59): 768
mk_rval_expr_v(2575): 3304
pp_define(1827): 1344
init_counters(125): 128
cnt_hash_add(341): 27856
str_hash_alloc(59): 1280
Total: 5708952
}
Module: permissions
{
get_pathname(243): 64
Total: 64
}
Module: auth
{
generate_random_secret(240): 32
generate_random_secret(239): 32
Total: 64
}
Module: dialog
{
dlg_bridge_init_hdrs(66): 80
Total: 80
}
Module: acc
{
parse_acc_extra(116): 6384
Total: 6384
}
Module: xlog
{
mod_init(212): 8200
xdbg_fixup_helper(498): 7736
xlog_fixup_helper(533): 36280
Total: 52216
}
Module: dmq
{
make_socket_str_from_uri(143): 24
Total: 24
}
Module: rr
{
register_rrcb(61): 96
Total: 96
}
Module: jsonrpcs
{
jsonrpc_dgram_mod_init(182): 40
jsonrpc_init_fifo_server(123): 128
jsonrpc_fifo_mod_init(525): 40
Total: 208
}
Module: sqlops
{
sqlops_tr_buffer_init(46): 2048
sql_init_con(83): 480
Total: 2528
}
Module: db_mysql
{
db_mysql_new_connection(78): 3560
db_mysql_new_connection(68): 96
Total: 3656
}
Module: phonenum
{
sr_phonenum_add_item(99): 16
sr_phonenum_add_item(93): 608
pv_parse_phonenum_name(131): 224
Total: 848
}
Module: pv
{
tr_parse_string(2717): 96
tr_parse_uri(2806): 64
tr_parse_string(2639): 336
tr_parse_string(2628): 424
tr_parse_paramlist(2978): 288
tr_parse_paramlist(2906): 192
tr_parse_string(2569): 120
pv_parse_xavp_name(541): 2664
pv_parse_xavp_name(515): 3720
tr_parse_string(2484): 864
tr_parse_string(2473): 904
add_var(58): 4384
tr_parse_string(2519): 2144
tr_parse_string(2509): 5184
add_var(65): 1544
Total: 22928
}
Module: textops
{
tr_txt_parse_re(212): 32
hname_fixup(3528): 1960
fixup_method(3598): 1648
Total: 3640
}
Module: tmx
{
pv_parse_t_var_name(390): 176
Total: 176
}
Module: rtpengine
{
build_rtpp_socks(1801): 24
child_init(1987): 240
Total: 264
}
Module: htable
{
fixup_ht_key(352): 352
pv_parse_ht_name(158): 2240
Total: 2592
}
Module: textopsx
{
fixup_hname_param(593): 320
Total: 320
}
Module: avpops
{
avpops_parse_pvar(57): 64
Total: 64
}
Module: ndb_redis
{
redisc_add_server(348): 216648
Total: 216648
}
Module: sl
{
sl_register_callback(514): 64
Total: 64
}
Module: siptrace
{
mod_init(341): 384
Total: 384
}
Module: tm
{
Total: 0
}
Module: kex
{
Total: 0
}
Module: topos
{
Total: 0
}
Module: mtree
{
Total: 0
}
Module: cfgutils
{
Total: 0
}
Module: dispatcher
{
Total: 0
}
Module: outbound
{
Total: 0
}
kamcmd mod.stats all shm
Module: core
{
dns_cache_mk_rd_entry(1110): 1288
counters_prefork_init(212): 65280
cfg_clone_str(130): 136
cfg_shmize(217): 824
main_loop(1341): 8
init_pt(107): 8
init_pt(106): 8
init_pt(105): 7552
build_req_buf_from_sip_req(2218): 791408
xavp_new_value(116): 186472
sip_msg_shm_clone(496): 4101904
msg_lump_cloner(986): 755320
create_avp(175): 145064
register_timer(995): 216
cfg_register_ctx(47): 64
init_tcp(4962): 8192
init_tcp(4956): 32768
init_tcp(4947): 8
init_tcp(4940): 8
init_tcp(4933): 8
init_tcp(4927): 8
init_tcp(4915): 8
init_avps(90): 8
init_avps(89): 8
init_dst_blacklist(438): 16384
init_dst_blacklist(430): 8
timer_alloc(498): 96
init_dns_cache(361): 8
init_dns_cache(352): 16384
init_dns_cache(344): 16
init_dns_cache(336): 8
init_timer(267): 8
init_timer(266): 16384
init_timer(265): 8
init_timer(264): 8
init_timer(253): 8
init_timer(221): 8
init_timer(210): 278544
init_timer(209): 8
init_timer(197): 8
cfg_child_cb_new(829): 64
sr_cfg_init(361): 8
sr_cfg_init(354): 8
sr_cfg_init(347): 8
sr_cfg_init(335): 8
sr_cfg_init(323): 8
ksr_shutdown_phase_init(120): 8
qm_shm_lock_init(1202): 8
Total: 6424576
}
Module: permissions
{
domain_name_table_insert(978): 96
addr_hash_table_insert(458): 2776
subnet_table_insert(670): 336
init_addresses(526): 8
new_domain_name_table(865): 2048
init_addresses(512): 8
new_subnet_table(635): 57456
init_addresses(498): 8
new_addr_hash_table(418): 2048
init_trusted(220): 8
new_hash_table(98): 2048
Total: 66840
}
Module: auth
{
Total: 0
}
Module: dialog
{
dlg_update_cseq(636): 456
dlg_get_iuid_shm_clone(308): 4688
set_dlg_profile(580): 139680
dlg_set_leg_info(553): 64608
dlg_set_leg_info(544): 16296
build_new_dlg(469): 356616
dlg_set_leg_info(536): 29176
new_dlg_var(127): 624976
new_dlg_var(108): 1675696
init_dlg_table(297): 131088
init_dlg_table(289): 8
init_dlg_table(282): 8
init_dlg_table(277): 8
init_dlg_timer(57): 8
init_dlg_timer(48): 32
new_dlg_profile(188): 2272
new_dlg_var(117): 642216
dlg_set_leg_info(562): 18208
init_dlg_callback(55): 32
register_dlgcb(124): 189728
Total: 3895800
}
Module: acc
{
Total: 0
}
Module: xlog
{
Total: 0
}
Module: dmq
{
alloc_job_queue(255): 64
shm_str_dup(802): 280
build_dmq_node(164): 3616
add_peer(68): 168
mod_init(236): 8
mod_init(229): 48
init_dmq_node_list(70): 24
init_peer_list(33): 24
Total: 4232
}
Module: rr
{
Total: 0
}
Module: jsonrpcs
{
Total: 0
}
Module: sqlops
{
Total: 0
}
Module: db_mysql
{
Total: 0
}
Module: phonenum
{
Total: 0
}
Module: pv
{
add_shvar(177): 8
add_shvar(170): 64
lock_set_alloc(66): 80
Total: 152
}
Module: textops
{
Total: 0
}
Module: tmx
{
tmx_check_pretran(271): 6280
tmx_check_pretran(250): 3320
tmx_init_pretran_table(90): 512
Total: 10112
}
Module: rtpengine
{
rtpengine_hash_table_init(111): 14336
rtpengine_hash_table_init(96): 2048
rtpengine_hash_table_init(85): 1024
rtpengine_hash_table_init(76): 2048
rtpengine_hash_table_init(67): 2048
rtpengine_hash_table_init(58): 32
add_rtpengine_socks(978): 96
add_rtpengine_socks(959): 256
get_rtpp_set(873): 8
get_rtpp_set(864): 56
mod_init(1606): 8
mod_init(1599): 24
mod_init(1586): 8
mod_init(1579): 8
mod_init(1565): 16
rtpp_function_call(2657): 1367600
shm_str_dup(802): 2205800
Total: 3595416
}
Module: htable
{
ht_init_tables(394): 2574336
ht_cell_new(186): 68875496
ht_add_table(315): 256
ht_add_table(289): 8424
Total: 71458512
}
Module: textopsx
{
Total: 0
}
Module: avpops
{
Total: 0
}
Module: ndb_redis
{
Total: 0
}
Module: sl
{
init_sl_stats_child(125): 9792
sl_startup(90): 8
init_sl_stats(110): 8
Total: 9808
}
Module: siptrace
{
mod_init(320): 8
Total: 8
}
Module: tm
{
cancel_reason_pack(1224): 104
update_totag_set(321): 656
build_local_reparse(328): 28384
update_totag_set(320): 864
_reply_light(500): 21480
init_tm_stats_child(56): 13056
build_cell(334): 8250680
relay_reply(2064): 439848
build_uac_req(1597): 118672
insert_tmcb(135): 49272
init_tmcb_lists(72): 16
init_tmcb_lists(70): 16
init_hash_table(482): 2097152
tm_rpc_response_list_init(77): 16
Total: 11020216
}
Module: kex
{
pkg_proc_stats_init(79): 2448
Total: 2448
}
Module: topos
{
lock_set_alloc(66): 2064
Total: 2064
}
Module: mtree
{
mt_add_to_tree(229): 2656
mod_init(247): 8
mt_add_to_tree(255): 9559984
mt_add_to_tree(291): 811224
mt_add_to_tree(301): 269848
mt_init_tree(149): 152
mt_init_tree(136): 256
mt_init_tree(125): 176
mt_init_tree(114): 2376
mt_table_spec(840): 8
Total: 10646688
}
Module: cfgutils
{
mod_init(898): 32
mod_init(892): 8
mod_init(884): 8
Total: 48
}
Module: dispatcher
{
mod_init(484): 80
reindex_dests(723): 7680
ds_avl_insert(3621): 11984
ds_set_attrs(301): 960
pack_dest(411): 1456
ds_init_data(262): 16
ds_init_data(254): 16
ds_parse_reply_codes(1112): 808
mod_init(339): 8
mod_init(337): 8
ds_ping_active_init(138): 8
Total: 23024
}
Module: outbound
{
mod_init(99): 24
Total: 24
}
