Re: [Kamailio-Users] Can openser.cfg lead to pkg memory problem?

Hello, we have openser 1.3.3 running in production (current rev.: 4943). For 3 times in 50 days we had to restart openser to correct pkg memory problem. After some time logging messages like this: /openser.log:Aug 19 10:39:18 ipx022 /usr/local/sbin/openser[16991]: ERROR:core:new_credentials: no pkg memory left, openser will eventually run out of pkg memory and refuse all subsequent requests. We are trying to recreate this in our lab so that we can follow memory troubleshooting instructions at http://kamailio.net/dokuwiki/doku.php/troubleshooting:memory, but so far we were unable to do it even when generating millions of calls and registration transactions (we are using SIPp to generate normal call flows and even abnormal call flows detected when reading openser.log, like 'invalid cseq for aor', malformed SIP messages etc). And this is much more than in our production environment, with just 600 subscribers and about 2000 calls a day. The frequency the problem happens is increasing with the number of subscribers, so we are performing periodic restart of openser (actually, what we do is to switch over to the standby server). We already recompiled openser with pkg memory pool size set to 4MB so that this will not have to be done frequently. Since we cannot recreate this in our lab, we suspect there is a situation happening in production that might not be having been properly handled by openser.cfg. So my question is: would it be possible to an overlooked detail in openser.cfg to cause pkg memory problem? In case someone could take a look at it, here's our cfg file: ####### Global Parameters ######### debug=0 log_stderror=no log_facility=LOG_LOCAL0 fork=yes children=4 /* uncomment the following lines to enable debugging */ #debug=6 #fork=no #log_stderror=yes /* uncomment the next line to disable TCP (default on) */ disable_tcp=yes /* uncomment the next line to enable the auto temporary blacklisting of not available destinations (default disabled) */ #disable_dns_blacklist=no /* uncomment the next line to enable IPv6 lookup after IPv4 dns lookup failures (default disabled) */ #dns_try_ipv6=yes /* uncomment the next line to disable the auto discovery of local aliases based on revers DNS on IPs (default on) */ #auto_aliases=no /* uncomment the following lines to enable TLS support (default off) */ #disable_tls = no #listen = tls:your_IP:5061 #tls_verify_server = 1 #tls_verify_client = 1 #tls_require_client_certificate = 0 #tls_method = TLSv1 #tls_certificate = "/usr/local/etc/openser/tls/user/user-cert.pem" #tls_private_key = "/usr/local/etc/openser/tls/user/user-privkey.pem" #tls_ca_list = "/usr/local/etc/openser/tls/user/user-calist.pem" port=5060 /* uncomment and configure the following line if you want openser to bind on a specific interface/port/proto (default bind on all available) */ #listen=udp:202.173.5.181:5060 ####### Modules Section ######## #set module path mpath="/usr/local/lib/openser/modules/" /* uncomment next line for MySQL DB support */ loadmodule "mysql.so" loadmodule "sl.so" loadmodule "tm.so" loadmodule "rr.so" loadmodule "maxfwd.so" loadmodule "usrloc.so" loadmodule "registrar.so" loadmodule "textops.so" loadmodule "mi_fifo.so" loadmodule "uri_db.so" loadmodule "uri.so" loadmodule "xlog.so" loadmodule "acc.so" loadmodule "carrierroute.so" loadmodule "nathelper.so" loadmodule "dialog.so" loadmodule "snmpstats.so" /* uncomment next lines for MySQL based authentication support NOTE: a DB (like mysql) module must be also loaded */ loadmodule "auth.so" loadmodule "auth_db.so" loadmodule "lcr.so" /* uncomment next line for aliases support NOTE: a DB (like mysql) module must be also loaded */ loadmodule "alias_db.so" /* uncomment next line for multi-domain support NOTE: a DB (like mysql) module must be also loaded NOTE: be sure and enable multi-domain support in all used modules (see "multi-module params" section ) */ loadmodule "domain.so" /* uncomment the next two lines for presence server support NOTE: a DB (like mysql) module must be also loaded */ #loadmodule "presence.so" #loadmodule "presence_xml.so" loadmodule "uac.so" loadmodule "avpops.so" # ----------------- setting module-specific parameters --------------- # ----- mi_fifo params ----- modparam("mi_fifo", "fifo_name", "/tmp/openser_fifo") # ----- rr params ----- # add value to ;lr param to cope with most of the UAs modparam("rr", "enable_full_lr", 1) # do not append from tag to the RR (no need for this script) modparam("rr", "append_fromtag", 1) # ----- rr params ----- modparam("registrar", "method_filtering", 1) /* uncomment the next line to disable parallel forking via location */ # modparam("registrar", "append_branches", 0) /* uncomment the next line not to allow more than 10 contacts per AOR */ modparam("registrar", "max_contacts", 10) modparam("registrar", "min_expires", 30) modparam("registrar", "max_expires", 40) modparam("registrar", "default_expires", 35) # ----- uri_db params ----- /* by default we disable the DB support in the module as we do not need it in this configuration */ modparam("uri_db", "use_uri_table", 0) modparam("uri_db", "db_url", "mysql://openser:openserrw@localhost/openser") modparam("uri_db", "use_domain", 1) # ----- acc params ----- /* what sepcial events should be accounted ? */ modparam("acc", "early_media", 1) modparam("acc", "report_ack", 1) modparam("acc", "report_cancels", 1) /* by default ww do not adjust the direct of the sequential requests. if you enable this parameter, be sure the enable "append_fromtag" in "rr" module */ modparam("acc", "detect_direction", 0) /* account triggers (flags) */ modparam("acc", "failed_transaction_flag", 3) modparam("acc", "log_flag", 1) modparam("acc", "log_missed_flag", 2) /* uncomment the following lines to enable DB accounting also */ modparam("acc", "db_flag", 1) modparam("acc", "db_missed_flag", 2) # ----- usrloc params ----- #modparam("usrloc", "db_mode", 0) /* uncomment the following lines if you want to enable DB persistency for location entries */ modparam("usrloc", "db_mode", 2) modparam("usrloc", "db_url", "mysql://openser:openserrw@localhost/openser") modparam("usrloc", "use_domain", 1) # ----- auth_db params ----- /* uncomment the following lines if you want to enable the DB based authentication */ modparam("auth_db", "calculate_ha1", yes) modparam("auth_db", "password_column", "password") modparam("auth_db", "db_url", "mysql://openser:openserrw@localhost/openser") modparam("auth_db", "load_credentials", "$avp(s:rpid)=rpid; $avp(s:blocked)=subscriber_status") # ----- alias_db params ----- /* uncomment the following lines if you want to enable the DB based aliases */ modparam("alias_db", "db_url", "mysql://openser:openserrw@localhost/openser") modparam("alias_db", "use_domain", 0) # ----- domain params ----- /* uncomment the following lines to enable multi-domain detection support */ modparam("domain", "db_url", "mysql://openser:openserrw@localhost/openser") modparam("domain", "db_mode", 1) # Use caching # ----- multi-module params ----- /* uncomment the following line if you want to enable multi-domain support in the modules (dafault off) */ #modparam("alias_db|auth_db|usrloc|uri_db", "use_domain", 1) # ----- presence params ----- /* uncomment the following lines if you want to enable presence */ #modparam("presence|presence_xml", "db_url", # "mysql://openser:openserrw@localhost/openser") #modparam("presence_xml", "force_active", 1) #modparam("presence", "server_address", "sip:192.168.1.2:5060") # ----- carrieroute params ----- modparam("carrierroute", "db_url", "mysql://openser:openserrw@localhost/openser") modparam("carrierroute", "config_source", "db") modparam("carrierroute", "use_domain", 1) # ----- NatHelper ----- #para versao a partir da versao 1.2 eh necessario esse paramtro para nao dar erro qdo usa a funcao "fix_nated_register();" modparam("nathelper|registrar", "received_avp", "$avp(i:42)") modparam("nathelper", "rtpproxy_sock", "udp:127.0.0.1:22222") # ----- LCR ----- modparam("lcr", "db_url", "mysql://openser:openserrw@localhost/openser") modparam("lcr|tm", "fr_inv_timer_avp", "$avp(i:704)") modparam("lcr", "gw_uri_avp", "$avp(i:709)") modparam("^auth$|lcr", "rpid_avp", "$avp(i:302)") modparam("lcr", "contact_avp", "$avp(i:711)") modparam("lcr", "ruri_user_avp", "$avp(i:500)") modparam("lcr", "dm_flag", 25) # ----- Dialog ---- modparam("dialog", "dlg_flag", 4) # ----- SnmpStat ----- modparam("snmpstats", "sipEntityType", "registrarServer") modparam("snmpstats", "sipEntityType", "proxyServer") modparam("snmpstats", "MsgQueueMinorThreshold", 2000) modparam("snmpstats", "MsgQueueMajorThreshold", 5000) modparam("snmpstats", "dlg_minor_threshold", 500) modparam("snmpstats", "dlg_major_threshold", 750) modparam("snmpstats", "snmpgetPath","/usr/bin/") modparam("snmpstats", "snmpCommunity","public") ####### Routing Logic ######## # main request routing logic route{ if (!mf_process_maxfwd_header("10")) { sl_send_reply("483","Too Many Hops"); exit; } ##nat route(2); if (has_totag()) { # sequential request withing a dialog should # take the path determined by record-routing if (loose_route()) { if (is_method("BYE")) { setflag(1); # do accouting ... setflag(3); # ... even if the transaction fails } route(1); } else { /* uncomment the following lines if you want to enable presence */ ##if (is_method("SUBSCRIBE") && $rd == "your.server.ip.address") { ## # in-dialog subscribe requests ## route(2); ## exit; ##} if ( is_method("ACK") ) { if ( t_check_trans() ) { # non loose-route, but stateful ACK; must be an ACK after a 487 or e.g. 404 from upstream server #t_relay(); #exit; route(1); } else { # ACK without matching transaction ... ignore and discard.\n"); exit; } } sl_send_reply("404","Not here"); } exit; } #initial requests setflag(4); #for dialog statistics # CANCEL processing if (is_method("CANCEL")) { if (t_check_trans()) route(1); # t_relay(); # exit; } #t_check_trans(); if (is_method("PUBLISH|SUBSCRIBE|REFER|OPTIONS|MESSAGE")) { sl_send_reply("405", "Method not allowed"); exit; } # authenticate if from local subscriber (uncomment to enable auth) if (!(method=="REGISTER") && (!from_gw())) { if (!proxy_authorize("", "subscriber")) { proxy_challenge("", "0"); exit; } if (!check_from()) { sl_send_reply("403","Forbidden auth ID"); exit; }else if (avp_check("$avp(s:blocked)", "eq/0")) { sl_send_reply("603","Subscriber disabled"); exit; }else if (avp_check("$avp(s:blocked)", "eq/1")) { sl_send_reply("603","Subscriber with outgoing blocked"); exit; } consume_credentials(); # caller authenticated } # record routing if (!is_method("REGISTER|MESSAGE")) record_route(); # account only INVITEs if (is_method("INVITE")) { setflag(1); # do accouting } if (is_method("REGISTER")) { # authenticate the REGISTER requests (uncomment to enable auth) if (!proxy_authorize("", "subscriber")) { proxy_challenge("", "0"); exit; } if (!check_to()) { sl_send_reply("403","Forbidden auth ID"); exit; }else if (avp_check("$avp(s:blocked)", "eq/0")) { sl_send_reply("403","Subscriber disabled"); exit; } if (!save("location")) sl_reply_error(); exit; } if ($rU==NULL) { # request with no Username in RURI sl_send_reply("484","Address Incomplete"); exit; } # apply DB based aliases (uncomment to enable) ##alias_db_lookup("dbaliases"); #if the call came from a known gateway it is not authenticated and we cannot use the function check_from() if (from_gw()) { route(4); }else if (!check_from()) {#if the check_from() returns false the call is not from a subscriber route(4); } else {#it is a subscriber, route using flip domain xlog("L_INFO", "routing using carrierroute $rm to $ru\n"); if (!cr_user_rewrite_uri("$fu", "flip")) { t_newtran(); t_reply("404", "No Route"); exit; } #replaces from by it's default DID uac_replace_from("sip:$avp(s:rpid)@$fd"); } # when routing via usrloc, log the missed calls also setflag(2); route(1); } route[1] { xlog("L_INFO", "ROUTE_1 $rm to $ru\n"); if (subst_uri('/(sip:.*);nat=yes/\1/')) { setflag(6); }; if (isflagset(5)||isflagset(6)) { route(3); } if (!t_relay()) { sl_reply_error(); }; exit; } route[2] { xlog("L_INFO", "ROUTE_2 $rm to $ru\n"); if (method=="REGISTER") { fix_nated_register(); } else if (!from_gw()){ fix_nated_contact(); }; setflag(5); } route[3] { xlog("L_INFO", "ROUTE_3 $rm to $ru\n"); if (is_method("BYE|CANCEL")) { unforce_rtp_proxy(); } else if (is_method("INVITE")) { xlog("L_INFO", "FORCE RTP w/ parameter.\n"); force_rtp_proxy("r"); t_on_failure("1"); }; if (isflagset(5)) search_append('Contact:.*sip:[^>[:cntrl:]]*', ';nat=yes'); t_on_reply("1"); } route[4] { xlog("L_INFO", "uri does exist $rm to $ru \n"); if (alias_db_lookup("dbaliases")){ if (!lookup("location")) { switch ($retcode) { case -1: t_newtran(); t_reply("404", "Subscriber not online"); exit; case -2: sl_send_reply("405", "Method Not Allowed"); exit; } } }else{#check if did is blocked $rU = "(BLK)" + $rU; if (alias_db_lookup("dbaliases")){ sl_send_reply("403", "DID blocked"); exit; }else{# if it is not a valid DID nor a blocked DID tries to route it using peering domain if (!cr_rewrite_uri("peering", "call_id")) { t_newtran(); t_reply("404", "Peering Not Found"); exit; } } } } failure_route[1] { xlog("L_INFO", "FAILURE $rm to $ru\n"); if (isflagset(6)||isflagset(5)) { unforce_rtp_proxy(); } } onreply_route[1] { xlog("L_INFO", "ONREPLY_1 - Status $rs from $si $rm .\n"); if (is_method("INVITE")) { if ((isflagset(5)||isflagset(6)) && status=~"(183)|(2[0-9][0-9])") { force_rtp_proxy(); } search_append('Contact:.*sip:[^>[:cntrl:]]*', ';nat=yes'); if (!from_gw()){ #if (isflagset(6)) { xlog("L_INFO", "ONREPLY_1 - ! from gw.\n"); fix_nated_contact(); } exit; } } Regards, takeshi _______________________________________________ Users mailing list Users(a)lists.kamailio.org http://lists.kamailio.org/cgi-bin/mailman/listinfo/users