On Tue, 2008-09-23 at 16:31 +0900, mayamatakeshi wrote:
> Hello,
> we have openser 1.3.3 running in production (current rev.: 4943).
> For 3 times in 50 days we had to restart openser to correct pkg memory
> problem.
> After some time logging messages like this:
> /openser.log:Aug 19 10:39:18 ipx022 /usr/local/sbin/openser[16991]:
> ERROR:core:new_credentials: no pkg memory left,
> openser will eventually run out of pkg memory and refuse all
> subsequent requests.
>
> We are trying to recreate this in our lab so that we can follow memory
> troubleshooting instructions at
>
http://kamailio.net/dokuwiki/doku.php/troubleshooting:memory, but so
> far we were unable to do it even when generating millions of calls and
> registration transactions (we are using SIPp to generate normal call
> flows and even abnormal call flows detected when reading openser.log,
> like 'invalid cseq for aor', malformed SIP messages etc).
> And this is much more than in our production environment, with just
> 600 subscribers and about 2000 calls a day.
>
> The frequency the problem happens is increasing with the number of
> subscribers, so we are performing periodic restart of openser
> (actually, what we do is to switch over to the standby server). We
> already recompiled openser with pkg memory pool size set to 4MB so
> that this will not have to be done frequently.
>
> Since we cannot recreate this in our lab, we suspect there is a
> situation happening in production that might not be having been
> properly handled by openser.cfg. So my question is: would it be
> possible to an overlooked detail in openser.cfg to cause pkg memory
> problem?
>
>
> In case someone could take a look at it, here's our cfg file:
>
> ####### Global Parameters #########
>
> debug=0
> log_stderror=no
> log_facility=LOG_LOCAL0
>
> fork=yes
> children=4
>
> /* uncomment the following lines to enable debugging */
> #debug=6
> #fork=no
> #log_stderror=yes
>
> /* uncomment the next line to disable TCP (default on) */
> disable_tcp=yes
>
> /* uncomment the next line to enable the auto temporary blacklisting
> of
> not available destinations (default disabled) */
> #disable_dns_blacklist=no
>
> /* uncomment the next line to enable IPv6 lookup after IPv4 dns
> lookup failures (default disabled) */
> #dns_try_ipv6=yes
>
> /* uncomment the next line to disable the auto discovery of local
> aliases
> based on revers DNS on IPs (default on) */
> #auto_aliases=no
>
> /* uncomment the following lines to enable TLS support (default off)
> */
> #disable_tls = no
> #listen = tls:your_IP:5061
> #tls_verify_server = 1
> #tls_verify_client = 1
> #tls_require_client_certificate = 0
> #tls_method = TLSv1
> #tls_certificate = "/usr/local/etc/openser/tls/user/user-cert.pem"
> #tls_private_key = "/usr/local/etc/openser/tls/user/user-privkey.pem"
> #tls_ca_list = "/usr/local/etc/openser/tls/user/user-calist.pem"
>
>
> port=5060
>
> /* uncomment and configure the following line if you want openser to
> bind on a specific interface/port/proto (default bind on all
> available) */
> #listen=udp:
202.173.5.181:5060
>
>
> ####### Modules Section ########
>
> #set module path
> mpath="/usr/local/lib/openser/modules/"
>
> /* uncomment next line for MySQL DB support */
> loadmodule "mysql.so"
> loadmodule "sl.so"
> loadmodule "tm.so"
> loadmodule "rr.so"
> loadmodule "maxfwd.so"
> loadmodule "usrloc.so"
> loadmodule "registrar.so"
> loadmodule "textops.so"
> loadmodule "mi_fifo.so"
> loadmodule "uri_db.so"
> loadmodule "uri.so"
> loadmodule "xlog.so"
> loadmodule "acc.so"
> loadmodule "carrierroute.so"
> loadmodule "nathelper.so"
> loadmodule "dialog.so"
> loadmodule "snmpstats.so"
> /* uncomment next lines for MySQL based authentication support
> NOTE: a DB (like mysql) module must be also loaded */
> loadmodule "auth.so"
> loadmodule "auth_db.so"
> loadmodule "lcr.so"
> /* uncomment next line for aliases support
> NOTE: a DB (like mysql) module must be also loaded */
> loadmodule "alias_db.so"
> /* uncomment next line for multi-domain support
> NOTE: a DB (like mysql) module must be also loaded
> NOTE: be sure and enable multi-domain support in all used modules
> (see "multi-module params" section ) */
> loadmodule "domain.so"
> /* uncomment the next two lines for presence server support
> NOTE: a DB (like mysql) module must be also loaded */
> #loadmodule "presence.so"
> #loadmodule "presence_xml.so"
> loadmodule "uac.so"
> loadmodule "avpops.so"
> # ----------------- setting module-specific parameters ---------------
>
>
> # ----- mi_fifo params -----
> modparam("mi_fifo", "fifo_name", "/tmp/openser_fifo")
>
>
> # ----- rr params -----
> # add value to ;lr param to cope with most of the UAs
> modparam("rr", "enable_full_lr", 1)
> # do not append from tag to the RR (no need for this script)
> modparam("rr", "append_fromtag", 1)
>
>
> # ----- rr params -----
> modparam("registrar", "method_filtering", 1)
> /* uncomment the next line to disable parallel forking via location */
> # modparam("registrar", "append_branches", 0)
> /* uncomment the next line not to allow more than 10 contacts per AOR
> */
> modparam("registrar", "max_contacts", 10)
> modparam("registrar", "min_expires", 30)
> modparam("registrar", "max_expires", 40)
> modparam("registrar", "default_expires", 35)
> # ----- uri_db params -----
> /* by default we disable the DB support in the module as we do not
> need it
> in this configuration */
> modparam("uri_db", "use_uri_table", 0)
> modparam("uri_db", "db_url",
> "mysql://openser:openserrw@localhost/openser")
> modparam("uri_db", "use_domain", 1)
>
> # ----- acc params -----
> /* what sepcial events should be accounted ? */
> modparam("acc", "early_media", 1)
> modparam("acc", "report_ack", 1)
> modparam("acc", "report_cancels", 1)
> /* by default ww do not adjust the direct of the sequential requests.
> if you enable this parameter, be sure the enable "append_fromtag"
> in "rr" module */
> modparam("acc", "detect_direction", 0)
> /* account triggers (flags) */
> modparam("acc", "failed_transaction_flag", 3)
> modparam("acc", "log_flag", 1)
> modparam("acc", "log_missed_flag", 2)
> /* uncomment the following lines to enable DB accounting also */
> modparam("acc", "db_flag", 1)
> modparam("acc", "db_missed_flag", 2)
>
>
> # ----- usrloc params -----
> #modparam("usrloc", "db_mode", 0)
> /* uncomment the following lines if you want to enable DB persistency
> for location entries */
> modparam("usrloc", "db_mode", 2)
> modparam("usrloc", "db_url",
> "mysql://openser:openserrw@localhost/openser")
> modparam("usrloc", "use_domain", 1)
>
> # ----- auth_db params -----
> /* uncomment the following lines if you want to enable the DB based
> authentication */
> modparam("auth_db", "calculate_ha1", yes)
> modparam("auth_db", "password_column", "password")
> modparam("auth_db", "db_url",
> "mysql://openser:openserrw@localhost/openser")
> modparam("auth_db", "load_credentials", "$avp(s:rpid)=rpid;
> $avp(s:blocked)=subscriber_status")
>
>
> # ----- alias_db params -----
> /* uncomment the following lines if you want to enable the DB based
> aliases */
> modparam("alias_db", "db_url",
> "mysql://openser:openserrw@localhost/openser")
> modparam("alias_db", "use_domain", 0)
>
> # ----- domain params -----
> /* uncomment the following lines to enable multi-domain detection
> support */
> modparam("domain", "db_url",
> "mysql://openser:openserrw@localhost/openser")
> modparam("domain", "db_mode", 1) # Use caching
>
>
> # ----- multi-module params -----
> /* uncomment the following line if you want to enable multi-domain
> support
> in the modules (dafault off) */
> #modparam("alias_db|auth_db|usrloc|uri_db", "use_domain", 1)
>
>
> # ----- presence params -----
> /* uncomment the following lines if you want to enable presence */
> #modparam("presence|presence_xml", "db_url",
> # "mysql://openser:openserrw@localhost/openser")
> #modparam("presence_xml", "force_active", 1)
> #modparam("presence", "server_address", "sip:
192.168.1.2:5060")
>
> # ----- carrieroute params -----
> modparam("carrierroute", "db_url",
> "mysql://openser:openserrw@localhost/openser")
> modparam("carrierroute", "config_source", "db")
> modparam("carrierroute", "use_domain", 1)
> # ----- NatHelper -----
> #para versao a partir da versao 1.2 eh necessario esse paramtro para
> nao dar erro qdo usa a funcao "fix_nated_register();"
> modparam("nathelper|registrar", "received_avp", "$avp(i:42)")
> modparam("nathelper", "rtpproxy_sock", "udp:
127.0.0.1:22222")
>
>
> # ----- LCR -----
> modparam("lcr", "db_url",
> "mysql://openser:openserrw@localhost/openser")
> modparam("lcr|tm", "fr_inv_timer_avp", "$avp(i:704)")
> modparam("lcr", "gw_uri_avp", "$avp(i:709)")
> modparam("^auth$|lcr", "rpid_avp", "$avp(i:302)")
> modparam("lcr", "contact_avp", "$avp(i:711)")
> modparam("lcr", "ruri_user_avp", "$avp(i:500)")
> modparam("lcr", "dm_flag", 25)
>
> # ----- Dialog ----
> modparam("dialog", "dlg_flag", 4)
>
> # ----- SnmpStat -----
> modparam("snmpstats", "sipEntityType", "registrarServer")
> modparam("snmpstats", "sipEntityType", "proxyServer")
> modparam("snmpstats", "MsgQueueMinorThreshold", 2000)
> modparam("snmpstats", "MsgQueueMajorThreshold", 5000)
> modparam("snmpstats", "dlg_minor_threshold", 500)
> modparam("snmpstats", "dlg_major_threshold", 750)
> modparam("snmpstats", "snmpgetPath","/usr/bin/")
> modparam("snmpstats", "snmpCommunity","public")
>
>
>
> ####### Routing Logic ########
>
>
> # main request routing logic
>
> route{
>
> if (!mf_process_maxfwd_header("10")) {
> sl_send_reply("483","Too Many Hops");
> exit;
> }
>
> ##nat
> route(2);
>
> if (has_totag()) {
> # sequential request withing a dialog should
> # take the path determined by record-routing
> if (loose_route()) {
> if (is_method("BYE")) {
> setflag(1); # do accouting ...
> setflag(3); # ... even if the transaction fails
> }
> route(1);
> } else {
> /* uncomment the following lines if you want to enable
> presence */
> ##if (is_method("SUBSCRIBE") && $rd ==
> "your.server.ip.address") {
> ## # in-dialog subscribe requests
> ## route(2);
> ## exit;
> ##}
> if ( is_method("ACK") ) {
> if ( t_check_trans() ) {
> # non loose-route, but stateful ACK; must be an
> ACK after a 487 or e.g. 404 from upstream server
> #t_relay();
> #exit;
> route(1);
> } else {
> # ACK without matching transaction ... ignore and
> discard.\n");
> exit;
> }
> }
> sl_send_reply("404","Not here");
> }
> exit;
> }
>
> #initial requests
> setflag(4); #for dialog statistics
>
>
> # CANCEL processing
> if (is_method("CANCEL"))
> {
> if (t_check_trans()) route(1);
> # t_relay();
> # exit;
> }
>
> #t_check_trans();
>
> if (is_method("PUBLISH|SUBSCRIBE|REFER|OPTIONS|MESSAGE"))
> {
> sl_send_reply("405", "Method not allowed");
> exit;
> }
>
> # authenticate if from local subscriber (uncomment to enable auth)
> if (!(method=="REGISTER") && (!from_gw()))
> {
> if (!proxy_authorize("", "subscriber")) {
> proxy_challenge("", "0");
> exit;
> }
> if (!check_from()) {
> sl_send_reply("403","Forbidden auth ID");
> exit;
> }else if (avp_check("$avp(s:blocked)", "eq/0")) {
> sl_send_reply("603","Subscriber disabled");
> exit;
> }else if (avp_check("$avp(s:blocked)", "eq/1")) {
> sl_send_reply("603","Subscriber with outgoing
> blocked");
> exit;
> }
>
> consume_credentials();
> # caller authenticated
> }
>
> # record routing
> if (!is_method("REGISTER|MESSAGE"))
> record_route();
>
> # account only INVITEs
> if (is_method("INVITE")) {
> setflag(1); # do accouting
> }
>
> if (is_method("REGISTER"))
> {
> # authenticate the REGISTER requests (uncomment to enable
> auth)
> if (!proxy_authorize("", "subscriber"))
> {
> proxy_challenge("", "0");
> exit;
> }
>
> if (!check_to())
> {
> sl_send_reply("403","Forbidden auth ID");
> exit;
> }else if (avp_check("$avp(s:blocked)", "eq/0")) {
> sl_send_reply("403","Subscriber disabled");
> exit;
> }
>
> if (!save("location"))
> sl_reply_error();
>
> exit;
> }
>
> if ($rU==NULL) {
> # request with no Username in RURI
> sl_send_reply("484","Address Incomplete");
> exit;
> }
>
> # apply DB based aliases (uncomment to enable)
> ##alias_db_lookup("dbaliases");
> #if the call came from a known gateway it is not authenticated and
> we cannot use the function check_from()
> if (from_gw()) {
> route(4);
> }else if (!check_from()) {#if the check_from() returns false the
> call is not from a subscriber
> route(4);
> } else {#it is a subscriber, route using flip domain
> xlog("L_INFO", "routing using carrierroute $rm to $ru\n");
> if (!cr_user_rewrite_uri("$fu", "flip"))
> {
> t_newtran();
> t_reply("404", "No Route");
> exit;
> }
> #replaces from by it's default DID
> uac_replace_from("sip:$avp(s:rpid)@$fd");
> }
>
> # when routing via usrloc, log the missed calls also
> setflag(2);
>
> route(1);
> }
>
> route[1] {
> xlog("L_INFO", "ROUTE_1 $rm to $ru\n");
> if (subst_uri('/(sip:.*);nat=yes/\1/'))
> {
> setflag(6);
> };
>
> if (isflagset(5)||isflagset(6)) {
> route(3);
> }
>
> if (!t_relay()) {
> sl_reply_error();
> };
> exit;
> }
>
> route[2] {
> xlog("L_INFO", "ROUTE_2 $rm to $ru\n");
> if (method=="REGISTER") {
> fix_nated_register();
> } else if (!from_gw()){
> fix_nated_contact();
> };
> setflag(5);
> }
> route[3] {
> xlog("L_INFO", "ROUTE_3 $rm to $ru\n");
> if (is_method("BYE|CANCEL")) {
> unforce_rtp_proxy();
> } else if (is_method("INVITE")) {
> xlog("L_INFO", "FORCE RTP w/ parameter.\n");
> force_rtp_proxy("r");
> t_on_failure("1");
> };
> if (isflagset(5))
> search_append('Contact:.*sip:[^>[:cntrl:]]*',
> ';nat=yes');
> t_on_reply("1");
> }
>
> route[4] {
>
> xlog("L_INFO", "uri does exist $rm to $ru \n");
> if (alias_db_lookup("dbaliases")){
>
> if (!lookup("location")) {
> switch ($retcode) {
> case -1:
> t_newtran();
> t_reply("404", "Subscriber not
> online");
> exit;
> case -2:
> sl_send_reply("405", "Method Not
> Allowed");
> exit;
> }
> }
> }else{#check if did is blocked
> $rU = "(BLK)" + $rU;
> if (alias_db_lookup("dbaliases")){
> sl_send_reply("403", "DID blocked");
> exit;
>
> }else{# if it is not a valid DID nor a blocked DID tries to
> route it using peering domain
> if (!cr_rewrite_uri("peering", "call_id"))
> {
> t_newtran();
> t_reply("404", "Peering Not Found");
> exit;
> }
>
> }
> }
> }
>
> failure_route[1] {
> xlog("L_INFO", "FAILURE $rm to $ru\n");
> if (isflagset(6)||isflagset(5)) {
> unforce_rtp_proxy();
> }
> }
>
> onreply_route[1] {
> xlog("L_INFO", "ONREPLY_1 - Status $rs from $si $rm .\n");
> if (is_method("INVITE")) {
> if ((isflagset(5)||isflagset(6)) &&
> status=~"(183)|(2[0-9][0-9])") {
> force_rtp_proxy();
> }
> search_append('Contact:.*sip:[^>[:cntrl:]]*',
> ';nat=yes');
>
> if (!from_gw()){ #if (isflagset(6)) {
> xlog("L_INFO", "ONREPLY_1 - ! from gw.\n");
> fix_nated_contact();
> }
> exit;
> }
> }
>
> Regards,
> takeshi
>
>
>