[Kamailio-Users] Can openser.cfg lead to pkg memory problem? - sr-users

23 Sep 2008


      Hello,
we have openser 1.3.3 running in production (current rev.: 4943).
For 3 times in 50 days we had to restart openser to correct pkg memory
problem.
After some time logging messages like this:
/openser.log:Aug 19 10:39:18 ipx022 /usr/local/sbin/openser[16991]:
ERROR:core:new_credentials: no pkg memory left,
openser will eventually run out of pkg memory and refuse all subsequent
requests.
We are trying to recreate this in our lab so that we can follow memory
troubleshooting instructions at
http://kamailio.net/dokuwiki/doku.php/troubleshooting:memory, but so far we
were unable to do it even when generating millions of calls and registration
transactions (we are using SIPp to generate normal call flows and even
abnormal call flows detected when reading openser.log, like 'invalid cseq
for aor', malformed SIP messages etc).
And this is much more than in our production environment, with just 600
subscribers and about 2000 calls a day.
The frequency the problem happens is increasing with the number of
subscribers, so we are performing periodic restart of openser (actually,
what we do is to switch over to the standby server). We already recompiled
openser with pkg memory  pool size set to 4MB so that this will not have to
be done frequently.
Since we cannot recreate this in our lab, we suspect there is a situation
happening in production that might not be having been properly handled by
openser.cfg. So my question is: would it be possible to an overlooked detail
in openser.cfg to cause pkg memory problem?
In case someone could take a look at it, here's our cfg file:
####### Global Parameters #########
debug=0
log_stderror=no
log_facility=LOG_LOCAL0
fork=yes
children=4
/* uncomment the following lines to enable debugging */
#debug=6
#fork=no
#log_stderror=yes
/* uncomment the next line to disable TCP (default on) */
disable_tcp=yes
/* uncomment the next line to enable the auto temporary blacklisting of
   not available destinations (default disabled) */
#disable_dns_blacklist=no
/* uncomment the next line to enable IPv6 lookup after IPv4 dns
   lookup failures (default disabled) */
#dns_try_ipv6=yes
/* uncomment the next line to disable the auto discovery of local aliases
   based on revers DNS on IPs (default on) */
#auto_aliases=no
/* uncomment the following lines to enable TLS support  (default off) */
#disable_tls = no
#listen = tls:your_IP:5061
#tls_verify_server = 1
#tls_verify_client = 1
#tls_require_client_certificate = 0
#tls_method = TLSv1
#tls_certificate = "/usr/local/etc/openser/tls/user/user-cert.pem"
#tls_private_key = "/usr/local/etc/openser/tls/user/user-privkey.pem"
#tls_ca_list = "/usr/local/etc/openser/tls/user/user-calist.pem"
port=5060
/* uncomment and configure the following line if you want openser to
   bind on a specific interface/port/proto (default bind on all available)
*/
#listen=udp:202.173.5.181:5060
####### Modules Section ########
#set module path
mpath="/usr/local/lib/openser/modules/"
/* uncomment next line for MySQL DB support */
loadmodule "mysql.so"
loadmodule "sl.so"
loadmodule "tm.so"
loadmodule "rr.so"
loadmodule "maxfwd.so"
loadmodule "usrloc.so"
loadmodule "registrar.so"
loadmodule "textops.so"
loadmodule "mi_fifo.so"
loadmodule "uri_db.so"
loadmodule "uri.so"
loadmodule "xlog.so"
loadmodule "acc.so"
loadmodule "carrierroute.so"
loadmodule "nathelper.so"
loadmodule "dialog.so"
loadmodule "snmpstats.so"
/* uncomment next lines for MySQL based authentication support
   NOTE: a DB (like mysql) module must be also loaded */
loadmodule "auth.so"
loadmodule "auth_db.so"
loadmodule "lcr.so"
/* uncomment next line for aliases support
   NOTE: a DB (like mysql) module must be also loaded */
loadmodule "alias_db.so"
/* uncomment next line for multi-domain support
   NOTE: a DB (like mysql) module must be also loaded
   NOTE: be sure and enable multi-domain support in all used modules
         (see "multi-module params" section ) */
loadmodule "domain.so"
/* uncomment the next two lines for presence server support
   NOTE: a DB (like mysql) module must be also loaded */
#loadmodule "presence.so"
#loadmodule "presence_xml.so"
loadmodule "uac.so"
loadmodule "avpops.so"
# ----------------- setting module-specific parameters ---------------
# ----- mi_fifo params -----
modparam("mi_fifo", "fifo_name", "/tmp/openser_fifo")
# ----- rr params -----
# add value to ;lr param to cope with most of the UAs
modparam("rr", "enable_full_lr", 1)
# do not append from tag to the RR (no need for this script)
modparam("rr", "append_fromtag", 1)
# ----- rr params -----
modparam("registrar", "method_filtering", 1)
/* uncomment the next line to disable parallel forking via location */
# modparam("registrar", "append_branches", 0)
/* uncomment the next line not to allow more than 10 contacts per AOR */
modparam("registrar", "max_contacts", 10)
modparam("registrar", "min_expires", 30)
modparam("registrar", "max_expires", 40)
modparam("registrar", "default_expires", 35)
# ----- uri_db params -----
/* by default we disable the DB support in the module as we do not need it
   in this configuration */
modparam("uri_db", "use_uri_table", 0)
modparam("uri_db", "db_url", "mysql://openser:openserrw@localhost/openser")
modparam("uri_db", "use_domain", 1)
# ----- acc params -----
/* what sepcial events should be accounted ? */
modparam("acc", "early_media", 1)
modparam("acc", "report_ack", 1)
modparam("acc", "report_cancels", 1)
/* by default ww do not adjust the direct of the sequential requests.
   if you enable this parameter, be sure the enable "append_fromtag"
   in "rr" module */
modparam("acc", "detect_direction", 0)
/* account triggers (flags) */
modparam("acc", "failed_transaction_flag", 3)
modparam("acc", "log_flag", 1)
modparam("acc", "log_missed_flag", 2)
/* uncomment the following lines to enable DB accounting also */
modparam("acc", "db_flag", 1)
modparam("acc", "db_missed_flag", 2)
# ----- usrloc params -----
#modparam("usrloc", "db_mode",   0)
/* uncomment the following lines if you want to enable DB persistency
   for location entries */
modparam("usrloc", "db_mode",   2)
modparam("usrloc", "db_url", "mysql://openser:openserrw@localhost/openser")
modparam("usrloc", "use_domain", 1)
# ----- auth_db params -----
/* uncomment the following lines if you want to enable the DB based
   authentication */
modparam("auth_db", "calculate_ha1", yes)
modparam("auth_db", "password_column", "password")
modparam("auth_db", "db_url", "mysql://openser:openserrw@localhost/openser")
modparam("auth_db", "load_credentials",
"$avp(s:rpid)=rpid;$avp(s:blocked)=subscriber_status")
# ----- alias_db params -----
/* uncomment the following lines if you want to enable the DB based
   aliases */
modparam("alias_db", "db_url",
    "mysql://openser:openserrw@localhost/openser")
modparam("alias_db", "use_domain", 0)
# ----- domain params -----
/* uncomment the following lines to enable multi-domain detection
   support */
modparam("domain", "db_url", "mysql://openser:openserrw@localhost/openser")
modparam("domain", "db_mode", 1)   # Use caching
# ----- multi-module params -----
/* uncomment the following line if you want to enable multi-domain support
   in the modules (dafault off) */
#modparam("alias_db|auth_db|usrloc|uri_db", "use_domain", 1)
# ----- presence params -----
/* uncomment the following lines if you want to enable presence */
#modparam("presence|presence_xml", "db_url",
#    "mysql://openser:openserrw@localhost/openser")
#modparam("presence_xml", "force_active", 1)
#modparam("presence", "server_address", "sip:192.168.1.2:5060")
# ----- carrieroute params -----
modparam("carrierroute", "db_url", "mysql://openser:openserrw@localhost
/openser")
modparam("carrierroute", "config_source", "db")
modparam("carrierroute", "use_domain", 1)
# ----- NatHelper -----
#para versao a partir da versao 1.2 eh necessario esse paramtro para nao dar
erro qdo usa a funcao "fix_nated_register();"
modparam("nathelper|registrar", "received_avp", "$avp(i:42)")
modparam("nathelper", "rtpproxy_sock", "udp:127.0.0.1:22222")
# ----- LCR -----
modparam("lcr", "db_url", "mysql://openser:openserrw@localhost/openser")
modparam("lcr|tm", "fr_inv_timer_avp", "$avp(i:704)")
modparam("lcr", "gw_uri_avp", "$avp(i:709)")
modparam("^auth$|lcr", "rpid_avp", "$avp(i:302)")
modparam("lcr", "contact_avp", "$avp(i:711)")
modparam("lcr", "ruri_user_avp", "$avp(i:500)")
modparam("lcr", "dm_flag", 25)
# ----- Dialog ----
modparam("dialog", "dlg_flag", 4)
# ----- SnmpStat -----
modparam("snmpstats", "sipEntityType", "registrarServer")
modparam("snmpstats", "sipEntityType", "proxyServer")
modparam("snmpstats", "MsgQueueMinorThreshold", 2000)
modparam("snmpstats", "MsgQueueMajorThreshold", 5000)
modparam("snmpstats", "dlg_minor_threshold", 500)
modparam("snmpstats", "dlg_major_threshold", 750)
modparam("snmpstats", "snmpgetPath","/usr/bin/")
modparam("snmpstats", "snmpCommunity","public")
####### Routing Logic ########
# main request routing logic
route{
if (!mf_process_maxfwd_header("10")) {
        sl_send_reply("483","Too Many Hops");
        exit;
    }
##nat
    route(2);
if (has_totag()) {
        # sequential request withing a dialog should
        # take the path determined by record-routing
        if (loose_route()) {
            if (is_method("BYE")) {
                setflag(1); # do accouting ...
                setflag(3); # ... even if the transaction fails
            }
            route(1);
        } else {
            /* uncomment the following lines if you want to enable presence
*/
            ##if (is_method("SUBSCRIBE") && $rd == "your.server.ip.address")
{
            ##    # in-dialog subscribe requests
            ##    route(2);
            ##    exit;
            ##}
            if ( is_method("ACK") ) {
                if ( t_check_trans() ) {
                    # non loose-route, but stateful ACK; must be an ACK
after a 487 or e.g. 404 from upstream server
                    #t_relay();
                    #exit;
                    route(1);
                } else {
                    # ACK without matching transaction ... ignore and
discard.\n");
                    exit;
                }
            }
            sl_send_reply("404","Not here");
        }
        exit;
    }
#initial requests
    setflag(4); #for  dialog statistics
# CANCEL processing
    if (is_method("CANCEL"))
    {
        if (t_check_trans()) route(1);
    #        t_relay();
    #    exit;
    }
#t_check_trans();
if (is_method("PUBLISH|SUBSCRIBE|REFER|OPTIONS|MESSAGE"))
        {
                sl_send_reply("405", "Method not allowed");
                exit;
        }
# authenticate if from local subscriber (uncomment to enable auth)
    if (!(method=="REGISTER") && (!from_gw()))
    {
        if (!proxy_authorize("", "subscriber")) {
            proxy_challenge("", "0");
            exit;
        }
        if (!check_from()) {
            sl_send_reply("403","Forbidden auth ID");
            exit;
        }else if (avp_check("$avp(s:blocked)", "eq/0")) {
                        sl_send_reply("603","Subscriber disabled");
            exit;
        }else if (avp_check("$avp(s:blocked)", "eq/1")) {
                        sl_send_reply("603","Subscriber with outgoing
blocked");
                        exit;
        }
consume_credentials();
        # caller authenticated
    }
# record routing
    if (!is_method("REGISTER|MESSAGE"))
        record_route();
# account only INVITEs
    if (is_method("INVITE")) {
        setflag(1); # do accouting
    }
if (is_method("REGISTER"))
    {
        # authenticate the REGISTER requests (uncomment to enable auth)
        if (!proxy_authorize("", "subscriber"))
        {
            proxy_challenge("", "0");
            exit;
        }
if (!check_to())
        {
            sl_send_reply("403","Forbidden auth ID");
            exit;
        }else if (avp_check("$avp(s:blocked)", "eq/0")) {
                        sl_send_reply("403","Subscriber disabled");
                        exit;
                }
if (!save("location"))
            sl_reply_error();
exit;
    }
if ($rU==NULL) {
        # request with no Username in RURI
        sl_send_reply("484","Address Incomplete");
        exit;
    }
# apply DB based aliases (uncomment to enable)
    ##alias_db_lookup("dbaliases");
    #if the call came from a known gateway it is not authenticated and we
cannot use the function check_from()
    if (from_gw()) {
        route(4);
    }else if (!check_from()) {#if the check_from() returns false the call is
not from a subscriber
            route(4);
    } else {#it is a subscriber, route using flip domain
        xlog("L_INFO", "routing using carrierroute $rm to $ru\n");
        if (!cr_user_rewrite_uri("$fu", "flip"))
        {
             t_newtran();
                         t_reply("404", "No Route");
                         exit;
        }
        #replaces from by it's default DID
            uac_replace_from("sip:$avp(s:rpid)@$fd");
    }
# when routing via usrloc, log the missed calls also
    setflag(2);
route(1);
}
route[1] {
        xlog("L_INFO", "ROUTE_1 $rm to $ru\n");
       if (subst_uri('/(sip:.*);nat=yes/\1/'))
        {
                setflag(6);
        };
if (isflagset(5)||isflagset(6)) {
                route(3);
        }
if (!t_relay()) {
               sl_reply_error();
           };
           exit;
}
route[2] {
        xlog("L_INFO", "ROUTE_2 $rm to $ru\n");
        if (method=="REGISTER") {
                fix_nated_register();
        } else if (!from_gw()){
                fix_nated_contact();
        };
        setflag(5);
}
route[3] {
        xlog("L_INFO", "ROUTE_3 $rm to $ru\n");
        if (is_method("BYE|CANCEL")) {
                unforce_rtp_proxy();
        } else if (is_method("INVITE")) {
                xlog("L_INFO", "FORCE RTP w/ parameter.\n");
                force_rtp_proxy("r");
                t_on_failure("1");
        };
        if (isflagset(5))
                search_append('Contact:.*sip:[^>[:cntrl:]]*', ';nat=yes');
        t_on_reply("1");
}
route[4] {
xlog("L_INFO", "uri does exist $rm to $ru \n");
    if (alias_db_lookup("dbaliases")){
if (!lookup("location")) {
                switch ($retcode) {
                        case -1:
                                    t_newtran();
                                    t_reply("404", "Subscriber not online");
                                    exit;
                            case -2:
                                    sl_send_reply("405", "Method Not
Allowed");
                                    exit;
                    }
            }
    }else{#check if did is blocked
        $rU = "(BLK)" + $rU;
        if (alias_db_lookup("dbaliases")){
            sl_send_reply("403", "DID blocked");
                        exit;
}else{# if it is not a valid DID nor a blocked DID tries to route it
using peering domain
             if (!cr_rewrite_uri("peering", "call_id"))
                         {
                             t_newtran();
                                t_reply("404", "Peering Not Found");
                                exit;
                         }
}
    }
}
failure_route[1] {
        xlog("L_INFO", "FAILURE $rm to $ru\n");
        if (isflagset(6)||isflagset(5)) {
                unforce_rtp_proxy();
        }
}
onreply_route[1] {
        xlog("L_INFO", "ONREPLY_1 - Status $rs from $si $rm .\n");
        if (is_method("INVITE")) {
                if ((isflagset(5)||isflagset(6)) &&
status=~"(183)|(2[0-9][0-9])") {
                        force_rtp_proxy();
                }
                search_append('Contact:.*sip:[^>[:cntrl:]]*', ';nat=yes');
if (!from_gw()){ #if (isflagset(6)) {
            xlog("L_INFO", "ONREPLY_1 - ! from gw.\n");
                        fix_nated_contact();
                }
        exit;
        }
}
Regards,
takeshi