Description

Kamailio segfaults possibly when trying to write to the DB when it can't.

Troubleshooting

Reproduction

Shutdown Kamailio when the DB is no longer connected (I see this most often with a system reboot)

Debugging Data

Stack trace of thread 6620:
#0  0x00007f8df3c54c4a get_event (pua.so)
#1  0x00007f8df3c4e450 send_publish (pua.so)
#2  0x00007f8df3a3c189 ul_publish (pua_usrloc.so)
#3  0x00007f8df71feda0 run_ul_callbacks (usrloc.so)
#4  0x00007f8df7201136 wb_timer (usrloc.so)
#5  0x00007f8df720233a timer_urecord (usrloc.so)
#6  0x00007f8df71fa644 mem_timer_udomain (usrloc.so)
#7  0x00007f8df7225ae9 synchronize_all_udomains (usrloc.so)
#8  0x00007f8df7207d18 destroy (usrloc.so)
#9  0x000000000051d55e destroy_modules (kamailio)
#10 0x0000000000418269 cleanup (kamailio)
#11 0x00000000004199cc shutdown_children (kamailio)
#12 0x000000000041a3f7 handle_sigs (kamailio)
#13 0x0000000000424062 main_loop (kamailio)
#14 0x0000000000429ca3 main (kamailio)
#15 0x00007f8e05b8a88a __libc_start_main (libc.so.6)
#16 0x000000000041795a _start (kamailio)


#0  0x00007f8df3c54c4a in get_event (ev_flag=1) at event_list.c:128
        event = 0x7f8dfb007073
#1  0x00007f8df3c4e450 in send_publish (publ=0x7f8e053b8fd8) at send_publish.c:510
        met = {s = 0x7f8df3ca0c62 "PUBLISH", len = 7}
        str_hdr = 0x0
        presentity = 0x0
        body = 0x0
        tuple_id = 0x0
        cb_param = 0x0
        hash_code = 0
        etag = {s = 0x0, len = 0}
        ver = 0
        result = 500171648
        ret_code = 0
        ev = 0x0
        uac_r = {method = 0x7f8dfe86ed88, headers = 0x7f8e05369010, body = 0x7ffd1dd002a0, ssock = 0x7f8df84a5d4f <pkg_proc_update_stats+115>, dialog = 0x7f8dffe6a138, cb_flags = 0, cb = 0x800000 <yystos+2976>, 
          cbp = 0x524c18 <get_is_fparam+2277>, callid = 0x208250}
        res = 0x0
        dbpres = {id = {s = 0x4 <error: Cannot access memory at address 0x4>, len = 0}, pres_uri = 0x7f8dfba03520, event = 15, expires = 0, desired_expires = 4, flag = 0, db_flag = 0, cb_param = 0x0, next = 0x0, ua_flag = 0, 
          outbound_proxy = 0x7ffd1dd00630, etag = {s = 0x1 <error: Cannot access memory at address 0x1>, len = 0}, tuple_id = {s = 0x0, len = 7817596}, body = 0x2, content_type = {s = 0x0, len = 0}, watcher_uri = 0xffffffffffffffff, 
          call_id = {s = 0x59f2890c <error: Cannot access memory at address 0x59f2890c>, len = 500172592}, to_tag = {s = 0x0, len = 0}, from_tag = {s = 0x1 <error: Cannot access memory at address 0x1>, len = 96725184}, cseq = 4, 
          version = 0, extra_headers = 0x7f8e05c3e8c0 <__libc_wait+32>, record_route = {s = 0x0, len = 0}, remote_contact = {s = 0x4 <error: Cannot access memory at address 0x4>, len = 0}, contact = {
            s = 0x7f8dffe6a14a "Yealink SIP-T48G 35.81.0.25", len = 6659449}}
        pres_uri = {s = 0x0, len = 0}
        watcher_uri = {s = 0x0, len = 0}
        extra_headers = {s = 0x0, len = 0}
        ret = -1
        __func__ = "send_publish"
#2  0x00007f8df3a3c189 in ul_publish (c=0x7f8dff996470, type=8, param=0x0) at ul_publish.c:324
        body = 0x0
        uri = {s = 0x7f8e054e5d68 "sip:user@example.com", len = 25}
        at = 0x7f8dfe937d78 "@example.com.0"
        publ = 0x7f8e053b8fd8
        size = 192
        content_type = {s = 0x7f8df3a4007c "application/pidf+xml", len = 20}
        error = 3097352
        __func__ = "ul_publish"
#3  0x00007f8df71feda0 in run_ul_callbacks (type=8, c=0x7f8dff996470) at ul_callback.h:85
        cbp = 0x7f8dfba02d50
        __func__ = "run_ul_callbacks"
#4  0x00007f8df7201136 in wb_timer (_r=0x7f8dff74f340) at urecord.c:382
        ptr = 0x7f8dff996470
        t = 0x208150
        old_state = (unknown: 5393688)
        op = 0
        res = 0
        __func__ = "wb_timer"
#5  0x00007f8df720233a in timer_urecord (_r=0x7f8dff74f340) at urecord.c:461
No locals.
#6  0x00007f8df71fa644 in mem_timer_udomain (_d=0x7f8dfbad5088, istart=0, istep=1) at udomain.c:1015
        ptr = 0x7f8dff74f340
        t = 0x7f8df6919048
        i = 20
#7  0x00007f8df7225ae9 in synchronize_all_udomains (istart=0, istep=1) at dlist.c:761
        res = 0
        ptr = 0x7f8dfbacd920
#8  0x00007f8df7207d18 in destroy () at usrloc_mod.c:442
        __func__ = "destroy"
#9  0x000000000051d55e in destroy_modules () at core/sr_module.c:811
        t = 0x7f8e053d38c0
        foo = 0x7f8e053d33b0
        __func__ = "destroy_modules"
#10 0x0000000000418269 in cleanup (show_status=1) at main.c:525
        memlog = 0
        __func__ = "cleanup"
#11 0x00000000004199cc in shutdown_children (sig=15, show_status=1) at main.c:667
        __func__ = "shutdown_children"
#12 0x000000000041a3f7 in handle_sigs () at main.c:698
        chld = 4292094
        chld_status = 5363279
        any_chld_stopped = 0
        memlog = -1
        __func__ = "handle_sigs"
#13 0x0000000000424062 in main_loop () at main.c:1756
        i = 2
        pid = 6639
        si = 0x0
        si_desc = "udp receiver child=1 sock=[2603:300A:134:50E0:0:0:0:3]:5060\000\000\000\000\000\220\b\320\035\375\177\000\000hBc\005\216\177\000\000\020\t\320\035\375\177\000\000\252\360\017\005\216\177\000\000ȉc\005\216\177\000\000\000\000\000\000\001\000\000\000HnN\005\216\177\000\000(9\255\373\215\177\000"
        nrprocs = 2
        woneinit = 1
        __func__ = "main_loop"
#14 0x0000000000429ca3 in main (argc=6, argv=0x7ffd1dd00bc8) at main.c:2643
        cfg_stream = 0x1ce0010
        c = -1
        r = 0
        tmp = 0x7ffd1dd01f06 ""
        tmp_len = 0
        port = 0
        proto = 0
        options = 0x72e440 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:x:X:Y:"
        ret = -1
        seed = 2969426471
        rfd = 4
        debug_save = 0
        debug_flag = 0
        dont_fork_cnt = 2
        n_lst = 0x7f8e065858d0
        p = 0x7ffd1dd00a20 "\377\377\377\377"
        st = {st_dev = 22, st_ino = 3045, st_nlink = 2, st_mode = 16872, st_uid = 986, st_gid = 983, __pad0 = 0, st_rdev = 0, st_size = 40, st_blksize = 4096, st_blocks = 0, st_atim = {tv_sec = 1508981501, tv_nsec = 204171040}, 
          st_mtim = {tv_sec = 1509020740, tv_nsec = 222461911}, st_ctim = {tv_sec = 1509020740, tv_nsec = 222461911}, __glibc_reserved = {0, 0, 0}}
        __func__ = "main"

info locals
cfg_stream = 0x1ce0010
c = -1
r = 0
tmp = 0x7ffd1dd01f06 ""
tmp_len = 0
port = 0
proto = 0
options = 0x72e440 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:x:X:Y:"
ret = -1
seed = 2969426471
rfd = 4
debug_save = 0
debug_flag = 0
dont_fork_cnt = 2
n_lst = 0x7f8e065858d0
p = 0x7ffd1dd00a20 "\377\377\377\377"
st = {st_dev = 22, st_ino = 3045, st_nlink = 2, st_mode = 16872, st_uid = 986, st_gid = 983, __pad0 = 0, st_rdev = 0, st_size = 40, st_blksize = 4096, st_blocks = 0, st_atim = {tv_sec = 1508981501, tv_nsec = 204171040}, st_mtim = {
    tv_sec = 1509020740, tv_nsec = 222461911}, st_ctim = {tv_sec = 1509020740, tv_nsec = 222461911}, __glibc_reserved = {0, 0, 0}}
__func__ = "main"
(gdb) list
123             pua_event_t* event;
124             event= pua_evlist->next;
125
126             while(event)
127             {
128                     if(event->ev_flag== ev_flag)
129                     {
130                             return event;
131                     }
132                     event= event->next;

Log Messages

systemd[1]: Stopping Kamailio SIP Server...
kamailio[6620]: WARNING: db_postgres [km_dbase.c:242]: db_postgres_submit_query(): postgres query command failed, connection status 1, error [SSL connection has been closed unexpectedly]
systemd[1]: kamailio.service: Main process exited, code=dumped, status=11/SEGV

SIP Traffic

(paste your sip traffic here)

Possible Solutions

Additional Information

The version is the 5.0.4 release, though it happened in previous releases as well.

version: kamailio 5.0.4 (x86_64/linux) 
flags: STATS: Off, USE_TCP, USE_TLS, USE_SCTP, TLS_HOOKS, DISABLE_NAGLE, USE_MCAST, DNS_IP_HACK, SHM_MEM, SHM_MMAP, PKG_MALLOC, Q_MALLOC, F_MALLOC, TLSF_MALLOC, DBG_SR_MEMORY, USE_FUTEX, FAST_LOCK-ADAPTIVE_WAIT, USE_DNS_CACHE, USE_DNS_FAILOVER, USE_NAPTR, USE_DST_BLACKLIST, HAVE_RESOLV_RES
ADAPTIVE_WAIT_LOOPS=1024, MAX_RECV_BUFFER_SIZE 262144, MAX_LISTEN 16, MAX_URI_SIZE 1024, BUF_SIZE 65535, DEFAULT PKG_SIZE 8MB
poll method support: poll, epoll_lt, epoll_et, sigio_rt, select.
id: unknown 
compiled on 18:37:03 Oct 25 2017 with gcc 7.2.1
Fedora 26 x86_64
Linux chicago.example.com 4.13.9-200.fc26.x86_64 #1 SMP Mon Oct 23 13:52:45 UTC 2017 x86_64 x86_64 x86_64 GNU/Linux


You are receiving this because you are subscribed to this thread.
Reply to this email directly, view it on GitHub, or mute the thread.