15 Jul
2016
15 Jul
'16
2:06 p.m.
(gdb) frame 1
#1 dlg_unref (dlg=dlg@entry=0x7f585c494b40, cnt=cnt@entry=1) at
dlg_hash.c:921
921 dlg_lock( d_table, d_entry);
(gdb) p *dlg
$1 = {ref = 793790803, next = 0xa0d4b4f20303032, prev =
0x504953203a616956, h_id = 808333871, h_entry = 1346655535, state =
774976288, lifetime = 775107122, init_ts = 775435825,
start_ts = 976303410, end_ts = 808857653, dflags = 1667592763, iflags
= 1702259045, sflags = 825441636, toroute = 858927662, toroute_name = {
s = 0x6172623b3135322e <Address 0x6172623b3135322e out of bounds>,
len = 1030251374}, from_rr_nb = 894132788, tl = {next =
0x726f70723b646262, prev = 0xa0d303630353d74,
timeout = 1836020294}, callid = {s = 0x20226e776f6e6b6e <Address
0x20226e776f6e6b6e out of bounds>, len = 1885958972}, from_uri = {
s = 0x7340444c4f74656e <Address 0x7340444c4f74656e out of bounds>,
len = 1999532137}, to_uri = {s = 0x743b3e74656e2e70 <Address
0x743b3e74656e2e70 out of bounds>,
len = 1631414113}, req_uri = {s = 0x540a0d3536343766 <Address
0x540a0d3536343766 out of bounds>, len = 1008745071}, tag = {{
s = 0x363233313431332b <Address 0x363233313431332b out of bounds>,
len = 892614711}, {s = 0x2e3836312e333232 <Address 0x2e3836312e333232
out of bounds>, len = 1043608370}},
cseq = {{s = 0x663330643473613d <Address 0x663330643473613d out of
bounds>, len = 224671543}, {s = 0x3534203a44492d6c <Address
0x3534203a44492d6c out of bounds>,
len = 909665638}}, route_set = {{s = 0x3433333435356635 <Address
0x3433333435356635 out of bounds>, len = 825582898}, {
s = 0x7340353762316435 <Address 0x7340353762316435 out of bounds>,
len = 1999532137}}, contact = {{s = 0x430a0d74656e2e70 <Address
0x430a0d74656e2e70 out of bounds>,
len = 980510035}, {s = 0x65530a0d45594220 <Address
0x65530a0d45594220 out of bounds>, len = 1919252082}}, bind_addr =
{0x70696f766c772e70, 0x6c410a0d74656e2e}, cbs = {
first = 0x564e49203a776f6c, types = 742741065}, profile_links =
0x4c45434e4143202c, vars = 0x4e4f4954504f202c}
On 07/15/2016 01:00 PM, Daniel-Constantin Mierla wrote:
From the second crash, can you get:
frame 1
p *dlg
So far it looks like either to a double free or some buffer overflow...
Cheers,
Daniel
On 15/07/16 10:51, Dirk Teurlings - Signet B.V. wrote:
Just got another segfault.
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
Core was generated by `/usr/sbin/kamailio -f /etc/kamailio/kamailio.cfg
-P /var/run/kamailio/kamailio.'.
Program terminated with signal 11, Segmentation fault.
#0 atomic_get (v=0x7f6264d11378) at ../../mem/../atomic/atomic_common.h:74
74 return atomic_get_int(&(v->val));
(gdb) bt
#0 atomic_get (v=0x7f6264d11378) at ../../mem/../atomic/atomic_common.h:74
#1 dlg_unref (dlg=dlg@entry=0x7f585c494b40, cnt=cnt@entry=1) at
dlg_hash.c:921
#2 0x00007f5855912802 in dlg_run_event_route
(dlg=dlg@entry=0x7f585c494b40, msg=msg@entry=0x7f587d4be8e8,
ostate=<optimized out>, nstate=<optimized out>) at dlg_handlers.c:1630
#3 0x00007f585591416a in dlg_onroute (req=0x7f587d4be8e8,
route_params=<optimized out>, param=<optimized out>) at dlg_handlers.c:1307
#4 0x00007f585965b0e2 in run_rr_callbacks
(req=req@entry=0x7f587d4be8e8, rr_param=rr_param@entry=0x7f58598677a0)
at rr_cb.c:96
#5 0x00007f58596452c5 in after_loose (_m=0x7f587d4be8e8, preloaded=0)
at loose.c:919
#6 0x000000000042b618 in do_action (h=h@entry=0x7ffd6e277fd0,
a=a@entry=0x7f587d264338, msg=msg@entry=0x7f587d4be8e8) at action.c:1060
#7 0x000000000042a10a in run_actions (h=h@entry=0x7ffd6e277fd0,
a=0x7f587d264338, msg=0x7f587d4be8e8) at action.c:1549
#8 0x0000000000437544 in run_actions_safe (h=h@entry=0x7ffd6e279500,
a=<optimized out>, msg=<optimized out>) at action.c:1614
#9 0x000000000053b2e8 in rval_get_int (h=0x7ffd6e279500, msg=<optimized
out>, i=0x7ffd6e278430, rv=rv@entry=0x7f587d264d58,
cache=cache@entry=0x0) at rvalue.c:912
#10 0x000000000054261c in rval_expr_eval_int (h=h@entry=0x7ffd6e279500,
msg=msg@entry=0x7f587d4be8e8, res=res@entry=0x7ffd6e278430,
rve=rve@entry=0x7f587d264d50) at rvalue.c:1910
#11 0x000000000042bc91 in do_action (h=h@entry=0x7ffd6e279500,
a=a@entry=0x7f587d268f88, msg=msg@entry=0x7f587d4be8e8) at action.c:1030
#12 0x000000000042a10a in run_actions (h=h@entry=0x7ffd6e279500,
a=0x7f587d268f88, msg=msg@entry=0x7f587d4be8e8) at action.c:1549
#13 0x000000000042bcf2 in do_action (h=h@entry=0x7ffd6e279500,
a=a@entry=0x7f587d2691e8, msg=msg@entry=0x7f587d4be8e8) at action.c:1049
#14 0x000000000042a10a in run_actions (h=h@entry=0x7ffd6e279500,
a=0x7f587d263f48, msg=msg@entry=0x7f587d4be8e8) at action.c:1549
#15 0x000000000042bde0 in do_action (h=h@entry=0x7ffd6e279500,
a=a@entry=0x7f587d073d70, msg=msg@entry=0x7f587d4be8e8) at action.c:678
#16 0x000000000042a10a in run_actions (h=h@entry=0x7ffd6e279500,
a=a@entry=0x7f587d071698, msg=msg@entry=0x7f587d4be8e8) at action.c:1549
#17 0x00000000004375d0 in run_top_route (a=0x7f587d071698,
msg=msg@entry=0x7f587d4be8e8, c=c@entry=0x0) at action.c:1635
#18 0x0000000000504386 in receive_msg (buf=<optimized out>,
len=<optimized out>, rcv_info=<optimized out>) at receive.c:240
#19 0x00000000005f5bd4 in udp_rcv_loop () at udp_server.c:495
#20 0x00000000004b2625 in main_loop () at main.c:1600
#21 0x0000000000427e2b in main (argc=<optimized out>, argv=<optimized
out>) at main.c:2616
Relevant logmessages before crash:
Jul 15 10:37:55 server /usr/sbin/kamailio[12426]: NOTICE: dialog
[dlg_hash.c:245]: dlg_clean_run(): dialog in delete state is too old
(0x7f585c4a6820 ref 4)
Jul 15 10:37:55 server /usr/sbin/kamailio[12397]: WARNING: dialog
[dlg_handlers.c:1219]: dlg_onroute(): unable to find dialog for BYE with
route param '70f.b9d1' [3847:7579]
Jul 15 10:37:55 server /usr/sbin/kamailio[12395]: WARNING: dialog
[dlg_handlers.c:1348]: dlg_onroute(): inconsitent dlg timer data on dlg
0x7f585c4a6820 [3847:7579] with clid
'4c41f08d317ecb9342b93f22738003f3@server' and tags 'as5f3a16b4'
'as71cb6036'
Jul 15 10:40:13 server /usr/sbin/kamailio[12378]: WARNING: dialog
[dlg_handlers.c:1219]: dlg_onroute(): unable to find dialog for BYE with
route param 'eb6.1e21' [1726:4833]
Jul 15 10:40:13 server /usr/sbin/kamailio[12376]: WARNING: dialog
[dlg_handlers.c:1219]: dlg_onroute(): unable to find dialog for BYE with
route param 'eb6.1e21' [1726:4833]
Jul 15 10:40:14 server /usr/sbin/kamailio[12377]: WARNING: dialog
[dlg_handlers.c:1219]: dlg_onroute(): unable to find dialog for BYE with
route param 'eb6.1e21' [1726:4833]
Jul 15 10:40:16 server /usr/sbin/kamailio[12377]: WARNING: dialog
[dlg_handlers.c:1219]: dlg_onroute(): unable to find dialog for BYE with
route param 'eb6.1e21' [1726:4833]
Jul 15 10:40:16 server /usr/sbin/kamailio[12396]: WARNING: dialog
[dlg_handlers.c:1219]: dlg_onroute(): unable to find dialog for BYE with
route param 'eb6.1e21' [1726:4833]
Jul 15 10:41:34 server /usr/sbin/kamailio[12396]: ERROR: sl
[sl_funcs.c:363]: sl_reply_error(): ERROR: sl_reply_error used: I'm
terribly sorry, server error occurred (1/SL)
Jul 15 10:41:34 server /usr/sbin/kamailio[12396]: ERROR: tm
[t_reply.c:533]: _reply_light(): ERROR: _reply_light: can't generate 487
reply when a final 487 was sent out
Jul 15 10:41:34 server /usr/sbin/kamailio[12396]: ERROR: tm
[t_lookup.c:1471]: t_unref(): ERROR: t_unref: generation of a delayed
stateful reply failed
Jul 15 10:42:25 server /usr/sbin/kamailio[12426]: NOTICE: dialog
[dlg_hash.c:245]: dlg_clean_run(): dialog in delete state is too old
(0x7f585c49d5b0 ref 4)
Jul 15 10:42:25 server /usr/sbin/kamailio[12426]: NOTICE: dialog
[dlg_hash.c:245]: dlg_clean_run(): dialog in delete state is too old
(0x7f585c604f18 ref 4)
Jul 15 10:42:25 server /usr/sbin/kamailio[12426]: NOTICE: dialog
[dlg_hash.c:245]: dlg_clean_run(): dialog in delete state is too old
(0x7f585c494b40 ref 4)
Jul 15 10:42:25 server /usr/sbin/kamailio[12383]: WARNING: dialog
[dlg_handlers.c:1348]: dlg_onroute(): inconsitent dlg timer data on dlg
0x7f585c604f18 [2396:9046] with clid
'1b3ff5f0246fb7e82ed949544bcccbba@192.168.10.233:5060' and tags
'as4d83d6f8' '5788A162-2557E04D-3E86ED15'
Jul 15 10:42:25 server /usr/sbin/kamailio[12395]: WARNING: dialog
[dlg_handlers.c:1219]: dlg_onroute(): unable to find dialog for BYE with
route param '6b3.c6b' [950:2924]
Jul 15 10:42:25 server kernel: [209851.262461] kamailio[12376]: segfault
at 7f6264d11378 ip 00007f585592a908 sp 00007ffd6e277330 error 4 in
dialog.so[7f58558e0000+88000]
Jul 15 10:42:25 server /usr/sbin/kamailio[12394]: WARNING: dialog
[dlg_handlers.c:1348]: dlg_onroute(): inconsitent dlg timer data on dlg
0x7f585c49d5b0 [950:2924] with clid
'45fe86ce065f5543342e51ad355d1b75@server' and tags 'as152f7465'
'as4d03f77d'
Jul 15 10:42:26 server /usr/sbin/kamailio[12431]: CRITICAL: <core>
[pass_fd.c:275]: receive_fd(): EOF on 32
Jul 15 10:42:26 server /usr/sbin/kamailio[12370]: ALERT: <core>
[main.c:739]: handle_sigs(): child process 12376 exited by a signal 11
Jul 15 10:42:26 server /usr/sbin/kamailio[12370]: ALERT: <core>
[main.c:742]: handle_sigs(): core was generated
Jul 15 10:42:26 server /usr/sbin/kamailio[12370]: INFO: <core>
[main.c:754]: handle_sigs(): terminating due to SIGCHLD
Cheers,
Dirk
On 07/15/2016 10:06 AM, Dirk Teurlings - Signet B.V. wrote:
Hi,
Running Kamailio on Debian from the Kamailio repository with 4.4.2
stable (unpatched). Getting some random segfaults with it now, here's
the relevant backtrace from the generated core.
Core was generated by `/usr/sbin/kamailio -f /etc/kamailio/kamailio.cfg
-P /var/run/kamailio/kamailio.'.
Program terminated with signal 11, Segmentation fault.
#0 run_dlg_callbacks (type=type@entry=64, dlg=dlg@entry=0x7fceb400e2f0,
req=req@entry=0x7fced4f093c8, rpl=rpl@entry=0x0, dir=<optimized out>,
dlg_data=dlg_data@entry=0x0) at dlg_cb.c:253
253 if ( (cb->types)&type ) {
(gdb) bt
#0 run_dlg_callbacks (type=type@entry=64, dlg=dlg@entry=0x7fceb400e2f0,
req=req@entry=0x7fced4f093c8, rpl=rpl@entry=0x0, dir=<optimized out>,
dlg_data=dlg_data@entry=0x0) at dlg_cb.c:253
#1 0x00007fcead3648f9 in dlg_terminated (dir=<optimized out>,
dlg=0x7fceb400e2f0, req=0x7fced4f093c8) at dlg_handlers.c:368
#2 dlg_onroute (req=0x7fced4f093c8, route_params=<optimized out>,
param=<optimized out>) at dlg_handlers.c:1354
#3 0x00007fceb10ab0e2 in run_rr_callbacks
(req=req@entry=0x7fced4f093c8, rr_param=rr_param@entry=0x7fceb12b77a0)
at rr_cb.c:96
#4 0x00007fceb10952c5 in after_loose (_m=0x7fced4f093c8, preloaded=0)
at loose.c:919
#5 0x000000000042b618 in do_action (h=h@entry=0x7ffeb0b3ed80,
a=a@entry=0x7fced4cb4338, msg=msg@entry=0x7fced4f093c8) at action.c:1060
#6 0x000000000042a10a in run_actions (h=h@entry=0x7ffeb0b3ed80,
a=0x7fced4cb4338, msg=0x7fced4f093c8) at action.c:1549
#7 0x0000000000437544 in run_actions_safe (h=h@entry=0x7ffeb0b402b0,
a=<optimized out>, msg=<optimized out>) at action.c:1614
#8 0x000000000053b2e8 in rval_get_int (h=0x7ffeb0b402b0, msg=<optimized
out>, i=0x7ffeb0b3f1e0, rv=rv@entry=0x7fced4cb4d58,
cache=cache@entry=0x0) at rvalue.c:912
#9 0x000000000054261c in rval_expr_eval_int (h=h@entry=0x7ffeb0b402b0,
msg=msg@entry=0x7fced4f093c8, res=res@entry=0x7ffeb0b3f1e0,
rve=rve@entry=0x7fced4cb4d50) at rvalue.c:1910
#10 0x000000000042bc91 in do_action (h=h@entry=0x7ffeb0b402b0,
a=a@entry=0x7fced4cb8f88, msg=msg@entry=0x7fced4f093c8) at action.c:1030
#11 0x000000000042a10a in run_actions (h=h@entry=0x7ffeb0b402b0,
a=0x7fced4cb8f88, msg=msg@entry=0x7fced4f093c8) at action.c:1549
#12 0x000000000042bcf2 in do_action (h=h@entry=0x7ffeb0b402b0,
a=a@entry=0x7fced4cb91e8, msg=msg@entry=0x7fced4f093c8) at action.c:1049
#13 0x000000000042a10a in run_actions (h=h@entry=0x7ffeb0b402b0,
a=0x7fced4cb3f48, msg=msg@entry=0x7fced4f093c8) at action.c:1549
#14 0x000000000042bde0 in do_action (h=h@entry=0x7ffeb0b402b0,
a=a@entry=0x7fced4ac3d70, msg=msg@entry=0x7fced4f093c8) at action.c:678
#15 0x000000000042a10a in run_actions (h=h@entry=0x7ffeb0b402b0,
a=a@entry=0x7fced4ac1698, msg=msg@entry=0x7fced4f093c8) at action.c:1549
#16 0x00000000004375d0 in run_top_route (a=0x7fced4ac1698,
msg=msg@entry=0x7fced4f093c8, c=c@entry=0x0) at action.c:1635
#17 0x0000000000504386 in receive_msg (buf=<optimized out>,
len=<optimized out>, rcv_info=<optimized out>) at receive.c:240
#18 0x00000000005f5bd4 in udp_rcv_loop () at udp_server.c:495
#19 0x00000000004b2625 in main_loop () at main.c:1600
#20 0x0000000000427e2b in main (argc=<optimized out>, argv=<optimized
out>) at main.c:2616
And from syslog the relevant messages before this dump:
Jul 15 08:55:03 server /usr/sbin/kamailio[16470]: WARNING: dialog
[dlg_handlers.c:1219]: dlg_onroute(): unable to find dialog for BYE with
route param 'd4c.26d1' [3149:7522]
Jul 15 08:56:01 server /usr/sbin/kamailio[16481]: WARNING: dialog
[dlg_handlers.c:1219]: dlg_onroute(): unable to find dialog for BYE with
route param 'fc.99f1' [207:8089]
Jul 15 08:56:27 server /usr/sbin/kamailio[16470]: CRITICAL: dialog
[dlg_timer.c:200]: update_dlg_timer(): Trying to update a bogus dlg
tl=0x7fceb3f7d920 tl->next=(nil) tl->prev=(nil)
Jul 15 08:56:27 server /usr/sbin/kamailio[16470]: ERROR: dialog
[dlg_handlers.c:1377]: dlg_onroute(): failed to update dialog lifetime
Jul 15 08:57:01 server /usr/sbin/kamailio[16482]: ERROR: db_mysql
[km_dbase.c:128]: db_mysql_submit_query(): driver error on query:
Duplicate entry '9584-3854-435' for key 'hash_index' (1062)
Jul 15 08:57:01 server /usr/sbin/kamailio[16482]: ERROR: <core>
[db_query.c:181]: db_do_raw_query(): error while submitting query
Jul 15 08:57:01 server /usr/sbin/kamailio[16482]: ERROR: sqlops
[sql_api.c:265]: sql_do_query(): cannot do the query [INSERT INTO
`dialog_extra` (`h_i]
Jul 15 08:57:01 server /usr/sbin/kamailio[16482]: ERROR: auth
[api.c:119]: auth_check_hdr_md5(): auth:pre_auth: Credentials are not
filled properly
Jul 15 08:57:01 server /usr/sbin/kamailio[16483]: ERROR: auth
[api.c:119]: auth_check_hdr_md5(): auth:pre_auth: Credentials are not
filled properly
Jul 15 08:57:54 server /usr/sbin/kamailio[16506]: NOTICE: dialog
[dlg_hash.c:245]: dlg_clean_run(): dialog in delete state is too old
(0x7fceb3f64470 ref 4)
Jul 15 08:57:54 server /usr/sbin/kamailio[16473]: WARNING: dialog
[dlg_handlers.c:1348]: dlg_onroute(): inconsitent dlg timer data on dlg
0x7fceb3f64470 [1182:5803] with clid
'09ad128753e2535d24bde58e3d7eda04@192.168.10.232:5060' and tags
'as1b497b34' '5788890C-EC6F55F-3E86ED0C'
Jul 15 08:57:54 server /usr/sbin/kamailio[16469]: ERROR: dialog
[dlg_handlers.c:334]: dlg_terminated_confirmed(): failed to get dialog
from params!
Jul 15 08:58:49 server /usr/sbin/kamailio[16467]: WARNING: dialog
[dlg_handlers.c:1219]: dlg_onroute(): unable to find dialog for BYE with
route param '6d2.2581' [726:6226]
Jul 15 08:59:24 server /usr/sbin/kamailio[16506]: NOTICE: dialog
[dlg_hash.c:245]: dlg_clean_run(): dialog in delete state is too old
(0x7fceb400e2f0 ref 4)
Jul 15 08:59:25 server /usr/sbin/kamailio[16464]: WARNING: dialog
[dlg_handlers.c:1219]: dlg_onroute(): unable to find dialog for BYE with
route param '3e4.b5c1' [1251:7259]
Jul 15 08:59:25 server /usr/sbin/kamailio[16465]: WARNING: dialog
[dlg_handlers.c:1348]: dlg_onroute(): inconsitent dlg timer data on dlg
0x7fceb400e2f0 [1251:7259] with clid '87791a#015#012Call-ID:
25750e286a5654361ef9405d72edbc' and tags '' 'as148f41b1'
Jul 15 08:59:25 server kernel: [203670.830521] kamailio[16465] general
protection ip:7fcead34b3a5 sp:7ffeb0b3e220 error:0 in
dialog.so[7fcead330000+88000]
Jul 15 08:59:26 server /usr/sbin/kamailio[16511]: CRITICAL: <core>
[pass_fd.c:275]: receive_fd(): EOF on 33
Jul 15 08:59:26 server /usr/sbin/kamailio[16458]: ALERT: <core>
[main.c:739]: handle_sigs(): child process 16465 exited by a signal 11
Jul 15 08:59:26 server /usr/sbin/kamailio[16458]: ALERT: <core>
[main.c:742]: handle_sigs(): core was generated
Jul 15 08:59:26 server /usr/sbin/kamailio[16458]: INFO: <core>
[main.c:754]: handle_sigs(): terminating due to SIGCHLD
Any insight would be appreciated!
Cheers,
Dirk
_______________________________________________
SIP Express Router (SER) and Kamailio (OpenSER) - sr-users mailing list
sr-users(a)lists.sip-router.org
http://lists.sip-router.org/cgi-bin/mailman/listinfo/sr-users
_______________________________________________
SIP Express Router (SER) and Kamailio (OpenSER) - sr-users mailing list
sr-users(a)lists.sip-router.org
http://lists.sip-router.org/cgi-bin/mailman/listinfo/sr-users