SIGSEGV on DMQ - sr-dev

10 Sep 2021


      Hi everyone,
I'm trying to use the DMQ module to share the user location info across our
Kamailio's cluster.  But when I start the service under docker I get a
segmentation fault violation.  After analysing the core dump with the *LLDB*
I get (if you want I can share the core dump with you using Google Drive):
...
(lldb) bt

thread #1, name = 'kamailio', stop reason = signal SIGSEGV
frame #0: 0x000000000071c63c kamailio`get_hdr_field(buf="", end="",


hdr=0x0000000000000000) at msg_parser.c:144:5
    frame #1: 0x0000000000575b02
kamailio`resolve_select(s=0x00007ffd470c6da0) at select.c:384:3
    frame #2: 0x00007fbbd7637fb9
    frame #3: 0x00000000005ad785
kamailio`clen_builder(msg=0x00007fbbd7f6e008, clen_len=0x0000000000000000,
diff=0, body_only=0) at msg_translator.c:451:3
    frame #4: 0x0000000000439aa7
kamailio`_dns_hash_find(name=0x00007fbbd7f6e498, type=32765,
h=0x0000000000000000, err=0x0000000000000000) at dns_cache.c:551:4
    frame #5: 0x0000000000440cb2
kamailio`dns_cache_do_request(name=0x0000000000000000, type=0) at
dns_cache.c:1767:3
    frame #6: 0x000000000043f8f6
kamailio`dns_cache_mk_rd_entry(name=0x0000000000000000, type=0,
rd_lst=0x0000000000000000) at dns_cache.c:1226:7
    frame #7: 0x000000000046501e
kamailio`print_action(t=0x0000000000000000) at route_struct.c:467:4
    frame #8: 0x0000000000471f2f
kamailio`xavp_print_list_content(head=0x00007ffd470c31a0, level=32699) at
xavp.c:584:2
    frame #9: 0x0000000000462047
kamailio`print_action(t=0x0000000000000000) at route_struct.c:392:4
    frame #10: 0x0000000000471f2f
kamailio`xavp_print_list_content(head=0x00007ffd470c31a0, level=32699) at
xavp.c:584:2
    frame #11: 0x00000000004654dc
kamailio`print_action(t=0x0000000000000000) at route_struct.c:470:4
    frame #12: 0x0000000000471f2f
kamailio`xavp_print_list_content(head=0x00007ffd470c31a0, level=32699) at
xavp.c:584:2
    frame #13: 0x0000000000465532
kamailio`print_action(t=0x0000000003863810) at route_struct.c:470:4
    frame #14: 0x0000000000471f2f
kamailio`xavp_print_list_content(head=0x00007ffd470c31a0, level=32699) at
xavp.c:584:2
    frame #15: 0x0000000000462047
kamailio`print_action(t=0x00000000006da9d3) at route_struct.c:392:4
    frame #16: 0x0000000000471f2f
kamailio`xavp_print_list_content(head=0x00007ffd470c31a0, level=32699) at
xavp.c:584:2
    frame #17: 0x0000000000472680
kamailio`xavp_get_list_key_names(xavp=0x00007ffd470c6da0) at xavp.c:622:3
    frame #18: 0x00007fbbb357da54
    frame #19: 0x00007fbbb3580370
    frame #20: 0x00007fbbb358317b
    frame #21: 0x00007fbbb3583147
    frame #22: 0x00007fbbb3585e8d
    frame #23: 0x00007fbbaec3ef23
    frame #24: 0x00007fbbaec3d5da
    frame #25: 0x00007fbbaec4e158
    frame #26: 0x00007fbbaec4aadd
    frame #27: 0x00007fbbaec53f05
    frame #28: 0x0000000000541563
kamailio`handle_tcp_child(tcp_c=0x00000004b4481868, fd_i=136) at
tcp_main.c:3477:1
    frame #29: 0x0000000000541223
kamailio`handle_tcp_child(tcp_c=0x0000000000000000, fd_i=32699) at
tcp_main.c:3470:5
    frame #30: 0x0000000000541223
kamailio`handle_tcp_child(tcp_c=0x00000000470c6da0, fd_i=0) at
tcp_main.c:3470:5
    frame #31: 0x0000000000541223
kamailio`handle_tcp_child(tcp_c=0x00000000b4481868, fd_i=1863578990) at
tcp_main.c:3470:5
    frame #32: 0x0000000000541223
kamailio`handle_tcp_child(tcp_c=0x00000000b4481868, fd_i=1863578990) at
tcp_main.c:3470:5
    frame #33: 0x0000000000541223
kamailio`handle_tcp_child(tcp_c=0x00007fbbb4481868, fd_i=1863578990) at
tcp_main.c:3470:5
    frame #34: 0x0000000000541223
kamailio`handle_tcp_child(tcp_c=0x00007ffd470c64d0, fd_i=32699) at
tcp_main.c:3470:5
    frame #35: 0x0000000000541223
kamailio`handle_tcp_child(tcp_c=0x00007ffd470c658c, fd_i=32765) at
tcp_main.c:3470:5
    frame #36: 0x0000000000541223
kamailio`handle_tcp_child(tcp_c=0x00000000b4481868, fd_i=1863578990) at
tcp_main.c:3470:5
    frame #37: 0x0000000000541223
kamailio`handle_tcp_child(tcp_c=0x00007fbbb4481868, fd_i=1863578990) at
tcp_main.c:3470:5
    frame #38: 0x0000000000541223
kamailio`handle_tcp_child(tcp_c=0x00007fbbd8059e38, fd_i=32699) at
tcp_main.c:3470:5
    frame #39: 0x0000000000541223
kamailio`handle_tcp_child(tcp_c=0x00007ffd470c6770, fd_i=32765) at
tcp_main.c:3470:5
    frame #40: 0x0000000000541223
kamailio`handle_tcp_child(tcp_c=0x0000000000000000, fd_i=32765) at
tcp_main.c:3470:5
    frame #41: 0x0000000000541223
kamailio`handle_tcp_child(tcp_c=0x00007fbbb5179a80, fd_i=0) at
tcp_main.c:3470:5
    frame #42: 0x0000000000541223
kamailio`handle_tcp_child(tcp_c=0x00000065dbb52b93, fd_i=32765) at
tcp_main.c:3470:5
    frame #43: 0x00000000005418fa
kamailio`handle_ser_child(p=0x00000065dbb52b93, fd_i=32765) at
tcp_main.c:3513:3
    frame #44: 0x0000000000425d6e kamailio`main_loop at main.c:1620:4
    frame #45: 0x000000000042c6af kamailio`main(argc=13,
argv=0x00007ffd470c6da8) at main.c:2568:3
Using *GDB* I get the following info:
...
#22 0x00007efcca978e8d in request () from /usr/lib64/kamailio/modules/tm.so
---Type <return> to continue, or q <return> to quit---
No symbol table info available.
#23 0x00007efcc6031f23 in dmq_send_message (peer=0x7efccc5545c0,
body=0x7efcef106ef8, node=0x7efccc5ded10, resp_cback=0x7efcc625d370
<notification_callback>, max_forwards=2,
    content_type=0x7efcc625d360 <notification_content_type>) at
dmq_funcs.c:251
        uac_r = {method = 0x7efcc625d3b0 <dmq_request_method>, headers =
0x7ffe13cb34b0, body = 0x7efcef106ef8, ssock = 0x7efcc625da20
<dmq_server_socket>, dialog = 0x7efccc5e0918,
          cb_flags = 1024, cb = 0x7efcc602eb7b <dmq_tm_callback>, cbp =
0x7efccc5def58, callid = 0x0}
        str_hdr = {s = 0x7efcef1004e8 "Max-Forwards: 2\r\nContent-Type:
text/plain\r\n", len = 43}
        from = {s = 0x7efcef107248 "sip:notification_peer@10.12.4.238:5090",
len = 38}
        to = {s = 0x7efcef11d100 "sip:notification_peer@10.12.4.170:5090",
len = 38}
        cb_param = 0x7efccc5def58
        result = 0
        len = 43
        __FUNCTION__ = "dmq_send_message"
#24 0x00007efcc60305da in bcast_dmq_message1 (peer=0x7efccc5545c0,
body=0x7efcef106ef8, except=0x0, resp_cback=0x7efcc625d370
<notification_callback>, max_forwards=2,
    content_type=0x7efcc625d360 <notification_content_type>,
incl_inactive=1) at dmq_funcs.c:170
        node = 0x7efccc5ded10
        __FUNCTION__ = "bcast_dmq_message1"
#25 0x00007efcc6041158 in request_nodelist (node=0x7efccc5deac8,
forward=2) at notification_peer.c:573
        body = 0x7efcef106ef8
        ret = 0
        __FUNCTION__ = "request_nodelist"
#26 0x00007efcc603dadd in add_server_and_notify (paddr=0x7efcc625da30
<dmq_notification_address>) at notification_peer.c:340
        puri_data = "sip:10.12.5.29:5090\000\376\177\000\000\250I+\002\000\000\000\000\230L+\002\000\000\000\000\260;\313\023\376\177\000\000\250c+\002\000\000\000\000`;\313\023\376\177",
'\000' <repeats 17 times>,
"\200`?\313\023\376\177\000\000\325\222\255\307\374~\000\000\024=\313\023\376\177\000\000\030=\313\023\376\177\000\000\034=\313\023\376\177\000\000\260:\313\023\376\177\000\000*\000\000\000\000\000\000\000]s\f\000\000\000\000\000\001\000\000\000\000\000\000\000\244\201\000\000\204\352",
'\000' <repeats 19 times>,
"\340\f\000\000\000\000\000\000\020\000\000\000\000\000\000p\006\000\000\000\000\000\000"...
        puri_list = {0x7ffe13cb3950 "sip:10.12.5.29:5090", 0x7ffe13cb3a50
"sip:10.12.4.170:5090", 0x7ffe13cb3b50 "x\321-\002", 0x7ffe13cb3c50 "",
0x7ffe13cb3d50 " @\313\023\376\177",
          0x7ffe13cb3e50 "", 0x7ffe13cb3f50 "", 0x7ffe13cb4050 "",
0x7ffe13cb4150 "2\247h", 0x7ffe13cb4250 "pB\313\023\376\177",
0x7ffe13cb4350 "\200S\237\312\374~",
          0x7ffe13cb4450 "8\341\006\357\374~", 0x7ffe13cb4550
"\320E\313\023\376\177", 0x7ffe13cb4650 "\210*,\002", 0x7ffe13cb4750
"\330N\313\023\376\177", 0x7ffe13cb4850 "",
          0x7ffe13cb4950 "", 0x7ffe13cb4a50 "\200J\313\023\376\177",
0x7ffe13cb4b50 "\035\362\067\037\003", 0x7ffe13cb4c50
"\320L\313\023\376\177", 0x7ffe13cb4d50 "\177", 0x7ffe13cb4e50 "n",
          0x7ffe13cb4f50 "1\325\\002", 0x7ffe13cb5050
"\260P\313\023\376\177", 0x7ffe13cb5150 "", 0x7ffe13cb5250
"\320\343\270\307\374~", 0x7ffe13cb5350 "", 0x7ffe13cb5450 "xV,\002",
          0x7ffe13cb5550 "2\247h", 0x7ffe13cb5650 "\320\235\005\306\374~"}
        pfirst = 0x7efccc5deac8
        pnode = 0x7efccc5ded10
        host_cnt = 2
        index = 2
        puri = {{user = {s = 0x0, len = 0}, passwd = {s = 0x0, len = 0},
host = {s = 0x7efceefdde8c "
kamailio-kazoo.svc.cpaas.eu-west-1.aws.prv.talkdeskstg.com:5090", len =
58}, port = {
              s = 0x7efceefddec7 "5090", len = 4}, params = {s = 0x0, len
= 0}, sip_params = {s = 0x0, len = 0}, headers = {s = 0x0, len = 0},
port_no = 5090, proto = 0, type = SIP_URI_T,
            flags = (unknown: 0), transport = {s = 0x0, len = 0}, ttl = {s
= 0x0, len = 0}, user_param = {s = 0x0, len = 0}, maddr = {s = 0x0, len =
0}, method = {s = 0x0, len = 0}, lr = {
              s = 0x0, len = 0}, r2 = {s = 0x0, len = 0}, gr = {s = 0x0,
len = 0}, transport_val = {s = 0x0, len = 0}, ttl_val = {s = 0x0, len = 0},
user_param_val = {s = 0x0, len = 0},
            maddr_val = {s = 0x0, len = 0}, method_val = {s = 0x0, len =
0}, lr_val = {s = 0x0, len = 0}, r2_val = {s = 0x0, len = 0}, gr_val = {s =
0x0, len = 0}}}
        pstr = {{s = 0x7ffe13cb3a50 "sip:10.12.4.170:5090", len = 20}}
        __FUNCTION__ = "add_server_and_notify"
#27 0x00007efcc6046f05 in child_init (rank=0) at dmq.c:302
        i = 4
        newpid = 160
---Type <return> to continue, or q <return> to quit---
        __FUNCTION__ = "child_init"
#28 0x0000000000541563 in init_mod_child ()
No symbol table info available.
#29 0x0000000000541223 in init_mod_child ()
No symbol table info available.
#30 0x0000000000541223 in init_mod_child ()
No symbol table info available.
#31 0x0000000000541223 in init_mod_child ()
No symbol table info available.
#32 0x0000000000541223 in init_mod_child ()
No symbol table info available.
#33 0x0000000000541223 in init_mod_child ()
No symbol table info available.
#34 0x0000000000541223 in init_mod_child ()
No symbol table info available.
#35 0x0000000000541223 in init_mod_child ()
No symbol table info available.
#36 0x0000000000541223 in init_mod_child ()
No symbol table info available.
#37 0x0000000000541223 in init_mod_child ()
No symbol table info available.
#38 0x0000000000541223 in init_mod_child ()
No symbol table info available.
#39 0x0000000000541223 in init_mod_child ()
No symbol table info available.
#40 0x0000000000541223 in init_mod_child ()
No symbol table info available.
#41 0x0000000000541223 in init_mod_child ()
No symbol table info available.
#42 0x0000000000541223 in init_mod_child ()
No symbol table info available.
#43 0x00000000005418fa in init_child ()
No symbol table info available.
#44 0x0000000000425d6e in main_loop ()
No symbol table info available.
#45 0x000000000042c6af in main ()
No symbol table info available.
DMQ module parameters are:
...
#!ifdef DMQ_ROLE
loadmodule "dmq.so"
loadmodule "dmq_usrloc.so"
# ---- dmq params ----
modparam("dmq", "server_address", "sip:DMQ_SERVER_ADDRESS")
modparam("dmq", "notification_address", "sip:DMQ_NOTIFY_ADDRESS")
modparam("dmq", "ping_interval", 5)
modparam("dmq", "multi_notify", 1)
modparam("dmq", "num_workers", 4)
modparam("dmq", "worker_usleep", 0)
modparam("dmq_usrloc", "enable", 1)
#!endif
Kamailio version and flags are:
...
version: kamailio 5.2.5 (x86_64/linux)
flags: STATS: Off, USE_TCP, USE_TLS, USE_SCTP, TLS_HOOKS, USE_RAW_SOCKS,
DISABLE_NAGLE, USE_MCAST, DNS_IP_HACK, SHM_MEM, SHM_MMAP, PKG_MALLOC,
Q_MALLOC, F_MALLOC, TLSF_MALLOC, DBG_SR_MEMORY, USE_FUTEX,
FAST_LOCK-ADAPTIVE_WAIT, USE_DNS_CACHE, USE_DNS_FAILOVER, USE_NAPTR,
USE_DST_BLACKLIST, HAVE_RESOLV_RES
ADAPTIVE_WAIT_LOOPS=1024, MAX_RECV_BUFFER_SIZE 262144 MAX_URI_SIZE 1024,
BUF_SIZE 65535, DEFAULT PKG_SIZE 8MB
poll method support: poll, epoll_lt, epoll_et, sigio_rt, select.
id: unknown
compiled on 01:14:59 Nov 19 2019 with gcc 7.3.1
Do you have any idea what might be causing this *SEGV*?
Best regards,
Sergio Carvalho