### Description
I am upgraded Kamailio from 5.4.5 to 5.5, but i have got segfault when I try start Kamailio with custom config, with default config Kamailio start work done.
#### Reproduction
/usr/sbin/kamailio -Ee -d -DD -P /var/run/kamailio/kamailio.pid -m 64 -M 32 -f /etc/kazoo/kamailio/kamailio.cfg -w /run/kamailio/
#### Debugging Data ``` [root@hostname ~]# gdb /usr/sbin/kamailio /run/kamailio/core.29437 GNU gdb (GDB) Red Hat Enterprise Linux 7.6.1-120.el7 Copyright (C) 2013 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later http://gnu.org/licenses/gpl.html This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "x86_64-redhat-linux-gnu". For bug reporting instructions, please see: http://www.gnu.org/software/gdb/bugs/... Reading symbols from /usr/sbin/kamailio...Reading symbols from /usr/lib/debug/usr/sbin/kamailio.debug...done. done. [New LWP 29437] [Thread debugging using libthread_db enabled] Using host libthread_db library "/lib64/libthread_db.so.1". Core was generated by `/usr/sbin/kamailio -Ee -d -DD -P /var/run/kamailio/kamailio.pid -m 64 -M 32 -f'. Program terminated with signal 11, Segmentation fault. #0 0x00007f31b5397d26 in __memcpy_ssse3_back () from /lib64/libc.so.6 Missing separate debuginfos, use: debuginfo-install bzip2-libs-1.0.6-13.el7.x86_64 cyrus-sasl-lib-2.1.26-23.el7.x86_64 elfutils-libelf-0.176-5.el7.x86_64 elfutils-libs-0.176-5.el7.x86_64 glibc-2.17-323.el7_9.x86_64 jansson-2.10-1.el7.x86_64 json-c-0.11-4.el7_0.x86_64 keyutils-libs-1.5.8-3.el7.x86_64 krb5-libs-1.15.1-50.el7.x86_64 libattr-2.4.46-13.el7.x86_64 libcap-2.22-11.el7.x86_64 libcom_err-1.42.9-19.el7.x86_64 libcurl-7.29.0-59.el7_9.1.x86_64 libevent-2.0.21-4.el7.x86_64 libgcc-4.8.5-44.el7.x86_64 libidn-1.28-4.el7.x86_64 librabbitmq-0.8.0-3.el7.x86_64 libselinux-2.5-15.el7.x86_64 libssh2-1.8.0-4.el7.x86_64 libstdc++-4.8.5-44.el7.x86_64 libuuid-2.23.2-65.el7_9.1.x86_64 libxml2-2.9.1-6.el7.5.x86_64 mariadb-libs-5.5.68-1.el7.x86_64 nspr-4.25.0-2.el7_9.x86_64 nss-3.53.1-3.el7_9.x86_64 nss-softokn-freebl-3.53.1-6.el7_9.x86_64 nss-util-3.53.1-1.el7_9.x86_64 openldap-2.4.44-22.el7.x86_64 openssl-libs-1.0.2k-21.el7_9.x86_64 pcre-8.32-17.el7.x86_64 systemd-libs-219-78.el7_9.3.x86_64 xz-libs-5.2.2-1.el7.x86_64 zlib-1.2.7-19.el7_9.x86_64 (gdb) (gdb) (gdb) (gdb) bt full #0 0x00007f31b5397d26 in __memcpy_ssse3_back () from /lib64/libc.so.6 No symbol table info available. #1 0x00007f31b2dcdfed in pv_parse_hdr_name (sp=0x7f31b38b27d0, in=0x7ffc78f0e1f0) at pv_core.c:3272 s = {s = 0x0, len = 29437} p = 0x0 nsp = 0x0 hdr = {type = 2029052412, name = {s = 0x7ffc78f0e200 "", len = -1282654080}, body = {s = 0x7ffc78f0e20c "\001", len = -1282725936}, len = 2029052384, parsed = 0x1006ccf31, next = 0x49500000001} __FUNCTION__ = "pv_parse_hdr_name" #2 0x000000000062ee65 in pv_parse_spec2 (in=0x7f31b38b27b8, e=0x7f31b38b27d0, silent=0) at core/pvapi.c:969 p = 0x7f31b38b283b ")[0])" s = {s = 0x7f31b38b2836 "X-CID)[0])", len = 5} pvname = {s = 0x7f31b38b2832 "hdr(X-CID)[0])", len = 3} pvstate = 2 tr = 0x0 pte = 0x7f31b32d3c30 n = 0 __FUNCTION__ = "pv_parse_spec2" #3 0x000000000062a71e in pv_cache_add (name=0x7ffc78f0e440) at core/pvapi.c:359 pvn = 0x7f31b38b27b8 pvid = 949637875 p = 0xffffffff000072fd <Address 0xffffffff000072fd out of bounds> __FUNCTION__ = "pv_cache_add" #4 0x000000000062bf56 in pv_spec_lookup (name=0x7ffc78f0e520, len=0x7ffc78f0e51c) at core/pvapi.c:498 pvs = 0x0 tname = {s = 0x7f31b38c9db8 "$(hdr(X-CID)[0])", len = 16} __FUNCTION__ = "pv_spec_lookup" #5 0x000000000063241d in pv_parse_format (in=0x7ffc78f0e660, el=0x7f31b38b26a0) at core/pvapi.c:1194 p = 0x7f31b38c9db8 "$(hdr(X-CID)[0])" p0 = 0x280007e2a15 <Address 0x280007e2a15 out of bounds> n = 1 e = 0x7f31b38b2730 e0 = 0x0 s = {s = 0x7f31b38c9db8 "$(hdr(X-CID)[0])", len = 16} len = 16 __FUNCTION__ = "pv_parse_format" #6 0x000000000059faa1 in fix_param (type=256, param=0x7f31b38c9bf8) at core/sr_module.c:1214 p = 0x7f31b38b2690 name = {s = 0x7f31b38c9db8 "$(hdr(X-CID)[0])", len = 16} s = {s = 0x20 <Address 0x20 out of bounds>, len = -1282726872} num = 0 err = 32764 __FUNCTION__ = "fix_param" #7 0x00000000005a00d8 in fix_param_types (types=256, param=0x7f31b38c9bf8) at core/sr_module.c:1336 ret = 2029056944 t = 256 #8 0x0000000000657f66 in fixup_spve_null (param=0x7f31b38c9bf8, param_no=1) at core/mod_fix.c:564 ret = 0 fp = 0x0 __FUNCTION__ = "fixup_spve_null" #9 0x00007f31aa22bfac in fixup_hvalue_param (param=0x7f31b38c9bf8, param_no=2) at textopsx.c:622 No locals. #10 0x00007f31aa22cd49 in fixup_hname_str (param=0x7f31b38c9bf8, param_no=2) at textopsx.c:719 ---Type <return> to continue, or q <return> to quit--- No locals. #11 0x00007f31aa2345ef in append_hf_value_fixup (param=0x7f31b38c9bf8, param_no=2) at textopsx.c:1644 res = 0 __FUNCTION__ = "append_hf_value_fixup" #12 0x00000000006743c9 in fix_actions (a=0x7f31b38c9b80) at core/route.c:932 t = 0x7f31b38c9b80 p = 0x7f31b380c668 tmp = 0x7f31b38c5ae8 "" tmp_p = 0x7f31b38c9db8 ret = 0 i = 1 cmd = 0x7f31b32fb050 s = {s = 0x7ffc78f0eb70 "\360\355\360x\374\177", len = 7141887} he = 0x41c380 <_start> ip = {af = 3012334440, len = 32561, u = {addrl = {139851442459504, 8589934592}, addr32 = {3012334448, 32561, 0, 2}, addr16 = {37744, 45964, 32561, 0, 0, 0, 2, 0}, addr = "p\223\214\263\061\177\000\000\000\000\000\000\002\000\000"}} si = 0x7ffc78f0eb70 lval = 0x7f31b38c6418 rve = 0x41c380 <_start> err_rve = 0x642e38 <sr_event_exec+415> rve_type = 32764 err_type = 2029054464 expected_type = 32764 rv = 0x4651dc <fix_rval_expr+783> rve_param_no = 0 __FUNCTION__ = "fix_actions" #13 0x000000000066fd5b in fix_actions (a=0x7f31b38c9e38) at core/route.c:723 t = 0x7f31b38c9e38 p = 0x7ffc78f0ee40 tmp = 0xb5605380 <Address 0xb5605380 out of bounds> tmp_p = 0x7f31b38c6040 ret = 0 i = 2 cmd = 0x7f31b32fe998 s = {s = 0x7f31b3895f40 "LIS_REPLY", len = 9} he = 0x7ffc78f0ed10 ip = {af = 2029055136, len = 32764, u = {addrl = {7947659, 140722337541280}, addr32 = {7947659, 0, 2029055136, 32764}, addr16 = {17803, 121, 0, 0, 60576, 30960, 32764, 0}, addr = "\213Ey\000\000\000\000\000\240\354\360x\374\177\000"}} si = 0x100000400 lval = 0x7f31b3894c78 rve = 0x7f31b38c9368 err_rve = 0x0 rve_type = RV_INT err_type = RV_NONE expected_type = RV_NONE rv = 0x7f31b3896938 rve_param_no = 0 __FUNCTION__ = "fix_actions" #14 0x0000000000680d97 in fix_rl (rt=0xb9e5a0 <main_rt>) at core/route.c:2102 i = 104 ret = 0 #15 0x0000000000680dce in fix_rls () at core/route.c:2118 ---Type <return> to continue, or q <return> to quit--- ret = 0 #16 0x0000000000436704 in main (argc=14, argv=0x7ffc78f0f3b8) at main.c:3047 cfg_stream = 0x2a7d040 c = -1 r = 0 tmp = 0x7ffc78f0f850 "" tmp_len = 1472 port = 960 proto = 32561 ahost = 0x0 aport = 0 options = 0x7dd1e8 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:x:X:Y:" ret = -1 seed = 539315467 rfd = 4 debug_save = 1 debug_flag = 1 dont_fork_cnt = 2 n_lst = 0x7ffc78f0f270 p = 0xf0b5ff <Address 0xf0b5ff out of bounds> st = {st_dev = 20, st_ino = 10213, st_nlink = 2, st_mode = 16832, st_uid = 997, st_gid = 1000, __pad0 = 0, st_rdev = 0, st_size = 160, st_blksize = 4096, st_blocks = 0, st_atim = {tv_sec = 1621281263, tv_nsec = 948503600}, st_mtim = {tv_sec = 1621281245, tv_nsec = 465387888}, st_ctim = {tv_sec = 1621281245, tv_nsec = 465387888}, __unused = {0, 0, 0}} tbuf = '\000' <repeats 392 times>... option_index = 0 long_options = {{name = 0x7df5ff "help", has_arg = 0, flag = 0x0, val = 104}, {name = 0x7da674 "version", has_arg = 0, flag = 0x0, val = 118}, {name = 0x7df604 "alias", has_arg = 1, flag = 0x0, val = 1024}, { name = 0x7df60a "subst", has_arg = 1, flag = 0x0, val = 1025}, {name = 0x7df610 "substdef", has_arg = 1, flag = 0x0, val = 1026}, {name = 0x7df619 "substdefs", has_arg = 1, flag = 0x0, val = 1027}, { name = 0x7df623 "server-id", has_arg = 1, flag = 0x0, val = 1028}, {name = 0x7df62d "loadmodule", has_arg = 1, flag = 0x0, val = 1029}, {name = 0x7df638 "modparam", has_arg = 1, flag = 0x0, val = 1030}, { name = 0x7df641 "log-engine", has_arg = 1, flag = 0x0, val = 1031}, {name = 0x7df64c "debug", has_arg = 1, flag = 0x0, val = 1032}, {name = 0x7df652 "cfg-print", has_arg = 0, flag = 0x0, val = 1033}, { name = 0x7df65c "atexit", has_arg = 1, flag = 0x0, val = 1034}, {name = 0x0, has_arg = 0, flag = 0x0, val = 0}} __FUNCTION__ = "main" (gdb) (gdb) (gdb) info locals cfg_stream = 0x2a7d040 c = -1 r = 0 tmp = 0x7ffc78f0f850 "" tmp_len = 1472 port = 960 proto = 32561 ahost = 0x0 aport = 0 options = 0x7dd1e8 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:x:X:Y:" ret = -1 seed = 539315467 rfd = 4 debug_save = 1 debug_flag = 1 dont_fork_cnt = 2 n_lst = 0x7ffc78f0f270 p = 0xf0b5ff <Address 0xf0b5ff out of bounds> st = {st_dev = 20, st_ino = 10213, st_nlink = 2, st_mode = 16832, st_uid = 997, st_gid = 1000, __pad0 = 0, st_rdev = 0, st_size = 160, st_blksize = 4096, st_blocks = 0, st_atim = {tv_sec = 1621281263, tv_nsec = 948503600}, st_mtim = {tv_sec = 1621281245, tv_nsec = 465387888}, st_ctim = {tv_sec = 1621281245, tv_nsec = 465387888}, __unused = {0, 0, 0}} tbuf = '\000' <repeats 392 times>... option_index = 0 long_options = {{name = 0x7df5ff "help", has_arg = 0, flag = 0x0, val = 104}, {name = 0x7da674 "version", has_arg = 0, flag = 0x0, val = 118}, {name = 0x7df604 "alias", has_arg = 1, flag = 0x0, val = 1024}, { name = 0x7df60a "subst", has_arg = 1, flag = 0x0, val = 1025}, {name = 0x7df610 "substdef", has_arg = 1, flag = 0x0, val = 1026}, {name = 0x7df619 "substdefs", has_arg = 1, flag = 0x0, val = 1027}, {name = 0x7df623 "server-id", has_arg = 1, flag = 0x0, val = 1028}, {name = 0x7df62d "loadmodule", has_arg = 1, flag = 0x0, val = 1029}, {name = 0x7df638 "modparam", has_arg = 1, flag = 0x0, val = 1030}, {name = 0x7df641 "log-engine", has_arg = 1, flag = 0x0, val = 1031}, {name = 0x7df64c "debug", has_arg = 1, flag = 0x0, val = 1032}, {name = 0x7df652 "cfg-print", has_arg = 0, flag = 0x0, val = 1033}, {name = 0x7df65c "atexit", has_arg = 1, flag = 0x0, val = 1034}, { name = 0x0, has_arg = 0, flag = 0x0, val = 0}} __FUNCTION__ = "main" (gdb) cfg_stream = 0x2a7d040 c = -1 r = 0 tmp = 0x7ffc78f0f850 "" tmp_len = 1472 port = 960 proto = 32561 ahost = 0x0 aport = 0 options = 0x7dd1e8 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:x:X:Y:" ret = -1 seed = 539315467 rfd = 4 debug_save = 1 debug_flag = 1 dont_fork_cnt = 2 n_lst = 0x7ffc78f0f270 p = 0xf0b5ff <Address 0xf0b5ff out of bounds> st = {st_dev = 20, st_ino = 10213, st_nlink = 2, st_mode = 16832, st_uid = 997, st_gid = 1000, __pad0 = 0, st_rdev = 0, st_size = 160, st_blksize = 4096, st_blocks = 0, st_atim = {tv_sec = 1621281263, tv_nsec = 948503600}, st_mtim = {tv_sec = 1621281245, tv_nsec = 465387888}, st_ctim = {tv_sec = 1621281245, tv_nsec = 465387888}, __unused = {0, 0, 0}} tbuf = '\000' <repeats 392 times>... option_index = 0 long_options = {{name = 0x7df5ff "help", has_arg = 0, flag = 0x0, val = 104}, {name = 0x7da674 "version", has_arg = 0, flag = 0x0, val = 118}, {name = 0x7df604 "alias", has_arg = 1, flag = 0x0, val = 1024}, { name = 0x7df60a "subst", has_arg = 1, flag = 0x0, val = 1025}, {name = 0x7df610 "substdef", has_arg = 1, flag = 0x0, val = 1026}, {name = 0x7df619 "substdefs", has_arg = 1, flag = 0x0, val = 1027}, {name = 0x7df623 "server-id", has_arg = 1, flag = 0x0, val = 1028}, {name = 0x7df62d "loadmodule", has_arg = 1, flag = 0x0, val = 1029}, {name = 0x7df638 "modparam", has_arg = 1, flag = 0x0, val = 1030}, {name = 0x7df641 "log-engine", has_arg = 1, flag = 0x0, val = 1031}, {name = 0x7df64c "debug", has_arg = 1, flag = 0x0, val = 1032}, {name = 0x7df652 "cfg-print", has_arg = 0, flag = 0x0, val = 1033}, {name = 0x7df65c "atexit", has_arg = 1, flag = 0x0, val = 1034}, { name = 0x0, has_arg = 0, flag = 0x0, val = 0}} __FUNCTION__ = "main" (gdb) cfg_stream = 0x2a7d040 c = -1 r = 0 tmp = 0x7ffc78f0f850 "" tmp_len = 1472 port = 960 proto = 32561 ahost = 0x0 aport = 0 options = 0x7dd1e8 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:x:X:Y:" ret = -1 seed = 539315467 rfd = 4 debug_save = 1 debug_flag = 1 dont_fork_cnt = 2 n_lst = 0x7ffc78f0f270 p = 0xf0b5ff <Address 0xf0b5ff out of bounds> st = {st_dev = 20, st_ino = 10213, st_nlink = 2, st_mode = 16832, st_uid = 997, st_gid = 1000, __pad0 = 0, st_rdev = 0, st_size = 160, st_blksize = 4096, st_blocks = 0, st_atim = {tv_sec = 1621281263, tv_nsec = 948503600}, st_mtim = {tv_sec = 1621281245, tv_nsec = 465387888}, st_ctim = {tv_sec = 1621281245, tv_nsec = 465387888}, __unused = {0, 0, 0}} tbuf = '\000' <repeats 392 times>... option_index = 0 long_options = {{name = 0x7df5ff "help", has_arg = 0, flag = 0x0, val = 104}, {name = 0x7da674 "version", has_arg = 0, flag = 0x0, val = 118}, {name = 0x7df604 "alias", has_arg = 1, flag = 0x0, val = 1024}, { name = 0x7df60a "subst", has_arg = 1, flag = 0x0, val = 1025}, {name = 0x7df610 "substdef", has_arg = 1, flag = 0x0, val = 1026}, {name = 0x7df619 "substdefs", has_arg = 1, flag = 0x0, val = 1027}, {name = 0x7df623 "server-id", has_arg = 1, flag = 0x0, val = 1028}, {name = 0x7df62d "loadmodule", has_arg = 1, flag = 0x0, val = 1029}, {name = 0x7df638 "modparam", has_arg = 1, flag = 0x0, val = 1030}, {name = 0x7df641 "log-engine", has_arg = 1, flag = 0x0, val = 1031}, {name = 0x7df64c "debug", has_arg = 1, flag = 0x0, val = 1032}, {name = 0x7df652 "cfg-print", has_arg = 0, flag = 0x0, val = 1033}, {name = 0x7df65c "atexit", has_arg = 1, flag = 0x0, val = 1034}, { name = 0x0, has_arg = 0, flag = 0x0, val = 0}} __FUNCTION__ = "main" (gdb) cfg_stream = 0x2a7d040 c = -1 r = 0 tmp = 0x7ffc78f0f850 "" tmp_len = 1472 port = 960 proto = 32561 ahost = 0x0 aport = 0 options = 0x7dd1e8 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:x:X:Y:" ret = -1 seed = 539315467 rfd = 4 debug_save = 1 debug_flag = 1 dont_fork_cnt = 2 n_lst = 0x7ffc78f0f270 p = 0xf0b5ff <Address 0xf0b5ff out of bounds> st = {st_dev = 20, st_ino = 10213, st_nlink = 2, st_mode = 16832, st_uid = 997, st_gid = 1000, __pad0 = 0, st_rdev = 0, st_size = 160, st_blksize = 4096, st_blocks = 0, st_atim = {tv_sec = 1621281263, tv_nsec = 948503600}, st_mtim = {tv_sec = 1621281245, tv_nsec = 465387888}, st_ctim = {tv_sec = 1621281245, tv_nsec = 465387888}, __unused = {0, 0, 0}} tbuf = '\000' <repeats 392 times>... option_index = 0 long_options = {{name = 0x7df5ff "help", has_arg = 0, flag = 0x0, val = 104}, {name = 0x7da674 "version", has_arg = 0, flag = 0x0, val = 118}, {name = 0x7df604 "alias", has_arg = 1, flag = 0x0, val = 1024}, { name = 0x7df60a "subst", has_arg = 1, flag = 0x0, val = 1025}, {name = 0x7df610 "substdef", has_arg = 1, flag = 0x0, val = 1026}, {name = 0x7df619 "substdefs", has_arg = 1, flag = 0x0, val = 1027}, {name = 0x7df623 "server-id", has_arg = 1, flag = 0x0, val = 1028}, {name = 0x7df62d "loadmodule", has_arg = 1, flag = 0x0, val = 1029}, {name = 0x7df638 "modparam", has_arg = 1, flag = 0x0, val = 1030}, {name = 0x7df641 "log-engine", has_arg = 1, flag = 0x0, val = 1031}, {name = 0x7df64c "debug", has_arg = 1, flag = 0x0, val = 1032}, {name = 0x7df652 "cfg-print", has_arg = 0, flag = 0x0, val = 1033}, {name = 0x7df65c "atexit", has_arg = 1, flag = 0x0, val = 1034}, { name = 0x0, has_arg = 0, flag = 0x0, val = 0}} __FUNCTION__ = "main" (gdb) cfg_stream = 0x2a7d040 c = -1 r = 0 tmp = 0x7ffc78f0f850 "" tmp_len = 1472 port = 960 proto = 32561 ahost = 0x0 aport = 0 options = 0x7dd1e8 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:x:X:Y:" ret = -1 seed = 539315467 rfd = 4 debug_save = 1 debug_flag = 1 dont_fork_cnt = 2 n_lst = 0x7ffc78f0f270 p = 0xf0b5ff <Address 0xf0b5ff out of bounds> st = {st_dev = 20, st_ino = 10213, st_nlink = 2, st_mode = 16832, st_uid = 997, st_gid = 1000, __pad0 = 0, st_rdev = 0, st_size = 160, st_blksize = 4096, st_blocks = 0, st_atim = {tv_sec = 1621281263, tv_nsec = 948503600}, st_mtim = {tv_sec = 1621281245, tv_nsec = 465387888}, st_ctim = {tv_sec = 1621281245, tv_nsec = 465387888}, __unused = {0, 0, 0}} tbuf = '\000' <repeats 392 times>... option_index = 0 long_options = {{name = 0x7df5ff "help", has_arg = 0, flag = 0x0, val = 104}, {name = 0x7da674 "version", has_arg = 0, flag = 0x0, val = 118}, {name = 0x7df604 "alias", has_arg = 1, flag = 0x0, val = 1024}, { name = 0x7df60a "subst", has_arg = 1, flag = 0x0, val = 1025}, {name = 0x7df610 "substdef", has_arg = 1, flag = 0x0, val = 1026}, {name = 0x7df619 "substdefs", has_arg = 1, flag = 0x0, val = 1027}, {name = 0x7df623 "server-id", has_arg = 1, flag = 0x0, val = 1028}, {name = 0x7df62d "loadmodule", has_arg = 1, flag = 0x0, val = 1029}, {name = 0x7df638 "modparam", has_arg = 1, flag = 0x0, val = 1030}, {name = 0x7df641 "log-engine", has_arg = 1, flag = 0x0, val = 1031}, {name = 0x7df64c "debug", has_arg = 1, flag = 0x0, val = 1032}, {name = 0x7df652 "cfg-print", has_arg = 0, flag = 0x0, val = 1033}, {name = 0x7df65c "atexit", has_arg = 1, flag = 0x0, val = 1034}, { name = 0x0, has_arg = 0, flag = 0x0, val = 0}} __FUNCTION__ = "main" (gdb) cfg_stream = 0x2a7d040 c = -1 r = 0 tmp = 0x7ffc78f0f850 "" tmp_len = 1472 port = 960 proto = 32561 ahost = 0x0 aport = 0 options = 0x7dd1e8 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:x:X:Y:" ret = -1 seed = 539315467 rfd = 4 debug_save = 1 debug_flag = 1 dont_fork_cnt = 2 n_lst = 0x7ffc78f0f270 p = 0xf0b5ff <Address 0xf0b5ff out of bounds> st = {st_dev = 20, st_ino = 10213, st_nlink = 2, st_mode = 16832, st_uid = 997, st_gid = 1000, __pad0 = 0, st_rdev = 0, st_size = 160, st_blksize = 4096, st_blocks = 0, st_atim = {tv_sec = 1621281263, tv_nsec = 948503600}, st_mtim = {tv_sec = 1621281245, tv_nsec = 465387888}, st_ctim = {tv_sec = 1621281245, tv_nsec = 465387888}, __unused = {0, 0, 0}} tbuf = '\000' <repeats 392 times>... option_index = 0 long_options = {{name = 0x7df5ff "help", has_arg = 0, flag = 0x0, val = 104}, {name = 0x7da674 "version", has_arg = 0, flag = 0x0, val = 118}, {name = 0x7df604 "alias", has_arg = 1, flag = 0x0, val = 1024}, { name = 0x7df60a "subst", has_arg = 1, flag = 0x0, val = 1025}, {name = 0x7df610 "substdef", has_arg = 1, flag = 0x0, val = 1026}, {name = 0x7df619 "substdefs", has_arg = 1, flag = 0x0, val = 1027}, {name = 0x7df623 "server-id", has_arg = 1, flag = 0x0, val = 1028}, {name = 0x7df62d "loadmodule", has_arg = 1, flag = 0x0, val = 1029}, {name = 0x7df638 "modparam", has_arg = 1, flag = 0x0, val = 1030}, {name = 0x7df641 "log-engine", has_arg = 1, flag = 0x0, val = 1031}, {name = 0x7df64c "debug", has_arg = 1, flag = 0x0, val = 1032}, {name = 0x7df652 "cfg-print", has_arg = 0, flag = 0x0, val = 1033}, {name = 0x7df65c "atexit", has_arg = 1, flag = 0x0, val = 1034}, { name = 0x0, has_arg = 0, flag = 0x0, val = 0}} __FUNCTION__ = "main" (gdb) list 1981 int main(int argc, char** argv) 1982 { 1983 1984 FILE* cfg_stream; 1985 int c,r; 1986 char *tmp; 1987 int tmp_len; 1988 int port; 1989 int proto; 1990 char *ahost = NULL; (gdb) ```
#### Log Messages
``` [root@hostname ~]# /usr/sbin/kamailio -Ee -d -DD -P /var/run/kamailio/kamailio.pid -m 64 -M 32 -f /etc/kazoo/kamailio/kamailio.cfg -w /run/kamailio/ 0(30249) INFO: tls [tls_init.c:503]: init_tls_compression(): disabling compression... 0(30249) ERROR: <core> [core/pvapi.c:924]: pv_parse_spec2(): error searching pvar "kzE" 0(30249) ERROR: <core> [core/pvapi.c:1128]: pv_parse_spec2(): wrong char [k/107] in [$(kzE{kz.json,From}{uri.user})] at [6 (4)] 0(30249) ERROR: <core> [core/pv_core.c:213]: pv_eval_str(): error in parsing src parameter 0(30249) ERROR: <core> [core/pvapi.c:924]: pv_parse_spec2(): error searching pvar "kzE" 0(30249) ERROR: <core> [core/pvapi.c:1128]: pv_parse_spec2(): wrong char [k/107] in [$(kzE{kz.json,Realm})] at [6 (4)] 0(30249) ERROR: <core> [core/pv_core.c:213]: pv_eval_str(): error in parsing src parameter 0(30249) ERROR: <core> [core/pvapi.c:924]: pv_parse_spec2(): error searching pvar "kzE" 0(30249) ERROR: <core> [core/pvapi.c:1128]: pv_parse_spec2(): wrong char [k/107] in [$(kzE{kz.json,Realm})] at [6 (4)] 0(30249) ERROR: <core> [core/pv_core.c:213]: pv_eval_str(): error in parsing src parameter 0(30249) ERROR: <core> [core/pvapi.c:924]: pv_parse_spec2(): error searching pvar "kzE" 0(30249) ERROR: <core> [core/pvapi.c:1128]: pv_parse_spec2(): wrong char [k/107] in [$(kzE{kz.json,Realm})] at [6 (4)] 0(30249) ERROR: <core> [core/pv_core.c:213]: pv_eval_str(): error in parsing src parameter 0(30249) ERROR: <core> [core/pvapi.c:924]: pv_parse_spec2(): error searching pvar "subs" 0(30249) ERROR: <core> [core/pvapi.c:1128]: pv_parse_spec2(): wrong char [t/116] in [$subs(to_user)] at [6 (5)] 0(30249) ERROR: <core> [core/pv_core.c:213]: pv_eval_str(): error in parsing src parameter 0(30249) INFO: pv [pv_shv.c:60]: shvar_init_locks(): locks array size 16 0(30249) INFO: mqueue [mqueue_mod.c:257]: mq_param(): mqueue param: [presence_last_notity|0] 0(30249) INFO: mqueue [mqueue_mod.c:257]: mq_param(): mqueue param: [node_track|0] 0(30249) INFO: mqueue [mqueue_mod.c:257]: mq_param(): mqueue param: [node_heartbeat|0] Listening on udp: myip1:5060 tcp: myip1:5060 Aliases: udp: myip1:5060 tcp: myip1:5060
0(30249) WARNING: <core> [core/daemonize.c:348]: daemonize(): pid file contains old pid, replacing pid 0(30249) NOTICE: nat_traversal [nat_traversal.c:1845]: mod_init(): keeping alive dialogs is disabled because the dialog module is not loaded 0(30249) NOTICE: regex [regex_mod.c:168]: mod_init(): 'file' parameter is not set, group matching disabled 0(30249) WARNING: tls [tls_init.c:796]: tls_h_mod_init_f(): openssl bug #1491 (crash/mem leaks on low memory) workaround enabled (on low memory tls operations will fail preemptively) with free memory thresholds 1024 and 1024 bytes Segmentation fault (core dumped) [root@hostname ~]#
```
### Additional Information
``` [root@hostname ~]# kamailio -v version: kamailio 5.5.0 (x86_64/linux) d4c1a1 flags: USE_TCP, USE_TLS, USE_SCTP, TLS_HOOKS, USE_RAW_SOCKS, DISABLE_NAGLE, USE_MCAST, DNS_IP_HACK, SHM_MMAP, PKG_MALLOC, Q_MALLOC, F_MALLOC, TLSF_MALLOC, DBG_SR_MEMORY, USE_FUTEX, FAST_LOCK-ADAPTIVE_WAIT, USE_DNS_CACHE, USE_DNS_FAILOVER, USE_NAPTR, USE_DST_BLOCKLIST, HAVE_RESOLV_RES ADAPTIVE_WAIT_LOOPS 1024, MAX_RECV_BUFFER_SIZE 262144, MAX_URI_SIZE 1024, BUF_SIZE 65535, DEFAULT PKG_SIZE 8MB poll method support: poll, epoll_lt, epoll_et, sigio_rt, select. id: d4c1a1 compiled on 07:50:25 May 6 2021 with gcc 4.8.5 ```
``` [root@hostname ~]# rpm -qa | grep kamailio kamailio-presence-5.5.0-0.el7.centos.x86_64 kamailio-mysql-5.5.0-0.el7.centos.x86_64 kamailio-websocket-5.5.0-0.el7.centos.x86_64 kamailio-http_async_client-5.5.0-0.el7.centos.x86_64 kamailio-xmpp-5.5.0-0.el7.centos.x86_64 kamailio-utils-5.5.0-0.el7.centos.x86_64 kamailio-uuid-5.5.0-0.el7.centos.x86_64 kamailio-kazoo-5.5.0-0.el7.centos.x86_64 kamailio-http_client-5.5.0-0.el7.centos.x86_64 kamailio-xmlops-5.5.0-0.el7.centos.x86_64 kamailio-outbound-5.5.0-0.el7.centos.x86_64 kamailio-debuginfo-5.5.0-0.el7.centos.x86_64 kamailio-5.5.0-0.el7.centos.x86_64 kamailio-tls-5.5.0-0.el7.centos.x86_64 kamailio-regex-5.5.0-0.el7.centos.x86_64 kamailio-jansson-5.5.0-0.el7.centos.x86_64
``` * **Operating System**:
<!-- Details about the operating system, the type: Linux (e.g.,: Debian 8.4, Ubuntu 16.04, CentOS 7.1, ...), MacOS, xBSD, Solaris, ...; Kernel details (output of `uname -a`) -->
``` [root@hostname ~]# cat /etc/redhat-release CentOS Linux release 7.9.2009 (Core)
```
Seems to be related to the fixup of `append_hf_value()` function -- can you paste here all the lines in your config with this function?
My configuration many lines func "append_hf_value". I am comment all lines for this function, but i still get segfault. I created new TT #2738 for this issue with new debug core and additional info about my configuration.
In the #2738, the backtrace indicated a crash due to fixup of xlog functions while processing the parameter:
``` "$ci|end|sent subscription $hdr(Subscription-State)\n" ```
I just tested with the next line in a kamailio.cfg used with kamailio 5.5:
``` xlog("$ci|end|sent subscription $hdr(Subscription-State)\n"); ```
and it starts ok. So the problem is somewhere else, potentially a buffer overflow in completely different part of the code.
Can you list all loaded modules in your configuration (the list of `loadmodule` lines)?
my list modules ``` accounting-role.cfg:loadmodule "acc.so" antiflood-role.cfg:loadmodule "pike.so" db_kazoo.cfg:loadmodule "db_kazoo.so" db_mysql.cfg:loadmodule "db_mysql.so" db_postgres.cfg:loadmodule "db_postgres.so" default.cfg:loadmodule "mqueue.so" default.cfg:loadmodule "outbound.so" default.cfg:loadmodule "stun.so" default.cfg:loadmodule "path.so" default.cfg:loadmodule "ctl.so" default.cfg:loadmodule "cfg_rpc.so" default.cfg:loadmodule "cfgutils.so" default.cfg:loadmodule "corex.so" default.cfg:loadmodule "uuid.so" default.cfg:loadmodule "kex.so" default.cfg:loadmodule "tm.so" default.cfg:loadmodule "tmx.so" default.cfg:loadmodule "sl.so" default.cfg:loadmodule "rr.so" default.cfg:loadmodule "maxfwd.so" default.cfg:loadmodule "siputils.so" default.cfg:loadmodule "textopsx.so" default.cfg:loadmodule "sdpops.so" default.cfg:loadmodule "htable.so" default.cfg:loadmodule "rtimer.so" default.cfg:loadmodule "evrexec.so" default.cfg:loadmodule "xlog.so" default.cfg:loadmodule "uac.so" default.cfg:loadmodule "avp.so" default.cfg:loadmodule "avpops.so" default.cfg:loadmodule "uac_redirect.so" default.cfg:loadmodule "jsonrpcs.so" default.cfg:loadmodule "sqlops.so" default.cfg:loadmodule "debugger.so" default.cfg:loadmodule "statistics.so" default.cfg:loadmodule "permissions.so" dispatcher-role-5.1.cfg:loadmodule "dispatcher.so" dispatcher-role-5.2.cfg:loadmodule "dispatcher.so" dispatcher-role-5.4.cfg:loadmodule "dispatcher.so" dispatcher-role-5.5.cfg:loadmodule "dispatcher.so" e911-role.cfg:loadmodule "regex.so" kamailio.cfg:loadmodule "ipops.so" kamailio.cfg:loadmodule "pv.so" kamailio.cfg:loadmodule "textops.so" kazoo-bindings.cfg:loadmodule "kazoo.so" lis-role.cfg:loadmodule "jansson.so" lis-role.cfg:loadmodule "http_async_client.so" lis-role.cfg:loadmodule "http_client.so" lis-role.cfg:loadmodule "xmlops.so" msrp-proxy.cfg:loadmodule "msrp.so" nat-traversal-role.cfg:loadmodule "nathelper.so" presence-role.cfg:loadmodule "nat_traversal.so" presence-role.cfg:loadmodule "presence.so" presence-role.cfg:loadmodule "presence_dialoginfo.so" presence-role.cfg:loadmodule "presence_mwi.so" presence-role.cfg:loadmodule "presence_xml.so" registrar-role.cfg:loadmodule "auth.so" registrar-role.cfg:loadmodule "usrloc.so" registrar-role.cfg:loadmodule "registrar.so" registrar-role.cfg:loadmodule "nathelper.so" sanity.cfg:loadmodule "sanity.so" sip_trace-role.cfg:loadmodule "siptrace.so" tls-role.cfg:loadmodule "tls.so" websockets-role.cfg:loadmodule "nathelper.so" websockets-role.cfg:loadmodule "xhttp.so" websockets-role.cfg:loadmodule "websocket.so" ```
Hi Daniel @miconda Alexei can find a commit where the issue is introduced. Should we make `git bisect`?
`loadmodule "db_kazoo.so"` - this is not a module offered by Kamailio project, we cannot help when private extensions are used.
@miconda This module is not used, he is not installed in system. Kamailio is default and installed from official repo. ``` [root@hostname kamailio]# ls -la /usr/lib64/kamailio/modules/ | grep db_ -rwxr-xr-x. 1 root root 192264 May 6 07:55 db_cluster.so -rwxr-xr-x. 1 root root 99168 May 6 07:55 db_flatstore.so -rwxr-xr-x. 1 root root 227840 May 6 07:55 db_mysql.so -rwxr-xr-x. 1 root root 247136 May 6 07:55 db_text.so ``` ``` kamailio -v version: kamailio 5.5.0 (x86_64/linux) d4c1a1 flags: USE_TCP, USE_TLS, USE_SCTP, TLS_HOOKS, USE_RAW_SOCKS, DISABLE_NAGLE, USE_MCAST, DNS_IP_HACK, SHM_MMAP, PKG_MALLOC, Q_MALLOC, F_MALLOC, TLSF_MALLOC, DBG_SR_MEMORY, USE_FUTEX, FAST_LOCK-ADAPTIVE_WAIT, USE_DNS_CACHE, USE_DNS_FAILOVER, USE_NAPTR, USE_DST_BLOCKLIST, HAVE_RESOLV_RES ADAPTIVE_WAIT_LOOPS 1024, MAX_RECV_BUFFER_SIZE 262144, MAX_URI_SIZE 1024, BUF_SIZE 65535, DEFAULT PKG_SIZE 8MB poll method support: poll, epoll_lt, epoll_et, sigio_rt, select. id: d4c1a1 compiled on 07:50:25 May 6 2021 with gcc 4.8.5 ```
Then try to isolate why happens by disabling loading modules one by one (and commenting the modparams and functions for those modules).The crash happens at startup, so should be easy to go this way till kamailio starts ok. It should be one of the modules not used in the default config files, which likely has a buffer overflow.
Minimal config ``` loadmodule "ipops.so" loadmodule "pv.so" loadmodule "textops.so" loadmodule "textopsx.so"
pv_buffer_slots = 30
#!substdef "!MAJOR!$(version(num){re.subst,/^(([^.])*.([^.])*)..*/\1/})!g" #!substdef "!MY_HOSTNAME!$HN(f)!g" #!substdef "!MY_WEBSOCKET_DOMAIN!$HN(d)!g" #!substdef "!KAMAILIO_DBMS!$def(KZ_DB_MODULE)!g"
#!substdef "!MY_IP_ADDRESS!$HN(i)!g" #!substdef "!SANITY_SUBST_CACHE_PERIOD!$def(SANITY_CACHE_PERIOD)!g"
#!substdef "!KZQ_CHECK_MEDIA_SERVER_INSERT!insert into dispatcher (setid, destination) select $var(SetId), "$var(MediaUrl)" from DUAL where not exists(select * from dispatcher where destination = "$var(MediaUrl)")!g" #!substdef "!KZQ_COUNT_SUBSCRIBERS!select event, (select count(*) from active_watchers b where presentity_uri = "$var(presentity)" and b.event = a.event) count from event_list a!g" #!substdef "!KZQ_HANDLE_NEW_SUBSCRIBE_DELETE1!delete from active_watchers where callid = "$ci"!g" #!substdef "!KZQ_HANDLE_NEW_SUBSCRIBE_DELETE2!delete from active_watchers where watcher_username="$fU" and presentity_uri="$var(presentity_uri)" and to_user="$tU" and watcher_domain="$fd" and event="$hdr(Event)"!g" #!substdef "!KZQ_RESET_PUBLISHER_UPDATE!update active_watchers set expires = $TS where id in (select * from (select b.id from presentity a inner join active_watchers b on a.username = b.to_user and a.domain = b.to_domain and a.event = b.event where a.sender = "$var(MediaUrl)") AS presentity_temp)!g" #!substdef "!KZQ_PRESENCE_SEARCH_DETAIL!select * from active_watchers_log where presentity_uri = "$var(presentity_uri)"!g" #!substdef "!KZQ_PRESENCE_SEARCH_SUMMARY!select * from active_watchers where watcher_domain = "$var(Domain)"!g" #!substdef "!KZQ_PRESENCE_RESET!delete from presentity where sender = "$var(MediaUrl)"!g"
listen=tcp:127.0.0.1:5090
####### Routing Logic ######## route { $avp(device_id) = $hdr(X-Device-Id); $avp(account_db) = $hdr(X-Account-Db); $var(text) = $hdr(Contact); $var(expires) = $hdr(Expires); $var(header) = $hdr(X-KAZOO-Respond-With); $var(xxxx) = $hdr(To); $var(rr_base) = $hdr(Record-Route); $xavp(hf=>X-AUTH-IP) = $hdr(X-AUTH-IP); $xavp(hf=>X-AUTH-PORT) = $hdr(X-AUTH-PORT); $var(LocalRoute) = $hdr(X-TM-Local); $ru = $hdr(X-URN-Service); append_hf_value("Call-Info", "$(hdr(X-NenaCallId)[0])");
if ($hdr(X-KAZOO-INVITE-FORMAT) == "route") { $var(referred_by) = $hdr(Referred-By); }
if ($hdr(X-Redirect-Server) != $null) { $avp(destination_uri) = $hdr(X-KAZOO-AOR); }
} ```
If `pv_buffer_slots` has value 18-24, 26, 27, 29, 30 then core created. Tested 0e51ce1075f206a4441333f72c69fcc56f8d6855
Also important, `-m` and `-M` Kamailio arguments affect how long Kamailio trying to start. But no matter how much memory provided, Kamailio will fail.
Issue added by commit 004190b2ebe62681ae1f4f65f18de1a9e430742d Trying revert on master.
Issue fixed after reverting 004190b2ebe62681ae1f4f65f18de1a9e430742d
Thanks for digging in further and narrowing it down. I will analyze to see what is the reason, like the references to the buffers become invalid as the needs for defines/substdefs can exceed/overwrite them. It is interesting that it didn't surface for defines which had this kind of capability/behaviour for long time before, the commit referenced above extended for substdef to make it coherent, because this one makes a define behind.
Can you try with master branch or the patch from the commit referenced above?
Tested current master (c146ef490e1d7d35add7d3ee593f6d3d20e327ad). Issue resolved.
Closed #2736.
-
Daniel-Constantin Mierla -
ReznikovAlexei
-
sergey-safarov