I tested with docker AlmaLinux 8 and default kamailio.cfg with tls enabled and it does not crash, so indeed the issue can be with snmpops module.
The "previous fragment tail overwritten" is very likely a buffer overflow, which is detected when tls makes some operations with shm, but the previous fragment can be from another module. But having the tail overwritten, makes it hard to jump one fragment back, so the list has to be walked from the beginning.
Is it a test system so that you can you provide ssh access to the system? I could try to look at core dump with gdb.
Cheers,
Daniel
Daniel,
Yes, with help of a collegaue was found it's SNMP dealing the problems. Turning it off helps.
bt full:
Core was generated by `/usr/sbin/kamailio -P /run/kamailio/kamailio.pid -f /etc/kamailio//kamailio.cfg'.
Program terminated with signal SIGABRT, Aborted.
#0 0x00007f5b722feaff in raise () from /lib64/libc.so.6
Missing separate debuginfos, use: yum debuginfo-install audit-libs-3.0.7-4.el8.x86_64 bzip2-libs-1.0.6-26.el8.x86_64 elfutils-libelf-0.187-4.el8.x86_64 glibc-2.28-211.el8.x86_64 libacl-2.2.53-1.el8.x86_64 libattr-2.4.48-3.el8.x86_64 libcap-2.48-4.el8.x86_64 libcap-ng-0.7.11-1.el8.x86_64 libdb-5.3.28-42.el8_4.x86_64 libunistring-0.9.9-3.el8.x86_64 libxcrypt-4.1.1-6.el8.x86_64 libxml2-2.9.7-15.el8_7.1.x86_64 libzstd-1.4.4-1.el8.x86_64 lm_sensors-libs-3.4.0-23.20180522git70f7e08.el8.x86_64 lua-libs-5.3.4-12.el8.x86_64 mariadb-connector-c-3.1.11-2.el8_3.x86_64 net-snmp-agent-libs-5.8-25.el8_7.1.x86_64 net-snmp-libs-5.8-25.el8_7.1.x86_64 openssl-libs-1.1.1k-7.el8_6.x86_64 pcre-8.42-6.el8.x86_64 perl-libs-5.26.3-421.el8.x86_64 popt-1.18-1.el8.x86_64 python2-libs-2.7.18-11.module_el8.7.0+3331+6c5bd851.alma.x86_64 sqlite-libs-3.26.0-17.el8_7.x86_64 xz-libs-5.2.4-4.el8_6.x86_64 zlib-1.2.11-21.el8_7.x86_64
(gdb) bt full
#0 0x00007f5b722feaff in raise () from /lib64/libc.so.6
No symbol table info available.
#1 0x00007f5b722d1ea5 in abort () from /lib64/libc.so.6
No symbol table info available.
#2 0x00000000006eec2f in qm_debug_check_frag (qm=0x7f5b678b4000, f=0x7f5b678ed820, file=0x7f5b6b926d24 "tls: tls_init.c", line=293, efile=0x87cc19 "core/mem/q_malloc.c", eline=391) at core/mem/q_malloc.c:158
p = 0x7f5b678ed790
__func__ = "qm_debug_check_frag"
#3 0x00000000006f09a6 in qm_malloc (qmp=0x7f5b678b4000, size=48, file=0x7f5b6b926d24 "tls: tls_init.c", func=0x7f5b6b928550 <__func__.23484> "ser_malloc", line=293, mname=0x7f5b6b926d20 "tls") at core/mem/q_malloc.c:391
qm = 0x7f5b678b4000
f = 0x7f5b678ed820
hash = 11
list_cntr = 1
__func__ = "qm_malloc"
#4 0x00000000006fc930 in qm_shm_malloc (qmp=0x7f5b678b4000, size=48, file=0x7f5b6b926d24 "tls: tls_init.c", func=0x7f5b6b928550 <__func__.23484> "ser_malloc", line=293, mname=0x7f5b6b926d20 "tls") at core/mem/q_malloc.c:1301
r = 0x7f5b667d1464 <sc_find_auth_alg_byoid+116>
#5 0x00007f5b6b8d3c31 in ser_malloc (size=48, fname=0x7f5b70676bcb "crypto/evp/digest.c", fline=54) at tls_init.c:293
__func__ = "ser_malloc"
#6 0x00007f5b705b890d in CRYPTO_zalloc () from /lib64/libcrypto.so.1.1
No symbol table info available.
#7 0x00007f5b667d2163 in sc_hash_type () from /lib64/libnetsnmp.so.35
No symbol table info available.
#8 0x00007f5b667ce9df in hash_engineID () from /lib64/libnetsnmp.so.35
No symbol table info available.
#9 0x00007f5b667ceb11 in search_enginetime_list () from /lib64/libnetsnmp.so.35
No symbol table info available.
#10 0x00007f5b667cef8e in set_enginetime () from /lib64/libnetsnmp.so.35
No symbol table info available.
#11 0x00007f5b667ce885 in init_snmpv3_post_config () from /lib64/libnetsnmp.so.35
No symbol table info available.
#12 0x00007f5b667d3c92 in snmp_call_callbacks () from /lib64/libnetsnmp.so.35
No symbol table info available.
#13 0x00007f5b6725001b in register_with_master_agent (name_to_register_under=0x7f5b6726dbad "snmpstats_alarm_agent") at sub_agent.c:196
__func__ = "register_with_master_agent"
#14 0x00007f5b67251f14 in run_alarm_check (ticks=111051753, attr=0x0) at alarm_checks.c:102
msg_queue_minor_threshold = 0
msg_queue_major_threshold = 0
dialog_minor_threshold = 0
dialog_major_threshold = 0
firstRun = 1 '\001'
bytesInMsgQueue = 0
numActiveDialogs = 8820307
#15 0x0000000000531cde in compat_old_handler (ti=1776828050, tl=0x7f5b67e00120, data=0x7f5b67e00120) at core/timer.c:980
t = 0x7f5b67e00120
#16 0x000000000053284e in slow_timer_main () at core/timer.c:1105
n = 12
ret = 4294967295
tl = 0x7f5b67e00120
i = 4
__func__ = "slow_timer_main"
#17 0x000000000042e3e1 in main_loop () at main.c:1815
i = 4
pid = 0
si = 0x0
si_desc = "udp receiver child=3 sock=188.185.121.232:5060 (dev.tone.cern.ch:5060)\000\000(\274tq[\177\000\000S\226\206\000\000\000\000\000\020\064\227q[\177\000\000\000\000\000\004\000\000\000\000\060\247=r[\177\000\000\060\000\000\000\060\000\000\000H\365$)\377\177\000"
nrprocs = 4
woneinit = 1
__func__ = "main_loop"
#18 0x0000000000438d14 in main (argc=5, argv=0x7fff2924fa68) at main.c:3078
cfg_stream = 0x24002a0
c = -1
r = 0
tmp = 0x7fff2924f8c0 "\377\377\377\377"
tmp_len = 0
port = 0
proto = 32603
ahost = 0x0
aport = 0
options = 0x8114d8 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:x:X:Y:"
ret = -1
seed = 1009486776
--Type <RET> for more, q to quit, c to continue without paging--
rfd = 4
debug_save = 0
debug_flag = 0
dont_fork_cnt = 0
n_lst = 0x7fff2924fa98
p = 0x7f5b7241a838 <_dl_vdso_vsym+104> "H\213t$\030ZYH\205\366t\026f\203~\006\361t-H\205\300t(H\213"
st = {st_dev = 24, st_ino = 84031, st_nlink = 2, st_mode = 16832, st_uid = 994, st_gid = 991, __pad0 = 0, st_rdev = 0, st_size = 40, st_blksize = 4096, st_blocks = 0, st_atim = {tv_sec = 1674662431, tv_nsec = 788813597}, st_mtim = {
tv_sec = 1674746240, tv_nsec = 961870546}, st_ctim = {tv_sec = 1674746240, tv_nsec = 961870546}, __glibc_reserved = {0, 0, 0}}
tbuf = "(O\003s[\177\000\000\000\000\000\000\000\000\000\000\370\210%s[\177\000\000(O\003s[\177\000\000\200\366$)\377\177\000\000`G\003s[\177\000\000x5&s[\177\000\000\000@\003s[\177\000\000\274I\003s[\177\000\000\263/\004s[\177\000\000\350/+r[\177\000\000\254\202\226\006\000\000\000\000\036\215\005s[\177\000\000\254\202\226\006\000\000\000\000\205\317c\t\000\000\000\000y0\004s[\177\000\000\020\215\005s[\177\000\000\354\032+r[\177\000\000\000\000\000\000\000\000\000\000\220\a&s[\177\000\000\001\000\000\000\000\000\000\000\002\035\326D\224\266\000\000\020\062&s[\177\000\000\200\224%s[\177\000\000\020\062&s[\177\000\000"...
option_index = 0
long_options = {{name = 0x813876 "help", has_arg = 0, flag = 0x0, val = 104}, {name = 0x80e6f4 "version", has_arg = 0, flag = 0x0, val = 118}, {name = 0x81387b "alias", has_arg = 1, flag = 0x0, val = 1024}, {name = 0x813881 "subst",
has_arg = 1, flag = 0x0, val = 1025}, {name = 0x813887 "substdef", has_arg = 1, flag = 0x0, val = 1026}, {name = 0x813890 "substdefs", has_arg = 1, flag = 0x0, val = 1027}, {name = 0x81389a "server-id", has_arg = 1, flag = 0x0,
val = 1028}, {name = 0x8138a4 "loadmodule", has_arg = 1, flag = 0x0, val = 1029}, {name = 0x8138af "modparam", has_arg = 1, flag = 0x0, val = 1030}, {name = 0x8138b8 "log-engine", has_arg = 1, flag = 0x0, val = 1031}, {
name = 0x8138c3 "debug", has_arg = 1, flag = 0x0, val = 1032}, {name = 0x8138c9 "cfg-print", has_arg = 0, flag = 0x0, val = 1033}, {name = 0x8138d3 "atexit", has_arg = 1, flag = 0x0, val = 1034}, {name = 0x0, has_arg = 0, flag = 0x0,
val = 0}}
__func__ = "main"
If it helps.Many thanks!
Le jeu. 26 janv. 2023 à 16:06, Daniel-Constantin Mierla <miconda@gmail.com> a écrit :
Hello,
it seems to be at start up, does it happen every time?
Cheers,
Daniel
On 26.01.23 14:43, Ihor Olkhovskyi wrote:
Hello!
Currently testing Kamailio on Almalinux 8 (stated to be RHEL 8 binary compatible) and got this while trying to get TLS working:
/usr/sbin/kamailio[32858]: INFO: tls [tls_domain.c:347]: ksr_tls_fill_missing(): TLSc<default>: ca_list='(null)'
/usr/sbin/kamailio[32858]: INFO: tls [tls_domain.c:354]: ksr_tls_fill_missing(): TLSc<default>: ca_path='(null)'
/usr/sbin/kamailio[32858]: INFO: tls [tls_domain.c:361]: ksr_tls_fill_missing(): TLSc<default>: crl='(null)'
/usr/sbin/kamailio[32858]: INFO: tls [tls_domain.c:365]: ksr_tls_fill_missing(): TLSc<default>: require_certificate=1
/usr/sbin/kamailio[32858]: INFO: tls [tls_domain.c:372]: ksr_tls_fill_missing(): TLSc<default>: cipher_list='(null)'
/usr/sbin/kamailio[32858]: INFO: tls [tls_domain.c:379]: ksr_tls_fill_missing(): TLSc<default>: private_key='(null)'
/usr/sbin/kamailio[32858]: INFO: tls [tls_domain.c:383]: ksr_tls_fill_missing(): TLSc<default>: verify_certificate=1
/usr/sbin/kamailio[32858]: INFO: tls [tls_domain.c:386]: ksr_tls_fill_missing(): TLSc<default>: verify_depth=9
/usr/sbin/kamailio[32858]: INFO: tls [tls_domain.c:389]: ksr_tls_fill_missing(): TLSc<default>: verify_client=0
/usr/sbin/kamailio[32858]: INFO: tls [tls_domain.c:736]: set_verification(): TLSc<default>: Server MUST present valid certificate
/usr/sbin/kamailio[32862]: ERROR: rtpengine [rtpengine.c:3070]: send_rtpp_command(): can't send command "ping" to RTPEngine <udp:127.0.0.1:19999>
/usr/sbin/kamailio[32862]: ERROR: rtpengine [rtpengine.c:2919]: rtpp_test(): proxy did not respond to ping
/usr/sbin/kamailio[32883]: INFO: snmpstats [snmpstats.c:519]: spawn_sysUpTime_child(): A snmpVersion parameter was not provided. Defaulting to 3
/usr/sbin/kamailio[32883]: ERROR: snmpstats [snmpstats.c:561]: spawn_sysUpTime_child(): snmpget failed to run. Did you supply the snmpstats module with a proper snmpgetPath parameter? The kamailioSIPServiceStartTime is defaulting to zero
/usr/sbin/kamailio[32881]: INFO: ctl [io_listener.c:214]: io_listen_loop(): io_listen_loop: using epoll_lt io watch method (config)
/usr/sbin/kamailio[32870]: CRITICAL: <core> [core/mem/q_malloc.c:145]: qm_debug_check_frag(): BUG: qm: prev. fragm. tail overwritten(0, abcdefed)[0x7fe8375d3820:0x7fe8375d3858]! Memory allocator was called from tls: tls_init.c:293. Fragment marked by tls: tls_init.c:293. Exec from core/mem/q_malloc.c:391.
/usr/sbin/kamailio[32870]: CRITICAL: <core> [core/mem/q_malloc.c:153]: qm_debug_check_frag(): BUG: qm: prev. fragm. tail overwritten [0x7fe8375d3790:0x7fe8375d37c8] - fragment marked by tls: tls_init.c:293
systemd[1]: Started Process Core Dump (PID 32923/UID 0).
systemd-coredump[32924]: Process 32870 (kamailio) of user 0 dumped core.#012#012Stack trace of thread 32870:#012#0 0x00007fe841786aff raise (libc.so.6)#012#1 0x00007fe841759ea5 abort (libc.so.6)#012#2 0x00000000006eec2f qm_debug_check_frag (kamailio)#012#3 0x00000000006f09a6 qm_malloc (kamailio)#012#4 0x00000000006fc930 qm_shm_malloc (kamailio)#012#5 0x00007fe83b5b9c31 ser_malloc (tls.so)#012#6 0x00007fe83fa4090d CRYPTO_zalloc (libcrypto.so.1.1)#012#7 0x00007fe8364b8163 sc_hash_type (libnetsnmp.so.35)#012#8 0x00007fe8364b49df hash_engineID (libnetsnmp.so.35)#012#9 0x00007fe8364b4b11 search_enginetime_list (libnetsnmp.so.35)#012#10 0x00007fe8364b4f8e set_enginetime (libnetsnmp.so.35)#012#11 0x00007fe8364b4885 init_snmpv3_post_config (libnetsnmp.so.35)#012#12 0x00007fe8364b9c92 snmp_call_callbacks (libnetsnmp.so.35)#012#13 0x00007fe836f3601b register_with_master_agent (snmpstats.so)#012#14 0x00007fe836f37f14 run_alarm_check (snmpstats.so)#012#15 0x0000000000531cde compat_old_handler (kamailio)#012#16 0x000000000053284e slow_timer_main (kamailio)#012#17 0x000000000042e3e1 main_loop (kamailio)#012#18 0x0000000000438d14 main (kamailio)#012#19 0x00007fe841772d85 __libc_start_main (libc.so.6)#012#20 0x000000000041cfce _start (kamailio)
/usr/sbin/kamailio[32915]: CRITICAL: <core> [core/pass_fd.c:277]: receive_fd(): EOF on 24
/usr/sbin/kamailio[32858]: ALERT: <core> [main.c:774]: handle_sigs(): child process 32870 exited by a signal 6
...version: kamailio 5.6.3 (x86_64/linux) fe0c4d
flags: USE_TCP, USE_TLS, USE_SCTP, TLS_HOOKS, USE_RAW_SOCKS, DISABLE_NAGLE, USE_MCAST, DNS_IP_HACK, SHM_MMAP, PKG_MALLOC, Q_MALLOC, F_MALLOC, TLSF_MALLOC, DBG_SR_MEMORY, USE_FUTEX, FAST_LOCK-ADAPTIVE_WAIT, USE_DNS_CACHE, USE_DNS_FAILOVER, USE_NAPTR, USE_DST_BLOCKLIST, HAVE_RESOLV_RES, TLS_PTHREAD_MUTEX_SHARED
ADAPTIVE_WAIT_LOOPS 1024, MAX_RECV_BUFFER_SIZE 262144, MAX_URI_SIZE 1024, BUF_SIZE 65535, DEFAULT PKG_SIZE 8MB
poll method support: poll, epoll_lt, epoll_et, sigio_rt, select.
id: fe0c4d
compiled on 11:02:26 Jan 19 2023 with gcc 8.5.0
Same config is starting without issues on CentOS 7.
OpenSSL version is 1.1.1kKamailio is installed from official RPM repos.
Is core dump would be useful here or best to compile from source and redo?
--
Best regards,Ihor (Igor)
__________________________________________________________ Kamailio - Users Mailing List - Non Commercial Discussions To unsubscribe send an email to sr-users-leave@lists.kamailio.org Important: keep the mailing list in the recipients, do not reply only to the sender! Edit mailing list options or unsubscribe:-- Daniel-Constantin Mierla -- www.asipto.com www.twitter.com/miconda -- www.linkedin.com/in/miconda Kamailio World Conference - June 5-7, 2023 - www.kamailioworld.com
--
Best regards,Ihor (Igor)
__________________________________________________________ Kamailio - Users Mailing List - Non Commercial Discussions To unsubscribe send an email to sr-users-leave@lists.kamailio.org Important: keep the mailing list in the recipients, do not reply only to the sender! Edit mailing list options or unsubscribe:
-- Daniel-Constantin Mierla -- www.asipto.com www.twitter.com/miconda -- www.linkedin.com/in/miconda Kamailio World Conference - June 5-7, 2023 - www.kamailioworld.com