I have seen a segmentation fault on a kamailio 5.7.0
``` Core was generated by `/usr/local/sbin/kamailio -P /run/kamailio/kamailio.pid -f /usr/local/etc/kamailio/kamailio.cfg -m 512 -M 12 --atexit=no'.
Stack trace of thread 221186: #0 0x00007fbc28ffb97d n/a (libc.so.6 + 0x19d97d) #1 0x00007fbc28ed4db1 n/a (libc.so.6 + 0x76db1) #2 0x00007fbc28f7c16a n/a (libc.so.6 + 0x11e16a) #3 0x00007fbc28f7c64a syslog (libc.so.6 + 0x11e64a) #4 0x000056472e67feeb qm_debug_check_frag (kamailio + 0x358eeb) #5 0x000056472e684788 qm_free (kamailio + 0x35d788) #6 0x000056472e69054e qm_shm_free (kamailio + 0x36954e) #7 0x00007fbc24832698 ser_free (tls.so + 0x34698) #8 0x00007fbc277028e6 ERR_pop_to_mark (libcrypto.so.3 + 0x16d8e6) #9 0x00007fbc2765119e asn1_d2i_read_bio (libcrypto.so.3 + 0xbc19e) #10 0x00007fbc277fd137 n/a (libcrypto.so.3 + 0x268137) #11 0x00007fbc277ede9d n/a (libcrypto.so.3 + 0x258e9d) #12 0x00007fbc276fe62b n/a (libcrypto.so.3 + 0x16962b) #13 0x00007fbc277ef718 n/a (libcrypto.so.3 + 0x25a718) #14 0x00007fbc276fe62b n/a (libcrypto.so.3 + 0x16962b) #15 0x00007fbc276fe901 OSSL_DECODER_from_bio (libcrypto.so.3 + 0x169901) #16 0x00007fbc276fec2e OSSL_DECODER_from_data (libcrypto.so.3 + 0x169c2e) #17 0x00007fbc277d45d4 n/a (libcrypto.so.3 + 0x23f5d4) #18 0x00007fbc27663b3b n/a (libcrypto.so.3 + 0xceb3b) #19 0x00007fbc27665388 n/a (libcrypto.so.3 + 0xd0388) #20 0x00007fbc27663cb9 n/a (libcrypto.so.3 + 0xcecb9) #21 0x00007fbc27665388 n/a (libcrypto.so.3 + 0xd0388) #22 0x00007fbc27663cb9 n/a (libcrypto.so.3 + 0xcecb9) #23 0x00007fbc27664433 ASN1_item_d2i_ex (libcrypto.so.3 + 0xcf433) #24 0x00007fbc27a44f14 n/a (libssl.so.3 + 0x6df14) #25 0x00007fbc27a3aa05 n/a (libssl.so.3 + 0x63a05) #26 0x00007fbc27a3c608 n/a (libssl.so.3 + 0x65608) #27 0x00007fbc2485b216 tls_accept (tls.so + 0x5d216) #28 0x00007fbc2486686d tls_h_read_f (tls.so + 0x6886d) #29 0x000056472e6289f3 tcp_read_headers (kamailio + 0x3019f3) #30 0x000056472e631c9d tcp_read_req (kamailio + 0x30ac9d) #31 0x000056472e638c23 handle_io (kamailio + 0x311c23) #32 0x000056472e622d70 io_wait_loop_epoll (kamailio + 0x2fbd70) #33 0x000056472e63a760 tcp_receive_loop (kamailio + 0x313760) #34 0x000056472e616dde tcp_init_children (kamailio + 0x2efdde) #35 0x000056472e36565c main_loop (kamailio + 0x3e65c) #36 0x000056472e3705b0 main (kamailio + 0x495b0) #37 0x00007fbc28e87d90 n/a (libc.so.6 + 0x29d90) #38 0x00007fbc28e87e40 __libc_start_main (libc.so.6 + 0x29e40) #39 0x000056472e351805 _start (kamailio + 0x2a805)
#0 __strlen_avx2 () at ../sysdeps/x86_64/multiarch/strlen-avx2.S:74 No locals. #1 0x00007fbc28ed4db1 in __vfprintf_internal (s=s@entry=0x56472ec58a00, format=format@entry=0x56472e8246e0 "%s: %.*s%s%s%sBUG: qm: fragm. %p (address %p) beginning overwritten (%lx)! Memory allocator was called from %s:%u. Fragment marked by %s:%lu. Exec from %s:%u.\n", ap=ap@entry=0x7fff8baf6450, mode_flags=mode_flags@entry=0) at ./stdio-common/vfprintf-internal.c:1517 len = <optimized out> ptrptr = 0x20 signed_number = <optimized out> step0_jumps = {0, 2488, 2400, 1600, 1512, 2624, 1408, 2728, 2976, 1864, 2888, 608, 528, 2224, 2016, 1968, 2352, 2240, 2376, -880, 48, -96, 1776, 1688, 272, -392, -876, -960, -956, 3088, 688} space = <optimized out> is_short = <optimized out> use_outdigits = <optimized out> step1_jumps = {0, 0, 0, 0, 0, 0, 0, 0, 0, 1864, 2888, 608, 528, 2224, 2016, 1968, 2352, 2240, 2376, -880, 48, -96, 1776, 1688, 272, -392, -876, -960, -956, 0, 688} group = <optimized out> prec = -1 step2_jumps = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2888, 608, 528, 2224, 2016, 1968, 2352, 2240, 2376, -880, 48, -96, 1776, 1688, 272, -392, -876, -960, -956, 0, 688} string = 0x6e6f73616e615020 <error: Cannot access memory at address 0x6e6f73616e615020> left = <optimized out> is_long_double = <optimized out> width = <optimized out> step3a_jumps = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2544, 0, 0, 0, 2016, 1968, 2352, 2240, 2376, 0, 0, 0, 0, 1688, 0, 0, 0, 0, 0, 0, 688} alt = <optimized out> showsign = <optimized out> is_long = <optimized out> is_char = <optimized out> pad = <optimized out> step3b_jumps = {0 <repeats 11 times>, 528, 0, 0, 2016, 1968, 2352, 2240, 2376, -880, 48, -96, 1776, 1688, 272, -392, -876, 0, 0, 0, 688} step4_jumps = {0 <repeats 14 times>, 2016, 1968, 2352, 2240, 2376, -880, 48, -96, 1776, 1688, 272, -392, -876, 0, 0, 0, 688} is_negative = <optimized out> number = <optimized out> base = <optimized out> the_arg = {pa_wchar = 89781848 L'\x559f658', pa_int = 89781848, pa_long_int = 140445520361048, pa_long_long_int = 140445520361048, pa_u_int = 89781848, pa_u_long_int = 140445520361048, pa_u_long_long_int = 140445520361048, pa_double = 6.9389306722691122e-310, pa_long_double = <invalid float value>, pa_float128 = 1.67756396290316875220685842695669771e-4932, pa_string = 0x7fbc0559f658 "\355\357ͫ", pa_wstring = 0x7fbc0559f658 L"\xabcdefed", pa_pointer = 0x7fbc0559f658, pa_user = 0x7fbc0559f658} spec = 115 's' _cleanup_start_doit = <optimized out> _buffer = {__routine = 0x7fff8baf5f10, __arg = 0x56472ec58a00, __canceltype = 89781560, __prev = 0x7fbc047fe000} _cleanup_routine = <optimized out> thousands_sep = 0x0 grouping = 0xffffffffffffffff <error: Cannot access memory at address 0xffffffffffffffff> done = 238 f = 0x56472e824767 "s:%lu. Exec from %s:%u.\n" lead_str_end = 0x56472e8246e0 "%s: %.*s%s%s%sBUG: qm: fragm. %p (address %p) beginning overwritten (%lx)! Memory allocator was called from %s:%u. Fragment marked by %s:%lu. Exec from %s:%u.\n" end_of_spec = <optimized out> work_buffer = "`\355\177\004\274\177\000\000`\355\177\004\274\177\000\000P\241\207$\274\177\000\000(Xh.C\001\000\000\220\271\207$\274\177\000\000T\241\207$\274\177\000\000p\365Y\005\274\177\000\000\000\340\177\004\274\177\000\000\220\271\207$\274\177\000\000T\241\207$\274\177\000\000\000\366Y\005\274\177\000\000\000\340\177\004\274\177\000\000(\000\000\000\000\000\000\000\070\365Y\005\274\177\000\000\200\001\000\000\000\000\000\000\000\340\177\004\274\177\000\000\260\364Y\005\274\177\000\000\310\365Y\005\274\177\000\000\360\000\000\000\000\000\000\000\000\340\177\004\274\177\000\000\070\365Y\005\274\177\000\000X\366Y\005\274\177\000\000\001\000\000\000\000\000\000\000\310v\203\004\274\177\000\000\001\000\000\000\000\000\000\000"... workend = 0x7fff8baf62e8 "" ap_save = {{gp_offset = 16, fp_offset = 48, overflow_arg_area = 0x7fff8baf6530, reg_save_area = 0x7fff8baf6470}} nspecs_done = 10 save_errno = 0 readonly_format = 0 do_longlong_number = <optimized out> __result = <optimized out> #2 0x00007fbc28f7c16a in __vsyslog_internal (pri=<optimized out>, fmt=0x56472e8246e0 "%s: %.*s%s%s%sBUG: qm: fragm. %p (address %p) beginning overwritten (%lx)! Memory allocator was called from %s:%u. Fragment marked by %s:%lu. Exec from %s:%u.\n", ap=0x7fff8baf6450, mode_flags=0) at ./misc/syslog.c:229 __clframe = {__cancel_routine = <optimized out>, __cancel_arg = <synthetic pointer>, __do_it = 1, __cancel_type = <optimized out>} now_tm = {tm_sec = 51, tm_min = 27, tm_hour = 0, tm_mday = 9, tm_mon = 9, tm_year = 122, tm_wday = 0, tm_yday = 281, tm_isdst = 0, tm_gmtoff = 0, tm_zone = 0x56472ec59350 "UTC"} now = 1665275271 fd = <optimized out> --Type <RET> for more, q to quit, c to continue without paging-- f = 0x56472ec58a00 buf = 0x0 bufsize = 0 msgoff = 21 saved_errno = <optimized out> failbuf = "\000\340\177\004\274\177\000\000\030\027r\005\274\177\000\000\220M^\005\274\177\000\000\000\340\177\004\274" clarg = {buf = 0x0, oldaction = <optimized out>} #3 0x00007fbc28f7c64a in __syslog (pri=<optimized out>, fmt=<optimized out>) at ./misc/syslog.c:109 ap = {{gp_offset = 48, fp_offset = 48, overflow_arg_area = 0x7fff8baf6570, reg_save_area = 0x7fff8baf6470}} #4 0x000056472e67feeb in qm_debug_check_frag (qm=0x7fbc047fe000, f=0x7fbc0562dfe8, file=0x7fbc2487a154 "tls: tls_init.c", line=323, efile=0x56472e824559 "core/mem/q_malloc.c", eline=511) at core/mem/q_malloc.c:123 __llevel = -3 p = 0x56472e824554 __func__ = "qm_debug_check_frag" #5 0x000056472e684788 in qm_free (qmp=0x7fbc047fe000, p=0x7fbc0562e020, file=0x7fbc2487a154 "tls: tls_init.c", func=0x7fbc2487b990 <__func__.0> "ser_free", line=323, mname=0x7fbc2487a150 "tls") at core/mem/q_malloc.c:511 qm = 0x7fbc047fe000 f = 0x7fbc0562dfe8 size = 1016 next = 0x7fbc0562e1e8 prev = 0x7fbc0562e0f8 __func__ = "qm_free" #6 0x000056472e69054e in qm_shm_free (qmp=0x7fbc047fe000, p=0x7fbc0562e020, file=0x7fbc2487a154 "tls: tls_init.c", func=0x7fbc2487b990 <__func__.0> "ser_free", line=323, mname=0x7fbc2487a150 "tls") at core/mem/q_malloc.c:1350 No locals. #7 0x00007fbc24832698 in ser_free (ptr=0x7fbc0562e020, fname=0x7fbc278ce6e5 "../crypto/err/err_local.h", fline=86) at tls_init.c:323 __func__ = "ser_free" #8 0x00007fbc277028e6 in ERR_pop_to_mark () from /lib/x86_64-linux-gnu/libcrypto.so.3 No symbol table info available. #9 0x00007fbc2765119e in asn1_d2i_read_bio () from /lib/x86_64-linux-gnu/libcrypto.so.3 No symbol table info available. #10 0x00007fbc277fd137 in ?? () from /lib/x86_64-linux-gnu/libcrypto.so.3 No symbol table info available. #11 0x00007fbc277ede9d in ?? () from /lib/x86_64-linux-gnu/libcrypto.so.3 No symbol table info available. #12 0x00007fbc276fe62b in ?? () from /lib/x86_64-linux-gnu/libcrypto.so.3 No symbol table info available. #13 0x00007fbc277ef718 in ?? () from /lib/x86_64-linux-gnu/libcrypto.so.3 No symbol table info available. #14 0x00007fbc276fe62b in ?? () from /lib/x86_64-linux-gnu/libcrypto.so.3 No symbol table info available. #15 0x00007fbc276fe901 in OSSL_DECODER_from_bio () from /lib/x86_64-linux-gnu/libcrypto.so.3 No symbol table info available. #16 0x00007fbc276fec2e in OSSL_DECODER_from_data () from /lib/x86_64-linux-gnu/libcrypto.so.3 No symbol table info available. #17 0x00007fbc277d45d4 in ?? () from /lib/x86_64-linux-gnu/libcrypto.so.3 No symbol table info available. #18 0x00007fbc27663b3b in ?? () from /lib/x86_64-linux-gnu/libcrypto.so.3 No symbol table info available. #19 0x00007fbc27665388 in ?? () from /lib/x86_64-linux-gnu/libcrypto.so.3 No symbol table info available. #20 0x00007fbc27663cb9 in ?? () from /lib/x86_64-linux-gnu/libcrypto.so.3 No symbol table info available. #21 0x00007fbc27665388 in ?? () from /lib/x86_64-linux-gnu/libcrypto.so.3 No symbol table info available. #22 0x00007fbc27663cb9 in ?? () from /lib/x86_64-linux-gnu/libcrypto.so.3 No symbol table info available. #23 0x00007fbc27664433 in ASN1_item_d2i_ex () from /lib/x86_64-linux-gnu/libcrypto.so.3 No symbol table info available. #24 0x00007fbc27a44f14 in ?? () from /lib/x86_64-linux-gnu/libssl.so.3 No symbol table info available. #25 0x00007fbc27a3aa05 in ?? () from /lib/x86_64-linux-gnu/libssl.so.3 No symbol table info available. #26 0x00007fbc27a3c608 in ?? () from /lib/x86_64-linux-gnu/libssl.so.3 No symbol table info available. --Type <RET> for more, q to quit, c to continue without paging-- #27 0x00007fbc2485b216 in tls_accept (c=0x7fbc056d3ef0, error=0x7fff8baf7994) at tls_server.c:468 ret = -1951434416 ssl = 0x7fbc056d82f8 cert = 0x7fbc056d2388 tls_c = 0x7fbc056d1fa0 tls_log = 612737204 __func__ = "tls_accept" pkey = 0x0 #28 0x00007fbc2486686d in tls_h_read_f (c=0x7fbc056d3ef0, flags=0x7fff8bb17e9c) at tls_server.c:1173 r = 0x7fbc056d4018 bytes_free = 16383 bytes_read = 1713 read_size = 16383 ssl_error = 0 ssl_read = 0 ssl = 0x7fbc056d82f8 rd_buf = "\026\003\003\004\071\v\000\004\065\000\004\062\000\004/0\202\004+0\202\003\023\240\003\002\001\002\002\t\000\267\207\251\066\225\277\372\335\060\r\006\t*\206H\206\367\r\001\001\v\005\000\060\201\217\061\v0\t\006\003U\004\006\023\002JP1\020\060\016\006\003U\004\b\f\aFukuoka1-0+\006\003U\004\n\f$ Panasonic System Networks co., Ltd.1?0=\006\003U\004\003\f6SIP Phone Client Authentication Self CA Certificate V2"... wr_buf = "\025\003\003\000\002\002\060\000-\003\003\210z\357o\213$\017\235\231\334~\207\233\365\026\261\307~lV\214A\277\337DOWNGRD\001\000\000=\000\000\005\377\001\000\001\000\026\003\003\026\035\v\000\026\031\000\026\026\000\006\070\060\202\006\064\060\202\005\034\240\003\002\001\002\002\020&~b\352RT\000\227\236Dq\264RL\271\304\060\r\006\t*\206H\206\367\r\001\001\v\005\000\060\201\217\061\v0\t\006\003U\004\006\023\002GB1\033\060\031\006\003U\004\b\023\022Greater Manchester1\020\060\016\006\003U\004\a\023\aSalford1\030\060\026\006\003U\004\n\023\017Sectigo Lim"... rd = {buf = 0x7fff8baf7af0 "\026\003\003\004\071\v", pos = 1086, used = 1713, size = 65536} wr = {buf = 0x7fff8bb07af0 "\025\003\003", pos = 0, used = 0, size = 65536} tls_c = 0x7fbc056d1fa0 enc_rd_buf = 0x0 n = 0 flush_flags = 0 err_src = 0x7fbc248839b0 "TLS read:" ip_buf = "10.20.0.100", '\000' <repeats 52 times> x = 0 tls_dbg = 0 __func__ = "tls_h_read_f" #29 0x000056472e6289f3 in tcp_read_headers (c=0x7fbc056d3ef0, read_flags=0x7fff8bb17e9c) at core/tcp_read.c:441 bytes = -1 remaining = 0 p = 0x7fbc054ebd60 "\002" r = 0x7fbc056d4018 mc = 2638013693 body_len = 50944 mfline = 0x7fbc054ec0e8 "REGISTER sip:voip.isip.ca:443 SIP/2.0\r\nv: SIP/2.0/TLS 89.28.64.156:52319;rport;branch=z9hG4bKPjr.w8BMId-6xgyjA5coZyVJeC2QNp6cni;alias\r\nMax-Forwards: 70\r\nf: sip:1217264@voip.isip.ca;tag=Dt75LCcpfD8Oe"... mtransid = {s = 0xffff00000000 <error: Cannot access memory at address 0xffff00000000>, len = 0} __func__ = "tcp_read_headers" #30 0x000056472e631c9d in tcp_read_req (con=0x7fbc056d3ef0, bytes_read=0x7fff8bb17e94, read_flags=0x7fff8bb17e9c) at core/tcp_read.c:1469 bytes = -1 total_bytes = 0 resp = 1 size = 2 req = 0x7fbc056d4018 dst = {send_sock = 0x7fff8bb17e10, to = {s = {sa_family = 27615, sa_data = "E.\001\000\000\000\300}\261\213\377\177\000"}, sin = {sin_family = 27615, sin_port = 11845, sin_addr = {s_addr = 1}, sin_zero = "\300}\261\213\377\177\000"}, sin6 = {sin6_family = 27615, sin6_port = 11845, sin6_flowinfo = 1, sin6_addr = {__in6_u = {__u6_addr8 = "\300}\261\213\377\177\000\000\030\000\000\000Q\000\000", __u6_addr16 = { 32192, 35761, 32767, 0, 24, 0, 81, 0}, __u6_addr32 = {2343665088, 32767, 24, 81}}}, sin6_scope_id = 4294967294}, sas = {ss_family = 27615, __ss_padding = "E.\001\000\000\000\300}\261\213\377\177\000\000\030\000\000\000Q\000\000\000\376\377\377\377\377\377\377\377\210\256p\005\274\177\000\000 \260p\005\274\177\000\000@{\221.GV\000\000p~\261\213\001 \000\000\340\204I(\274\177\000\000\320}\261\213\001\000\000\000\020\200\203\004Q\000\000\000\001\000\000\000\000\000\000\000H\275N\005\274\177\000\000\340~\261\213\377\177\000\000\b\000\000\000\000\000\000", __ss_align = 140445521849992}}, id = -2, send_flags = {f = 65535, blst_imask = 65535}, proto = 36 '$', proto_pad0 = 0 '\000', proto_pad1 = 0} c = 46 '.' ret = 778163298 __func__ = "tcp_read_req" #31 0x000056472e638c23 in handle_io (fm=0x7fbc284984e0, events=1, idx=-1) at core/tcp_read.c:1855 ret = -1 n = 0 read_flags = RD_CONN_SHORT_READ --Type <RET> for more, q to quit, c to continue without paging-- con = 0x7fbc056d3ef0 s = 36 resp = -2 t = 359788347 ee = 0x0 __func__ = "handle_io" #32 0x000056472e622d70 in io_wait_loop_epoll (h=0x56472e917980 <io_w>, t=2, repeat=0) at core/io_wait.h:1070 n = 1 r = 0 fm = 0x7fbc284984e0 revents = 1 __func__ = "io_wait_loop_epoll" #33 0x000056472e63a760 in tcp_receive_loop (unix_sock=81) at core/tcp_read.c:1976 __func__ = "tcp_receive_loop" #34 0x000056472e616dde in tcp_init_children (woneinit=0x7fff8bb18288) at core/tcp_main.c:5236 r = 0 i = 7 reader_fd_1 = 81 pid = 0 si_desc = "tcp receiver (generic)\000\000\317jZ.GV\000\000`\202\261\213\377\177\000\000\036Xd.\000\000\000\000\340\345.(\000\000\000\000\253\066\200.GV\000\000\250\000\261\213\377\177\000\000\f\332v.GV\000\000\000\000\000\000\000\000\000\000\020\200\203\004\274\177\000\000\067\000\000\000\000\000\000\000\320\066\302\004\001\000\000\000P\202\261\213\377\177\000\000\036\334v.GV\000" si = 0x0 __func__ = "tcp_init_children" #35 0x000056472e36565c in main_loop () at main.c:1851 i = 8 pid = 221167 si = 0x0 si_desc = "udp receiver child=7 sock=10.20.20.100:5060\000\063.124.220.146:5080)\000X7\211.GV\000\000\210`{.GV\000\000\000\000\000\000\000\000\000\000s\314|.GV\000\000\067\000\000\000\000\000\000\000\260\305\367(\274\177\000\000 \204\261\213\377\177\000\000z\271Q.GV\000" nrprocs = 8 woneinit = 1 __func__ = "main_loop" #36 0x000056472e3705b0 in main (argc=10, argv=0x7fff8bb18998) at main.c:3085 cfg_stream = 0x56472eb33380 c = -1 r = 0 tmp = 0x7fff8bb18e39 "" tmp_len = 0 port = 0 proto = 0 ahost = 0x0 aport = 0 options = 0x56472e7b9278 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:x:X:Y:" ret = -1 seed = 3559158303 rfd = 4 debug_save = 0 debug_flag = 0 dont_fork_cnt = 0 n_lst = 0x7fff8bb18998 p = 0x7fff8bb18910 "" st = {st_dev = 26, st_ino = 85318, st_nlink = 2, st_mode = 16888, st_uid = 114, st_gid = 121, __pad0 = 0, st_rdev = 0, st_size = 40, st_blksize = 4096, st_blocks = 0, st_atim = {tv_sec = 1665275250, tv_nsec = 283708644}, st_mtim = {tv_sec = 1665275250, tv_nsec = 283708644}, st_ctim = {tv_sec = 1665275250, tv_nsec = 283708644}, __glibc_reserved = {0, 0, 0}} tbuf = "@\003\000\000@\003\000\000@\003\000\000@\003\000\000@\003\000\000@\003\000\000@\003\000\000@\003", '\000' <repeats 50 times>, "\377\000\000\000\377\000\000\000\000\000\377\000\000\000\000\377", '/' <repeats 16 times>, "\230\r", '\000' <repeats 14 times>, "`", '\000' <repeats 15 times>, "\001", '\000' <repeats 145 times>... option_index = 12 long_options = {{name = 0x56472e7bb6b6 "help", has_arg = 0, flag = 0x0, val = 104}, {name = 0x56472e7b6521 "version", has_arg = 0, flag = 0x0, val = 118}, {name = 0x56472e7bb6bb "alias", has_arg = 1, flag = 0x0, val = 1024}, { name = 0x56472e7bb6c1 "subst", has_arg = 1, flag = 0x0, val = 1025}, {name = 0x56472e7bb6c7 "substdef", has_arg = 1, flag = 0x0, val = 1026}, {name = 0x56472e7bb6d0 "substdefs", has_arg = 1, flag = 0x0, val = 1027}, { name = 0x56472e7bb6da "server-id", has_arg = 1, flag = 0x0, val = 1028}, {name = 0x56472e7bb6e4 "loadmodule", has_arg = 1, flag = 0x0, val = 1029}, {name = 0x56472e7bb6ef "modparam", has_arg = 1, flag = 0x0, val = 1030}, { name = 0x56472e7bb6f8 "log-engine", has_arg = 1, flag = 0x0, val = 1031}, {name = 0x56472e7bb703 "debug", has_arg = 1, flag = 0x0, val = 1032}, {name = 0x56472e7bb709 "cfg-print", has_arg = 0, flag = 0x0, val = 1033}, { name = 0x56472e7bb713 "atexit", has_arg = 1, flag = 0x0, val = 1034}, {name = 0x56472e7bb71a "all-errors", has_arg = 0, flag = 0x0, val = 1035}, {name = 0x0, has_arg = 0, flag = 0x0, val = 0}} __func__ = "main" (gdb) (gdb) info locals No locals. (gdb) list 69 in ../sysdeps/x86_64/multiarch/strlen-avx2.S (gdb)
```
### Additional Information
* version: kamailio 5.7.0-dev0 (x86_64/linux) da0178 flags: USE_TCP, USE_TLS, USE_SCTP, TLS_HOOKS, USE_RAW_SOCKS, DISABLE_NAGLE, USE_MCAST, DNS_IP_HACK, SHM_MMAP, PKG_MALLOC, Q_MALLOC, F_MALLOC, TLSF_MALLOC, DBG_SR_MEMORY, USE_FUTEX, FAST_LOCK-ADAPTIVE_WAIT, USE_DNS_CACHE, USE_DNS_FAILOVER, USE_NAPTR, USE_DST_BLOCKLIST, HAVE_RESOLV_RES, TLS_PTHREAD_MUTEX_SHARED ADAPTIVE_WAIT_LOOPS 1024, MAX_RECV_BUFFER_SIZE 262144, MAX_URI_SIZE 1024, BUF_SIZE 65535, DEFAULT PKG_SIZE 8MB poll method support: poll, epoll_lt, epoll_et, sigio_rt, select. id: da0178 compiled on 19:15:18 Aug 9 2022 with gcc 11.2.0
```
* **Operating System**: ``` Distributor ID: Ubuntu Description: Ubuntu 22.04.1 LTS Release: 22.04 Codename: jammy
Linux kamailio-prod 5.15.0-1019-aws #23-Ubuntu SMP Wed Aug 17 18:33:13 UTC 2022 x86_64 x86_64 x86_64 GNU/Linux ```
Hello, thanks for the report. Please note that the development version is not meant for productive use, as new code gets introduced that is work in progress and can crash for that reason. Does it happen randomly or can be reproduced with a special workload?
It happen randomly when there was a lot of TLS clients
It seems to be a case of using libssl 3.0, which is somehow known to not be supported because of other similar reports, like:
- https://github.com/kamailio/kamailio/issues/3222#issuecomment-1222104843
Same problem on kamailio 5.6.2:
``` Stack trace of thread 3249: #0 0x00007f1ad021aa7c __pthread_kill_implementation (libc.so.6 + 0x96a7c) #1 0x00007f1ad01c6476 __GI_raise (libc.so.6 + 0x42476) #2 0x00007f1ad01ac7f3 __GI_abort (libc.so.6 + 0x287f3) #3 0x0000555930b2b4a3 qm_debug_check_frag (kamailio + 0x3594a3) #4 0x0000555930b2fd19 qm_free (kamailio + 0x35dd19) #5 0x0000555930b3badf qm_shm_free (kamailio + 0x369adf) #6 0x00007f1acbb66698 ser_free (tls.so + 0x34698) #7 0x00007f1acea27fe8 ERR_clear_error (libcrypto.so.3 + 0x16cfe8) #8 0x00007f1aced6214a n/a (libssl.so.3 + 0x6514a) #9 0x00007f1acbb8f216 tls_accept (tls.so + 0x5d216) #10 0x00007f1acbb9a86d tls_h_read_f (tls.so + 0x6886d) #11 0x0000555930ad4241 tcp_read_headers (kamailio + 0x302241) #12 0x0000555930add4eb tcp_read_req (kamailio + 0x30b4eb) #13 0x0000555930ae2f7a handle_io (kamailio + 0x310f7a) #14 0x0000555930ace5be io_wait_loop_epoll (kamailio + 0x2fc5be) #15 0x0000555930ae5fae tcp_receive_loop (kamailio + 0x313fae) #16 0x0000555930ac263c tcp_init_children (kamailio + 0x2f063c) #17 0x000055593081065c main_loop (kamailio + 0x3e65c) #18 0x000055593081b5cd main (kamailio + 0x495cd) #19 0x00007f1ad01add90 __libc_start_call_main (libc.so.6 + 0x29d90) #20 0x00007f1ad01ade40 __libc_start_main_impl (libc.so.6 + 0x29e40) #21 0x00005559307fc805 _start (kamailio + 0x2a805)
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1". Core was generated by `/usr/local/sbin/kamailio -P /run/kamailio/kamailio.pid -f /usr/local/etc/kamail'. Program terminated with signal SIGABRT, Aborted. #0 __pthread_kill_implementation (no_tid=0, signo=6, threadid=139753137116992) at ./nptl/pthread_kill.c:44 44 ./nptl/pthread_kill.c: No such file or directory. (gdb) bt full #0 __pthread_kill_implementation (no_tid=0, signo=6, threadid=139753137116992) at ./nptl/pthread_kill.c:44 tid = <optimized out> ret = 0 pd = 0x7f1ad0181740 old_mask = {__val = {523986010121, 1236950581248, 1, 139752526756840, 93841567167696, 818738900, 6399889808, 139752526756840, 140731003343248, 139752525920989, 93841559189208, 139752526756840, 93841559189418, 5113706410, 140731003343312, 139752525963041}} ret = <optimized out> pd = <optimized out> old_mask = <optimized out> ret = <optimized out> tid = <optimized out> ret = <optimized out> resultvar = <optimized out> resultvar = <optimized out> __arg3 = <optimized out> __arg2 = <optimized out> __arg1 = <optimized out> _a3 = <optimized out> _a2 = <optimized out> _a1 = <optimized out> __futex = <optimized out> resultvar = <optimized out> __arg3 = <optimized out> __arg2 = <optimized out> __arg1 = <optimized out> _a3 = <optimized out> _a2 = <optimized out> _a1 = <optimized out> __futex = <optimized out> __private = <optimized out> __oldval = <optimized out> result = <optimized out> #1 __pthread_kill_internal (signo=6, threadid=139753137116992) at ./nptl/pthread_kill.c:78 No locals. #2 __GI___pthread_kill (threadid=139753137116992, signo=signo@entry=6) at ./nptl/pthread_kill.c:89 No locals. #3 0x00007f1ad01c6476 in __GI_raise (sig=sig@entry=6) at ../sysdeps/posix/raise.c:26 ret = <optimized out> #4 0x00007f1ad01ac7f3 in __GI_abort () at ./stdlib/abort.c:79 save_stage = 1 act = {__sigaction_handler = {sa_handler = 0x100000000, sa_sigaction = 0x100000000}, sa_mask = {__val = {139752526756840, 140731003343536, 139752525920989, 139752526755360, 139752526520320, 1, 4294967297, 140731003343600, 139752525963041, 139753063899472, 93841557468284, 93841559189184, 93841559189592, 139752545381128, 4792273281145066240, 0}}, sa_flags = 818739114, sa_restorer = 0x0} sigs = {__val = {32, 4792273281145066240, 140731003343648, 93841557521914, 752, 139752526520320, 93841558830132, 0, 93841559189418, 139752526520320, 23, 21474838579, 12, 139752526756840, 36, 2880643072}} #5 0x0000555930b2b4a3 in qm_debug_check_frag (qm=0x7f1aabb32000, f=0x7f1aacd2eb08, file=0x7f1acbbae154 "tls: tls_init.c", line=323, efile=0x555930ccf6d9 "core/mem/q_malloc.c", eline=511) at core/mem/q_malloc.c:129 p = 0x7f1aabb6bbe8 __func__ = "qm_debug_check_frag" #6 0x0000555930b2fd19 in qm_free (qmp=0x7f1aabb32000, p=0x7f1aacd2eb40, file=0x7f1acbbae154 "tls: tls_init.c", func=0x7f1acbbaf990 <__func__.0> "ser_free", line=323, mname=0x7f1acbbae150 "tls") at core/mem/q_malloc.c:511 qm = 0x7f1aabb32000 f = 0x7f1aacd2eb08 size = 896 next = 0x7f1aacd2eb08 prev = 0x7f1aacd2e720 __func__ = "qm_free" #7 0x0000555930b3badf in qm_shm_free (qmp=0x7f1aabb32000, p=0x7f1aacd2eb40, file=0x7f1acbbae154 "tls: tls_init.c", func=0x7f1acbbaf990 <__func__.0> "ser_free", line=323, mname=0x7f1acbbae150 "tls") at core/mem/q_malloc.c:1350 No locals. #8 0x00007f1acbb66698 in ser_free (ptr=0x7f1aacd2eb40, fname=0x7f1acebf46e5 "../crypto/err/err_local.h", fline=88) at tls_init.c:323 __func__ = "ser_free" #9 0x00007f1acea27fe8 in ERR_clear_error () from /lib/x86_64-linux-gnu/libcrypto.so.3 No symbol table info available. #10 0x00007f1aced6214a in ?? () from /lib/x86_64-linux-gnu/libssl.so.3 No symbol table info available. #11 0x00007f1acbb8f216 in tls_accept (c=0x7f1aacf042d0, error=0x7ffe7d7697c4) at tls_server.c:468 --Type <RET> for more, q to quit, c to continue without paging-- ret = 2104924032 ssl = 0x7f1aaccf7210 cert = 0x7f1aacae6bc8 tls_c = 0x7f1aacf13a58 tls_log = -877076300 __func__ = "tls_accept" pkey = 0x0 #12 0x00007f1acbb9a86d in tls_h_read_f (c=0x7f1aacf042d0, flags=0x7ffe7d789ccc) at tls_server.c:1173 r = 0x7f1aacf043f8 bytes_free = 16383 bytes_read = 227 read_size = 16383 ssl_error = 0 ssl_read = 0 ssl = 0x7f1aaccf7210 rd_buf = "\026\003\001\000\336\001\000\000\332\003\001\271\264颗\017\004\233C/\036<\225\027\206\215\tG\"@\214Kp\341\027\374\317\066\f\024\000\000\000h\300\024\300\n\300"\300!\000\071\000\070\000\210\000\207\300\017\300\005\000\065\000\204\300\022\300\b\300\034\300\033\000\026\000\023\300\r\300\003\000\n\300\023\300\t\300\037\300\036\000\063\000\062\000\232\000\231\000E\000D\300\016\300\004\000/\000\226\000A\000\a\300\021\300\a\300\f\300\002\000\005\000\004\000\025\000\022\000\t\000\024\000\021\000\b\000\006\000\003\000\377\001\000\000I\000\v\000\004\003\000\001\002\000\n\000\064\000\062\000\016\000\r\000\031\000\v\000\f\000\030\000\t\000\n\000\026\000\027\000\b\000\006\000\a\000\024\000\025\000\004\000\005"... wr_buf = "\025\003\001\000\002\002F\000\306\000\000\034 \000\300\067\204\237t';\252^?\254\312\066|\bkȟ\337\360\204{%>&»\nw\373݉\206L-Z\362\200\001\325.0O\353\317\r͞\333\366\032\342\001t\\215\361\006{?f\027`\204\342\016.\355\263\257\376\211N\331\aL\246\032\346\240\317@V\216\324\337\063\222\212\273\353M\244\204\026T\204\277\334\177B(\260pA\035z\037$ݖ\022\244ؠ%\313w\272d\245\315\372\365\026\034\222\nue.\256\023\215\214ڲ\245V\241\247o\237\376n\027?\304poVo\002\221X\267\227\265\210\247@\347BÇ\260\026a\377e\330\\314w,"\027\070\317S\235\247X\361r\r!LO\306>\026\362\024"... rd = {buf = 0x7ffe7d769920 "\026\003\001", pos = 0, used = 227, size = 65536} wr = {buf = 0x7ffe7d779920 "\025\003\001", pos = 0, used = 0, size = 65536} tls_c = 0x7f1aacf13a58 enc_rd_buf = 0x0 n = 0 flush_flags = 0 err_src = 0x7f1acbbb79a0 "TLS read:" ip_buf = "10.20.0.100", '\000' <repeats 52 times> x = 0 tls_dbg = 0 __func__ = "tls_h_read_f" #13 0x0000555930ad4241 in tcp_read_headers (c=0x7f1aacf042d0, read_flags=0x7ffe7d789ccc) at core/tcp_read.c:441 bytes = 0 remaining = 0 p = 0x7f1aacd3aae0 "\002" r = 0x7f1aacf043f8 mc = 0 body_len = 0 mfline = 0x7f1aacd3ae68 "POST /RPC HTTP/1.1\r\nHost: 127.0.0.1:8080\r\nUser-Agent: python-requests/2.25.1\r\nAccept-Encoding: gzip, deflate\r\nAccept: */*\r\nConnection: keep-alive\r\nContent-Length: 38\r\n\r\n{"jsonrpc":"2.0", "method": "ul"... mtransid = {s = 0xffff30cbfa24 <error: Cannot access memory at address 0xffff30cbfa24>, len = 0} __func__ = "tcp_read_headers" #14 0x0000555930add4eb in tcp_read_req (con=0x7f1aacf042d0, bytes_read=0x7ffe7d789cc8, read_flags=0x7ffe7d789ccc) at core/tcp_read.c:1469 bytes = -1 total_bytes = 0 resp = 1 size = 408021893128 req = 0x7f1aacf043f8 dst = {send_sock = 0x1, to = {s = {sa_family = 48104, sa_data = "\266\253\032\177\000\000\360\233x}\376\177\000"}, sin = {sin_family = 48104, sin_port = 43958, sin_addr = {s_addr = 32538}, sin_zero = "\360\233x}\376\177\000"}, sin6 = {sin6_family = 48104, sin6_port = 43958, sin6_flowinfo = 32538, sin6_addr = {__in6_u = {__u6_addr8 = "\360\233x}\376\177\000\000\030\000\000\000\000\000\000", __u6_addr16 = {39920, 32120, 32766, 0, 24, 0, 0, 0}, __u6_addr32 = {2105056240, 32766, 24, 0}}}, sin6_scope_id = 2105056208}, sas = {ss_family = 48104, __ss_padding = "\266\253\032\177\000\000\360\233x}\376\177\000\000\030\000\000\000\000\000\000\000Лx}\001\000\000\000軶\253\032\177\000\000Лx}\376\177\000\000\335\372\251\253\032\177\000\000$\372\313\060YU\000\000軶\253\032\177\000\000\002\000\000\000\000\000\000\000\002\000\000\000\001\000\000\000\020\234x}\376\177\000\000!\237\252\253\032\177\000\000\020\235x}\376\177\000\000\064\234x}\376\177\000", __ss_align = 17179869204}}, id = 818674208, send_flags = {f = 21849, blst_imask = 0}, proto = 110 'n', proto_pad0 = -11 '\365', proto_pad1 = 28435} c = -85 '\253' ret = 2105056068 __func__ = "tcp_read_req" #15 0x0000555930ae2f7a in handle_io (fm=0x7f1acf7e70a0, events=1, idx=-1) at core/tcp_read.c:1780 ret = 8 n = 8 --Type <RET> for more, q to quit, c to continue without paging-- read_flags = RD_CONN_SHORT_READ con = 0x7f1aacf042d0 s = 35 resp = -2 t = 589705696 ee = 0x0 __func__ = "handle_io" #16 0x0000555930ace5be in io_wait_loop_epoll (h=0x555930dc18a0 <io_w>, t=2, repeat=0) at core/io_wait.h:1070 n = 1 r = 0 fm = 0x7f1acf7e70a0 revents = 1 __func__ = "io_wait_loop_epoll" #17 0x0000555930ae5fae in tcp_receive_loop (unix_sock=95) at core/tcp_read.c:1976 __func__ = "tcp_receive_loop" #18 0x0000555930ac263c in tcp_init_children (woneinit=0x7ffe7d78a0b8) at core/tcp_main.c:5227 r = 7 i = 7 reader_fd_1 = 95 pid = 0 si_desc = "tcp receiver (generic)\000\000\020'\245\060YU\000\000\220\240x}\376\177\000\000l\020\257\060\000\000\000\000\060Ha\317\000\000\000\000\033\351\312\060YU\000\000\250\000x}\376\177\000\000@\217\301\060YU\000\000\000\000\000\000\000\000\000\000\bŶ\253\032\177\000\000\067\000\000\000\000\000\000\000hn\365\253\001\000\000\000\200\240x}\376\177\000\000R\221\301\060YU\000" si = 0x0 __func__ = "tcp_init_children" #19 0x000055593081065c in main_loop () at main.c:1849 i = 8 pid = 3223 si = 0x0 si_desc = "udp receiver child=7 sock=10.20.20.100:5060\000\0653.14.220.146:5080)\000X\327\323\060YU\000\000\210\020\306\060YU\000\000\000\000\000\000\000\000\000\000S|\307\060YU\000\000\067\000\000\000\000\000\000\000\260%*\320\032\177\000\000P\242x}\376\177\000\000\315z\234\060YU\000" nrprocs = 8 woneinit = 1 __func__ = "main_loop" #20 0x000055593081b5cd in main (argc=10, argv=0x7ffe7d78a7c8) at main.c:3078 cfg_stream = 0x555931347380 c = -1 r = 0 tmp = 0x7ffe7d78ae3d "" tmp_len = 0 port = 1 proto = -800478768 ahost = 0x0 aport = 0 options = 0x555930c64268 ":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:x:X:Y:" ret = -1 seed = 3102110895 rfd = 4 debug_save = 0 debug_flag = 0 dont_fork_cnt = 0 n_lst = 0x7ffe7d78a740 p = 0x7f1ad04b7680 <_dl_audit_preinit> "\363\017\036\372\213\005\356\347\001" st = {st_dev = 26, st_ino = 949, st_nlink = 2, st_mode = 16888, st_uid = 114, st_gid = 121, __pad0 = 0, st_rdev = 0, st_size = 40, st_blksize = 4096, st_blocks = 0, st_atim = {tv_sec = 1665949927, tv_nsec = 977110727}, st_mtim = {tv_sec = 1665949927, tv_nsec = 977110727}, st_ctim = {tv_sec = 1665949927, tv_nsec = 977110727}, __glibc_reserved = {0, 0, 0}} tbuf = "@\003\000\000@\003\000\000@\003\000\000@\003\000\000@\003\000\000@\003\000\000@\003\000\000@\003", '\000' <repeats 11 times>, "\001", '\000' <repeats 54 times>, "\377\000\000\000\377\000\000\000\000\000\377\000\000\000\000\377", '/' <repeats 16 times>, "\230\r", '\000' <repeats 14 times>, "`", '\000' <repeats 15 times>, "\001", '\000' <repeats 143 times>... option_index = 12 long_options = {{name = 0x555930c66696 "help", has_arg = 0, flag = 0x0, val = 104}, {name = 0x555930c61514 "version", has_arg = 0, flag = 0x0, val = 118}, {name = 0x555930c6669b "alias", has_arg = 1, flag = 0x0, val = 1024}, { name = 0x555930c666a1 "subst", has_arg = 1, flag = 0x0, val = 1025}, {name = 0x555930c666a7 "substdef", has_arg = 1, flag = 0x0, val = 1026}, {name = 0x555930c666b0 "substdefs", has_arg = 1, flag = 0x0, val = 1027}, { name = 0x555930c666ba "server-id", has_arg = 1, flag = 0x0, val = 1028}, {name = 0x555930c666c4 "loadmodule", has_arg = 1, flag = 0x0, val = 1029}, {name = 0x555930c666cf "modparam", has_arg = 1, flag = 0x0, val = 1030}, { name = 0x555930c666d8 "log-engine", has_arg = 1, flag = 0x0, val = 1031}, {name = 0x555930c666e3 "debug", has_arg = 1, flag = 0x0, val = 1032}, {name = 0x555930c666e9 "cfg-print", has_arg = 0, flag = 0x0, val = 1033}, { name = 0x555930c666f3 "atexit", has_arg = 1, flag = 0x0, val = 1034}, {name = 0x0, has_arg = 0, flag = 0x0, val = 0}} --Type <RET> for more, q to quit, c to continue without paging-- __func__ = "main" (gdb) info locals tid = <optimized out> ret = 0 pd = 0x7f1ad0181740 old_mask = {__val = {523986010121, 1236950581248, 1, 139752526756840, 93841567167696, 818738900, 6399889808, 139752526756840, 140731003343248, 139752525920989, 93841559189208, 139752526756840, 93841559189418, 5113706410, 140731003343312, 139752525963041}} ret = <optimized out> pd = <optimized out> old_mask = <optimized out> ret = <optimized out> tid = <optimized out> ret = <optimized out> resultvar = <optimized out> resultvar = <optimized out> __arg3 = <optimized out> __arg2 = <optimized out> __arg1 = <optimized out> _a3 = <optimized out> _a2 = <optimized out> _a1 = <optimized out> __futex = <optimized out> resultvar = <optimized out> __arg3 = <optimized out> __arg2 = <optimized out> __arg1 = <optimized out> _a3 = <optimized out> _a2 = <optimized out> _a1 = <optimized out> __futex = <optimized out> __private = <optimized out> __oldval = <optimized out> result = <optimized out> (gdb) list 39 in ./nptl/pthread_kill.c ```
### Additional Information ``` version: kamailio 5.6.2 (x86_64/linux) 54a9c1 flags: USE_TCP, USE_TLS, USE_SCTP, TLS_HOOKS, USE_RAW_SOCKS, DISABLE_NAGLE, USE_MCAST, DNS_IP_HACK, SHM_MMAP, PKG_MALLOC, Q_MALLOC, F_MALLOC, TLSF_MALLOC, DBG_SR_MEMORY, USE_FUTEX, FAST_LOCK-ADAPTIVE_WAIT, USE_DNS_CACHE, USE_DNS_FAILOVER, USE_NAPTR, USE_DST_BLOCKLIST, HAVE_RESOLV_RES, TLS_PTHREAD_MUTEX_SHARED ADAPTIVE_WAIT_LOOPS 1024, MAX_RECV_BUFFER_SIZE 262144, MAX_URI_SIZE 1024, BUF_SIZE 65535, DEFAULT PKG_SIZE 8MB poll method support: poll, epoll_lt, epoll_et, sigio_rt, select. id: 54a9c1 compiled on 18:59:28 Oct 13 2022 with gcc 11.2.0 ```
@admin-toneca can you please try master now? We introduced some changes related to openssl 3.0 recently
Closing this one as master and 5.7 branches have new code related to support of libssl 3.0, if there are problems that pop up now, then new issues should be created.
Closed #3259 as completed.