Unfortunately that didn't fix the problem, however the core dump has changed.
``` Reading symbols from /usr/local/kamailio/sbin/kamailio...done. [New LWP 26088] [Thread debugging using libthread_db enabled] Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1". Core was generated by `/usr/local/kamailio/sbin/kamailio -P /var/run/kamailio/kamailio.pid -f /usr/loc'. Program terminated with signal SIGSEGV, Segmentation fault. #0 aesni_ecb_encrypt () at crypto/aes/aesni-x86_64.s:624 624 movups (%rcx),%xmm1 ```
``` (gdb) disas aesni_ecb_encrypt Dump of assembler code for function aesni_ecb_encrypt: 0x00007ffba23869e0 <+0>: and $0xfffffffffffffff0,%rdx 0x00007ffba23869e4 <+4>: je 0x7ffba2386f34 <aesni_ecb_encrypt+1364> 0x00007ffba23869ea <+10>: mov 0xf0(%rcx),%eax 0x00007ffba23869f0 <+16>: movups (%rcx),%xmm0 0x00007ffba23869f3 <+19>: mov %rcx,%r11 0x00007ffba23869f6 <+22>: mov %eax,%r10d 0x00007ffba23869f9 <+25>: test %r8d,%r8d 0x00007ffba23869fc <+28>: je 0x7ffba2386c50 <aesni_ecb_encrypt+624> 0x00007ffba2386a02 <+34>: cmp $0x80,%rdx 0x00007ffba2386a09 <+41>: jb 0x7ffba2386af7 <aesni_ecb_encrypt+279> 0x00007ffba2386a0f <+47>: movdqu (%rdi),%xmm2 0x00007ffba2386a13 <+51>: movdqu 0x10(%rdi),%xmm3 0x00007ffba2386a18 <+56>: movdqu 0x20(%rdi),%xmm4 0x00007ffba2386a1d <+61>: movdqu 0x30(%rdi),%xmm5 0x00007ffba2386a22 <+66>: movdqu 0x40(%rdi),%xmm6 0x00007ffba2386a27 <+71>: movdqu 0x50(%rdi),%xmm7 0x00007ffba2386a2c <+76>: movdqu 0x60(%rdi),%xmm8 0x00007ffba2386a32 <+82>: movdqu 0x70(%rdi),%xmm9 0x00007ffba2386a38 <+88>: lea 0x80(%rdi),%rdi 0x00007ffba2386a3f <+95>: sub $0x80,%rdx 0x00007ffba2386a46 <+102>: jmp 0x7ffba2386aae <aesni_ecb_encrypt+206> 0x00007ffba2386a48 <+104>: nopl 0x0(%rax,%rax,1) 0x00007ffba2386a50 <+112>: movups %xmm2,(%rsi) 0x00007ffba2386a53 <+115>: mov %r11,%rcx 0x00007ffba2386a56 <+118>: movdqu (%rdi),%xmm2 0x00007ffba2386a5a <+122>: mov %r10d,%eax 0x00007ffba2386a5d <+125>: movups %xmm3,0x10(%rsi) 0x00007ffba2386a61 <+129>: movdqu 0x10(%rdi),%xmm3 0x00007ffba2386a66 <+134>: movups %xmm4,0x20(%rsi) 0x00007ffba2386a6a <+138>: movdqu 0x20(%rdi),%xmm4 0x00007ffba2386a6f <+143>: movups %xmm5,0x30(%rsi) 0x00007ffba2386a73 <+147>: movdqu 0x30(%rdi),%xmm5 0x00007ffba2386a78 <+152>: movups %xmm6,0x40(%rsi) 0x00007ffba2386a7c <+156>: movdqu 0x40(%rdi),%xmm6 0x00007ffba2386a81 <+161>: movups %xmm7,0x50(%rsi) 0x00007ffba2386a85 <+165>: movdqu 0x50(%rdi),%xmm7 0x00007ffba2386a8a <+170>: movups %xmm8,0x60(%rsi) 0x00007ffba2386a8f <+175>: movdqu 0x60(%rdi),%xmm8 0x00007ffba2386a95 <+181>: movups %xmm9,0x70(%rsi) 0x00007ffba2386a9a <+186>: lea 0x80(%rsi),%rsi 0x00007ffba2386aa1 <+193>: movdqu 0x70(%rdi),%xmm9 0x00007ffba2386aa7 <+199>: lea 0x80(%rdi),%rdi 0x00007ffba2386aae <+206>: callq 0x7ffba23867c0 <_aesni_encrypt8> 0x00007ffba2386ab3 <+211>: sub $0x80,%rdx 0x00007ffba2386aba <+218>: jae 0x7ffba2386a50 <aesni_ecb_encrypt+112> 0x00007ffba2386abc <+220>: movups %xmm2,(%rsi) 0x00007ffba2386abf <+223>: mov %r11,%rcx 0x00007ffba2386ac2 <+226>: movups %xmm3,0x10(%rsi) 0x00007ffba2386ac6 <+230>: mov %r10d,%eax 0x00007ffba2386ac9 <+233>: movups %xmm4,0x20(%rsi) 0x00007ffba2386acd <+237>: movups %xmm5,0x30(%rsi) 0x00007ffba2386ad1 <+241>: movups %xmm6,0x40(%rsi) 0x00007ffba2386ad5 <+245>: movups %xmm7,0x50(%rsi) 0x00007ffba2386ad9 <+249>: movups %xmm8,0x60(%rsi) 0x00007ffba2386ade <+254>: movups %xmm9,0x70(%rsi) 0x00007ffba2386ae3 <+259>: lea 0x80(%rsi),%rsi 0x00007ffba2386aea <+266>: add $0x80,%rdx 0x00007ffba2386af1 <+273>: je 0x7ffba2386f34 <aesni_ecb_encrypt+1364> 0x00007ffba2386af7 <+279>: movups (%rdi),%xmm2 0x00007ffba2386afa <+282>: cmp $0x20,%rdx 0x00007ffba2386afe <+286>: jb 0x7ffba2386b70 <aesni_ecb_encrypt+400> 0x00007ffba2386b00 <+288>: movups 0x10(%rdi),%xmm3 0x00007ffba2386b04 <+292>: je 0x7ffba2386ba0 <aesni_ecb_encrypt+448> 0x00007ffba2386b0a <+298>: movups 0x20(%rdi),%xmm4 0x00007ffba2386b0e <+302>: cmp $0x40,%rdx 0x00007ffba2386b12 <+306>: jb 0x7ffba2386bc0 <aesni_ecb_encrypt+480> 0x00007ffba2386b18 <+312>: movups 0x30(%rdi),%xmm5 0x00007ffba2386b1c <+316>: je 0x7ffba2386be0 <aesni_ecb_encrypt+512> 0x00007ffba2386b22 <+322>: movups 0x40(%rdi),%xmm6 0x00007ffba2386b26 <+326>: cmp $0x60,%rdx 0x00007ffba2386b2a <+330>: jb 0x7ffba2386c00 <aesni_ecb_encrypt+544> 0x00007ffba2386b30 <+336>: movups 0x50(%rdi),%xmm7 0x00007ffba2386b34 <+340>: je 0x7ffba2386c20 <aesni_ecb_encrypt+576> 0x00007ffba2386b3a <+346>: movdqu 0x60(%rdi),%xmm8 0x00007ffba2386b40 <+352>: xorps %xmm9,%xmm9 0x00007ffba2386b44 <+356>: callq 0x7ffba23867c0 <_aesni_encrypt8> 0x00007ffba2386b49 <+361>: movups %xmm2,(%rsi) 0x00007ffba2386b4c <+364>: movups %xmm3,0x10(%rsi) 0x00007ffba2386b50 <+368>: movups %xmm4,0x20(%rsi) 0x00007ffba2386b54 <+372>: movups %xmm5,0x30(%rsi) 0x00007ffba2386b58 <+376>: movups %xmm6,0x40(%rsi) 0x00007ffba2386b5c <+380>: movups %xmm7,0x50(%rsi) 0x00007ffba2386b60 <+384>: movups %xmm8,0x60(%rsi) 0x00007ffba2386b65 <+389>: jmpq 0x7ffba2386f34 <aesni_ecb_encrypt+1364> 0x00007ffba2386b6a <+394>: nopw 0x0(%rax,%rax,1) 0x00007ffba2386b70 <+400>: movups (%rcx),%xmm0 0x00007ffba2386b73 <+403>: movups 0x10(%rcx),%xmm1 0x00007ffba2386b77 <+407>: lea 0x20(%rcx),%rcx 0x00007ffba2386b7b <+411>: xorps %xmm0,%xmm2 0x00007ffba2386b7e <+414>: aesenc %xmm1,%xmm2 0x00007ffba2386b83 <+419>: dec %eax => 0x00007ffba2386b85 <+421>: movups (%rcx),%xmm1 0x00007ffba2386b88 <+424>: lea 0x10(%rcx),%rcx 0x00007ffba2386b8c <+428>: jne 0x7ffba2386b7e <aesni_ecb_encrypt+414> 0x00007ffba2386b8e <+430>: aesenclast %xmm1,%xmm2 0x00007ffba2386b93 <+435>: movups %xmm2,(%rsi) 0x00007ffba2386b96 <+438>: jmpq 0x7ffba2386f34 <aesni_ecb_encrypt+1364> 0x00007ffba2386b9b <+443>: nopl 0x0(%rax,%rax,1) 0x00007ffba2386ba0 <+448>: callq 0x7ffba2386340 <_aesni_encrypt2> 0x00007ffba2386ba5 <+453>: movups %xmm2,(%rsi) 0x00007ffba2386ba8 <+456>: movups %xmm3,0x10(%rsi) 0x00007ffba2386bac <+460>: jmpq 0x7ffba2386f34 <aesni_ecb_encrypt+1364> 0x00007ffba2386bb1 <+465>: nopl 0x0(%rax,%rax,1) 0x00007ffba2386bb6 <+470>: nopw %cs:0x0(%rax,%rax,1) 0x00007ffba2386bc0 <+480>: callq 0x7ffba2386400 <_aesni_encrypt3> 0x00007ffba2386bc5 <+485>: movups %xmm2,(%rsi) 0x00007ffba2386bc8 <+488>: movups %xmm3,0x10(%rsi) 0x00007ffba2386bcc <+492>: movups %xmm4,0x20(%rsi) 0x00007ffba2386bd0 <+496>: jmpq 0x7ffba2386f34 <aesni_ecb_encrypt+1364> ```
``` (gdb) i r rax 0xfc3ba328 4231766824 rbx 0x1 1 rcx 0x7ffb96805000 140718538510336 rdx 0x10 16 rsi 0x7ffb58c04108 140717502513416 rdi 0x7ffb56b0c898 140717467945112 rbp 0x7ffcff3bd080 0x7ffcff3bd080 rsp 0x7ffcff3bd038 0x7ffcff3bd038 r8 0x1 1 r9 0x0 0 r10 0x0 0 r11 0x7ffb5a3a8270 140717527302768 r12 0x55e1430f52d3 94425981080275 r13 0x40000000 1073741824 r14 0x10000000 268435456 r15 0x6 6 rip 0x7ffba2386b85 0x7ffba2386b85 <aesni_ecb_encrypt+421> eflags 0x10287 [ CF PF SF IF RF ] cs 0x33 51 ss 0x2b 43 ds 0x0 0 es 0x0 0 fs 0x0 0 gs 0x0 0 ```
I've been looking at buffers and limits to see if we're perhaps hitting some kind of limit that could be triggering this and have found that increasing tls_max_connections and tcp_max_connections to 32768 appears to increase the threshold before the segmentation fault occurs. We're now able to get >20000 presence subscriptions with-out any segmentation faults.
I have more testing to do before I can reach a conclusion but it appears there may be a correlation between tls_max_connections and the segmentation fault.