16 Apr
2019
16 Apr
'19
10:09 p.m.
Unfortunately that didn't fix the problem, however the core dump has changed.
```
Reading symbols from /usr/local/kamailio/sbin/kamailio...done.
[New LWP 26088]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
Core was generated by `/usr/local/kamailio/sbin/kamailio -P /var/run/kamailio/kamailio.pid
-f /usr/loc'.
Program terminated with signal SIGSEGV, Segmentation fault.
#0 aesni_ecb_encrypt () at crypto/aes/aesni-x86_64.s:624
624 movups (%rcx),%xmm1
```
```
(gdb) disas aesni_ecb_encrypt
Dump of assembler code for function aesni_ecb_encrypt:
0x00007ffba23869e0 <+0>: and $0xfffffffffffffff0,%rdx
0x00007ffba23869e4 <+4>: je 0x7ffba2386f34
<aesni_ecb_encrypt+1364>
0x00007ffba23869ea <+10>: mov 0xf0(%rcx),%eax
0x00007ffba23869f0 <+16>: movups (%rcx),%xmm0
0x00007ffba23869f3 <+19>: mov %rcx,%r11
0x00007ffba23869f6 <+22>: mov %eax,%r10d
0x00007ffba23869f9 <+25>: test %r8d,%r8d
0x00007ffba23869fc <+28>: je 0x7ffba2386c50 <aesni_ecb_encrypt+624>
0x00007ffba2386a02 <+34>: cmp $0x80,%rdx
0x00007ffba2386a09 <+41>: jb 0x7ffba2386af7 <aesni_ecb_encrypt+279>
0x00007ffba2386a0f <+47>: movdqu (%rdi),%xmm2
0x00007ffba2386a13 <+51>: movdqu 0x10(%rdi),%xmm3
0x00007ffba2386a18 <+56>: movdqu 0x20(%rdi),%xmm4
0x00007ffba2386a1d <+61>: movdqu 0x30(%rdi),%xmm5
0x00007ffba2386a22 <+66>: movdqu 0x40(%rdi),%xmm6
0x00007ffba2386a27 <+71>: movdqu 0x50(%rdi),%xmm7
0x00007ffba2386a2c <+76>: movdqu 0x60(%rdi),%xmm8
0x00007ffba2386a32 <+82>: movdqu 0x70(%rdi),%xmm9
0x00007ffba2386a38 <+88>: lea 0x80(%rdi),%rdi
0x00007ffba2386a3f <+95>: sub $0x80,%rdx
0x00007ffba2386a46 <+102>: jmp 0x7ffba2386aae <aesni_ecb_encrypt+206>
0x00007ffba2386a48 <+104>: nopl 0x0(%rax,%rax,1)
0x00007ffba2386a50 <+112>: movups %xmm2,(%rsi)
0x00007ffba2386a53 <+115>: mov %r11,%rcx
0x00007ffba2386a56 <+118>: movdqu (%rdi),%xmm2
0x00007ffba2386a5a <+122>: mov %r10d,%eax
0x00007ffba2386a5d <+125>: movups %xmm3,0x10(%rsi)
0x00007ffba2386a61 <+129>: movdqu 0x10(%rdi),%xmm3
0x00007ffba2386a66 <+134>: movups %xmm4,0x20(%rsi)
0x00007ffba2386a6a <+138>: movdqu 0x20(%rdi),%xmm4
0x00007ffba2386a6f <+143>: movups %xmm5,0x30(%rsi)
0x00007ffba2386a73 <+147>: movdqu 0x30(%rdi),%xmm5
0x00007ffba2386a78 <+152>: movups %xmm6,0x40(%rsi)
0x00007ffba2386a7c <+156>: movdqu 0x40(%rdi),%xmm6
0x00007ffba2386a81 <+161>: movups %xmm7,0x50(%rsi)
0x00007ffba2386a85 <+165>: movdqu 0x50(%rdi),%xmm7
0x00007ffba2386a8a <+170>: movups %xmm8,0x60(%rsi)
0x00007ffba2386a8f <+175>: movdqu 0x60(%rdi),%xmm8
0x00007ffba2386a95 <+181>: movups %xmm9,0x70(%rsi)
0x00007ffba2386a9a <+186>: lea 0x80(%rsi),%rsi
0x00007ffba2386aa1 <+193>: movdqu 0x70(%rdi),%xmm9
0x00007ffba2386aa7 <+199>: lea 0x80(%rdi),%rdi
0x00007ffba2386aae <+206>: callq 0x7ffba23867c0 <_aesni_encrypt8>
0x00007ffba2386ab3 <+211>: sub $0x80,%rdx
0x00007ffba2386aba <+218>: jae 0x7ffba2386a50 <aesni_ecb_encrypt+112>
0x00007ffba2386abc <+220>: movups %xmm2,(%rsi)
0x00007ffba2386abf <+223>: mov %r11,%rcx
0x00007ffba2386ac2 <+226>: movups %xmm3,0x10(%rsi)
0x00007ffba2386ac6 <+230>: mov %r10d,%eax
0x00007ffba2386ac9 <+233>: movups %xmm4,0x20(%rsi)
0x00007ffba2386acd <+237>: movups %xmm5,0x30(%rsi)
0x00007ffba2386ad1 <+241>: movups %xmm6,0x40(%rsi)
0x00007ffba2386ad5 <+245>: movups %xmm7,0x50(%rsi)
0x00007ffba2386ad9 <+249>: movups %xmm8,0x60(%rsi)
0x00007ffba2386ade <+254>: movups %xmm9,0x70(%rsi)
0x00007ffba2386ae3 <+259>: lea 0x80(%rsi),%rsi
0x00007ffba2386aea <+266>: add $0x80,%rdx
0x00007ffba2386af1 <+273>: je 0x7ffba2386f34
<aesni_ecb_encrypt+1364>
0x00007ffba2386af7 <+279>: movups (%rdi),%xmm2
0x00007ffba2386afa <+282>: cmp $0x20,%rdx
0x00007ffba2386afe <+286>: jb 0x7ffba2386b70 <aesni_ecb_encrypt+400>
0x00007ffba2386b00 <+288>: movups 0x10(%rdi),%xmm3
0x00007ffba2386b04 <+292>: je 0x7ffba2386ba0 <aesni_ecb_encrypt+448>
0x00007ffba2386b0a <+298>: movups 0x20(%rdi),%xmm4
0x00007ffba2386b0e <+302>: cmp $0x40,%rdx
0x00007ffba2386b12 <+306>: jb 0x7ffba2386bc0 <aesni_ecb_encrypt+480>
0x00007ffba2386b18 <+312>: movups 0x30(%rdi),%xmm5
0x00007ffba2386b1c <+316>: je 0x7ffba2386be0 <aesni_ecb_encrypt+512>
0x00007ffba2386b22 <+322>: movups 0x40(%rdi),%xmm6
0x00007ffba2386b26 <+326>: cmp $0x60,%rdx
0x00007ffba2386b2a <+330>: jb 0x7ffba2386c00 <aesni_ecb_encrypt+544>
0x00007ffba2386b30 <+336>: movups 0x50(%rdi),%xmm7
0x00007ffba2386b34 <+340>: je 0x7ffba2386c20 <aesni_ecb_encrypt+576>
0x00007ffba2386b3a <+346>: movdqu 0x60(%rdi),%xmm8
0x00007ffba2386b40 <+352>: xorps %xmm9,%xmm9
0x00007ffba2386b44 <+356>: callq 0x7ffba23867c0 <_aesni_encrypt8>
0x00007ffba2386b49 <+361>: movups %xmm2,(%rsi)
0x00007ffba2386b4c <+364>: movups %xmm3,0x10(%rsi)
0x00007ffba2386b50 <+368>: movups %xmm4,0x20(%rsi)
0x00007ffba2386b54 <+372>: movups %xmm5,0x30(%rsi)
0x00007ffba2386b58 <+376>: movups %xmm6,0x40(%rsi)
0x00007ffba2386b5c <+380>: movups %xmm7,0x50(%rsi)
0x00007ffba2386b60 <+384>: movups %xmm8,0x60(%rsi)
0x00007ffba2386b65 <+389>: jmpq 0x7ffba2386f34
<aesni_ecb_encrypt+1364>
0x00007ffba2386b6a <+394>: nopw 0x0(%rax,%rax,1)
0x00007ffba2386b70 <+400>: movups (%rcx),%xmm0
0x00007ffba2386b73 <+403>: movups 0x10(%rcx),%xmm1
0x00007ffba2386b77 <+407>: lea 0x20(%rcx),%rcx
0x00007ffba2386b7b <+411>: xorps %xmm0,%xmm2
0x00007ffba2386b7e <+414>: aesenc %xmm1,%xmm2
0x00007ffba2386b83 <+419>: dec %eax
=> 0x00007ffba2386b85 <+421>: movups (%rcx),%xmm1
0x00007ffba2386b88 <+424>: lea 0x10(%rcx),%rcx
0x00007ffba2386b8c <+428>: jne 0x7ffba2386b7e <aesni_ecb_encrypt+414>
0x00007ffba2386b8e <+430>: aesenclast %xmm1,%xmm2
0x00007ffba2386b93 <+435>: movups %xmm2,(%rsi)
0x00007ffba2386b96 <+438>: jmpq 0x7ffba2386f34
<aesni_ecb_encrypt+1364>
0x00007ffba2386b9b <+443>: nopl 0x0(%rax,%rax,1)
0x00007ffba2386ba0 <+448>: callq 0x7ffba2386340 <_aesni_encrypt2>
0x00007ffba2386ba5 <+453>: movups %xmm2,(%rsi)
0x00007ffba2386ba8 <+456>: movups %xmm3,0x10(%rsi)
0x00007ffba2386bac <+460>: jmpq 0x7ffba2386f34
<aesni_ecb_encrypt+1364>
0x00007ffba2386bb1 <+465>: nopl 0x0(%rax,%rax,1)
0x00007ffba2386bb6 <+470>: nopw %cs:0x0(%rax,%rax,1)
0x00007ffba2386bc0 <+480>: callq 0x7ffba2386400 <_aesni_encrypt3>
0x00007ffba2386bc5 <+485>: movups %xmm2,(%rsi)
0x00007ffba2386bc8 <+488>: movups %xmm3,0x10(%rsi)
0x00007ffba2386bcc <+492>: movups %xmm4,0x20(%rsi)
0x00007ffba2386bd0 <+496>: jmpq 0x7ffba2386f34
<aesni_ecb_encrypt+1364>
```
```
(gdb) i r
rax 0xfc3ba328 4231766824
rbx 0x1 1
rcx 0x7ffb96805000 140718538510336
rdx 0x10 16
rsi 0x7ffb58c04108 140717502513416
rdi 0x7ffb56b0c898 140717467945112
rbp 0x7ffcff3bd080 0x7ffcff3bd080
rsp 0x7ffcff3bd038 0x7ffcff3bd038
r8 0x1 1
r9 0x0 0
r10 0x0 0
r11 0x7ffb5a3a8270 140717527302768
r12 0x55e1430f52d3 94425981080275
r13 0x40000000 1073741824
r14 0x10000000 268435456
r15 0x6 6
rip 0x7ffba2386b85 0x7ffba2386b85 <aesni_ecb_encrypt+421>
eflags 0x10287 [ CF PF SF IF RF ]
cs 0x33 51
ss 0x2b 43
ds 0x0 0
es 0x0 0
fs 0x0 0
gs 0x0 0
```
I've been looking at buffers and limits to see if we're perhaps hitting some kind
of limit that could be triggering this and have found that increasing tls_max_connections
and tcp_max_connections to 32768 appears to increase the threshold before the segmentation
fault occurs. We're now able to get >20000 presence subscriptions with-out any
segmentation faults.
I have more testing to do before I can reach a conclusion but it appears there may be a
correlation between tls_max_connections and the segmentation fault.
--
You are receiving this because you are subscribed to this thread.
Reply to this email directly or view it on GitHub:
https://github.com/kamailio/kamailio/issues/1860#issuecomment-483804879