25 Sep
2015
25 Sep
'15
8:34 p.m.
I have one kamailio server listening in 2 network interfaces, kamailio
receives sip messages from interface A and then send it to destine through
interface B.
When we start kamailio service then run gdb for each kamailio network
interface process, execute the command "bt full" the result is the
following:
*tmp = 0x5f801b994c4715bd <Address 0x5f801b994c4715bd out of bounds>*
interface - A
bt full
#0 0x00007fb01bfa40c3 in recvfrom () from /lib64/libc.so.6
No symbol table info available.
#1 0x000000000047f2ba in udp_rcv_loop () at udp_server.c:421
len = 13
buf = '\000' <repeats 65535 times>
tmp = 0x5f801b994c4715bd <Address 0x5f801b994c4715bd out of bounds>
from = 0x7faf9bf38bb8
fromlen = 16
ri = {src_ip = {af = 2309178400, len = 32765, u = {addrl =
{5458819, 4284160}, addr32 = {5458819, 0, 4284160, 0}, addr16 = {19331, 83,
0, 0, 24320, 65, 0, 0},
addr = "\203KS\000\000\000\000\000\000_A\000\000\000\000"}},
dst_ip = {af = 2, len = 4, u = {addrl = {80026686, 0}, addr32 = {80026686,
0, 0, 0},
addr16 = {7230, 1221, 0, 0, 0, 0, 0, 0}, addr =
">\034\305\004", '\000' <repeats 11 times>}}, src_port = 55576,
dst_port =
5060, proto_reserved1 = 0,
proto_reserved2 = 0, src_su = {s = {sa_family = 32687, sa_data =
"\000\000\200D\243\211\375\177\000\000\213\337D"}, sin = {sin_family =
32687, sin_port = 0,
sin_addr = {s_addr = 2309178496}, sin_zero =
"\375\177\000\000\213\337D"}, sin6 = {sin6_family = 32687, sin6_port = 0,
sin6_flowinfo = 2309178496,
sin6_addr = {__in6_u = {__u6_addr8 =
"\375\177\000\000\213\337D\000\000\000\000\000p", <incomplete sequence
\372\233>, __u6_addr16 = {32765, 0, 57227,
68, 0, 0, 21616, 39930}, __u6_addr32 = {32765, 4513675,
0, 2616874096}}}, sin6_scope_id = 32687}}, bind_address = 0x7faf9becd7d8,
proto = 1 '\001'}
__FUNCTION__ = "udp_rcv_loop"
#2 0x0000000000525eae in main_loop () at main.c:1573
i = 0
pid = 0
si = 0x7faf9becd7d8
si_desc = "udp receiver child=0 sock=62.28.197.4:5060
\000\000\000\000\000\000\001\000\000\000\000\000\000\000\260\244r\023n\000\000\000\071\263q\000\000\000\000\000\020\243q\000\000\000\000\000\b\000\000\000\001\000\000\000\260\373\371\233\257\177\000\000\220E\243\211\375\177\000\000|\351O\000\000\000\000\000\000\266\253\023\257\177\000\000\260\244r\023\257\177\000"
---Type <return> to continue, or q <return> to quit---
nrprocs = 32
__FUNCTION__ = "main_loop"
#3 0x000000000052c395 in main (argc=11, argv=0x7ffd89a34848) at main.c:2533
cfg_stream = 0x2296010
c = -1
r = 0
tmp = 0x7ffd89a35f61 ""
tmp_len = 0
port = 0
proto = 32688
options = 0x725010
":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
ret = -1
seed = 1641321572
rfd = 4
debug_save = 0
debug_flag = 0
dont_fork_cnt = 0
n_lst = 0x7ffd89a34710
p = 0x7fb01bf4c646 <intel_check_word+166>
"H\205\300t\025\017\266P\003\017\266\312D9\371\017\204", <incomplete
sequence \305>
__FUNCTION__ = "main"
interface B
bt full
#0 0x00007fb01bfa40c3 in recvfrom () from /lib64/libc.so.6
No symbol table info available.
#1 0x000000000047f2ba in udp_rcv_loop () at udp_server.c:421
len = 45
buf = '\000' <repeats 65535 times>
tmp = 0x13cb2f24296000e9 <Address 0x13cb2f24296000e9 out of bounds>
from = 0x7faf9bf38bb8
fromlen = 16
ri = {src_ip = {af = 2309178400, len = 32765, u = {addrl =
{5458819, 4284160}, addr32 = {5458819, 0, 4284160, 0}, addr16 = {19331, 83,
0, 0, 24320, 65, 0, 0}, addr =
"\203KS\000\000\000\000\000\000_A\000\000\000\000"}},
dst_ip = {af = 2, len = 4, u = {addrl = {2764381246, 0}, addr32 =
{2764381246, 0, 0, 0}, addr16 = {7230, 42181, 0, 0, 0, 0, 0, 0}, addr =
">\034Ť", '\000' <repeats 11 times>}}, src_port = 56136,
dst_port = 5060,
proto_reserved1 = 0, proto_reserved2 = 0, src_su = {s =
{sa_family = 32687, sa_data =
"\000\000\200D\243\211\375\177\000\000\213\337D"}, sin = {sin_family =
32687, sin_port = 0, sin_addr = {s_addr = 2309178496},
sin_zero = "\375\177\000\000\213\337D"}, sin6 = {sin6_family
= 32687, sin6_port = 0, sin6_flowinfo = 2309178496, sin6_addr = {__in6_u = {
__u6_addr8 =
"\375\177\000\000\213\337D\000\000\000\000\000p", <incomplete sequence
\372\233>, __u6_addr16 = {32765, 0, 57227, 68, 0, 0, 21616, 39930},
__u6_addr32 = {32765, 4513675, 0, 2616874096}}},
sin6_scope_id = 32687}}, bind_address = 0x7faf9becda08, proto
= 1 '\001'}
__FUNCTION__ = "udp_rcv_loop"
#2 0x0000000000525eae in main_loop () at main.c:1573
---Type <return> to continue, or q <return> to quit---
i = 0
pid = 0
si = 0x7faf9becda08
si_desc = "udp receiver child=0 sock=62.28.197.164:5060
\000\000\000\000\001\000\000\000\000\000\000\000\260\244r\023n\000\000\000\071\263q\000\000\000\000\000\020\243q\000\000\000\000\000\b\000\000\000\001\000\000\000\260\373\371\233\257\177\000\000\220E\243\211\375\177\000\000|\351O\000\000\000\000\000\000\266\253\023\257\177\000\000\260\244r\023\257\177\000"
nrprocs = 32
__FUNCTION__ = "main_loop"
#3 0x000000000052c395 in main (argc=11, argv=0x7ffd89a34848) at main.c:2533
cfg_stream = 0x2296010
c = -1
r = 0
tmp = 0x7ffd89a35f61 ""
tmp_len = 0
port = 0
proto = 32688
options = 0x725010
":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
---Type <return> to continue, or q <return> to quit---
ret = -1
seed = 1641321572
rfd = 4
debug_save = 0
debug_flag = 0
dont_fork_cnt = 0
n_lst = 0x7ffd89a34710
p = 0x7fb01bf4c646 <intel_check_word+166>
"H\205\300t\025\017\266P\003\017\266\312D9\371\017\204", <incomplete
sequence \305>
__FUNCTION__ = "main"
[root@c4-srv1 kamailio]# kamctl stats shmem
shmem:fragments = 2552
shmem:free_size = 2082764040
shmem:max_used_size = 66449600
shmem:real_used_size = 64719608
shmem:total_size = 2147483648
shmem:used_size = 54355992
Can anyone explain the meaning of
*tmp = 0x5f801b994c4715bd <Address 0x5f801b994c4715bd out of bounds>*
Best Regards
José Seabra
25 Sep
25 Sep
8:53 p.m.
Adding more information to my last email, when kamailio starts receiving
calls, after more or less 40 seconds it stops processing some sip messages.
I'm using the kamailio version 4.3.2
Best Regards
2015-09-25 18:34 GMT+01:00 José Seabra <joseseabra4(a)gmail.com>om>:
I have one kamailio server listening in 2 network
interfaces, kamailio
receives sip messages from interface A and then send it to destine through
interface B.
When we start kamailio service then run gdb for each kamailio network
interface process, execute the command "bt full" the result is the
following:
*tmp = 0x5f801b994c4715bd <Address 0x5f801b994c4715bd out of bounds>*
interface - A
bt full
#0 0x00007fb01bfa40c3 in recvfrom () from /lib64/libc.so.6
No symbol table info available.
#1 0x000000000047f2ba in udp_rcv_loop () at udp_server.c:421
len = 13
buf = '\000' <repeats 65535 times>
tmp = 0x5f801b994c4715bd <Address 0x5f801b994c4715bd out of bounds>
from = 0x7faf9bf38bb8
fromlen = 16
ri = {src_ip = {af = 2309178400, len = 32765, u = {addrl =
{5458819, 4284160}, addr32 = {5458819, 0, 4284160, 0}, addr16 = {19331, 83,
0, 0, 24320, 65, 0, 0},
addr = "\203KS\000\000\000\000\000\000_A\000\000\000\000"}},
dst_ip = {af = 2, len = 4, u = {addrl = {80026686, 0}, addr32 = {80026686,
0, 0, 0},
addr16 = {7230, 1221, 0, 0, 0, 0, 0, 0}, addr =
">\034\305\004", '\000' <repeats 11 times>}}, src_port =
55576, dst_port =
5060, proto_reserved1 = 0,
proto_reserved2 = 0, src_su = {s = {sa_family = 32687, sa_data =
"\000\000\200D\243\211\375\177\000\000\213\337D"}, sin = {sin_family =
32687, sin_port = 0,
sin_addr = {s_addr = 2309178496}, sin_zero =
"\375\177\000\000\213\337D"}, sin6 = {sin6_family = 32687, sin6_port = 0,
sin6_flowinfo = 2309178496,
sin6_addr = {__in6_u = {__u6_addr8 =
"\375\177\000\000\213\337D\000\000\000\000\000p", <incomplete sequence
\372\233>, __u6_addr16 = {32765, 0, 57227,
68, 0, 0, 21616, 39930}, __u6_addr32 = {32765,
4513675, 0, 2616874096}}}, sin6_scope_id = 32687}}, bind_address =
0x7faf9becd7d8, proto = 1 '\001'}
__FUNCTION__ = "udp_rcv_loop"
#2 0x0000000000525eae in main_loop () at main.c:1573
i = 0
pid = 0
si = 0x7faf9becd7d8
si_desc = "udp receiver child=0 sock=62.28.197.4:5060
\000\000\000\000\000\000\001\000\000\000\000\000\000\000\260\244r\023n\000\000\000\071\263q\000\000\000\000\000\020\243q\000\000\000\000\000\b\000\000\000\001\000\000\000\260\373\371\233\257\177\000\000\220E\243\211\375\177\000\000|\351O\000\000\000\000\000\000\266\253\023\257\177\000\000\260\244r\023\257\177\000"
---Type <return> to continue, or q <return> to quit---
nrprocs = 32
__FUNCTION__ = "main_loop"
#3 0x000000000052c395 in main (argc=11, argv=0x7ffd89a34848) at
main.c:2533
cfg_stream = 0x2296010
c = -1
r = 0
tmp = 0x7ffd89a35f61 ""
tmp_len = 0
port = 0
proto = 32688
options = 0x725010
":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
ret = -1
seed = 1641321572
rfd = 4
debug_save = 0
debug_flag = 0
dont_fork_cnt = 0
n_lst = 0x7ffd89a34710
p = 0x7fb01bf4c646 <intel_check_word+166>
"H\205\300t\025\017\266P\003\017\266\312D9\371\017\204", <incomplete
sequence \305>
__FUNCTION__ = "main"
interface B
bt full
#0 0x00007fb01bfa40c3 in recvfrom () from /lib64/libc.so.6
No symbol table info available.
#1 0x000000000047f2ba in udp_rcv_loop () at udp_server.c:421
len = 45
buf = '\000' <repeats 65535 times>
tmp = 0x13cb2f24296000e9 <Address 0x13cb2f24296000e9 out of bounds>
from = 0x7faf9bf38bb8
fromlen = 16
ri = {src_ip = {af = 2309178400, len = 32765, u = {addrl =
{5458819, 4284160}, addr32 = {5458819, 0, 4284160, 0}, addr16 = {19331, 83,
0, 0, 24320, 65, 0, 0}, addr =
"\203KS\000\000\000\000\000\000_A\000\000\000\000"}},
dst_ip = {af = 2, len = 4, u = {addrl = {2764381246, 0}, addr32
= {2764381246, 0, 0, 0}, addr16 = {7230, 42181, 0, 0, 0, 0, 0, 0}, addr =
">\034Ť", '\000' <repeats 11 times>}}, src_port = 56136,
dst_port = 5060,
proto_reserved1 = 0, proto_reserved2 = 0, src_su = {s =
{sa_family = 32687, sa_data =
"\000\000\200D\243\211\375\177\000\000\213\337D"}, sin = {sin_family =
32687, sin_port = 0, sin_addr = {s_addr = 2309178496},
sin_zero = "\375\177\000\000\213\337D"}, sin6 = {sin6_family
= 32687, sin6_port = 0, sin6_flowinfo = 2309178496, sin6_addr = {__in6_u = {
__u6_addr8 =
"\375\177\000\000\213\337D\000\000\000\000\000p", <incomplete sequence
\372\233>, __u6_addr16 = {32765, 0, 57227, 68, 0, 0, 21616, 39930},
__u6_addr32 = {32765, 4513675, 0, 2616874096}}},
sin6_scope_id = 32687}}, bind_address = 0x7faf9becda08,
proto = 1 '\001'}
__FUNCTION__ = "udp_rcv_loop"
#2 0x0000000000525eae in main_loop () at main.c:1573
---Type <return> to continue, or q <return> to quit---
i = 0
pid = 0
si = 0x7faf9becda08
si_desc = "udp receiver child=0 sock=62.28.197.164:5060
\000\000\000\000\001\000\000\000\000\000\000\000\260\244r\023n\000\000\000\071\263q\000\000\000\000\000\020\243q\000\000\000\000\000\b\000\000\000\001\000\000\000\260\373\371\233\257\177\000\000\220E\243\211\375\177\000\000|\351O\000\000\000\000\000\000\266\253\023\257\177\000\000\260\244r\023\257\177\000"
nrprocs = 32
__FUNCTION__ = "main_loop"
#3 0x000000000052c395 in main (argc=11, argv=0x7ffd89a34848) at
main.c:2533
cfg_stream = 0x2296010
c = -1
r = 0
tmp = 0x7ffd89a35f61 ""
tmp_len = 0
port = 0
proto = 32688
options = 0x725010
":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
---Type <return> to continue, or q <return> to quit---
ret = -1
seed = 1641321572
rfd = 4
debug_save = 0
debug_flag = 0
dont_fork_cnt = 0
n_lst = 0x7ffd89a34710
p = 0x7fb01bf4c646 <intel_check_word+166>
"H\205\300t\025\017\266P\003\017\266\312D9\371\017\204", <incomplete
sequence \305>
__FUNCTION__ = "main"
[root@c4-srv1 kamailio]# kamctl stats shmem
shmem:fragments = 2552
shmem:free_size = 2082764040
shmem:max_used_size = 66449600
shmem:real_used_size = 64719608
shmem:total_size = 2147483648
shmem:used_size = 54355992
Can anyone explain the meaning of
*tmp = 0x5f801b994c4715bd <Address 0x5f801b994c4715bd out of bounds>*
Best Regards
José Seabra
--
Cumprimentos
José Seabra
27 Sep
27 Sep
3:52 p.m.
The tmp variable is not intialized by default, so it can be anything
therefore the out of bounds value. It used only when printing some
errors, so it didn't get a proper value as no such error was printed for
you case.
The backtrace shows the process in in recvfrom(), so it waits for
traffic from network. That is the case when the processing of former sip
message was finished and now Kamailio waits to receive another one.
If you run centos, be sure you don't have limits in selinux rules -
better disable it completely and test again. Also, be sure you don't
have limits in the firewall, especially contrack module and sip firewall
modules in the kernel.
In kamailio.cfg, if you have pike module, be sure you don't drop good
packet with this module.
Cheers,
Daniel
On 25/09/15 19:53, José Seabra wrote:
Adding more information to my last email, when
kamailio starts
receiving calls, after more or less 40 seconds it stops processing
some sip messages.
I'm using the kamailio version 4.3.2
Best Regards
2015-09-25 18:34 GMT+01:00 José Seabra <joseseabra4(a)gmail.com
<mailto:joseseabra4@gmail.com>>:
I have one kamailio server listening in 2 network interfaces,
kamailio receives sip messages from interface A and then send it
to destine through interface B.
When we start kamailio service then run gdb for each kamailio
network interface process, execute the command "bt full" the
result is the following:
*tmp = 0x5f801b994c4715bd <Address 0x5f801b994c4715bd out of bounds>*
interface - A
bt full
#0 0x00007fb01bfa40c3 in recvfrom () from /lib64/libc.so.6
No symbol table info available.
#1 0x000000000047f2ba in udp_rcv_loop () at udp_server.c:421
len = 13
buf = '\000' <repeats 65535 times>
tmp = 0x5f801b994c4715bd <Address 0x5f801b994c4715bd out
of bounds>
from = 0x7faf9bf38bb8
fromlen = 16
ri = {src_ip = {af = 2309178400, len = 32765, u = {addrl =
{5458819, 4284160}, addr32 = {5458819, 0, 4284160, 0}, addr16 =
{19331, 83, 0, 0, 24320, 65, 0, 0},
addr =
"\203KS\000\000\000\000\000\000_A\000\000\000\000"}}, dst_ip = {af
= 2, len = 4, u = {addrl = {80026686, 0}, addr32 = {80026686, 0,
0, 0},
addr16 = {7230, 1221, 0, 0, 0, 0, 0, 0}, addr =
">\034\305\004", '\000' <repeats 11 times>}}, src_port =
55576,
dst_port = 5060, proto_reserved1 = 0,
proto_reserved2 = 0, src_su = {s = {sa_family = 32687,
sa_data = "\000\000\200D\243\211\375\177\000\000\213\337D"}, sin =
{sin_family = 32687, sin_port = 0,
sin_addr = {s_addr = 2309178496}, sin_zero =
"\375\177\000\000\213\337D"}, sin6 = {sin6_family = 32687,
sin6_port = 0, sin6_flowinfo = 2309178496,
sin6_addr = {__in6_u = {__u6_addr8 =
"\375\177\000\000\213\337D\000\000\000\000\000p", <incomplete
sequence \372\233>, __u6_addr16 = {32765, 0, 57227,
68, 0, 0, 21616, 39930}, __u6_addr32 = {32765,
4513675, 0, 2616874096}}}, sin6_scope_id = 32687}}, bind_address =
0x7faf9becd7d8, proto = 1 '\001'}
__FUNCTION__ = "udp_rcv_loop"
#2 0x0000000000525eae in main_loop () at main.c:1573
i = 0
pid = 0
si = 0x7faf9becd7d8
si_desc = "udp receiver child=0 sock=62.28.197.4:5060
<http://62.28.197.4:5060>\000\000\000\000\000\000\001\000\000\000\000\000\000\000\260\244r\023n\000\000\000\071\263q\000\000\000\000\000\020\243q\000\000\000\000\000\b\000\000\000\001\000\000\000\260\373\371\233\257\177\000\000\220E\243\211\375\177\000\000|\351O\000\000\000\000\000\000\266\253\023\257\177\000\000\260\244r\023\257\177\000"
---Type <return> to continue, or q <return> to quit---
nrprocs = 32
__FUNCTION__ = "main_loop"
#3 0x000000000052c395 in main (argc=11, argv=0x7ffd89a34848) at
main.c:2533
cfg_stream = 0x2296010
c = -1
r = 0
tmp = 0x7ffd89a35f61 ""
tmp_len = 0
port = 0
proto = 32688
options = 0x725010
":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
ret = -1
seed = 1641321572
rfd = 4
debug_save = 0
debug_flag = 0
dont_fork_cnt = 0
n_lst = 0x7ffd89a34710
p = 0x7fb01bf4c646 <intel_check_word+166>
"H\205\300t\025\017\266P\003\017\266\312D9\371\017\204",
<incomplete sequence \305>
__FUNCTION__ = "main"
interface B
bt full
#0 0x00007fb01bfa40c3 in recvfrom () from /lib64/libc.so.6
No symbol table info available.
#1 0x000000000047f2ba in udp_rcv_loop () at udp_server.c:421
len = 45
buf = '\000' <repeats 65535 times>
tmp = 0x13cb2f24296000e9 <Address 0x13cb2f24296000e9 out
of bounds>
from = 0x7faf9bf38bb8
fromlen = 16
ri = {src_ip = {af = 2309178400, len = 32765, u = {addrl =
{5458819, 4284160}, addr32 = {5458819, 0, 4284160, 0}, addr16 =
{19331, 83, 0, 0, 24320, 65, 0, 0}, addr =
"\203KS\000\000\000\000\000\000_A\000\000\000\000"}},
dst_ip = {af = 2, len = 4, u = {addrl = {2764381246, 0},
addr32 = {2764381246, 0, 0, 0}, addr16 = {7230, 42181, 0, 0, 0, 0,
0, 0}, addr = ">\034Ť", '\000' <repeats 11 times>}},
src_port =
56136, dst_port = 5060,
proto_reserved1 = 0, proto_reserved2 = 0, src_su = {s =
{sa_family = 32687, sa_data =
"\000\000\200D\243\211\375\177\000\000\213\337D"}, sin =
{sin_family = 32687, sin_port = 0, sin_addr = {s_addr = 2309178496},
sin_zero = "\375\177\000\000\213\337D"}, sin6 =
{sin6_family = 32687, sin6_port = 0, sin6_flowinfo = 2309178496,
sin6_addr = {__in6_u = {
__u6_addr8 =
"\375\177\000\000\213\337D\000\000\000\000\000p", <incomplete
sequence \372\233>, __u6_addr16 = {32765, 0, 57227, 68, 0, 0,
21616, 39930}, __u6_addr32 = {32765, 4513675, 0, 2616874096}}},
sin6_scope_id = 32687}}, bind_address =
0x7faf9becda08, proto = 1 '\001'}
__FUNCTION__ = "udp_rcv_loop"
#2 0x0000000000525eae in main_loop () at main.c:1573
---Type <return> to continue, or q <return> to quit---
i = 0
pid = 0
si = 0x7faf9becda08
si_desc = "udp receiver child=0 sock=62.28.197.164:5060
<http://62.28.197.164:5060>\000\000\000\000\001\000\000\000\000\000\000\000\260\244r\023n\000\000\000\071\263q\000\000\000\000\000\020\243q\000\000\000\000\000\b\000\000\000\001\000\000\000\260\373\371\233\257\177\000\000\220E\243\211\375\177\000\000|\351O\000\000\000\000\000\000\266\253\023\257\177\000\000\260\244r\023\257\177\000"
nrprocs = 32
__FUNCTION__ = "main_loop"
#3 0x000000000052c395 in main (argc=11, argv=0x7ffd89a34848) at
main.c:2533
cfg_stream = 0x2296010
c = -1
r = 0
tmp = 0x7ffd89a35f61 ""
tmp_len = 0
port = 0
proto = 32688
options = 0x725010
":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
---Type <return> to continue, or q <return> to quit---
ret = -1
seed = 1641321572
rfd = 4
debug_save = 0
debug_flag = 0
dont_fork_cnt = 0
n_lst = 0x7ffd89a34710
p = 0x7fb01bf4c646 <intel_check_word+166>
"H\205\300t\025\017\266P\003\017\266\312D9\371\017\204",
<incomplete sequence \305>
__FUNCTION__ = "main"
[root@c4-srv1 kamailio]# kamctl stats shmem
shmem:fragments = 2552
shmem:free_size = 2082764040
shmem:max_used_size = 66449600
shmem:real_used_size = 64719608
shmem:total_size = 2147483648
shmem:used_size = 54355992
Can anyone explain the meaning of
*tmp = 0x5f801b994c4715bd <Address 0x5f801b994c4715bd out of bounds>*
Best Regards
José Seabra
--
Cumprimentos
José Seabra
_______________________________________________
SIP Express Router (SER) and Kamailio (OpenSER) - sr-users mailing list
sr-users(a)lists.sip-router.org
http://lists.sip-router.org/cgi-bin/mailman/listinfo/sr-users
--
Daniel-Constantin Mierla
http://twitter.com/#!/miconda - http://www.linkedin.com/in/miconda
Book: SIP Routing With Kamailio - http://www.asipto.com
Kamailio Advanced Training, Sep 28-30, 2015, in Berlin - http://asipto.com/u/kat
4:52 p.m.
Hello Danel,
Thank you for your response,
Well i don't have SELinux enabled and IPTables isn't running.
About kamailio configuration, Pike module isn't also enabled, the only
thing that i have enabled to limit the number of concurrent calls is the
dialog module, counting the number of active dialogs and reject next new
invites only after it has reached the 10.000 simultaneous calls.
The strange behavior is that kamailio only stop processing some sip
messages like Byes and 200 Ok after more or less 40 seconds after siptester
start sending calls.
And i can see several cases in wireshark that kamailio receives BYES/200OK
from interface B(Carriers interface) but it isn't forward to Interface
A(customer interface), if i make for example 20cps i don't have this issue,
all packets are handled correctly and there isn't 408 requests timeouts.
Best Regards
José Seabra
2015-09-27 13:52 GMT+01:00 Daniel-Constantin Mierla <miconda(a)gmail.com>om>:
The tmp variable is not intialized by default, so it
can be anything
therefore the out of bounds value. It used only when printing some errors,
so it didn't get a proper value as no such error was printed for you case.
The backtrace shows the process in in recvfrom(), so it waits for traffic
from network. That is the case when the processing of former sip message
was finished and now Kamailio waits to receive another one.
If you run centos, be sure you don't have limits in selinux rules - better
disable it completely and test again. Also, be sure you don't have limits
in the firewall, especially contrack module and sip firewall modules in the
kernel.
In kamailio.cfg, if you have pike module, be sure you don't drop good
packet with this module.
Cheers,
Daniel
On 25/09/15 19:53, José Seabra wrote:
Adding more information to my last email, when kamailio starts receiving
calls, after more or less 40 seconds it stops processing some sip messages.
I'm using the kamailio version 4.3.2
Best Regards
2015-09-25 18:34 GMT+01:00 José Seabra <joseseabra4(a)gmail.com>om>:
--
Cumprimentos
José Seabra
I have one kamailio server listening in 2 network
interfaces, kamailio
receives sip messages from interface A and then send it to destine through
interface B.
When we start kamailio service then run gdb for each kamailio network
interface process, execute the command "bt full" the result is the
following:
*tmp = 0x5f801b994c4715bd <Address 0x5f801b994c4715bd out of bounds>*
interface - A
bt full
#0 0x00007fb01bfa40c3 in recvfrom () from /lib64/libc.so.6
No symbol table info available.
#1 0x000000000047f2ba in udp_rcv_loop () at udp_server.c:421
len = 13
buf = '\000' <repeats 65535 times>
tmp = 0x5f801b994c4715bd <Address 0x5f801b994c4715bd out of
bounds>
from = 0x7faf9bf38bb8
fromlen = 16
ri = {src_ip = {af = 2309178400, len = 32765, u = {addrl =
{5458819, 4284160}, addr32 = {5458819, 0, 4284160, 0}, addr16 = {19331, 83,
0, 0, 24320, 65, 0, 0},
addr =
"\203KS\000\000\000\000\000\000_A\000\000\000\000"}}, dst_ip = {af = 2, len
= 4, u = {addrl = {80026686, 0}, addr32 = {80026686, 0, 0, 0},
addr16 = {7230, 1221, 0, 0, 0, 0, 0, 0}, addr =
">\034\305\004", '\000' <repeats 11 times>}}, src_port =
55576, dst_port =
5060, proto_reserved1 = 0,
proto_reserved2 = 0, src_su = {s = {sa_family = 32687, sa_data
= "\000\000\200D\243\211\375\177\000\000\213\337D"}, sin = {sin_family =
32687, sin_port = 0,
sin_addr = {s_addr = 2309178496}, sin_zero =
"\375\177\000\000\213\337D"}, sin6 = {sin6_family = 32687, sin6_port = 0,
sin6_flowinfo = 2309178496,
sin6_addr = {__in6_u = {__u6_addr8 =
"\375\177\000\000\213\337D\000\000\000\000\000p", <incomplete sequence
\372\233>, __u6_addr16 = {32765, 0, 57227,
68, 0, 0, 21616, 39930}, __u6_addr32 = {32765,
4513675, 0, 2616874096}}}, sin6_scope_id = 32687}}, bind_address =
0x7faf9becd7d8, proto = 1 '\001'}
__FUNCTION__ = "udp_rcv_loop"
#2 0x0000000000525eae in main_loop () at main.c:1573
i = 0
pid = 0
si = 0x7faf9becd7d8
si_desc = "udp receiver child=0 sock=62.28.197.4:5060
\000\000\000\000\000\000\001\000\000\000\000\000\000\000\260\244r\023n\000\000\000\071\263q\000\000\000\000\000\020\243q\000\000\000\000\000\b\000\000\000\001\000\000\000\260\373\371\233\257\177\000\000\220E\243\211\375\177\000\000|\351O\000\000\000\000\000\000\266\253\023\257\177\000\000\260\244r\023\257\177\000"
---Type <return> to continue, or q <return> to quit---
nrprocs = 32
__FUNCTION__ = "main_loop"
#3 0x000000000052c395 in main (argc=11, argv=0x7ffd89a34848) at
main.c:2533
cfg_stream = 0x2296010
c = -1
r = 0
tmp = 0x7ffd89a35f61 ""
tmp_len = 0
port = 0
proto = 32688
options = 0x725010
":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
ret = -1
seed = 1641321572
rfd = 4
debug_save = 0
debug_flag = 0
dont_fork_cnt = 0
n_lst = 0x7ffd89a34710
p = 0x7fb01bf4c646 <intel_check_word+166>
"H\205\300t\025\017\266P\003\017\266\312D9\371\017\204", <incomplete
sequence \305>
__FUNCTION__ = "main"
interface B
bt full
#0 0x00007fb01bfa40c3 in recvfrom () from /lib64/libc.so.6
No symbol table info available.
#1 0x000000000047f2ba in udp_rcv_loop () at udp_server.c:421
len = 45
buf = '\000' <repeats 65535 times>
tmp = 0x13cb2f24296000e9 <Address 0x13cb2f24296000e9 out of
bounds>
from = 0x7faf9bf38bb8
fromlen = 16
ri = {src_ip = {af = 2309178400, len = 32765, u = {addrl =
{5458819, 4284160}, addr32 = {5458819, 0, 4284160, 0}, addr16 = {19331, 83,
0, 0, 24320, 65, 0, 0}, addr =
"\203KS\000\000\000\000\000\000_A\000\000\000\000"}},
dst_ip = {af = 2, len = 4, u = {addrl = {2764381246, 0}, addr32
= {2764381246, 0, 0, 0}, addr16 = {7230, 42181, 0, 0, 0, 0, 0, 0}, addr =
">\034Ť", '\000' <repeats 11 times>}}, src_port = 56136,
dst_port = 5060,
proto_reserved1 = 0, proto_reserved2 = 0, src_su = {s =
{sa_family = 32687, sa_data =
"\000\000\200D\243\211\375\177\000\000\213\337D"}, sin = {sin_family =
32687, sin_port = 0, sin_addr = {s_addr = 2309178496},
sin_zero = "\375\177\000\000\213\337D"}, sin6 =
{sin6_family = 32687, sin6_port = 0, sin6_flowinfo = 2309178496, sin6_addr
= {__in6_u = {
__u6_addr8 =
"\375\177\000\000\213\337D\000\000\000\000\000p", <incomplete sequence
\372\233>, __u6_addr16 = {32765, 0, 57227, 68, 0, 0, 21616, 39930},
__u6_addr32 = {32765, 4513675, 0, 2616874096}}},
sin6_scope_id = 32687}}, bind_address = 0x7faf9becda08,
proto = 1 '\001'}
__FUNCTION__ = "udp_rcv_loop"
#2 0x0000000000525eae in main_loop () at main.c:1573
---Type <return> to continue, or q <return> to quit---
i = 0
pid = 0
si = 0x7faf9becda08
si_desc = "udp receiver child=0 sock=62.28.197.164:5060
\000\000\000\000\001\000\000\000\000\000\000\000\260\244r\023n\000\000\000\071\263q\000\000\000\000\000\020\243q\000\000\000\000\000\b\000\000\000\001\000\000\000\260\373\371\233\257\177\000\000\220E\243\211\375\177\000\000|\351O\000\000\000\000\000\000\266\253\023\257\177\000\000\260\244r\023\257\177\000"
nrprocs = 32
__FUNCTION__ = "main_loop"
#3 0x000000000052c395 in main (argc=11, argv=0x7ffd89a34848) at
main.c:2533
cfg_stream = 0x2296010
c = -1
r = 0
tmp = 0x7ffd89a35f61 ""
tmp_len = 0
port = 0
proto = 32688
options = 0x725010
":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
---Type <return> to continue, or q <return> to quit---
ret = -1
seed = 1641321572
rfd = 4
debug_save = 0
debug_flag = 0
dont_fork_cnt = 0
n_lst = 0x7ffd89a34710
p = 0x7fb01bf4c646 <intel_check_word+166>
"H\205\300t\025\017\266P\003\017\266\312D9\371\017\204", <incomplete
sequence \305>
__FUNCTION__ = "main"
[root@c4-srv1 kamailio]# kamctl stats shmem
shmem:fragments = 2552
shmem:free_size = 2082764040
shmem:max_used_size = 66449600
shmem:real_used_size = 64719608
shmem:total_size = 2147483648
shmem:used_size = 54355992
Can anyone explain the meaning of
*tmp = 0x5f801b994c4715bd <Address 0x5f801b994c4715bd out of bounds>*
Best Regards
José Seabra
--
Cumprimentos
José Seabra
_______________________________________________
SIP Express Router (SER) and Kamailio (OpenSER) - sr-users mailing
listsr-users@lists.sip-router.orghttp://lists.sip-router.org/cgi-bin/mailman/listinfo/sr-users
--
Daniel-Constantin Mierlahttp://twitter.com/#!/miconda -
http://www.linkedin.com/in/miconda
Book: SIP Routing With Kamailio - http://www.asipto.com
Kamailio Advanced Training, Sep 28-30, 2015, in Berlin - http://asipto.com/u/kat
_______________________________________________
SIP Express Router (SER) and Kamailio (OpenSER) - sr-users mailing list
sr-users(a)lists.sip-router.org
http://lists.sip-router.org/cgi-bin/mailman/listinfo/sr-users
7:08 p.m.
Hello,
first, be sure you run latest version of branch 4.3, there was a fix for
dialog since 4.3.2.
If a process in shown to run recvfrom() that means it waits for kernel
to pass a new upd packet to it -- recvfrom() is a function from standard
lib c.
Wireshark (like other sniffing tools) is capturing the network traffic
before the kernel passes it to application layer. If wireshark see it,
doesn't mean the application has received it from kernel, it can still
be dropped by firewall or other kernel modules.
You can add a xlog() at the beginning of request_route to see what
requests are received by kamailio.
Cheers,
Daniel
On 27/09/15 15:52, José Seabra wrote:
Hello Danel,
Thank you for your response,
Well i don't have SELinux enabled and IPTables isn't running.
About kamailio configuration, Pike module isn't also enabled, the only
thing that i have enabled to limit the number of concurrent calls is
the dialog module, counting the number of active dialogs and reject
next new invites only after it has reached the 10.000 simultaneous calls.
The strange behavior is that kamailio only stop processing some sip
messages like Byes and 200 Ok after more or less 40 seconds after
siptester start sending calls.
And i can see several cases in wireshark that kamailio receives
BYES/200OK from interface B(Carriers interface) but it isn't forward
to Interface A(customer interface), if i make for example 20cps i
don't have this issue, all packets are handled correctly and there
isn't 408 requests timeouts.
Best Regards
José Seabra
2015-09-27 13:52 GMT+01:00 Daniel-Constantin Mierla <miconda(a)gmail.com
<mailto:miconda@gmail.com>>:
The tmp variable is not intialized by default, so it can be
anything therefore the out of bounds value. It used only when
printing some errors, so it didn't get a proper value as no such
error was printed for you case.
The backtrace shows the process in in recvfrom(), so it waits for
traffic from network. That is the case when the processing of
former sip message was finished and now Kamailio waits to receive
another one.
If you run centos, be sure you don't have limits in selinux rules
- better disable it completely and test again. Also, be sure you
don't have limits in the firewall, especially contrack module and
sip firewall modules in the kernel.
In kamailio.cfg, if you have pike module, be sure you don't drop
good packet with this module.
Cheers,
Daniel
On 25/09/15 19:53, José Seabra wrote:
--
Daniel-Constantin Mierla
http://twitter.com/#!/miconda - http://www.linkedin.com/in/miconda
Book: SIP Routing With Kamailio - http://www.asipto.com
Kamailio Advanced Training, Sep 28-30, 2015, in Berlin - http://asipto.com/u/kat
Adding more information to my last email,
when kamailio starts
receiving calls, after more or less 40 seconds it stops
processing some sip messages.
I'm using the kamailio version 4.3.2
Best Regards
2015-09-25 18:34 GMT+01:00 José Seabra <joseseabra4(a)gmail.com
<mailto:joseseabra4@gmail.com>>:
I have one kamailio server listening in 2 network interfaces,
kamailio receives sip messages from interface A and then send
it to destine through interface B.
When we start kamailio service then run gdb for each
kamailio network interface process, execute the command "bt
full" the result is the following:
*tmp = 0x5f801b994c4715bd <Address 0x5f801b994c4715bd out of
bounds>*
interface - A
bt full
#0 0x00007fb01bfa40c3 in recvfrom () from /lib64/libc.so.6
No symbol table info available.
#1 0x000000000047f2ba in udp_rcv_loop () at udp_server.c:421
len = 13
buf = '\000' <repeats 65535 times>
tmp = 0x5f801b994c4715bd <Address 0x5f801b994c4715bd
out of bounds>
from = 0x7faf9bf38bb8
fromlen = 16
ri = {src_ip = {af = 2309178400, len = 32765, u =
{addrl = {5458819, 4284160}, addr32 = {5458819, 0, 4284160,
0}, addr16 = {19331, 83, 0, 0, 24320, 65, 0, 0},
addr =
"\203KS\000\000\000\000\000\000_A\000\000\000\000"}}, dst_ip
= {af = 2, len = 4, u = {addrl = {80026686, 0}, addr32 =
{80026686, 0, 0, 0},
addr16 = {7230, 1221, 0, 0, 0, 0, 0, 0}, addr =
">\034\305\004", '\000' <repeats 11 times>}}, src_port
=
55576, dst_port = 5060, proto_reserved1 = 0,
proto_reserved2 = 0, src_su = {s = {sa_family =
32687, sa_data =
"\000\000\200D\243\211\375\177\000\000\213\337D"}, sin =
{sin_family = 32687, sin_port = 0,
sin_addr = {s_addr = 2309178496}, sin_zero =
"\375\177\000\000\213\337D"}, sin6 = {sin6_family = 32687,
sin6_port = 0, sin6_flowinfo = 2309178496,
sin6_addr = {__in6_u = {__u6_addr8 =
"\375\177\000\000\213\337D\000\000\000\000\000p", <incomplete
sequence \372\233>, __u6_addr16 = {32765, 0, 57227,
68, 0, 0, 21616, 39930}, __u6_addr32 =
{32765, 4513675, 0, 2616874096}}}, sin6_scope_id = 32687}},
bind_address = 0x7faf9becd7d8, proto = 1 '\001'}
__FUNCTION__ = "udp_rcv_loop"
#2 0x0000000000525eae in main_loop () at main.c:1573
i = 0
pid = 0
si = 0x7faf9becd7d8
si_desc = "udp receiver child=0 sock=62.28.197.4:5060
<http://62.28.197.4:5060>\000\000\000\000\000\000\001\000\000\000\000\000\000\000\260\244r\023n\000\000\000\071\263q\000\000\000\000\000\020\243q\000\000\000\000\000\b\000\000\000\001\000\000\000\260\373\371\233\257\177\000\000\220E\243\211\375\177\000\000|\351O\000\000\000\000\000\000\266\253\023\257\177\000\000\260\244r\023\257\177\000"
---Type <return> to continue, or q <return> to quit---
nrprocs = 32
__FUNCTION__ = "main_loop"
#3 0x000000000052c395 in main (argc=11, argv=0x7ffd89a34848)
at main.c:2533
cfg_stream = 0x2296010
c = -1
r = 0
tmp = 0x7ffd89a35f61 ""
tmp_len = 0
port = 0
proto = 32688
options = 0x725010
":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
ret = -1
seed = 1641321572
rfd = 4
debug_save = 0
debug_flag = 0
dont_fork_cnt = 0
n_lst = 0x7ffd89a34710
p = 0x7fb01bf4c646 <intel_check_word+166>
"H\205\300t\025\017\266P\003\017\266\312D9\371\017\204",
<incomplete sequence \305>
__FUNCTION__ = "main"
interface B
bt full
#0 0x00007fb01bfa40c3 in recvfrom () from /lib64/libc.so.6
No symbol table info available.
#1 0x000000000047f2ba in udp_rcv_loop () at udp_server.c:421
len = 45
buf = '\000' <repeats 65535 times>
tmp = 0x13cb2f24296000e9 <Address 0x13cb2f24296000e9
out of bounds>
from = 0x7faf9bf38bb8
fromlen = 16
ri = {src_ip = {af = 2309178400, len = 32765, u =
{addrl = {5458819, 4284160}, addr32 = {5458819, 0, 4284160,
0}, addr16 = {19331, 83, 0, 0, 24320, 65, 0, 0}, addr =
"\203KS\000\000\000\000\000\000_A\000\000\000\000"}},
dst_ip = {af = 2, len = 4, u = {addrl =
{2764381246, 0}, addr32 = {2764381246, 0, 0, 0}, addr16 =
{7230, 42181, 0, 0, 0, 0, 0, 0}, addr = ">\034Ť", '\000'
<repeats 11 times>}}, src_port = 56136, dst_port = 5060,
proto_reserved1 = 0, proto_reserved2 = 0, src_su =
{s = {sa_family = 32687, sa_data =
"\000\000\200D\243\211\375\177\000\000\213\337D"}, sin =
{sin_family = 32687, sin_port = 0, sin_addr = {s_addr =
2309178496},
sin_zero = "\375\177\000\000\213\337D"}, sin6 =
{sin6_family = 32687, sin6_port = 0, sin6_flowinfo =
2309178496, sin6_addr = {__in6_u = {
__u6_addr8 =
"\375\177\000\000\213\337D\000\000\000\000\000p", <incomplete
sequence \372\233>, __u6_addr16 = {32765, 0, 57227, 68, 0, 0,
21616, 39930}, __u6_addr32 = {32765, 4513675, 0, 2616874096}}},
sin6_scope_id = 32687}}, bind_address =
0x7faf9becda08, proto = 1 '\001'}
__FUNCTION__ = "udp_rcv_loop"
#2 0x0000000000525eae in main_loop () at main.c:1573
---Type <return> to continue, or q <return> to quit---
i = 0
pid = 0
si = 0x7faf9becda08
si_desc = "udp receiver child=0
sock=62.28.197.164:5060
<http://62.28.197.164:5060>\000\000\000\000\001\000\000\000\000\000\000\000\260\244r\023n\000\000\000\071\263q\000\000\000\000\000\020\243q\000\000\000\000\000\b\000\000\000\001\000\000\000\260\373\371\233\257\177\000\000\220E\243\211\375\177\000\000|\351O\000\000\000\000\000\000\266\253\023\257\177\000\000\260\244r\023\257\177\000"
nrprocs = 32
__FUNCTION__ = "main_loop"
#3 0x000000000052c395 in main (argc=11, argv=0x7ffd89a34848)
at main.c:2533
cfg_stream = 0x2296010
c = -1
r = 0
tmp = 0x7ffd89a35f61 ""
tmp_len = 0
port = 0
proto = 32688
options = 0x725010
":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
---Type <return> to continue, or q <return> to quit---
ret = -1
seed = 1641321572
rfd = 4
debug_save = 0
debug_flag = 0
dont_fork_cnt = 0
n_lst = 0x7ffd89a34710
p = 0x7fb01bf4c646 <intel_check_word+166>
"H\205\300t\025\017\266P\003\017\266\312D9\371\017\204",
<incomplete sequence \305>
__FUNCTION__ = "main"
[root@c4-srv1 kamailio]# kamctl stats shmem
shmem:fragments = 2552
shmem:free_size = 2082764040
shmem:max_used_size = 66449600
shmem:real_used_size = 64719608
shmem:total_size = 2147483648
shmem:used_size = 54355992
Can anyone explain the meaning of
*tmp = 0x5f801b994c4715bd <Address 0x5f801b994c4715bd out of
bounds>*
Best Regards
José Seabra
--
Cumprimentos
José Seabra
_______________________________________________
SIP Express Router (SER) and Kamailio (OpenSER) - sr-users mailing list
sr-users(a)lists.sip-router.org <mailto:sr-users@lists.sip-router.org>
http://lists.sip-router.org/cgi-bin/mailman/listinfo/sr-users
--
Daniel-Constantin Mierla
http://twitter.com/#!/miconda <http://twitter.com/#%21/miconda> -
http://www.linkedin.com/in/miconda
Book: SIP Routing With Kamailio - http://www.asipto.com
Kamailio Advanced Training, Sep 28-30, 2015, in Berlin - http://asipto.com/u/kat
_______________________________________________
SIP Express Router (SER) and Kamailio (OpenSER) - sr-users mailing
list
sr-users(a)lists.sip-router.org <mailto:sr-users@lists.sip-router.org>
http://lists.sip-router.org/cgi-bin/mailman/listinfo/sr-users
--
Cumprimentos
José Seabra
28 Sep
28 Sep
8:07 p.m.
Hello Daniel,
Indeed xlog doesn't print in syslog the sip message received and wireshak
got it, that means that packet was received in network interface but
kamailio didn't received the packet(as xlog didn't print that packet).
Anyway i have configured the following parameters in sysctl
net.ipv4.udp_rmem_min = 131072
net.ipv4.udp_wmem_min = 131072
net.core.netdev_max_backlog=1000
net.core.rmem_max=67108864
net.ipv4.udp_mem = 192576 256768 385152
and also set maxbuffer=134190336 in my kamailio configuration.
With these configurations I got an improvement but still below of what we
need and even the CPU is between 65% and 70% sleeping.
Do you have any suggestion? the traffic average that interface receives now
is 200mbs.
The server network interface is 10GB.
Any suggestion is welcome.
Best Regards
José Seabra
2015-09-27 17:08 GMT+01:00 Daniel-Constantin Mierla <miconda(a)gmail.com>om>:
Hello,
first, be sure you run latest version of branch 4.3, there was a fix for
dialog since 4.3.2.
If a process in shown to run recvfrom() that means it waits for kernel to
pass a new upd packet to it -- recvfrom() is a function from standard lib c.
Wireshark (like other sniffing tools) is capturing the network traffic
before the kernel passes it to application layer. If wireshark see it,
doesn't mean the application has received it from kernel, it can still be
dropped by firewall or other kernel modules.
You can add a xlog() at the beginning of request_route to see what
requests are received by kamailio.
Cheers,
Daniel
On 27/09/15 15:52, José Seabra wrote:
Hello Danel,
Thank you for your response,
Well i don't have SELinux enabled and IPTables isn't running.
About kamailio configuration, Pike module isn't also enabled, the only
thing that i have enabled to limit the number of concurrent calls is the
dialog module, counting the number of active dialogs and reject next new
invites only after it has reached the 10.000 simultaneous calls.
The strange behavior is that kamailio only stop processing some sip
messages like Byes and 200 Ok after more or less 40 seconds after siptester
start sending calls.
And i can see several cases in wireshark that kamailio receives BYES/200OK
from interface B(Carriers interface) but it isn't forward to Interface
A(customer interface), if i make for example 20cps i don't have this issue,
all packets are handled correctly and there isn't 408 requests timeouts.
Best Regards
José Seabra
2015-09-27 13:52 GMT+01:00 Daniel-Constantin Mierla < <miconda(a)gmail.com>
miconda(a)gmail.com>gt;:
The tmp variable is not intialized by default, so
it can be anything
therefore the out of bounds value. It used only when printing some errors,
so it didn't get a proper value as no such error was printed for you case.
The backtrace shows the process in in recvfrom(), so it waits for traffic
from network. That is the case when the processing of former sip message
was finished and now Kamailio waits to receive another one.
If you run centos, be sure you don't have limits in selinux rules -
better disable it completely and test again. Also, be sure you don't have
limits in the firewall, especially contrack module and sip firewall modules
in the kernel.
In kamailio.cfg, if you have pike module, be sure you don't drop good
packet with this module.
Cheers,
Daniel
On 25/09/15 19:53, José Seabra wrote:
Adding more information to my last email, when kamailio starts receiving
calls, after more or less 40 seconds it stops processing some sip messages.
I'm using the kamailio version 4.3.2
Best Regards
2015-09-25 18:34 GMT+01:00 José Seabra < <joseseabra4(a)gmail.com>
joseseabra4(a)gmail.com>gt;:
--
Cumprimentos
José Seabra
--
Daniel-Constantin Mierlahttp://twitter.com/#!/miconda -
http://www.linkedin.com/in/miconda
Book: SIP Routing With Kamailio - http://www.asipto.com
Kamailio Advanced Training, Sep 28-30, 2015, in Berlin - http://asipto.com/u/kat
--
Cumprimentos
José Seabra
<http://asipto.com/u/kat>[image: ITCENTER Store]
<http://store.itcenter.com.pt/>[image: ITCENTER Helpdesk]
<http://support.itcenter.com.pt/otrs/customer.pl>[image: ITCENTER Facebook]
<https://www.facebook.com/itcenterpt>[image: ITCENTER Linkedin]
<https://www.linkedin.com/company/itcenter>[image: ITCENTER Twitter]
<https://twitter.com/ITCenter_PT>
I have one kamailio server listening in 2 network
interfaces, kamailio
receives sip messages from interface A and then send it to destine through
interface B.
When we start kamailio service then run gdb for each kamailio network
interface process, execute the command "bt full" the result is the
following:
*tmp = 0x5f801b994c4715bd <Address 0x5f801b994c4715bd out of bounds>*
interface - A
bt full
#0 0x00007fb01bfa40c3 in recvfrom () from /lib64/libc.so.6
No symbol table info available.
#1 0x000000000047f2ba in udp_rcv_loop () at udp_server.c:421
len = 13
buf = '\000' <repeats 65535 times>
tmp = 0x5f801b994c4715bd <Address 0x5f801b994c4715bd out of
bounds>
from = 0x7faf9bf38bb8
fromlen = 16
ri = {src_ip = {af = 2309178400, len = 32765, u = {addrl =
{5458819, 4284160}, addr32 = {5458819, 0, 4284160, 0}, addr16 = {19331, 83,
0, 0, 24320, 65, 0, 0},
addr =
"\203KS\000\000\000\000\000\000_A\000\000\000\000"}}, dst_ip = {af = 2, len
= 4, u = {addrl = {80026686, 0}, addr32 = {80026686, 0, 0, 0},
addr16 = {7230, 1221, 0, 0, 0, 0, 0, 0}, addr =
">\034\305\004", '\000' <repeats 11 times>}}, src_port =
55576, dst_port =
5060, proto_reserved1 = 0,
proto_reserved2 = 0, src_su = {s = {sa_family = 32687, sa_data
= "\000\000\200D\243\211\375\177\000\000\213\337D"}, sin = {sin_family =
32687, sin_port = 0,
sin_addr = {s_addr = 2309178496}, sin_zero =
"\375\177\000\000\213\337D"}, sin6 = {sin6_family = 32687, sin6_port = 0,
sin6_flowinfo = 2309178496,
sin6_addr = {__in6_u = {__u6_addr8 =
"\375\177\000\000\213\337D\000\000\000\000\000p", <incomplete sequence
\372\233>, __u6_addr16 = {32765, 0, 57227,
68, 0, 0, 21616, 39930}, __u6_addr32 = {32765,
4513675, 0, 2616874096}}}, sin6_scope_id = 32687}}, bind_address =
0x7faf9becd7d8, proto = 1 '\001'}
__FUNCTION__ = "udp_rcv_loop"
#2 0x0000000000525eae in main_loop () at main.c:1573
i = 0
pid = 0
si = 0x7faf9becd7d8
si_desc = "udp receiver child=0 sock=62.28.197.4:5060
\000\000\000\000\000\000\001\000\000\000\000\000\000\000\260\244r\023n\000\000\000\071\263q\000\000\000\000\000\020\243q\000\000\000\000\000\b\000\000\000\001\000\000\000\260\373\371\233\257\177\000\000\220E\243\211\375\177\000\000|\351O\000\000\000\000\000\000\266\253\023\257\177\000\000\260\244r\023\257\177\000"
---Type <return> to continue, or q <return> to quit---
nrprocs = 32
__FUNCTION__ = "main_loop"
#3 0x000000000052c395 in main (argc=11, argv=0x7ffd89a34848) at
main.c:2533
cfg_stream = 0x2296010
c = -1
r = 0
tmp = 0x7ffd89a35f61 ""
tmp_len = 0
port = 0
proto = 32688
options = 0x725010
":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
ret = -1
seed = 1641321572
rfd = 4
debug_save = 0
debug_flag = 0
dont_fork_cnt = 0
n_lst = 0x7ffd89a34710
p = 0x7fb01bf4c646 <intel_check_word+166>
"H\205\300t\025\017\266P\003\017\266\312D9\371\017\204", <incomplete
sequence \305>
__FUNCTION__ = "main"
interface B
bt full
#0 0x00007fb01bfa40c3 in recvfrom () from /lib64/libc.so.6
No symbol table info available.
#1 0x000000000047f2ba in udp_rcv_loop () at udp_server.c:421
len = 45
buf = '\000' <repeats 65535 times>
tmp = 0x13cb2f24296000e9 <Address 0x13cb2f24296000e9 out of
bounds>
from = 0x7faf9bf38bb8
fromlen = 16
ri = {src_ip = {af = 2309178400, len = 32765, u = {addrl =
{5458819, 4284160}, addr32 = {5458819, 0, 4284160, 0}, addr16 = {19331, 83,
0, 0, 24320, 65, 0, 0}, addr =
"\203KS\000\000\000\000\000\000_A\000\000\000\000"}},
dst_ip = {af = 2, len = 4, u = {addrl = {2764381246, 0},
addr32 = {2764381246, 0, 0, 0}, addr16 = {7230, 42181, 0, 0, 0, 0, 0, 0},
addr = ">\034Ť", '\000' <repeats 11 times>}}, src_port =
56136, dst_port =
5060,
proto_reserved1 = 0, proto_reserved2 = 0, src_su = {s =
{sa_family = 32687, sa_data =
"\000\000\200D\243\211\375\177\000\000\213\337D"}, sin = {sin_family =
32687, sin_port = 0, sin_addr = {s_addr = 2309178496},
sin_zero = "\375\177\000\000\213\337D"}, sin6 =
{sin6_family = 32687, sin6_port = 0, sin6_flowinfo = 2309178496, sin6_addr
= {__in6_u = {
__u6_addr8 =
"\375\177\000\000\213\337D\000\000\000\000\000p", <incomplete sequence
\372\233>, __u6_addr16 = {32765, 0, 57227, 68, 0, 0, 21616, 39930},
__u6_addr32 = {32765, 4513675, 0, 2616874096}}},
sin6_scope_id = 32687}}, bind_address = 0x7faf9becda08,
proto = 1 '\001'}
__FUNCTION__ = "udp_rcv_loop"
#2 0x0000000000525eae in main_loop () at main.c:1573
---Type <return> to continue, or q <return> to quit---
i = 0
pid = 0
si = 0x7faf9becda08
si_desc = "udp receiver child=0 sock=62.28.197.164:5060
\000\000\000\000\001\000\000\000\000\000\000\000\260\244r\023n\000\000\000\071\263q\000\000\000\000\000\020\243q\000\000\000\000\000\b\000\000\000\001\000\000\000\260\373\371\233\257\177\000\000\220E\243\211\375\177\000\000|\351O\000\000\000\000\000\000\266\253\023\257\177\000\000\260\244r\023\257\177\000"
nrprocs = 32
__FUNCTION__ = "main_loop"
#3 0x000000000052c395 in main (argc=11, argv=0x7ffd89a34848) at
main.c:2533
cfg_stream = 0x2296010
c = -1
r = 0
tmp = 0x7ffd89a35f61 ""
tmp_len = 0
port = 0
proto = 32688
options = 0x725010
":f:cm:M:dVIhEeb:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:"
---Type <return> to continue, or q <return> to quit---
ret = -1
seed = 1641321572
rfd = 4
debug_save = 0
debug_flag = 0
dont_fork_cnt = 0
n_lst = 0x7ffd89a34710
p = 0x7fb01bf4c646 <intel_check_word+166>
"H\205\300t\025\017\266P\003\017\266\312D9\371\017\204", <incomplete
sequence \305>
__FUNCTION__ = "main"
[root@c4-srv1 kamailio]# kamctl stats shmem
shmem:fragments = 2552
shmem:free_size = 2082764040
shmem:max_used_size = 66449600
shmem:real_used_size = 64719608
shmem:total_size = 2147483648
shmem:used_size = 54355992
Can anyone explain the meaning of
*tmp = 0x5f801b994c4715bd <Address 0x5f801b994c4715bd out of bounds>*
Best Regards
José Seabra
--
Cumprimentos
José Seabra
_______________________________________________
SIP Express Router (SER) and Kamailio (OpenSER) - sr-users mailing
listsr-users@lists.sip-router.orghttp://lists.sip-router.org/cgi-bin/mailman/listinfo/sr-users
--
Daniel-Constantin Mierlahttp://twitter.com/#!/miconda -
http://www.linkedin.com/in/miconda
Book: SIP Routing With Kamailio - http://www.asipto.com
Kamailio Advanced Training, Sep 28-30, 2015, in Berlin - http://asipto.com/u/kat
_______________________________________________
SIP Express Router (SER) and Kamailio (OpenSER) - sr-users mailing list
sr-users(a)lists.sip-router.org
http://lists.sip-router.org/cgi-bin/mailman/listinfo/sr-users
29 Sep
29 Sep
12:44 a.m.
Hello,
maxbuffer doesn't influence processing of udp traffic, because each SIP
message over UDP is handled at once, not internal buffering of many udp
packets.
You look at the statistics for network interface to see if there are
dropped received packets (ifconfig should show that).
What is the value for children parameter in kamailio.cfg?
CentOS has lots of hidden and strange limits, the next suggestion I
could do it to try on a Debian server
Cheers,
Daniel
On 28/09/15 19:07, José Seabra wrote:
--
Daniel-Constantin Mierla
http://twitter.com/#!/miconda - http://www.linkedin.com/in/miconda
Book: SIP Routing With Kamailio - http://www.asipto.com
Hello Daniel,
Indeed xlog doesn't print in syslog the sip message received and
wireshak got it, that means that packet was received in network
interface but kamailio didn't received the packet(as xlog didn't print
that packet).
Anyway i have configured the following parameters in sysctl
net.ipv4.udp_rmem_min = 131072
net.ipv4.udp_wmem_min = 131072
net.core.netdev_max_backlog=1000
net.core.rmem_max=67108864
net.ipv4.udp_mem = 192576 256768 385152
and also set maxbuffer=134190336 in my kamailio configuration.
With these configurations I got an improvement but still below of what
we need and even the CPU is between 65% and 70% sleeping.
Do you have any suggestion? the traffic average that interface
receives now is 200mbs.
The server network interface is 10GB.
Any suggestion is welcome.
Best Regards
José Seabra
2015-09-27 17:08 GMT+01:00 Daniel-Constantin Mierla <miconda(a)gmail.com
<mailto:miconda@gmail.com>>:
Hello,
first, be sure you run latest version of branch 4.3, there was a
fix for dialog since 4.3.2.
If a process in shown to run recvfrom() that means it waits for
kernel to pass a new upd packet to it -- recvfrom() is a function
from standard lib c.
Wireshark (like other sniffing tools) is capturing the network
traffic before the kernel passes it to application layer. If
wireshark see it, doesn't mean the application has received it
from kernel, it can still be dropped by firewall or other kernel
modules.
You can add a xlog() at the beginning of request_route to see what
requests are received by kamailio.
Cheers,
Daniel
On 27/09/15 15:52, José Seabra wrote:
> Hello Danel,
> Thank you for your response,
>
> Well i don't have SELinux enabled and IPTables isn't running.
>
> About kamailio configuration, Pike module isn't also enabled, the
> only thing that i have enabled to limit the number of concurrent
> calls is the dialog module, counting the number of active
> dialogs and reject next new invites only after it has reached
> the 10.000 simultaneous calls.
>
> The strange behavior is that kamailio only stop processing some
> sip messages like Byes and 200 Ok after more or less 40 seconds
> after siptester start sending calls.
>
> And i can see several cases in wireshark that kamailio receives
> BYES/200OK from interface B(Carriers interface) but it isn't
> forward to Interface A(customer interface), if i make for example
> 20cps i don't have this issue, all packets are handled correctly
> and there isn't 408 requests timeouts.
>
>
> Best Regards
> José Seabra
>
>
>
> 2015-09-27 13:52 GMT+01:00 Daniel-Constantin Mierla
> <miconda(a)gmail.com <mailto:miconda@gmail.com>>:
>
> The tmp variable is not intialized by default, so it can be
> anything therefore the out of bounds value. It used only when
> printing some errors, so it didn't get a proper value as no
> such error was printed for you case.
>
> The backtrace shows the process in in recvfrom(), so it waits
> for traffic from network. That is the case when the
> processing of former sip message was finished and now
> Kamailio waits to receive another one.
>
> If you run centos, be sure you don't have limits in selinux
> rules - better disable it completely and test again. Also, be
> sure you don't have limits in the firewall, especially
> contrack module and sip firewall modules in the kernel.
>
> In kamailio.cfg, if you have pike module, be sure you don't
> drop good packet with this module.
>
> Cheers,
> Daniel
>
>
> On 25/09/15 19:53, José Seabra wrote:
>> Adding more information to my last email, when kamailio
>> starts receiving calls, after more or less 40 seconds it
>> stops processing some sip messages.
>>
>> I'm using the kamailio version 4.3.2
>>
>> Best Regards
>>
3343
days inactive
3346
days old
6 comments
2 participants
participants (2)
-
Daniel-Constantin Mierla -
José Seabra