litespeed gone haywire

MentaL

MentaL

Well-Known Member
#1
#1
Had an attack earlier sending several thousand requests. I've managed to lock the load at around 6 but strangely now my main website takes around 30 seconds to a minute to load a whilst the other on the server loads instant.

I've enabled it and now have all the following settings (and locks at around 250 requests instead of 2000);








strace
Code: 
root@domain [/home/domain]# strace -c /usr/local/lsws/fcgi-bin/lsphp5
% time     seconds  usecs/call     calls    errors syscall
------ ----------- ----------- --------- --------- ----------------
 83.51    0.000238           3        93           mmap
 12.28    0.000035           0       104        39 open
  4.21    0.000012           0        57           read
  0.00    0.000000           0        50           close
  0.00    0.000000           0         8         7 stat
  0.00    0.000000           0        41           fstat
  0.00    0.000000           0        12           lstat
  0.00    0.000000           0         1           lseek
  0.00    0.000000           0        39           mprotect
  0.00    0.000000           0        11           munmap
  0.00    0.000000           0        17           brk
  0.00    0.000000           0         8           rt_sigaction
  0.00    0.000000           0         1           rt_sigprocmask
  0.00    0.000000           0         1         1 ioctl
  0.00    0.000000           0         2           readv
  0.00    0.000000           0         3         1 access
  0.00    0.000000           0        85         1 select
  0.00    0.000000           0         1           dup2
  0.00    0.000000           0         1           socket
  0.00    0.000000           0         1           connect
  0.00    0.000000           0         1           sendmsg
  0.00    0.000000           0         1         1 getpeername
  0.00    0.000000           0         1           execve
  0.00    0.000000           0         6           fcntl
  0.00    0.000000           0         1         1 ftruncate
  0.00    0.000000           0         1           getcwd
  0.00    0.000000           0        16           unlink
  0.00    0.000000           0         1           readlink
  0.00    0.000000           0         1           getrlimit
  0.00    0.000000           0        16           getuid
  0.00    0.000000           0        85           getppid
  0.00    0.000000           0         1           arch_prctl
  0.00    0.000000           0         1           setrlimit
  0.00    0.000000           0         3           futex
  0.00    0.000000           0         1           set_tid_address
  0.00    0.000000           0         1           set_robust_list
------ ----------- ----------- --------- --------- ----------------
100.00    0.000285                   673        51 total
Code: 
poll([{fd=20, events=POLLIN|POLLPRI}], 1, 0) = 0 (Timeout)
write(20, "O\r\0\0\3select t.forumid, t.threadi"..., 3411) = 3411
read(20, "\1\0\0\1\n:\0\0\2\3def\17domain_forums\1t\6"..., 16384) = 16384
read(20, "s - Works Perfectly\00234\7Lithium\0076"..., 16384) = 16384
read(20, "5804\0010\n1321884977\0011l\0\0w\003721\0067956"..., 16384) = 9317
poll([{fd=20, events=POLLIN|POLLPRI}], 1, 0) = 0 (Timeout)
write(20, "\20\0\0\0\2domain_forums", 20) = 20
read(20, "\7\0\0\1\0\0\0\2\0\0\0", 16384) = 11
poll([{fd=20, events=POLLIN|POLLPRI}], 1, 0) = 0 (Timeout)
write(20, "\177\0\0\0\3\nselect p.postid, t.threadi"..., 131) = 131
read(20, "\1\0\0\1\0046\0\0\2\3def\17domain_forums\1p\4"..., 16384) = 340
poll([{fd=20, events=POLLIN|POLLPRI}], 1, 0) = 0 (Timeout)
write(20, "\314\0\0\0\3\n\t\t\t\t\tUPDATE session\n\t\t\t\t\tS"..., 208) = 208
read(20, "0\0\0\1\0\1\0\2\0\0\0(Rows matched: 1  Cha"..., 16384) = 52
poll([{fd=20, events=POLLIN|POLLPRI}], 1, 0) = 0 (Timeout)
write(20, "8\0\0\0\3\n\t\tINSERT INTO threadviews "..., 60) = 60
read(20, "\7\0\0\1\0\1\0\2\0\0\0", 16384) = 11
writev(19, [{"LS\3\0O\1\0\0\7\0\0\0\0\0\0\0\31\0b\0\27\0\20\0'\0\26\0R\0", 30}, {"X-Powered-By: PHP/5.2.17\0Set-Coo"..., 305}, {"LS\4\0\10@\0\0", 8}, {"<!DOCTYPE html PUBLIC \"-//W3C//D"..., 16384}, {"LS\4\0\10@\0\0", 8}, {"m/arcade/images/trophy.gif' alt="..., 16384}, {"LS\4\0\10@\0\0", 8}, {"der vbseo_like_postbit\" cellpadd"..., 16384}, {"LS\4\0O\f\0\0", 8}, {"/tr> <tr> <td class=\"thead\">Book"..., 3143}], 10) = 52662
chdir("/usr/local/lsws/fcgi-bin")       = 0
rt_sigaction(SIGPIPE, {0x1, [PIPE], SA_RESTORER|SA_RESTART, 0x32c50302d0}, {0x1, [PIPE], SA_RESTORER|SA_RESTART, 0x32c50302d0}, 8) = 0
write(20, "\1\0\0\0\1", 5)              = 5
shutdown(20, 2 /* send and receive */)  = 0
close(20)                               = 0
rt_sigaction(SIGPIPE, {0x1, [PIPE], SA_RESTORER|SA_RESTART, 0x32c50302d0}, {0x1, [PIPE], SA_RESTORER|SA_RESTART, 0x32c50302d0}, 8) = 0
fcntl(3, F_SETLKW, {type=F_WRLCK, whence=SEEK_SET, start=0, len=1}) = 0
fcntl(3, F_SETLKW, {type=F_UNLCK, whence=SEEK_SET, start=0, len=1}) = 0
fcntl(4, F_SETLKW, {type=F_WRLCK, whence=SEEK_SET, start=0, len=1}) = 0
fcntl(4, F_SETLKW, {type=F_UNLCK, whence=SEEK_SET, start=0, len=1}) = 0
fcntl(5, F_SETLKW, {type=F_WRLCK, whence=SEEK_SET, start=0, len=1}) = 0
fcntl(5, F_SETLKW, {type=F_UNLCK, whence=SEEK_SET, start=0, len=1}) = 0
fcntl(6, F_SETLKW, {type=F_WRLCK, whence=SEEK_SET, start=0, len=1}) = 0
fcntl(6, F_SETLKW, {type=F_UNLCK, whence=SEEK_SET, start=0, len=1}) = 0
fcntl(7, F_SETLKW, {type=F_WRLCK, whence=SEEK_SET, start=0, len=1}) = 0
fcntl(7, F_SETLKW, {type=F_UNLCK, whence=SEEK_SET, start=0, len=1}) = 0
fcntl(8, F_SETLKW, {type=F_WRLCK, whence=SEEK_SET, start=0, len=1}) = 0
fcntl(8, F_SETLKW, {type=F_UNLCK, whence=SEEK_SET, start=0, len=1}) = 0
fcntl(9, F_SETLKW, {type=F_WRLCK, whence=SEEK_SET, start=0, len=1}) = 0
fcntl(9, F_SETLKW, {type=F_UNLCK, whence=SEEK_SET, start=0, len=1}) = 0
fcntl(10, F_SETLKW, {type=F_WRLCK, whence=SEEK_SET, start=0, len=1}) = 0
fcntl(10, F_SETLKW, {type=F_UNLCK, whence=SEEK_SET, start=0, len=1}) = 0
open("/dev/urandom", O_RDONLY)          = 20
read(20, "\v\"aC\370\177\242\273", 8)   = 8
close(20)                               = 0
open("/dev/urandom", O_RDONLY)          = 20
read(20, "\361O_\226?\331O\361", 8)     = 8
close(20)                               = 0
open("/dev/urandom", O_RDONLY)          = 20
read(20, "\307\34~T&\36a\10", 8)        = 8
close(20)                               = 0
setitimer(ITIMER_PROF, {it_interval={0, 0}, it_value={0, 0}}, NULL) = 0
writev(19, [{"LS\5\0\10\0\0\0", 8}], 1) = 8
close(21)                               = 0
munmap(0x2b3a5c918000, 2154256)         = 0
close(3)                                = 0
close(4)                                = 0
close(5)                                = 0
close(6)                                = 0
close(7)                                = 0
close(8)                                = 0
close(9)                                = 0
close(10)                               = 0
close(11)                               = 0
close(12)                               = 0
close(13)                               = 0
close(14)                               = 0
close(15)                               = 0
close(16)                               = 0
close(17)                               = 0
close(18)                               = 0
munmap(0x2b3a5cd3e000, 167772160)       = 0
munmap(0x2b3a5c6fb000, 2214456)         = 0
brk(0x9f29000)                          = 0x9f29000
exit_group(0)                           = ?
netstat

Code: 
root@domain [/home/domain]# netstat -nt|awk '{print $5;}'|awk -F ':' '{print $1;}'|sort|uniq -c|sort -r|headnetstat -nt|grep ESTABLISHED|wc
    274    1644   24386
Connections attached although its compressed into a winzip file since its over 20kb.
 

Attachments

Last edited:
MentaL

MentaL

Well-Known Member
#2
#2
This is what happens when php suexec is disabled;


And CloudFlare stats;



Current top stats

Code: 
top - 15:50:18 up  3:07,  1 user,  load average: 5.16, 5.11, 5.14
Tasks: 184 total,   6 running, 176 sleeping,   2 stopped,   0 zombie
Cpu(s): 69.6%us,  2.9%sy,  0.0%ni, 26.1%id,  0.4%wa,  0.0%hi,  0.9%si,  0.0%st
Mem:   8181024k total,  5761360k used,  2419664k free,    66504k buffers
Swap: 16771576k total,        0k used, 16771576k free,  4633292k cached

  PID USER      PR  NI  VIRT  RES  SHR S %CPU %MEM    TIME+  COMMAND
23103 domain  18   0  272m  58m  20m R 96.2  0.7   1:11.75 lsphp5
23104 domain  18   0  278m  75m  32m R 96.2  1.0   0:58.23 lsphp5
23033 domain  17   0  273m  95m  56m R 94.3  1.2   1:44.55 lsphp5
23072 domain  17   0  269m  62m  29m R 94.3  0.8   1:17.80 lsphp5
23074 domain  18   0  282m  94m  47m R 94.3  1.2   1:13.68 lsphp5
 3988 mysql     10  -5  839m 371m 3980 S 11.8  4.6  23:44.38 mysqld
 4615 nobody    15   0 99.0m  33m  780 S  3.9  0.4   7:50.54 memcached
20378 nobody     0 -19 34084  12m  696 S  2.0  0.2   0:43.54 litespeed
23266 root      15   0 12756 1048  728 R  2.0  0.0   0:00.01 top
    1 root      15   0 10320  684  572 S  0.0  0.0   0:02.62 init
Code: 
top - 16:36:18 up  3:53,  1 user,  load average: 5.07, 5.08, 5.04
Tasks: 185 total,   6 running, 176 sleeping,   3 stopped,   0 zombie
Cpu(s): 59.2%us,  2.3%sy,  0.0%ni, 37.8%id,  0.0%wa,  0.0%hi,  0.6%si,  0.0%st
Mem:   8181024k total,  5935692k used,  2245332k free,    74276k buffers
Swap: 16771576k total,        0k used, 16771576k free,  4735644k cached

  PID USER      PR  NI  VIRT  RES  SHR S %CPU %MEM    TIME+  COMMAND
26019 domain  18   0  283m  74m  26m R 97.7  0.9   1:49.66 lsphp5:/home/domain/public_html/forum/index.php
25988 domain  18   0  284m 107m  59m R 96.4  1.3   2:09.78 lsphp5:/home/domain/public_html/forum/index.php
25986 domain  18   0  275m  90m  51m R 95.4  1.1   2:19.98 lsphp5:/home/domain/public_html/forum/index.php
26023 domain  18   0  281m  73m  26m R 94.7  0.9   1:30.99 lsphp5:/home/domain/public_html/forum/index.php
25987 domain  17   0  275m 100m  61m R 94.1  1.3   2:18.17 lsphp5:/home/domain/public_html/forum/index.php
 3988 mysql     10  -5  842m 408m 3984 S 11.6  5.1  30:42.48 /usr/sbin/mysqld --basedir=/ --datadir=/var/lib/mysql --user=mysql --log-error=/var/lib/mysql/dmca.domain.com.err
 4615 nobody    15   0 99.0m  33m  780 S  4.0  0.4   9:37.14 /usr/local/bin/memcached -u root -m 2048 -p 11211 -u nobody -l 127.0.0.1
20378 nobody     0 -19 36740  16m  620 S  1.7  0.2   1:30.37 litespeed (
Specs ; Dual Quad E5045 (8 CPU) w/ 8GB Ram and Raid 10 setup.

Help = APPRECIATED!
 
Last edited:
M

mistwang

LiteSpeed Staff
#3
#3
You can increase the PHP suEXEC max conn, right now is 5, and the WaitQ is at >200.
Try 50, then increase it gradually if want. remember, the high the "max conn", the higher the load. it is normal.

You can try our antiDDoS service to filter the attack.
 
MentaL

MentaL

Well-Known Member
#4
#4
mistwang said:
You can increase the PHP suEXEC max conn, right now is 5, and the WaitQ is at >200.
Try 50, then increase it gradually if want. remember, the high the "max conn", the higher the load. it is normal.

You can try our antiDDoS service to filter the attack.
Click to expand...
I've made minor alterations none that are effective. An increase of 5 = to an additional 5 load. I'm also using CloudFlare that has built in protection so unsure how that would conflict. When I set to 30 it just cripples the load, too many connections being sent. The stats below are with it set to 15.

Code: 
top - 19:32:08 up  6:49,  1 user,  load average: 27.07, 23.98, 14.47
Tasks: 195 total,  17 running, 175 sleeping,   3 stopped,   0 zombie
Cpu(s): 64.9%us,  2.7%sy,  0.0%ni, 31.3%id,  0.3%wa,  0.0%hi,  0.8%si,  0.0%st
Mem:   8181024k total,  6222480k used,  1958544k free,    45424k buffers
Swap: 16771576k total,       16k used, 16771560k free,  4572012k cached

  PID USER      PR  NI  VIRT  RES  SHR S %CPU %MEM    TIME+  COMMAND
 7873 domain  16   0  281m  72m  24m R 84.1  0.9   0:36.20 lsphp5:/home/domain/public_html/forum/index.php
 7879 domain  16   0  282m  71m  24m R 82.3  0.9   0:35.76 lsphp5:/home/domain/public_html/forum/index.php
 7882 domain  16   0  274m  60m  20m R 78.6  0.8   0:35.69 lsphp5:/home/domain/public_html/forum/vbseo.php
 7866 domain  16   0  281m  67m  20m R 76.8  0.8   0:39.56 lsphp5:/home/domain/public_html/forum/index.php
 7870 domain  17   0  267m  53m  19m R 71.3  0.7   0:36.53 lsphp5:/home/domain/public_html/forum/index.php
 7876 domain  16   0  272m  58m  21m R 64.0  0.7   0:33.65 lsphp5:/home/domain/public_html/forum/index.php
 7878 domain  17   0  272m  58m  21m R 58.5  0.7   0:38.01 lsphp5:/home/domain/public_html/forum/index.php
 7872 domain  16   0  271m  56m  19m R 56.7  0.7   0:30.73 lsphp5:/home/domain/public_html/forum/index.php
 7867 domain  16   0  269m  55m  19m R 42.1  0.7   0:37.05 lsphp5:/home/domain/public_html/forum/index.php
 7874 domain  16   0  272m  57m  19m R 29.3  0.7   0:37.34 lsphp5:/home/domain/public_html/forum/index.php
 7865 domain  16   0  272m  59m  21m R 27.4  0.7   0:37.79 lsphp5:/home/domain/public_html/forum/index.php
 3988 mysql     10  -5  860m 485m 4096 S 25.6  6.1  60:31.87 /usr/sbin/mysqld --basedir=/ --datadir=/var/lib/mysql --user=mysql --log-error=/var/lib/mysql/dmca.domain.com.err
 7871 domain  16   0  266m  57m  24m R 25.6  0.7   0:36.32 lsphp5:/home/domain/public_html/forum/vbseo.php
 7875 domain  15   0  270m  55m  19m R 16.5  0.7   0:37.78 lsphp5:/home/domain/public_html/forum/index.php
 4615 nobody    15   0 99.1m  33m  780 S  7.3  0.4  16:15.80 /usr/local/bin/memcached -u root -m 2048 -p 11211 -u nobody -l 127.0.0.1
 7877 domain  16   0  266m  53m  21m R  3.7  0.7   0:35.52 lsphp5:/home/domain/public_html/forum/index.php
 7863 nobody     0 -19 33156  11m  596 S  1.8  0.1   0:02.13 litespeed (lshttpd)
 7869 domain  17   0  258m  43m  19m R  1.8  0.5   0:36.35 lsphp5:/home/domain/public_html/forum/index.php
I've attached the connections which are spamming "GET / HTTP/1.1". EAProc WaitQ is now over 500.
 

Attachments

MentaL

MentaL

Well-Known Member
#7
#7
Attack stopped. Cloudflare managed to stop a bit of it but not all. What is the best way to create a static page when being flooded?
 
W

webizen

Well-Known Member
#8
#8
if require no user login, you may run a cronjob to generate the result of index.php and save to a file, say default.html. point to directory index to default.html.

if user login is required, then page cache (for public) is a way to go.

anti-ddos solution is another way to go.
 
You must log in or register to reply here.
Top