Block IP Abuse

From DreamHost
Jump to: navigation, search
The instructions provided in this article or section are considered advanced.

You are expected to be knowledgeable in the UNIX shell.
Support for these instructions is not available from DreamHost tech support.
Server changes may cause this to break. Be prepared to troubleshoot this yourself if this happens.
We seriously aren't kidding about this.

IP Abuse Detection Script

This shell script checks the access and error logs generated by apache for a particular domain, looking for the IP addresses that have connected to your site the most. It checks for IP addresses that trigger a Concurrent Connection Limit Exceeded error, which is a good sign they are an automated bot of some kind, making over 20 requests to your site at the same time. This script also checks for Internal Recursion Errors which can have very negative effects on your speed and resources, and are basically internal looping problems generally caused by improperly configured Htaccess setups.

Once the script finishes scanning your logs for those events, it automatically generates .htaccess code that you may add to your sites root .htaccess file to block those IP addresses the script identified as abusive. The only IP addresses included in the generated .htaccess file are those that have no reverse dns.

alt text


Installation

  1. Log in to your account using SSH
  2. Save this code in your $HOME directory as ip-abuse-lookup.sh
    1. Run pico $HOME/ip-abuse-lookup.sh
    2. Copy the code to the screen by clicking the right-mouse-button
    3. Hold down the Ctrl button and then press x to save
  3. Run the command dos2unix -dv $HOME/ip-abuse-lookup.sh to fix line break issues (NOTE: fromdos replaces dos2unix on Debian 6)
  4. Run the command chmod -v 744 $HOME/ip-abuse-lookup.sh to make executable

Running the Script

From your $HOME directory (cd $HOME), run ./ip-abuse-lookup.sh to execute the program.


Example Generated .htaccess

This script will also generate code that you can place in your .htaccess file to block specific abusers.

## IP-ABUSE-LOOKUP
Order Allow,Deny
Allow from All
Deny from 6.132.177.129 27.67.117.178 6.135.166.102 8.93.225.133
Deny from 21.194.136.15 22.120.61.3 6.252.139.246 9.64.50.83
Deny from 8.123.144.98 21.249.83.87 29.85.238.28 25.214.237.62
Deny from 22.115.130.23 13.57.156.241 14.121.4.82 6.208.172.177


ip-abuse-lookup.sh

#!/bin/sh
# Version 0.2, 2008-04-20

# User-contributed script. Not sponsored by DreamHost.
# Script created 2008-01-16 by AskApache 


### SHELL OPTIONS
set +o noclobber  # allowed to clobber files
set +o noglob     # globbing on
set +o xtrace     # change to - to enable tracing
set +o verbose    # change to - to enable verbose debugging
set -e            # abort on first error


# directory where log files, reports, and generated .htaccess files will be saved
TMPDI="$HOME/ip_abuse"




function exitt(){
 # RESET WINDOW TITLE
 case $TERM in
  xterm*|vt*|ansi|rxvt|gnome*) echo -e "\033]0;$USER@`hostname`: $HOME\007" ;;
 esac
 
 # RESET TERMINAL
 tput sgr0
}

function ok_continue()
{ echo -en "\n\033[?25l\033[30;42m[ Press any key to continue ]\033[0m\n" ;read -n 1 ans;echo -en "\033[?25h"; }

function test_title()
{ echo -en "\n\n\033[0;32m\n>>>"; echo -e "\033[1;37m $1 \033[0m \n"; echo -e "\n\n[ ${1} ]\n" >> $REPORT; }


function error_abuse(){
 clear; title
 case "$2" in
  connlimit|1*) test_title "CONCURRENT CONNECTION TEST"; future=2
   echo -e "Shows IP's making more than 20 requests concurrently.\n\n"
   cat $TMPDI/$YD/logs/e* |grep 'concurrent\ connection\ limit'|awk -F ']' '{print $3}' |
   awk '{print $2}'|sort|uniq -c |sort -nr|sed 's/^ *//'|egrep "[0-9]{2}+\ " > $TMPDI/$YD/logs/out.txt ;;
  access|2*) test_title "TOP 50 IP TEST"; future=3
   echo -e "Displays the top 50 unique IP addresses that access your site."
   echo -e "If they don't have a reverse DNS maybe they should be blocked.\n\n"
   cat $TMPDI/$YD/logs/a* |awk '{print $1|"sort|uniq -dc|sort -nr"}' |egrep "[0-9]{3}+\ "|
   awk '{print $1,$2}' > $TMPDI/$YD/logs/out.txt ;;
  internal|3*) test_title "INTERNAL RECURSION TEST"; future="report"
   echo -e "Shows the IP's that triggered an Internal Recursion Error,"
   echo -e "meaning that their is a looping problem on your server.\n"
   cat $TMPDI/$YD/logs/e* |grep 'LimitInternalRecursion'|awk -F ']' '{print $3}'|awk '{print $2}' |
  sort|uniq -c|sort -nr|sed 's/^ *//'|egrep "[0-9]{2}+\ " > $TMPDI/$YD/logs/out.txt ;;
 esac

 echo -e "\033[?25l";t=0; h=0; cat $TMPDI/$YD/logs/out.txt |
 while read a
 do
 if [ $t -lt 50 ];then
  n=`echo "$a"|awk '{print $1}'`; ip=`echo "$a"|awk '{print $2}'`
  host=`nice -n 19 host -qQ -s 4 "${ip}" 2>&1|tr '\n' '\t'|awk '{print $2}'`;
  case "$host" in
   does) echo -en "\033[0;33m"; echo -e "$ip" >> $REPORT
    host=${host/does/!!!POSSIBLE-ABUSE!!!}; (( t++ )); (( h++ )); ips=" $ip$ips" ;;
   not) echo -en "\033[0;33m"; echo -e "$ip" >> $REPORT
    host=${host/not/!!!POSSIBLE-ABUSE!!!}; (( t++ )); (( h++ )); ips=" $ip$ips" ;;
   PTR) echo -en "\033[0;33m"; echo -e "$ip" >> $REPORT
    host=${host/PTR/!!!POSSIBLE-ABUSE!!!}; (( t++ )); (( h++ )); ips=" $ip$ips" ;;
   .) echo -en "\033[0;33m"; echo -e "$ip" >> $REPORT
    host=${host/\./!!!POSSIBLE-ABUSE!!!}; (( t++ )); (( h++ )); ips=" $ip$ips" ;;
  esac
  echo -en " ${n}\t${ip}\t${host}\033[0m\n"
  if [ $h -gt 3 ]; then
  echo -e "Deny from$ips" >> $TMPDI/$YD/.htaccess; h=0; ips=" ";
  fi
fi
done
[ $h -gt 0 ] && echo -e "Deny from$ips" >> $TMPDI/$YD/.htaccess
ok_continue; exec sh $0 "$1" "$future"
}





function menu(){
 PS3="`echo -e '\n\033[0;36m'`Please Select a Domain To Test: `echo -e '\033[0m'`"; echo -ne "\033[0m"
 select v
 do
  YD="$v"
  mkdir -p -m 0755 $TMPDI/$YD/reports
  mkdir -p -m 0755 $TMPDI/$YD/logs
  echo "" > $TMPDI/$YD/logs/out.txt
  echo "" > $TMPDI/$YD/.htaccess
  echo -e "## IP-ABUSE-LOOKUP\nOrder Allow,Deny\nAllow from All" > $TMPDI/$YD/.htaccess
  ELOG="$TMPDI/$YD/logs/error.log"
  ALOG="$TMPDI/$YD/logs/access.log"
  REPORT=$(echo -en "$TMPDI/$YD/reports/`date +%mx%dx%y`.txt");
  echo "" > $REPORT; echo -e "GENERATED REPORT FOR $YD \n`date`\n" >> $REPORT
  clear; title; cd ~/logs/$YD/http
  if [ ! -f $ELOG ]; then
   test_title "Creating Error log"
   `nice -n 19 gunzip -dc e*.gz | split -b 1m -a 4 -d - $TMPDI/$YD/logs/e.`; wait
   cp error.log $TMPDI/$YD/logs; echo -e "\033[0;31m [ DONE ]\n\n"
  fi
  if [ ! -f $ALOG ]; then
   test_title "Creating access log"
   `nice -n 19 gunzip -dc a*.gz | split -b 5m -a 4 -d - $TMPDI/$YD/logs/a.`; wait
   cp access.log $TMPDI/$YD/logs; echo -e "\033[0;31m [ DONE ]\n\n"
  fi
  cd $HOME; exec sh $0 "$YD" "1"
 done
}

function show_report(){
 clear; title;
 PS3="`echo -e '\n\033[0;36m'`Please select a course of action: `echo -e '\033[0m'`"; echo -ne "\033[0m"
 select v in "View Report" "View .htaccess" "Quit"
 do
  case "$v" in
   *Report) clear; title; test_title "VIEWING $REPORT"; cat $REPORT ;;
   *htaccess) clear; title; test_title "VIEWING $TMPDI/$YD/.htaccess"; cat $TMPDI/$YD/.htaccess ;;
   Quit) break;;
  esac
 done
exit 0
}

function title(){
 # pretty sweet!
 echo -e "\033[1;30m __________________________________________________________________________ "
 echo -e "|\033[1;32m                 ___                     __ __         __                 \033[1;30m|"
 echo -e "|\033[1;32m                / _ \_______ ___ ___ _  / // /__  ___ / /_                \033[1;30m|"
 echo -e '|\033[1;32m               / // / __/ -_) _ `/  ` \/ _  / _ \(_-</ __/                \033[1;30m|'
 echo -e "|\033[0;32m              /____/_/  \__/\_,_/_/_/_/_//_/\___/___/\__/                 \033[1;30m|"
 echo -e "|                                                                          |"
 echo -e "|            \033[1;37mDREAMHOST IP ABUSE DETECTION SCRIPT VERSION 0.2\033[1;30m               |";
 echo -e "\033[1;30m __________________________________________________________________________ \033[0m\n\n"
}


# catch non-kill exit to reset term / ncurses
trap exitt EXIT

# SET WINDOW TITLE IF CLIENT CAPABLE
case ${TERM:-dummy} in
 xterm*|vt*|ansi|rxvt|gnome*) echo -e "\033]0;DREAMHOST IP ABUSE DETECTION SCRIPT\007" ;;
esac

# MAKE NICE
renice 19 -p $$ &>/dev/null

if [ $# -lt 1 ]; then
 clear; title; [ -d $TMPDI ] || mkdir -m 755 $TMPDI
 cd ~/logs/; DOMAINS=( `command ls ~/logs/|grep -xv resources` ); cd $OLDPWD
 menu ${DOMAINS[@]}
else
 YD="${1}"
 mkdir -p -m 0755 $TMPDI/$YD/reports; mkdir -p -m 0755 $TMPDI/$YD/logs
 touch $TMPDI/$YD/logs/out.txt;
 ELOG="$TMPDI/$YD/logs/error.log"
 ALOG="$TMPDI/$YD/logs/access.log"
 REPORT=$(echo -en "$TMPDI/$YD/reports/`date +%mx%dx%y`.txt");

 if [ "$2" == "report" ]; then
  show_report "$1" "$2"
 else
  error_abuse "$1" "$2"
 fi
fi


exit 0

See Also