geimist/Update_Blocklist
1
0
Fork 0
Code Issues Pull requests Releases 1 Wiki Activity

„update_blocklist.sh“ ändern

Browse source
This commit is contained in:
Stephan 2022-10-17 17:42:48 +00:00
parent d40a54e564
commit b9e2581709
1 changed files with 144 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

161
update_blocklist.sh
View file

@ -1,9 +1,9 @@
#!/bin/sh #!/bin/sh
############################################################################################################################################# #############################################################################################################################################
# /volume1/homes/user/script/update_blocklist.sh #
# # # #
# Script import IP's from blocklist.de # # Script import IP's from blocklist.de #
# #
# https://www.synology-forum.de/showthread.html?103687-Freigabe-Blockierliste-automatisch-updaten&p=837478&viewfull=1#post837478 # # https://www.synology-forum.de/showthread.html?103687-Freigabe-Blockierliste-automatisch-updaten&p=837478&viewfull=1#post837478 #
# version 0.1 by Ruedi61, 15.11.2016 / DSM 6.0.3 # # version 0.1 by Ruedi61, 15.11.2016 / DSM 6.0.3 #
# version 0.2 by AndiHeitzer, 18.09.2019 / DSM 6.2.1 > add further Vars for DB # # version 0.2 by AndiHeitzer, 18.09.2019 / DSM 6.2.1 > add further Vars for DB #
@ -11,6 +11,7 @@
# version 0.4 by geimist, 24.05.2022 / DSM 7.1 > speed improvement over 5x # # version 0.4 by geimist, 24.05.2022 / DSM 7.1 > speed improvement over 5x #
# (for 10000 IPs only 107 seconds are needed instead of 658 seconds) # # (for 10000 IPs only 107 seconds are needed instead of 658 seconds) #
# version 0.5 by geimist, 16.10.2022 / DSM 7.1 > permanent block does not work properly # # version 0.5 by geimist, 16.10.2022 / DSM 7.1 > permanent block does not work properly #
# version 0.6 by geimist, 17.10.2022 / DSM 7.1 > GeoIP verification added (ATTENTION: this reduces the speed significantly) #
# # # #
############################################################################################################################################# #############################################################################################################################################
@ -20,25 +21,128 @@
# Download from www.blocklist.de | Select Typ: {all} {ssh} {mail} {apache} {imap} {ftp} {sip} {bots} {strongips} {ircbot} {bruteforcelogin} # Download from www.blocklist.de | Select Typ: {all} {ssh} {mail} {apache} {imap} {ftp} {sip} {bots} {strongips} {ircbot} {bruteforcelogin}
BLOCKLIST_TYP="all" BLOCKLIST_TYP="all"
# Delete IP after x Day's OR use 0 for permanent block # Delete IP after x Day's OR use 0 for permanent block
DELETE_IP_AFTER="5" DELETE_IP_AFTER=5
# Loglevel 1: Show Stats at the bottom / Loglevel 2: Show all / Loglevel 0: disable # Loglevel 1: Show Stats at the bottom / Loglevel 2: Show all / Loglevel 0: disable
LOGLEVEL=1 LOGLEVEL=1
PROGRESSBAR=1
# 0=Single Host / 1=? / 2=IP-Range (META must be set) / 3=subnetmask (META must be set) # 0=Single Host / 1=? / 2=IP-Range (META must be set) / 3=subnetmask (META must be set)
TYPE=0 TYPE=0
# e.g. subnetmask / upper IP-Range # e.g. subnetmask / upper IP-Range
META='' META=''
# GeoIP implementation:
# defined countrys are: blockonly | blockother | off
# blockonly only the defined countries should be included
# blockother only all other countries are to be included
useGeoIP="off"
GeoIP_DB="/var/db/geoip-database/GeoLite2-City.mmdb"
countrys=(DE CN) # ISO style - example: countrys=(DE CN)
############################################################################################################################################# #############################################################################################################################################
# Do NOT change after here! # Do NOT change after here!
skipByGeoIP=0
# SQL Create-Statement for restore: # SQL Create-Statement for clean restore:
# 'CREATE TABLE AutoBlockIP(IP varchar(50) PRIMARY KEY,RecordTime date NOT NULL,ExpireTime date NOT NULL,Deny boolean NOT NULL,IPStd varchr(50) NOT NULL,Type INTEGER,Meta varchar(256))' sqlCreateDB="'CREATE TABLE AutoBlockIP(IP varchar(50) PRIMARY KEY,RecordTime date NOT NULL,ExpireTime date NOT NULL,Deny boolean NOT NULL,IPStd varchr(50) NOT NULL,Type INTEGER,Meta varchar(256))'"
if [ $(whoami) != "root" ]; then if [ $(whoami) != "root" ]; then
echo "WARNING: this script must run from root!" >&2 echo "WARNING: this script must run from root!" >&2
exit 1 exit 1
fi fi
# Python3 Environment for GeoIP:
own_path="${0%/*}"
python3_env="${own_path}/update_blocklist_python3_env"
python_env_version=1 # is written to an info file after setting up the python env to skip a full check of the python env on each run
python_module_list=( geoip2 )
if [ "$useGeoIP" = blockonly ] || [ "$useGeoIP" = blockother ]; then
if [ ! $(which python3) ]; then
echo "(Python3 is not installed / GeoIP filter disabled)"
useGeoIP=off
else
[ ! -d "$python3_env" ] && python3 -m venv "$python3_env"
source "${python3_env}/bin/activate"
if [ "$(cat "${python3_env}/python_env_version" 2>/dev/null | head -n1)" != "$python_env_version" ]; then
# check / install pip:
# ---------------------------------------------------------------------
if ! python3 -m pip --version > /dev/null 2>&1 ; then
printf " Python3 pip was not found and will be now installed ➜ "
# install pip:
tmp_log1=$(python3 -m ensurepip --default-pip)
# upgrade pip:
tmp_log2=$(python3 -m pip install --upgrade pip)
# check install:
if python3 -m pip --version > /dev/null 2>&1 ; then
echo "ok"
else
echo "failed ! ! ! (please install Python3 pip manually)"
echo " install log:"
echo "$tmp_log1" | sed -e "s/^/ /g"
echo "$tmp_log2" | sed -e "s/^/ /g"
python_check=failed
return 1
fi
else
if python3 -m pip list 2>&1 | grep -q "version.*is available" ; then
printf " pip already installed ($(python3 -m pip --version)) / upgrade available ...\n"
python3 -m pip install --upgrade pip | sed -e "s/^/ /g"
fi
fi
printf "\nread installed python modules:\n"
moduleList=$(python3 -m pip list 2>/dev/null)
# check / install python modules:
# ---------------------------------------------------------------------
for module in ${python_module_list[@]}; do
moduleName=$(echo "$module" | awk -F'=' '{print $1}' )
unset tmp_log1
printf " ➜ check python module \"$module\": ➜ "
if ! grep -qi "$moduleName" <<<"$moduleList"; then
printf "$module was not found and will be installed ➜ "
# install module:
tmp_log1=$(python3 -m pip install "$module")
# check install:
if grep -qi "$moduleName" <<<"$(python3 -m pip list 2>/dev/null)" ; then
echo "ok"
else
echo "failed ! ! ! (please install $module manually)"
echo " install log:" && echo "$tmp_log1" | sed -e "s/^/ /g"
python_check=failed
return 1
fi
else
printf "ok\n"
fi
done
echo "$python_env_version" > "${python3_env}/python_env_version"
source "${python3_env}/bin/activate"
printf "\n"
fi
source "${python3_env}/bin/activate"
fi
# python3 -m pip list 2>/dev/null
fi
request_GeoIP() {
{ echo 'import geoip2.database'
echo "with geoip2.database.Reader('$GeoIP_DB') as reader:"
echo " response = reader.city('$1')"
echo " print(response.country.iso_code)"
# echo " print(response.country.name)"
# echo " print(response.city.name)"
} | python3 2>&1
}
progressbar() { progressbar() {
# https://blog.cscholz.io/bash-progress-or-spinner/ # https://blog.cscholz.io/bash-progress-or-spinner/
# Um die Progressbar darzustellen, muss ein Zähler (_start) und der Maximalwert (_end) definiert werden. # Um die Progressbar darzustellen, muss ein Zähler (_start) und der Maximalwert (_end) definiert werden.
@ -54,15 +158,15 @@ progressbar() {
####################################### #######################################
# Process data # Process data
let _progress=(${1}*100/${2}*100)/100 let _progress=(${1}*100/${2}*100)/100
let _done=(${_progress}*4)/10 let _done=(${_progress}*4)/10
let _left=40-$_done let _left=40-$_done
# Build progressbar string lengths # Build progressbar string lengths
_fill=$(printf "%${_done}s") _fill=$(printf "%${_done}s")
_empty=$(printf "%${_left}s") _empty=$(printf "%${_left}s")
printf "\rProgress : [${_fill// /#}${_empty// /-}] ${_progress}%% ($1/$2)" printf "\rProgress : [${_fill// /#}${_empty// /-}] ${_progress}%% ($1/$2)"
} }
sec_to_time() { sec_to_time() {
@ -105,7 +209,7 @@ sec_to_time() {
countbefore=$(sqlite3 "$db_path" "SELECT count(IP) FROM AutoBlockIP WHERE Deny='1' " ) countbefore=$(sqlite3 "$db_path" "SELECT count(IP) FROM AutoBlockIP WHERE Deny='1' " )
# delete IP if expired: # delete IP if expired:
CountExpiredIP=$(sqlite3 "$db_path" "SELECT count(IP) FROM AutoBlockIP WHERE ExpireTime <= $UNIXTIME AND Deny='1' AND NOT ExpireTime='0' ") CountExpiredIP=$(sqlite3 "$db_path" "SELECT count(IP) FROM AutoBlockIP WHERE ExpireTime <= $UNIXTIME AND Deny='1' AND NOT ExpireTime='0' AND NOT ExpireTime='0' ")
sqlite3 "$db_path" "DELETE FROM AutoBlockIP WHERE ExpireTime <= $UNIXTIME AND Deny='1' AND NOT ExpireTime='0' " sqlite3 "$db_path" "DELETE FROM AutoBlockIP WHERE ExpireTime <= $UNIXTIME AND Deny='1' AND NOT ExpireTime='0' "
# current IP-list: # current IP-list:
@ -130,9 +234,9 @@ sec_to_time() {
# count of diffs: # count of diffs:
countofdiffs=$(cat "$blocklist_list" | grep -Eo "^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$" | wc -l) countofdiffs=$(cat "$blocklist_list" | grep -Eo "^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$" | wc -l)
echo "$countofdiffs IPs must be importet" echo "$countofdiffs IPs are different do local block list"
# progressbar: # progressbar / stats:
progress_start=0 progress_start=0
progress_end=$countofdiffs progress_end=$countofdiffs
@ -141,9 +245,31 @@ sec_to_time() {
echo "INSERT OR IGNORE INTO AutoBlockIP ('IP', 'RecordTime', 'ExpireTime', 'Deny', 'IPStd', 'Type', 'Meta') VALUES " > "$sql_statement" echo "INSERT OR IGNORE INTO AutoBlockIP ('IP', 'RecordTime', 'ExpireTime', 'Deny', 'IPStd', 'Type', 'Meta') VALUES " > "$sql_statement"
while read BLOCKED_IP ; do while read BLOCKED_IP ; do
let progress_start=progress_start+1
# Check if IP valid # Check if IP valid
VALID_IPv4=$(echo "$BLOCKED_IP" | grep -Eo "^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$" | wc -l) VALID_IPv4=$(echo "$BLOCKED_IP" | grep -Eo "^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$" | wc -l)
# check GeoIP
if [ "$useGeoIP" = blockonly ] || [ "$useGeoIP" = blockother ]; then
request_GeoIP_result="$(request_GeoIP "$BLOCKED_IP")"
if grep -qi "is not in the database" <<<"$request_GeoIP_result" ; then
# is not in the database
request_GeoIP_result="empty"
fi
fi
if [ "$useGeoIP" = blockonly ] && echo "${countrys[@]}" | grep -qiv "$request_GeoIP_result" ; then
let skipByGeoIP=$skipByGeoIP+1
[ "$LOGLEVEL" -eq 2 ] && echo "continue - $BLOCKED_IP - country: $request_GeoIP_result"
continue
elif [ "$useGeoIP" = blockother ] && echo "${countrys[@]}" | grep -qi "$request_GeoIP_result" ; then
let skipByGeoIP=$skipByGeoIP+1
[ "$LOGLEVEL" -eq 2 ] && echo "continue - $BLOCKED_IP - country: $request_GeoIP_result"
continue
fi
# prepare sql statement
if [[ "$VALID_IPv4" -eq 1 ]]; then if [[ "$VALID_IPv4" -eq 1 ]]; then
# Convert IPv4 to IPv6 :) # Convert IPv4 to IPv6 :)
IPv4=$(echo $BLOCKED_IP | sed 's/\./ /g') IPv4=$(echo $BLOCKED_IP | sed 's/\./ /g')
@ -152,17 +278,15 @@ while read BLOCKED_IP ; do
echo "('$BLOCKED_IP','$UNIXTIME','$UNIXTIME_DELETE_IP','$Deny','$IPv6','$TYPE','$META')," >> "$sql_statement" echo "('$BLOCKED_IP','$UNIXTIME','$UNIXTIME_DELETE_IP','$Deny','$IPv6','$TYPE','$META')," >> "$sql_statement"
countadded=$(( $countadded + 1 )) countadded=$(( $countadded + 1 ))
if [[ $LOGLEVEL -eq 2 ]]; then if [ "$LOGLEVEL" -eq 2 ]; then
echo "IP added to Database! --> $BLOCKED_IP" echo "IP added to Database! --> $BLOCKED_IP"
elif [[ $LOGLEVEL -eq 1 ]]; then elif [ "$LOGLEVEL" -eq 1 ] && [ "$PROGRESSBAR" -eq 1 ]; then
# progressbar: # progressbar:
let progress_start=progress_start+1
progressbar ${progress_start} ${progress_end} progressbar ${progress_start} ${progress_end}
fi fi
fi fi
done < "$blocklist_list" done < "$blocklist_list"
if [ "$countofdiffs" -ge 1 ] ; then if [ "$countofdiffs" -ge 1 ] ; then
last_entry=$(cat "$sql_statement" | tail -n1) last_entry=$(cat "$sql_statement" | tail -n1)
sed -i "s/$last_entry/${last_entry%,};/g" "$sql_statement" sed -i "s/$last_entry/${last_entry%,};/g" "$sql_statement"
@ -171,13 +295,14 @@ if [ "$countofdiffs" -ge 1 ] ; then
fi fi
# stats … # stats …
if [[ $LOGLEVEL -eq 1 ]] || [[ $LOGLEVEL -eq 2 ]]; then if [ "$LOGLEVEL" -eq 1 ] || [ "$LOGLEVEL" -eq 2 ]; then
echo -e; echo -e; echo -e; echo -e;
echo "stats:--------------------------------" echo "stats:--------------------------------"
echo "duration of the process: $(sec_to_time $(expr $(date +%s)-${UNIXTIME}) )" echo "duration of the process: $(sec_to_time $(expr $(date +%s)-${UNIXTIME}) )"
echo "count of IPs in list: $(cat "$online_list" | grep -Eo "^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$" | wc -l)" echo "count of IPs in list: $(cat "$online_list" | grep -Eo "^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$" | wc -l)"
echo "count of diffs: $countofdiffs" echo "count of diffs: $countofdiffs"
echo "added IPs: $countadded" echo "added IPs: $countadded"
echo "IP skipped by GeoIP: $skipByGeoIP"
echo "expired IPs (deleted): $CountExpiredIP (set expiry time: $DELETE_IP_AFTER days)" echo "expired IPs (deleted): $CountExpiredIP (set expiry time: $DELETE_IP_AFTER days)"
echo "blocked IPs: before: $countbefore / current: $(sqlite3 "$db_path" "SELECT count(IP) FROM AutoBlockIP WHERE Deny='1' " )" echo "blocked IPs: before: $countbefore / current: $(sqlite3 "$db_path" "SELECT count(IP) FROM AutoBlockIP WHERE Deny='1' " )"
fi fi