diff --git a/update_blocklist.sh b/update_blocklist.sh index 91bbbf3..20a8568 100644 --- a/update_blocklist.sh +++ b/update_blocklist.sh @@ -1,17 +1,26 @@ #!/bin/sh -# /volume1/homes/admin/script/update_blocklist.sh -# Script import IP's from blocklist.de -# https://www.synology-forum.de/showthread.html?103687-Freigabe-Blockierliste-automatisch-updaten&p=837478&viewfull=1#post837478 -# version 0.1 by Ruedi61, 15.11.2016 / DSM 6.0.3 -# version 0.2 by AndiHeitzer, 18.09.2019 / DSM 6.2.1 > add further Vars for DB -# version 0.3 by geimist, 28.09.2019 / DSM 6.2.1 > add Stats / Loglevel / speed improvement / delete expired IPs +# /volume3/DEV/update_blocklist.sh | tee /volume3/DEV/update_blocklist_$(date +%s).log + +############################################################################################################################################# +# /volume1/homes/admin/script/update_blocklist.sh # +# # +# Script import IP's from blocklist.de # +# https://www.synology-forum.de/showthread.html?103687-Freigabe-Blockierliste-automatisch-updaten&p=837478&viewfull=1#post837478 # +# version 0.1 by Ruedi61, 15.11.2016 / DSM 6.0.3 # +# version 0.2 by AndiHeitzer, 18.09.2019 / DSM 6.2.1 > add further Vars for DB # +# version 0.3 by geimist, 28.09.2019 / DSM 6.2.1 > add stats / loglevel / speed improvement / delete expired IPs # +# version 0.4 by geimist, 24.05.2022 / DSM 7.1 > speed improvement over 5x # +# (for 10000 IPs only 107 seconds are needed instead of 658 seconds) # +# # +############################################################################################################################################# + # Deny=1 > Blacklist / Deny=0 > Whitelist Deny=1 # Download from www.blocklist.de | Select Typ: {all} {ssh} {mail} {apache} {imap} {ftp} {sip} {bots} {strongips} {ircbot} {bruteforcelogin} BLOCKLIST_TYP="all" # Delete IP after x Day's OR use 0 for permanent block - DELETE_IP_AFTER="7" + DELETE_IP_AFTER="5" # Loglevel 1: Show Stats at the bottom / Loglevel 2: Show all / Loglevel 0: disable LOGLEVEL=1 # 0=Single Host / 1=? / 2=IP-Range (META must be set) / 3=subnetmask (META must be set) @@ -19,16 +28,43 @@ # e.g. subnetmask / upper IP-Range META='' -############################################################################################################### +############################################################################################################################################# # Do NOT change after here! # SQL Create-Statement for restore: -# 'CREATE TABLE AutoBlockIP(IP varchar(50) PRIMARY KEY,RecordTime date NOT NULL,ExpireTime date NOT NULL,Deny boolean NOT NULL,IPStd varchr(50) NOT NULL,Type INTEGER,Meta varchar(256))' +# 'CREATE TABLE AutoBlockIP(IP varchar(50) PRIMARY KEY,RecordTime date NOT NULL,ExpireTime date NOT NULL,Deny boolean NOT NULL,IPStd varchr(50) NOT NULL,Type INTEGER,Meta varchar(256))' + if [ $(whoami) != "root" ]; then echo "WARNING: this script must run from root!" >&2 exit 1 fi +progressbar() { +# https://blog.cscholz.io/bash-progress-or-spinner/ +# Um die Progressbar darzustellen, muss ein Zähler (_start) und der Maximalwert (_end) definiert werden. +# _start=0 +# _end=$(wc -l $1) +####################################### +# Display a progress bar +# Arguments: +# $1 Current loop number +# $2 max. no of loops (1005) +# Returns: +# None +####################################### + +# Process data +let _progress=(${1}*100/${2}*100)/100 +let _done=(${_progress}*4)/10 +let _left=40-$_done + +# Build progressbar string lengths +_fill=$(printf "%${_done}s") +_empty=$(printf "%${_left}s") + +printf "\rProgress : [${_fill// /#}${_empty// /-}] ${_progress}%% ($1/$2)" +} + sec_to_time() { local seconds=$1 local sign="" @@ -42,68 +78,102 @@ sec_to_time() { printf "%s%02d:%02d:%02d" "$sign" $hours $minutes $seconds } -countadded=0 -countskipped=0 -UNIXTIME=$(date +%s) -UNIXTIME_DELETE_IP=$(date -d "+$DELETE_IP_AFTER days" +%s) +# create temporary working directory & prepare variables +# --------------------------------------------------------------------- + work_tmp=$(mktemp -d -t tmp.XXXXXXXXXX) + trap 'rm -rf "$work_tmp"; exit' EXIT + + before_list="${work_tmp}/before.txt" + online_list="${work_tmp}/online_list.txt" + blocklist_list="${work_tmp}/blocklist.txt" + sql_statement="${work_tmp}/insert_statement.sql" + + countadded=0 + db_path="/etc/synoautoblock.db" + + UNIXTIME=$(date +%s) + UNIXTIME_DELETE_IP=$(date -d "+$DELETE_IP_AFTER days" +%s) + [ ! -f "$db_path" ] && sqlite3 "$db_path" 'CREATE TABLE AutoBlockIP(IP varchar(50) PRIMARY KEY,RecordTime date NOT NULL,ExpireTime date NOT NULL,Deny boolean NOT NULL,IPStd varchr(50) NOT NULL,Type INTEGER,Meta varchar(256))' # count blocked IPs before: - countbefore=$(sqlite3 /etc/synoautoblock.db "SELECT count(IP) FROM AutoBlockIP WHERE Deny='1' " ) + countbefore=$(sqlite3 "$db_path" "SELECT count(IP) FROM AutoBlockIP WHERE Deny='1' " ) + # delete IP if expired: - CountExpiredIP=$(sqlite3 /etc/synoautoblock.db "SELECT count(IP) FROM AutoBlockIP WHERE ExpireTime <= $UNIXTIME AND Deny='1'") - sqlite3 /etc/synoautoblock.db "DELETE FROM AutoBlockIP WHERE ExpireTime <= $UNIXTIME AND Deny='1' " + CountExpiredIP=$(sqlite3 "$db_path" "SELECT count(IP) FROM AutoBlockIP WHERE ExpireTime <= $UNIXTIME AND Deny='1'") + sqlite3 "$db_path" "DELETE FROM AutoBlockIP WHERE ExpireTime <= $UNIXTIME AND Deny='1' " + # current IP-list: - sqlite3 -header -csv /etc/synoautoblock.db "select IP FROM AutoBlockIP WHERE Deny='1' ORDER BY 'IP' ASC;" | sed -e '1d' | sort > /tmp/before.txt + sqlite3 -header -csv "$db_path" "select IP FROM AutoBlockIP WHERE Deny='1' ORDER BY 'IP' ASC;" | sed -e '1d' | sort > "$before_list" + # load online IP-list: - curl --max-time 30 -s "https://lists.blocklist.de/lists/${BLOCKLIST_TYP}.txt" | sort > /tmp/onlinelist.txt -# filter diffs: - diff "/tmp/before.txt" "/tmp/onlinelist.txt" | grep '^>' | sed -e 's/> //' > /tmp/blocklist.txt # only diffs from left to right + wget -q --timeout=30 --tries=2 -nv -O - "https://lists.blocklist.de/lists/${BLOCKLIST_TYP}.txt" | sort | uniq > "$online_list" + + if [ $(stat -c %s "$online_list") -eq 0 ] || [ ! -f "$online_list" ];then + echo -n "WARNING: The server blocklist.de is not available! Use alternative list -> " + wget -q --timeout=30 --tries=2 -nv -O - "https://mariushosting.com/wp-content/uploads/$(date +%Y/%m)/deny-ip-list.txt" | sort | uniq > "$online_list" + if echo "$wgetlog" | grep -q "failed" ; then + echo "failed!" + exit 1 + else + echo "OK" + fi + fi + +# filter diffs - only diffs from left to right: + diff "$before_list" "$online_list" | grep '^>' | sed -e 's/> //' > "$blocklist_list" + # count of diffs: - countofdiffs=$(cat "/tmp/blocklist.txt" | grep -Eo "^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$" | wc -l) + countofdiffs=$(cat "$blocklist_list" | grep -Eo "^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$" | wc -l) echo "$countofdiffs IPs must be importet" - -while read BLOCKED_IP - do - # Check if IP valid - VALID_IPv4=$(echo "$BLOCKED_IP" | grep -Eo "^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$" | wc -l) - - if [[ $VALID_IPv4 -eq 1 ]]; then - # Convert IPv4 to IPv6 :) - IPv4=$(echo $BLOCKED_IP | sed 's/\./ /g') - IPv6=$(printf "0000:0000:0000:0000:0000:FFFF:%02X%02X:%02X%02X" $IPv4) - CHECK_IF_EXISTS=$(sqlite3 /etc/synoautoblock.db "SELECT DENY FROM AutoBlockIP WHERE IP = '$BLOCKED_IP'" | wc -l) - if [[ $CHECK_IF_EXISTS -lt 1 ]]; then - INSERT=$(sqlite3 /etc/synoautoblock.db "INSERT INTO AutoBlockIP VALUES ('$BLOCKED_IP','$UNIXTIME','$UNIXTIME_DELETE_IP','$Deny','$IPv6','$TYPE','$META')") - countadded=$(( $countadded + 1 )) - if [[ $LOGLEVEL -eq 2 ]]; then - echo "IP added to Database! --> $BLOCKED_IP" - elif [[ $LOGLEVEL -eq 1 ]]; then - echo -n "." - fi - else - countskipped=$(( $countskipped + 1 )) - if [[ $LOGLEVEL -eq 1 ]] || [[ $LOGLEVEL -eq 2 ]]; then - echo -e; echo "IP already in Database! --> $BLOCKED_IP" - fi - fi - fi - done < /tmp/blocklist.txt + +# progressbar: + progress_start=0 + progress_end=$countofdiffs + +# beginn sql statement:: + echo "INSERT OR IGNORE INTO AutoBlockIP ('IP', 'RecordTime', 'ExpireTime', 'Deny', 'IPStd', 'Type', 'Meta') VALUES " > "$sql_statement" + +while read BLOCKED_IP ; do + # Check if IP valid + VALID_IPv4=$(echo "$BLOCKED_IP" | grep -Eo "^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$" | wc -l) + + if [[ "$VALID_IPv4" -eq 1 ]]; then + # Convert IPv4 to IPv6 :) + IPv4=$(echo $BLOCKED_IP | sed 's/\./ /g') + IPv6=$(printf "0000:0000:0000:0000:0000:FFFF:%02X%02X:%02X%02X" $IPv4) + + echo "('$BLOCKED_IP','$UNIXTIME','$UNIXTIME_DELETE_IP','$Deny','$IPv6','$TYPE','$META')," >> "$sql_statement" + + countadded=$(( $countadded + 1 )) + if [[ $LOGLEVEL -eq 2 ]]; then + echo "IP added to Database! --> $BLOCKED_IP" + elif [[ $LOGLEVEL -eq 1 ]]; then + # progressbar: + let progress_start=progress_start+1 + progressbar ${progress_start} ${progress_end} + fi + fi +done < "$blocklist_list" + + +if [ "$countofdiffs" -ge 1 ] ; then + last_entry=$(cat "$sql_statement" | tail -n1) + sed -i "s/$last_entry/${last_entry%,};/g" "$sql_statement" + printf "\n\nwrite DB ...\n" + sqlite3 "$db_path" < "$sql_statement" +fi # stats … if [[ $LOGLEVEL -eq 1 ]] || [[ $LOGLEVEL -eq 2 ]]; then echo -e; echo -e; - echo "stats:----------------------------------" + echo "stats:--------------------------------" echo "duration of the process: $(sec_to_time $(expr $(date +%s)-${UNIXTIME}) )" - echo "count of IPs in list: $(cat "/tmp/onlinelist.txt" | grep -Eo "^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$" | wc -l)" + echo "count of IPs in list: $(cat "$online_list" | grep -Eo "^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$" | wc -l)" echo "count of diffs: $countofdiffs" echo "added IPs: $countadded" echo "expired IPs (deleted): $CountExpiredIP (set expiry time: $DELETE_IP_AFTER days)" - echo "skipped IPs: $countskipped" - echo "blocked IPs: before: $countbefore / current: $(sqlite3 /etc/synoautoblock.db "SELECT count(IP) FROM AutoBlockIP WHERE Deny='1' " )" + echo "blocked IPs: before: $countbefore / current: $(sqlite3 "$db_path" "SELECT count(IP) FROM AutoBlockIP WHERE Deny='1' " )" fi -rm /tmp/blocklist.txt -rm /tmp/before.txt -rm /tmp/onlinelist.txt exit 0