diff --git a/ipset/antifilter.helper b/ipset/antifilter.helper new file mode 100644 index 0000000..16b2903 --- /dev/null +++ b/ipset/antifilter.helper @@ -0,0 +1,19 @@ +get_antifilter() +{ + # $1 - list url + # $2 - target file + local ZIPLISTTMP="$TMPDIR/zapret-ip.txt" + + [ "$DISABLE_IPV4" != "1" ] && { + curl --fail --max-time 150 --connect-timeout 20 --max-filesize 41943040 -k -L "$1" | cut_local >"$ZIPLISTTMP" && + { + dlsize=$(LC_ALL=C LANG=C wc -c "$ZIPLISTTMP" | xargs | cut -f 1 -d ' ') + if [ $dlsize -lt 102400 ]; then + echo list file is too small. can be bad. + exit 2 + fi + ip2net4 <"$ZIPLISTTMP" | zz "$2" + rm -f "$ZIPLISTTMP" + } + } +} diff --git a/ipset/clear_lists.sh b/ipset/clear_lists.sh new file mode 100755 index 0000000..80c1531 --- /dev/null +++ b/ipset/clear_lists.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +IPSET_DIR="$(dirname "$0")" +IPSET_DIR="$(cd "$IPSET_DIR"; pwd)" + +. "$IPSET_DIR/def.sh" + +rm -f "$ZIPLIST"* "$ZIPLIST6"* "$ZIPLIST_USER" "$ZIPLIST_USER6" "$ZIPLIST_IPBAN"* "$ZIPLIST_IPBAN6"* "$ZIPLIST_USER_IPBAN" "$ZIPLIST_USER_IPBAN6" "$ZIPLIST_EXCLUDE" "$ZIPLIST_EXCLUDE6" "$ZHOSTLIST"* diff --git a/ipset/create_ipset.sh b/ipset/create_ipset.sh new file mode 100755 index 0000000..a88137d --- /dev/null +++ b/ipset/create_ipset.sh @@ -0,0 +1,308 @@ +#!/bin/sh + +# create ipset or ipfw table from resolved ip's +# $1=no-update - do not update ipset, only create if its absent +# $1=clear - clear ipset + +EXEDIR="$(dirname "$0")" +EXEDIR="$(cd "$EXEDIR"; pwd)" + +. "$EXEDIR/def.sh" +. "$ZAPRET_BASE/common/fwtype.sh" +. "$ZAPRET_BASE/common/nft.sh" + +IPSET_CMD="$TMPDIR/ipset_cmd.txt" +IPSET_SAVERAM_CHUNK_SIZE=20000 +IPSET_SAVERAM_MIN_FILESIZE=131072 + +NFSET_TEMP="$TMPDIR/nfset_temp.txt" +NFSET_SAVERAM_MIN_FILESIZE=16384 +NFSET_SAVERAM_CHUNK_SIZE=1000 + +IPSET_HOOK_TEMP="$TMPDIR/ipset_hook.txt" + +while [ -n "$1" ]; do + [ "$1" = "no-update" ] && NO_UPDATE=1 + [ "$1" = "clear" ] && DO_CLEAR=1 + shift +done + + +file_extract_lines() +{ + # $1 - filename + # $2 - from line (starting with 0) + # $3 - line count + # awk "{ err=1 } NR < $(($2+1)) { next } { print; err=0 } NR == $(($2+$3)) { exit err } END {exit err}" "$1" + $AWK "NR < $(($2+1)) { next } { print } NR == $(($2+$3)) { exit }" "$1" +} +ipset_restore_chunked() +{ + # $1 - filename + # $2 - chunk size + local pos lines + [ -f "$1" ] || return + lines=$(wc -l <"$1") + pos=$lines + while [ "$pos" -gt "0" ]; do + pos=$((pos-$2)) + [ "$pos" -lt "0" ] && pos=0 + file_extract_lines "$1" $pos $2 | ipset -! restore + sed -i "$(($pos+1)),$ d" "$1" + done +} + + +ipset_get_script() +{ + # $1 - ipset name + sed -nEe "s/^.+$/add $1 &/p" +} +ipset_get_script_from_file() +{ + # $1 - filename + # $2 - ipset name + zzcat "$1" | sort -u | ipset_get_script $2 +} +ipset_restore() +{ + # $1 - ipset name + # $2 - filename + + zzexist "$2" || return + local fsize=$(zzsize "$2") + local svram=0 + # do not saveram small files. file can also be gzipped + [ "$SAVERAM" = "1" ] && [ "$fsize" -ge "$IPSET_SAVERAM_MIN_FILESIZE" ] && svram=1 + + local T="Adding to ipset $1 " + [ "$svram" = "1" ] && T="$T (saveram)" + T="$T : $f" + echo $T + + if [ "$svram" = "1" ]; then + ipset_get_script_from_file "$2" "$1" >"$IPSET_CMD" + ipset_restore_chunked "$IPSET_CMD" $IPSET_SAVERAM_CHUNK_SIZE + rm -f "$IPSET_CMD" + else + ipset_get_script_from_file "$2" "$1" | ipset -! restore + fi +} +create_ipset() +{ + if [ "$1" -eq "6" ]; then + FAMILY=inet6 + else + FAMILY=inet + fi + ipset create $2 $3 $4 family $FAMILY 2>/dev/null || { + [ "$NO_UPDATE" = "1" ] && return 0 + } + ipset flush $2 + [ "$DO_CLEAR" = "1" ] || { + for f in "$5" "$6" ; do + ipset_restore "$2" "$f" + done + [ -n "$IPSET_HOOK" ] && $IPSET_HOOK $2 | ipset_get_script $2 | ipset -! restore + } + return 0 +} + +nfset_get_script_multi() +{ + # $1 - set name + # $2,$3,... - filenames + + # all in one shot. this allows to merge overlapping ranges + # good but eats lots of RAM + + local set=$1 nonempty N=1 f + + shift + # first we need to make sure at least one element exists or nft will fail + while : + do + eval f=\$$N + [ -n "$f" ] || break + nonempty=$(zzexist "$f" && zzcat "$f" 2>/dev/null | head -n 1) + [ -n "$nonempty" ] && break + N=$(($N+1)) + done + + [ -n "$nonempty" ] && { + echo "add element inet $ZAPRET_NFT_TABLE $set {" + while [ -n "$1" ]; do + zzexist "$1" && zzcat "$1" | sed -nEe "s/^.+$/&,/p" + shift + done + echo "}" + } +} +nfset_restore() +{ + # $1 - set name + # $2,$3,... - filenames + + echo "Adding to nfset $1 : $2 $3 $4 $5" + local hookfile + [ -n "$IPSET_HOOK" ] && { + $IPSET_HOOK $1 >"$IPSET_HOOK_TEMP" + [ -s "$IPSET_HOOK_TEMP" ] && hookfile=$IPSET_HOOK_TEMP + } + nfset_get_script_multi "$@" $hookfile | nft -f - + rm -f "$IPSET_HOOK_TEMP" +} +create_nfset() +{ + # $1 - family + # $2 - set name + # $3 - maxelem + # $4,$5 - list files + + local policy + [ $SAVERAM = "1" ] && policy="policy memory;" + nft_create_set $2 "type ipv${1}_addr; size $3; flags interval; auto-merge; $policy" || { + [ "$NO_UPDATE" = "1" ] && return 0 + nft flush set inet $ZAPRET_NFT_TABLE $2 + } + [ "$DO_CLEAR" = "1" ] || { + nfset_restore $2 $4 $5 + } + return 0 +} + +add_ipfw_table() +{ + # $1 - table name + sed -nEe "s/^.+$/table $1 add &/p" | ipfw -q /dev/stdin +} +populate_ipfw_table() +{ + # $1 - table name + # $2 - ip list file + zzexist "$2" || return + zzcat "$2" | sort -u | add_ipfw_table $1 +} +create_ipfw_table() +{ + # $1 - table name + # $2 - table options + # $3,$4, ... - ip list files. can be v4,v6 or mixed + + local name=$1 + ipfw table "$name" create $2 2>/dev/null || { + [ "$NO_UPDATE" = "1" ] && return 0 + } + ipfw -q table $1 flush + shift + shift + [ "$DO_CLEAR" = "1" ] || { + while [ -n "$1" ]; do + echo "Adding to ipfw table $name : $1" + populate_ipfw_table $name "$1" + shift + done + [ -n "$IPSET_HOOK" ] && $IPSET_HOOK $name | add_ipfw_table $name + } + return 0 +} + +print_reloading_backend() +{ + # $1 - backend name + local s="reloading $1 backend" + if [ "$NO_UPDATE" = 1 ]; then + s="$s (no-update)" + elif [ "$DO_CLEAR" = 1 ]; then + s="$s (clear)" + else + s="$s (forced-update)" + fi + echo $s +} + + +oom_adjust_high +get_fwtype + +if [ -n "$LISTS_RELOAD" ] ; then + if [ "$LISTS_RELOAD" = "-" ] ; then + echo not reloading ip list backend + true + else + echo executing custom ip list reload command : $LISTS_RELOAD + $LISTS_RELOAD + [ -n "$IPSET_HOOK" ] && $IPSET_HOOK + fi +else + case "$FWTYPE" in + iptables) + # ipset seem to buffer the whole script to memory + # on low RAM system this can cause oom errors + # in SAVERAM mode we feed script lines in portions starting from the end, while truncating source file to free /tmp space + # only /tmp is considered tmpfs. other locations mean tmpdir was redirected to a disk + SAVERAM=0 + [ "$TMPDIR" = "/tmp" ] && { + RAMSIZE=$($GREP MemTotal /proc/meminfo | $AWK '{print $2}') + [ "$RAMSIZE" -lt "110000" ] && SAVERAM=1 + } + print_reloading_backend ipset + [ "$DISABLE_IPV4" != "1" ] && { + create_ipset 4 $ZIPSET hash:net "$IPSET_OPT" "$ZIPLIST" "$ZIPLIST_USER" + create_ipset 4 $ZIPSET_IPBAN hash:net "$IPSET_OPT" "$ZIPLIST_IPBAN" "$ZIPLIST_USER_IPBAN" + create_ipset 4 $ZIPSET_EXCLUDE hash:net "$IPSET_OPT_EXCLUDE" "$ZIPLIST_EXCLUDE" + } + [ "$DISABLE_IPV6" != "1" ] && { + create_ipset 6 $ZIPSET6 hash:net "$IPSET_OPT" "$ZIPLIST6" "$ZIPLIST_USER6" + create_ipset 6 $ZIPSET_IPBAN6 hash:net "$IPSET_OPT" "$ZIPLIST_IPBAN6" "$ZIPLIST_USER_IPBAN6" + create_ipset 6 $ZIPSET_EXCLUDE6 hash:net "$IPSET_OPT_EXCLUDE" "$ZIPLIST_EXCLUDE6" + } + true + ;; + nftables) + nft_create_table && { + SAVERAM=0 + RAMSIZE=$($GREP MemTotal /proc/meminfo | $AWK '{print $2}') + [ "$RAMSIZE" -lt "420000" ] && SAVERAM=1 + print_reloading_backend "nftables set" + [ "$DISABLE_IPV4" != "1" ] && { + create_nfset 4 $ZIPSET $SET_MAXELEM "$ZIPLIST" "$ZIPLIST_USER" + create_nfset 4 $ZIPSET_IPBAN $SET_MAXELEM "$ZIPLIST_IPBAN" "$ZIPLIST_USER_IPBAN" + create_nfset 4 $ZIPSET_EXCLUDE $SET_MAXELEM_EXCLUDE "$ZIPLIST_EXCLUDE" + } + [ "$DISABLE_IPV6" != "1" ] && { + create_nfset 6 $ZIPSET6 $SET_MAXELEM "$ZIPLIST6" "$ZIPLIST_USER6" + create_nfset 6 $ZIPSET_IPBAN6 $SET_MAXELEM "$ZIPLIST_IPBAN6" "$ZIPLIST_USER_IPBAN6" + create_nfset 6 $ZIPSET_EXCLUDE6 $SET_MAXELEM_EXCLUDE "$ZIPLIST_EXCLUDE6" + } + true + } + ;; + ipfw) + print_reloading_backend "ipfw table" + if [ "$DISABLE_IPV4" != "1" ] && [ "$DISABLE_IPV6" != "1" ]; then + create_ipfw_table $ZIPSET "$IPFW_TABLE_OPT" "$ZIPLIST" "$ZIPLIST_USER" "$ZIPLIST6" "$ZIPLIST_USER6" + create_ipfw_table $ZIPSET_IPBAN "$IPFW_TABLE_OPT" "$ZIPLIST_IPBAN" "$ZIPLIST_USER_IPBAN" "$ZIPLIST_IPBAN6" "$ZIPLIST_USER_IPBAN6" + create_ipfw_table $ZIPSET_EXCLUDE "$IPFW_TABLE_OPT_EXCLUDE" "$ZIPLIST_EXCLUDE" "$ZIPLIST_EXCLUDE6" + elif [ "$DISABLE_IPV4" != "1" ]; then + create_ipfw_table $ZIPSET "$IPFW_TABLE_OPT" "$ZIPLIST" "$ZIPLIST_USER" + create_ipfw_table $ZIPSET_IPBAN "$IPFW_TABLE_OPT" "$ZIPLIST_IPBAN" "$ZIPLIST_USER_IPBAN" + create_ipfw_table $ZIPSET_EXCLUDE "$IPFW_TABLE_OPT_EXCLUDE" "$ZIPLIST_EXCLUDE" + elif [ "$DISABLE_IPV6" != "1" ]; then + create_ipfw_table $ZIPSET "$IPFW_TABLE_OPT" "$ZIPLIST6" "$ZIPLIST_USER6" + create_ipfw_table $ZIPSET_IPBAN "$IPFW_TABLE_OPT" "$ZIPLIST_IPBAN6" "$ZIPLIST_USER_IPBAN6" + create_ipfw_table $ZIPSET_EXCLUDE "$IPFW_TABLE_OPT_EXCLUDE" "$ZIPLIST_EXCLUDE6" + else + create_ipfw_table $ZIPSET "$IPFW_TABLE_OPT" + create_ipfw_table $ZIPSET_IPBAN "$IPFW_TABLE_OPT" + create_ipfw_table $ZIPSET_EXCLUDE "$IPFW_TABLE_OPT_EXCLUDE" + fi + true + ;; + *) + echo no supported ip list backend found + true + ;; + esac + +fi diff --git a/ipset/def.sh b/ipset/def.sh new file mode 100644 index 0000000..f5b20f1 --- /dev/null +++ b/ipset/def.sh @@ -0,0 +1,283 @@ +EXEDIR="$(dirname "$0")" +EXEDIR="$(cd "$EXEDIR"; pwd)" +ZAPRET_BASE=${ZAPRET_BASE:-"$(cd "$EXEDIR/.."; pwd)"} +ZAPRET_RW=${ZAPRET_RW:-"$ZAPRET_BASE"} +ZAPRET_CONFIG=${ZAPRET_CONFIG:-"$ZAPRET_RW/config"} +IPSET_RW_DIR="$ZAPRET_RW/ipset" + +[ -f "$ZAPRET_CONFIG" ] && . "$ZAPRET_CONFIG" +. "$ZAPRET_BASE/common/base.sh" + +[ -z "$TMPDIR" ] && TMPDIR=/tmp +[ -z "$GZIP_LISTS" ] && GZIP_LISTS=1 + +[ -z "$SET_MAXELEM" ] && SET_MAXELEM=262144 +[ -z "$IPSET_OPT" ] && IPSET_OPT="hashsize 262144 maxelem $SET_MAXELEM" +[ -z "$SET_MAXELEM_EXCLUDE" ] && SET_MAXELEM_EXCLUDE=65536 +[ -z "$IPSET_OPT_EXCLUDE" ] && IPSET_OPT_EXCLUDE="hashsize 1024 maxelem $SET_MAXELEM_EXCLUDE" + +[ -z "$IPFW_TABLE_OPT" ] && IPFW_TABLE_OPT="algo addr:radix" +[ -z "$IPFW_TABLE_OPT_EXCLUDE" ] && IPFW_TABLE_OPT_EXCLUDE="algo addr:radix" + +ZIPSET=zapret +ZIPSET6=zapret6 +ZIPSET_EXCLUDE=nozapret +ZIPSET_EXCLUDE6=nozapret6 +ZIPLIST="$IPSET_RW_DIR/zapret-ip.txt" +ZIPLIST6="$IPSET_RW_DIR/zapret-ip6.txt" +ZIPLIST_EXCLUDE="$IPSET_RW_DIR/zapret-ip-exclude.txt" +ZIPLIST_EXCLUDE6="$IPSET_RW_DIR/zapret-ip-exclude6.txt" +ZIPLIST_USER="$IPSET_RW_DIR/zapret-ip-user.txt" +ZIPLIST_USER6="$IPSET_RW_DIR/zapret-ip-user6.txt" +ZUSERLIST="$IPSET_RW_DIR/zapret-hosts-user.txt" +ZHOSTLIST="$IPSET_RW_DIR/zapret-hosts.txt" + +ZIPSET_IPBAN=ipban +ZIPSET_IPBAN6=ipban6 +ZIPLIST_IPBAN="$IPSET_RW_DIR/zapret-ip-ipban.txt" +ZIPLIST_IPBAN6="$IPSET_RW_DIR/zapret-ip-ipban6.txt" +ZIPLIST_USER_IPBAN="$IPSET_RW_DIR/zapret-ip-user-ipban.txt" +ZIPLIST_USER_IPBAN6="$IPSET_RW_DIR/zapret-ip-user-ipban6.txt" +ZUSERLIST_IPBAN="$IPSET_RW_DIR/zapret-hosts-user-ipban.txt" +ZUSERLIST_EXCLUDE="$IPSET_RW_DIR/zapret-hosts-user-exclude.txt" + + +[ -n "$IP2NET" ] || IP2NET="$ZAPRET_BASE/ip2net/ip2net" +[ -n "$MDIG" ] || MDIG="$ZAPRET_BASE/mdig/mdig" +[ -z "$MDIG_THREADS" ] && MDIG_THREADS=30 + + + +# BSD grep is damn slow with -f option. prefer GNU grep (ggrep) if present +# MacoS in cron does not include /usr/local/bin to PATH +if [ -x /usr/local/bin/ggrep ] ; then + GREP=/usr/local/bin/ggrep +elif [ -x /usr/local/bin/grep ] ; then + GREP=/usr/local/bin/grep +elif exists ggrep; then + GREP=$(whichq ggrep) +else + GREP=$(whichq grep) +fi + +# GNU awk is faster +if exists gawk; then + AWK=gawk +else + AWK=awk +fi + +grep_supports_b() +{ + # \b does not work with BSD grep + $GREP --version 2>&1 | $GREP -qE "BusyBox|GNU" +} +get_ip_regex() +{ + REG_IPV4='((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\/([0-9]|[12][0-9]|3[012]))?' + REG_IPV6='[0-9a-fA-F]{1,4}:([0-9a-fA-F]{1,4}|:)+(\/([0-9][0-9]?|1[01][0-9]|12[0-8]))?' + # good but too slow + # REG_IPV6='([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}(/[0-9]+)?|([0-9a-fA-F]{1,4}:){1,7}:(/[0-9]+)?|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}(/[0-9]+)?|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}(/[0-9]+)?|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}(/[0-9]+)?|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}(/[0-9]+)?|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}(/[0-9]+)?|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})(/[0-9]+)?|:((:[0-9a-fA-F]{1,4}){1,7}|:)(/([0-9][0-9]?|1[01][0-9]|12[0-8]))?' +# grep_supports_b && { +# REG_IPV4="\b$REG_IPV4\b" +# REG_IPV6="\b$REG_IPV6\b" +# } +} + +ip2net4() +{ + if [ -x "$IP2NET" ]; then + "$IP2NET" -4 $IP2NET_OPT4 + else + sort -u + fi +} +ip2net6() +{ + if [ -x "$IP2NET" ]; then + "$IP2NET" -6 $IP2NET_OPT6 + else + sort -u + fi +} + +zzexist() +{ + [ -f "$1.gz" ] || [ -f "$1" ] +} +zztest() +{ + gzip -t "$1" 2>/dev/null +} +zzcat() +{ + if [ -f "$1.gz" ]; then + gunzip -c "$1.gz" + elif [ -f "$1" ]; then + if zztest "$1"; then + gunzip -c "$1" + else + cat "$1" + fi + fi +} +zz() +{ + if [ "$GZIP_LISTS" = "1" ]; then + gzip -c >"$1.gz" + rm -f "$1" + else + cat >"$1" + rm -f "$1.gz" + fi +} +zzsize() +{ + local f="$1" + [ -f "$1.gz" ] && f="$1.gz" + if [ -f "$f" ]; then + wc -c <"$f" | xargs + else + printf 0 + fi +} +zzcopy() +{ + local is_gz=0 + zztest "$1" && is_gz=1 + if [ "$GZIP_LISTS" = 1 -a $is_gz = 1 ]; then + cp "$1" "${2}.gz" + elif [ "$GZIP_LISTS" != 1 -a $is_gz != 1 ]; then + cp "$1" "$2" + else + zzcat "$1" | zz "$2" + fi +} + +digger() +{ + # $1 - family (4|6) + # $2 - s=enable mdig stats + if [ -x "$MDIG" ]; then + local cmd + [ "$2" = "s" ] && cmd=--stats=1000 + "$MDIG" --family=$1 --threads=$MDIG_THREADS $cmd + else + local A=A + [ "$1" = "6" ] && A=AAAA + dig $A +short +time=8 +tries=2 -f - | $GREP -E '^[^;].*[^\.]$' + fi +} +filedigger() +{ + # $1 - hostlist + # $2 - family (4|6) + >&2 echo digging $(wc -l <"$1" | xargs) ipv$2 domains : "$1" + zzcat "$1" | digger $2 s +} +flush_dns_cache() +{ + echo clearing all known DNS caches + + if exists killall; then + killall -HUP dnsmasq 2>/dev/null + # MacOS + killall -HUP mDNSResponder 2>/dev/null + elif exists pkill; then + pkill -HUP ^dnsmasq$ + else + echo no mass killer available ! cant flush dnsmasq + fi + + if exists rndc; then + rndc flush + fi + + if exists systemd-resolve; then + systemd-resolve --flush-caches + fi + +} +dnstest() +{ + local ip="$(echo w3.org | digger 46)" + [ -n "$ip" ] +} +dnstest_with_cache_clear() +{ + flush_dns_cache + if dnstest ; then + echo DNS is working + return 0 + else + echo "! DNS is not working" + return 1 + fi +} + + +cut_local() +{ + $GREP -vE '^192\.168\.|^127\.|^10\.' +} +cut_local6() +{ + $GREP -vE '^::|^fc..:|^fd..:|^fe8.:|^fe9.:|^fea.:|^feb.:|^FC..:|^FD..:|^FE8.:|^FE9.:|^FEA.:|^FEB.:' +} + +oom_adjust_high() +{ + [ -f /proc/$$/oom_score_adj ] && { + echo setting high oom kill priority + echo -n 100 >/proc/$$/oom_score_adj + } +} + +getexclude() +{ + oom_adjust_high + dnstest_with_cache_clear || return + [ -f "$ZUSERLIST_EXCLUDE" ] && { + [ "$DISABLE_IPV4" != "1" ] && filedigger "$ZUSERLIST_EXCLUDE" 4 | sort -u > "$ZIPLIST_EXCLUDE" + [ "$DISABLE_IPV6" != "1" ] && filedigger "$ZUSERLIST_EXCLUDE" 6 | sort -u > "$ZIPLIST_EXCLUDE6" + } + return 0 +} + +_get_ipban() +{ + [ -f "$ZUSERLIST_IPBAN" ] && { + [ "$DISABLE_IPV4" != "1" ] && filedigger "$ZUSERLIST_IPBAN" 4 | cut_local | sort -u > "$ZIPLIST_USER_IPBAN" + [ "$DISABLE_IPV6" != "1" ] && filedigger "$ZUSERLIST_IPBAN" 6 | cut_local6 | sort -u > "$ZIPLIST_USER_IPBAN6" + } +} +getuser() +{ + getexclude || return + [ -f "$ZUSERLIST" ] && { + [ "$DISABLE_IPV4" != "1" ] && filedigger "$ZUSERLIST" 4 | cut_local | sort -u > "$ZIPLIST_USER" + [ "$DISABLE_IPV6" != "1" ] && filedigger "$ZUSERLIST" 6 | cut_local6 | sort -u > "$ZIPLIST_USER6" + } + _get_ipban + return 0 +} +getipban() +{ + getexclude || return + _get_ipban + return 0 +} + +hup_zapret_daemons() +{ + echo forcing zapret daemons to reload their hostlist + if exists killall; then + killall -HUP tpws nfqws dvtws 2>/dev/null + elif exists pkill; then + pkill -HUP ^tpws$ + pkill -HUP ^nfqws$ + pkill -HUP ^dvtws$ + else + echo no mass killer available ! cant HUP zapret daemons + fi +} diff --git a/ipset/get_antifilter_allyouneed.sh b/ipset/get_antifilter_allyouneed.sh new file mode 100755 index 0000000..a5b3d22 --- /dev/null +++ b/ipset/get_antifilter_allyouneed.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +IPSET_DIR="$(dirname "$0")" +IPSET_DIR="$(cd "$IPSET_DIR"; pwd)" + +. "$IPSET_DIR/def.sh" + +getuser && { + . "$IPSET_DIR/antifilter.helper" + get_antifilter https://antifilter.download/list/allyouneed.lst "$ZIPLIST" +} + +"$IPSET_DIR/create_ipset.sh" diff --git a/ipset/get_antifilter_ip.sh b/ipset/get_antifilter_ip.sh new file mode 100755 index 0000000..e2cd085 --- /dev/null +++ b/ipset/get_antifilter_ip.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +IPSET_DIR="$(dirname "$0")" +IPSET_DIR="$(cd "$IPSET_DIR"; pwd)" + +. "$IPSET_DIR/def.sh" + +getuser && { + . "$IPSET_DIR/antifilter.helper" + get_antifilter https://antifilter.download/list/ip.lst "$ZIPLIST" +} + +"$IPSET_DIR/create_ipset.sh" diff --git a/ipset/get_antifilter_ipresolve.sh b/ipset/get_antifilter_ipresolve.sh new file mode 100755 index 0000000..de08e28 --- /dev/null +++ b/ipset/get_antifilter_ipresolve.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +IPSET_DIR="$(dirname "$0")" +IPSET_DIR="$(cd "$IPSET_DIR"; pwd)" + +. "$IPSET_DIR/def.sh" + +getuser && { + . "$IPSET_DIR/antifilter.helper" + get_antifilter https://antifilter.download/list/ipresolve.lst "$ZIPLIST" +} + +"$IPSET_DIR/create_ipset.sh" diff --git a/ipset/get_antifilter_ipsmart.sh b/ipset/get_antifilter_ipsmart.sh new file mode 100755 index 0000000..9f0d671 --- /dev/null +++ b/ipset/get_antifilter_ipsmart.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +IPSET_DIR="$(dirname "$0")" +IPSET_DIR="$(cd "$IPSET_DIR"; pwd)" + +. "$IPSET_DIR/def.sh" + +getuser && { + . "$IPSET_DIR/antifilter.helper" + get_antifilter https://antifilter.network/download/ipsmart.lst "$ZIPLIST" +} + +"$IPSET_DIR/create_ipset.sh" diff --git a/ipset/get_antifilter_ipsum.sh b/ipset/get_antifilter_ipsum.sh new file mode 100755 index 0000000..ccf1c8f --- /dev/null +++ b/ipset/get_antifilter_ipsum.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +IPSET_DIR="$(dirname "$0")" +IPSET_DIR="$(cd "$IPSET_DIR"; pwd)" + +. "$IPSET_DIR/def.sh" + +getuser && { + . "$IPSET_DIR/antifilter.helper" + get_antifilter https://antifilter.download/list/ipsum.lst "$ZIPLIST" +} + +"$IPSET_DIR/create_ipset.sh" diff --git a/ipset/get_antizapret_domains.sh b/ipset/get_antizapret_domains.sh new file mode 100755 index 0000000..12583a8 --- /dev/null +++ b/ipset/get_antizapret_domains.sh @@ -0,0 +1,36 @@ +#!/bin/sh + +IPSET_DIR="$(dirname "$0")" +IPSET_DIR="$(cd "$IPSET_DIR"; pwd)" + +. "$IPSET_DIR/def.sh" + +# useful in case ipban set is used in custom scripts +FAIL= +getipban || FAIL=1 +"$IPSET_DIR/create_ipset.sh" +[ -n "$FAIL" ] && exit + +ZURL=https://antizapret.prostovpn.org:8443/domains-export.txt +ZDOM="$TMPDIR/zapret.txt" + + +curl -H "Accept-Encoding: gzip" -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL" | gunzip - >"$ZDOM" || +{ + echo domain list download failed + exit 2 +} + +dlsize=$(LC_ALL=C LANG=C wc -c "$ZDOM" | xargs | cut -f 1 -d ' ') +if test $dlsize -lt 102400; then + echo list file is too small. can be bad. + exit 2 +fi + +sort -u "$ZDOM" | zz "$ZHOSTLIST" + +rm -f "$ZDOM" + +hup_zapret_daemons + +exit 0 diff --git a/ipset/get_config.sh b/ipset/get_config.sh new file mode 100755 index 0000000..8c665a9 --- /dev/null +++ b/ipset/get_config.sh @@ -0,0 +1,10 @@ +#!/bin/sh +# run script specified in config + +IPSET_DIR="$(dirname "$0")" +IPSET_DIR="$(cd "$IPSET_DIR"; pwd)" + +[ -f "$IPSET_DIR/../config" ] && . "$IPSET_DIR/../config" + +[ -z "$GETLIST" ] && GETLIST=get_ipban.sh +[ -x "$IPSET_DIR/$GETLIST" ] && exec "$IPSET_DIR/$GETLIST" diff --git a/ipset/get_exclude.sh b/ipset/get_exclude.sh new file mode 100755 index 0000000..ab5dd38 --- /dev/null +++ b/ipset/get_exclude.sh @@ -0,0 +1,11 @@ +#!/bin/sh +# resolve user host list + +IPSET_DIR="$(dirname "$0")" +IPSET_DIR="$(cd "$IPSET_DIR"; pwd)" + +. "$IPSET_DIR/def.sh" + +getexclude + +"$IPSET_DIR/create_ipset.sh" diff --git a/ipset/get_ipban.sh b/ipset/get_ipban.sh new file mode 100755 index 0000000..825f342 --- /dev/null +++ b/ipset/get_ipban.sh @@ -0,0 +1,11 @@ +#!/bin/sh +# resolve only ipban user host list + +IPSET_DIR="$(dirname "$0")" +IPSET_DIR="$(cd "$IPSET_DIR"; pwd)" + +. "$IPSET_DIR/def.sh" + +getipban + +"$IPSET_DIR/create_ipset.sh" diff --git a/ipset/get_reestr_hostlist.sh b/ipset/get_reestr_hostlist.sh new file mode 100755 index 0000000..0054cbc --- /dev/null +++ b/ipset/get_reestr_hostlist.sh @@ -0,0 +1,65 @@ +#!/bin/sh + +IPSET_DIR="$(dirname "$0")" +IPSET_DIR="$(cd "$IPSET_DIR"; pwd)" + +. "$IPSET_DIR/def.sh" + +ZREESTR="$TMPDIR/zapret.txt.gz" +IPB="$TMPDIR/ipb.txt" +ZURL_REESTR=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv.gz + +dl_checked() +{ + # $1 - url + # $2 - file + # $3 - minsize + # $4 - maxsize + # $5 - maxtime + curl -k --fail --max-time $5 --connect-timeout 10 --retry 4 --max-filesize $4 -o "$2" "$1" || + { + echo list download failed : $1 + return 2 + } + dlsize=$(LC_ALL=C LANG=C wc -c "$2" | xargs | cut -f 1 -d ' ') + if test $dlsize -lt $3; then + echo list is too small : $dlsize bytes. can be bad. + return 2 + fi + return 0 +} + +reestr_list() +{ + LC_ALL=C LANG=C gunzip -c "$ZREESTR" | cut -s -f2 -d';' | LC_ALL=C LANG=C nice -n 5 sed -Ee 's/^\*\.(.+)$/\1/' -ne 's/^[a-z0-9A-Z._-]+$/&/p' | $AWK '{ print tolower($0) }' +} +reestr_extract_ip() +{ + LC_ALL=C LANG=C gunzip -c | nice -n 5 $AWK -F ';' '($1 ~ /^([0-9]{1,3}\.){3}[0-9]{1,3}/) && (($2 == "" && $3 == "") || ($1 == $2)) {gsub(/ \| /, RS); print $1}' | LC_ALL=C LANG=C $AWK '{split($1, a, /\|/); for (i in a) {print a[i]}}' +} + +ipban_fin() +{ + getipban + "$IPSET_DIR/create_ipset.sh" +} + +dl_checked "$ZURL_REESTR" "$ZREESTR" 204800 251658240 600 || { + ipban_fin + exit 2 +} + +reestr_list | sort -u | zz "$ZHOSTLIST" + +reestr_extract_ip <"$ZREESTR" >"$IPB" + +rm -f "$ZREESTR" +[ "$DISABLE_IPV4" != "1" ] && $AWK '/^([0-9]{1,3}\.){3}[0-9]{1,3}($|(\/[0-9]{2}$))/' "$IPB" | cut_local | ip2net4 | zz "$ZIPLIST_IPBAN" +[ "$DISABLE_IPV6" != "1" ] && $AWK '/^([0-9a-fA-F]{0,4}:){1,7}[0-9a-fA-F]{0,4}($|(\/[0-9]{2,3}$))/' "$IPB" | cut_local6 | ip2net6 | zz "$ZIPLIST_IPBAN6" +rm -f "$IPB" + +hup_zapret_daemons + +ipban_fin + +exit 0 diff --git a/ipset/get_reestr_preresolved.sh b/ipset/get_reestr_preresolved.sh new file mode 100755 index 0000000..e6ff585 --- /dev/null +++ b/ipset/get_reestr_preresolved.sh @@ -0,0 +1,47 @@ +#!/bin/sh + +IPSET_DIR="$(dirname "$0")" +IPSET_DIR="$(cd "$IPSET_DIR"; pwd)" + +. "$IPSET_DIR/def.sh" + +TMPLIST="$TMPDIR/list.txt" + +BASEURL="https://raw.githubusercontent.com/bol-van/rulist/main" +URL4="$BASEURL/reestr_resolved4.txt" +URL6="$BASEURL/reestr_resolved6.txt" +IPB4="$BASEURL/reestr_ipban4.txt" +IPB6="$BASEURL/reestr_ipban6.txt" + +dl() +{ + # $1 - url + # $2 - file + # $3 - minsize + # $4 - maxsize + curl -H "Accept-Encoding: gzip" -k --fail --max-time 120 --connect-timeout 10 --retry 4 --max-filesize $4 -o "$TMPLIST" "$1" || + { + echo list download failed : $1 + exit 2 + } + dlsize=$(LC_ALL=C LANG=C wc -c "$TMPLIST" | xargs | cut -f 1 -d ' ') + if test $dlsize -lt $3; then + echo list is too small : $dlsize bytes. can be bad. + exit 2 + fi + zzcopy "$TMPLIST" "$2" + rm -f "$TMPLIST" +} + +getuser && { + [ "$DISABLE_IPV4" != "1" ] && { + dl "$URL4" "$ZIPLIST" 32768 4194304 + dl "$IPB4" "$ZIPLIST_IPBAN" 8192 1048576 + } + [ "$DISABLE_IPV6" != "1" ] && { + dl "$URL6" "$ZIPLIST6" 8192 4194304 + dl "$IPB6" "$ZIPLIST_IPBAN6" 128 1048576 + } +} + +"$IPSET_DIR/create_ipset.sh" diff --git a/ipset/get_reestr_preresolved_smart.sh b/ipset/get_reestr_preresolved_smart.sh new file mode 100755 index 0000000..0310556 --- /dev/null +++ b/ipset/get_reestr_preresolved_smart.sh @@ -0,0 +1,47 @@ +#!/bin/sh + +IPSET_DIR="$(dirname "$0")" +IPSET_DIR="$(cd "$IPSET_DIR"; pwd)" + +. "$IPSET_DIR/def.sh" + +TMPLIST="$TMPDIR/list.txt" + +BASEURL="https://raw.githubusercontent.com/bol-van/rulist/main" +URL4="$BASEURL/reestr_smart4.txt" +URL6="$BASEURL/reestr_smart6.txt" +IPB4="$BASEURL/reestr_ipban4.txt" +IPB6="$BASEURL/reestr_ipban6.txt" + +dl() +{ + # $1 - url + # $2 - file + # $3 - minsize + # $4 - maxsize + curl -H "Accept-Encoding: gzip" -k --fail --max-time 120 --connect-timeout 10 --retry 4 --max-filesize $4 -o "$TMPLIST" "$1" || + { + echo list download failed : $1 + exit 2 + } + dlsize=$(LC_ALL=C LANG=C wc -c "$TMPLIST" | xargs | cut -f 1 -d ' ') + if test $dlsize -lt $3; then + echo list is too small : $dlsize bytes. can be bad. + exit 2 + fi + zzcopy "$TMPLIST" "$2" + rm -f "$TMPLIST" +} + +getuser && { + [ "$DISABLE_IPV4" != "1" ] && { + dl "$URL4" "$ZIPLIST" 32768 4194304 + dl "$IPB4" "$ZIPLIST_IPBAN" 8192 1048576 + } + [ "$DISABLE_IPV6" != "1" ] && { + dl "$URL6" "$ZIPLIST6" 8192 4194304 + dl "$IPB6" "$ZIPLIST_IPBAN6" 128 1048576 + } +} + +"$IPSET_DIR/create_ipset.sh" diff --git a/ipset/get_reestr_resolvable_domains.sh b/ipset/get_reestr_resolvable_domains.sh new file mode 100755 index 0000000..fa00869 --- /dev/null +++ b/ipset/get_reestr_resolvable_domains.sh @@ -0,0 +1,45 @@ +#!/bin/sh + +IPSET_DIR="$(dirname "$0")" +IPSET_DIR="$(cd "$IPSET_DIR"; pwd)" + +. "$IPSET_DIR/def.sh" + +TMPLIST="$TMPDIR/list_nethub.txt" + +BASEURL="https://raw.githubusercontent.com/bol-van/rulist/main" +URL="$BASEURL/reestr_hostname_resolvable.txt" +IPB4="$BASEURL/reestr_ipban4.txt" +IPB6="$BASEURL/reestr_ipban6.txt" + +dl() +{ + # $1 - url + # $2 - file + # $3 - minsize + # $4 - maxsize + curl -H "Accept-Encoding: gzip" -k --fail --max-time 120 --connect-timeout 10 --retry 4 --max-filesize $4 -o "$TMPLIST" "$1" || + { + echo list download failed : $1 + exit 2 + } + dlsize=$(LC_ALL=C LANG=C wc -c "$TMPLIST" | xargs | cut -f 1 -d ' ') + if test $dlsize -lt $3; then + echo list is too small : $dlsize bytes. can be bad. + exit 2 + fi + zzcopy "$TMPLIST" "$2" + rm -f "$TMPLIST" +} + +dl "$URL" "$ZHOSTLIST" 65536 67108864 + +hup_zapret_daemons + +[ "$DISABLE_IPV4" != "1" ] && dl "$IPB4" "$ZIPLIST_IPBAN" 8192 1048576 +[ "$DISABLE_IPV6" != "1" ] && dl "$IPB6" "$ZIPLIST_IPBAN6" 128 1048576 + +getipban +"$IPSET_DIR/create_ipset.sh" + +exit 0 diff --git a/ipset/get_reestr_resolve.sh b/ipset/get_reestr_resolve.sh new file mode 100755 index 0000000..c94e15f --- /dev/null +++ b/ipset/get_reestr_resolve.sh @@ -0,0 +1,83 @@ +#!/bin/sh + +IPSET_DIR="$(dirname "$0")" +IPSET_DIR="$(cd "$IPSET_DIR"; pwd)" + +. "$IPSET_DIR/def.sh" + +ZREESTR="$TMPDIR/zapret.txt.gz" +ZDIG="$TMPDIR/zapret-dig.txt" +IPB="$TMPDIR/ipb.txt" +ZIPLISTTMP="$TMPDIR/zapret-ip.txt" +#ZURL=https://reestr.rublacklist.net/api/current +ZURL_REESTR=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv.gz + +dl_checked() +{ + # $1 - url + # $2 - file + # $3 - minsize + # $4 - maxsize + # $5 - maxtime + curl -k --fail --max-time $5 --connect-timeout 10 --retry 4 --max-filesize $4 -o "$2" "$1" || + { + echo list download failed : $1 + return 2 + } + dlsize=$(LC_ALL=C LANG=C wc -c "$2" | xargs | cut -f 1 -d ' ') + if test $dlsize -lt $3; then + echo list is too small : $dlsize bytes. can be bad. + return 2 + fi + return 0 +} + +reestr_list() +{ + LC_ALL=C LANG=C gunzip -c "$ZREESTR" | cut -s -f2 -d';' | LC_ALL=C LANG=C nice -n 5 sed -Ee 's/^\*\.(.+)$/\1/' -ne 's/^[a-z0-9A-Z._-]+$/&/p' | $AWK '{ print tolower($0) }' +} +reestr_extract_ip() +{ + LC_ALL=C LANG=C gunzip -c | nice -n 5 $AWK -F ';' '($1 ~ /^([0-9]{1,3}\.){3}[0-9]{1,3}/) && (($2 == "" && $3 == "") || ($1 == $2)) {gsub(/ \| /, RS); print $1}' | LC_ALL=C LANG=C $AWK '{split($1, a, /\|/); for (i in a) {print a[i]}}' +} + +getuser && { + # both disabled + [ "$DISABLE_IPV4" = "1" ] && [ "$DISABLE_IPV6" = "1" ] && exit 0 + + dl_checked "$ZURL_REESTR" "$ZREESTR" 204800 251658240 600 || exit 2 + + echo preparing ipban list .. + + reestr_extract_ip <"$ZREESTR" >"$IPB" + [ "$DISABLE_IPV4" != "1" ] && $AWK '/^([0-9]{1,3}\.){3}[0-9]{1,3}($|(\/[0-9]{2}$))/' "$IPB" | cut_local | ip2net4 | zz "$ZIPLIST_IPBAN" + [ "$DISABLE_IPV6" != "1" ] && $AWK '/^([0-9a-fA-F]{0,4}:){1,7}[0-9a-fA-F]{0,4}($|(\/[0-9]{2,3}$))/' "$IPB" | cut_local6 | ip2net6 | zz "$ZIPLIST_IPBAN6" + rm -f "$IPB" + + echo preparing dig list .. + reestr_list | sort -u >"$ZDIG" + + rm -f "$ZREESTR" + + echo digging started. this can take long ... + + [ "$DISABLE_IPV4" != "1" ] && { + filedigger "$ZDIG" 4 | cut_local >"$ZIPLISTTMP" || { + rm -f "$ZDIG" + exit 1 + } + ip2net4 <"$ZIPLISTTMP" | zz "$ZIPLIST" + rm -f "$ZIPLISTTMP" + } + [ "$DISABLE_IPV6" != "1" ] && { + filedigger "$ZDIG" 6 | cut_local6 >"$ZIPLISTTMP" || { + rm -f "$ZDIG" + exit 1 + } + ip2net6 <"$ZIPLISTTMP" | zz "$ZIPLIST6" + rm -f "$ZIPLISTTMP" + } + rm -f "$ZDIG" +} + +"$IPSET_DIR/create_ipset.sh" diff --git a/ipset/get_refilter_domains.sh b/ipset/get_refilter_domains.sh new file mode 100755 index 0000000..b11d26d --- /dev/null +++ b/ipset/get_refilter_domains.sh @@ -0,0 +1,42 @@ +#!/bin/sh + +IPSET_DIR="$(dirname "$0")" +IPSET_DIR="$(cd "$IPSET_DIR"; pwd)" + +. "$IPSET_DIR/def.sh" + +TMPLIST="$TMPDIR/list.txt" + +URL="https://github.com/1andrevich/Re-filter-lists/releases/latest/download/domains_all.lst" + +dl() +{ + # $1 - url + # $2 - file + # $3 - minsize + # $4 - maxsize + curl -L -H "Accept-Encoding: gzip" -k --fail --max-time 60 --connect-timeout 10 --retry 4 --max-filesize $4 -o "$TMPLIST" "$1" || + { + echo list download failed : $1 + exit 2 + } + dlsize=$(LC_ALL=C LANG=C wc -c "$TMPLIST" | xargs | cut -f 1 -d ' ') + if test $dlsize -lt $3; then + echo list is too small : $dlsize bytes. can be bad. + exit 2 + fi + zzcopy "$TMPLIST" "$2" + rm -f "$TMPLIST" +} + +# useful in case ipban set is used in custom scripts +FAIL= +getipban || FAIL=1 +"$IPSET_DIR/create_ipset.sh" +[ -n "$FAIL" ] && exit + +dl "$URL" "$ZHOSTLIST" 32768 4194304 + +hup_zapret_daemons + +exit 0 diff --git a/ipset/get_refilter_ipsum.sh b/ipset/get_refilter_ipsum.sh new file mode 100755 index 0000000..58c646f --- /dev/null +++ b/ipset/get_refilter_ipsum.sh @@ -0,0 +1,38 @@ +#!/bin/sh + +IPSET_DIR="$(dirname "$0")" +IPSET_DIR="$(cd "$IPSET_DIR"; pwd)" + +. "$IPSET_DIR/def.sh" + +TMPLIST="$TMPDIR/list.txt" + +URL="https://github.com/1andrevich/Re-filter-lists/releases/latest/download/ipsum.lst" + +dl() +{ + # $1 - url + # $2 - file + # $3 - minsize + # $4 - maxsize + curl -L -H "Accept-Encoding: gzip" -k --fail --max-time 60 --connect-timeout 10 --retry 4 --max-filesize $4 -o "$TMPLIST" "$1" || + { + echo list download failed : $1 + exit 2 + } + dlsize=$(LC_ALL=C LANG=C wc -c "$TMPLIST" | xargs | cut -f 1 -d ' ') + if test $dlsize -lt $3; then + echo list is too small : $dlsize bytes. can be bad. + exit 2 + fi + zzcopy "$TMPLIST" "$2" + rm -f "$TMPLIST" +} + +getuser && { + [ "$DISABLE_IPV4" != "1" ] && { + dl "$URL" "$ZIPLIST" 32768 4194304 + } +} + +"$IPSET_DIR/create_ipset.sh" diff --git a/ipset/get_user.sh b/ipset/get_user.sh new file mode 100755 index 0000000..2d98981 --- /dev/null +++ b/ipset/get_user.sh @@ -0,0 +1,11 @@ +#!/bin/sh +# resolve user host list + +IPSET_DIR="$(dirname "$0")" +IPSET_DIR="$(cd "$IPSET_DIR"; pwd)" + +. "$IPSET_DIR/def.sh" + +getuser + +"$IPSET_DIR/create_ipset.sh" diff --git a/ipset/zapret-hosts-user-exclude.txt.default b/ipset/zapret-hosts-user-exclude.txt.default new file mode 100644 index 0000000..c14c04b --- /dev/null +++ b/ipset/zapret-hosts-user-exclude.txt.default @@ -0,0 +1,9 @@ +127.0.0.0/8 +10.0.0.0/8 +172.16.0.0/12 +192.168.0.0/16 +169.254.0.0/16 +100.64.0.0/10 +::1 +fc00::/7 +fe80::/10