update docs

common/dialog.sh

@@ -37,8 +37,7 @@ ask_list()
 	local M_DEFAULT
 	eval M_DEFAULT="\$$1"
 	local M_ALL=$M_DEFAULT
-	local M=""
-	local m
+	local M="" m M_OLD
 
 	[ -n "$3" ] && { find_str_in_list "$M_DEFAULT" "$2" || M_DEFAULT="$3" ;}
 
@@ -54,5 +53,5 @@ ask_list()
 	echo selected : $M
 	eval $1="\"$M\""
 
-	[ "$M" != "$M_OLD" ]
+	[ "$M" != "$M_DEFAULT" ]
 }

common/list.sh

@@ -25,7 +25,7 @@ filter_apply_hostlist_target()
 {
 	# $1 - var name of nfqws params
 
-	local v parm parm1 parm2 parm3 parm4 parm5 parm6 parm7 parm8 parm9 parm10 param11 param12 param13 parmNA
+	local v parm parm1 parm2 parm3 parm4 parm5 parm6 parm7 parm8 parm9 parm10 parm11 parm12 parm13 parmNA
 	eval v="\$$1"
 	if contains "$v" "$HOSTLIST_MARKER" || contains "$v" "$HOSTLIST_NOAUTO_MARKER"; then
 		[ "$MODE_FILTER" = hostlist -o "$MODE_FILTER" = autohostlist ] &&

common/nft.sh

@@ -24,7 +24,7 @@ nft_add_chain()
 	# $2 - params
 	nft add chain inet $ZAPRET_NFT_TABLE $1 "{ $2 }"
 }
-nft_delete_chain()
+nft_del_chain()
 {
 	# $1 - chain
 	nft delete chain inet $ZAPRET_NFT_TABLE $1

config.default

@@ -41,6 +41,8 @@ AUTOHOSTLIST_DEBUGLOG=0
 
 # number of parallel threads for domain list resolves
 MDIG_THREADS=30
+MDIG_EAGAIN=10
+MDIG_EAGAIN_DELAY=500
 
 # ipset/*.sh can compress large lists
 GZIP_LISTS=1
@@ -54,7 +56,7 @@ GZIP_LISTS=1
 DESYNC_MARK=0x40000000
 DESYNC_MARK_POSTNAT=0x20000000
 
-# do not pass outgoing traffic to tpws/nfqws not marked with this bit
+# do not pass outgoing traffic to nfqws not marked with this bit
 # this setting allows to write your own rules to limit traffic that should be fooled
 # for example based on source IP or incoming interface name
 # no filter if not defined

docs/changes.txt

@@ -162,3 +162,10 @@ v0.8.1
 * init.d: 80-dns-intercept
 * winws2: --wf-filter-loopback
 * blockcheck2: NOTEST_MISC_HTTP[S], NOTEST_SYNDATA_HTTP[S]
+
+0.8.3
+
+* nfqws2, zapret-lib: gzip compression and decompression
+* nfqws2: ignore trailing spaces and tabs in hostlists and ipsets. "host.com   " or "1.2.3.4 " are ok now
+* init.d: 99-lan-filter custom script
+* mdig: --eagain, --eagain-delay

docs/manual.md

@@ -60,6 +60,9 @@
       - [aes\_gcm](#aes_gcm)
       - [aes\_ctr](#aes_ctr)
       - [hkdf](#hkdf)
+    - [Компрессия](#компрессия)
+      - [gunzip](#gunzip)
+      - [gzip](#gzip)
     - [Системные функции](#системные-функции)
       - [uname](#uname)
       - [clock\_gettime](#clock_gettime)
@@ -119,6 +122,8 @@
     - [genhost](#genhost)
     - [host\_ip](#host_ip)
   - [Операции с именами файлов и путями](#операции-с-именами-файлов-и-путями)
+  - [Чтение и запись файлов](#чтение-и-запись-файлов)
+  - [Компрессия данных](#компрессия-данных)
   - [autottl](#autottl)
   - [Операции с диссектами](#операции-с-диссектами)
     - [standard ipid](#standard-ipid)
@@ -1919,6 +1924,34 @@ HKDF - HMAC-based Key Derivation Function. Генератор ключей на
 - okm_len - требуемая длина okm - output keying material
 - возвращается raw строка - okm
 
+### Компрессия
+
+#### gunzip
+
+```
+function gunzip_init(windowBits)
+function gunzip_end(zstream)
+function gunzip_inflate(zstream, compressed_data, expected_uncompressed_chunk_size)
+```
+
+* gunzip_init создает и возвращает контекст gzip потока для последующих вызовов других функций. Значение windowBits см. в документации по zlib (по умолчанию 47).
+* gunzip_end освобождает контекст gzip. может быть освобожден сборщиком мусора, но лучше вызывать явно.
+* gunzip_inflate разжимает очередную часть зипованных данных. Данные можно скармливать частями. Разжатые части конкатенируются для получения полных данных. Возвращается 2 аргумента : расжатые данные и bool признак конца gzip. В случае испорченных данных или при нехватке памяти возвращается nil.
+* expected_uncompressed_chunk_size - необязательный параметр для оптимизации выделения памяти под разжимаемые данные. Если буфера не хватает, вызываются realloc, копирующие блоки памяти и влияющие на производительность. Размер следует выбирать согласно ожидаемой степени сжатия с небольшим запасом. По умолчанию - четырехкратный размер compressed_data.
+
+#### gzip
+
+```
+function gzip_init(windowBits, level, memlevel)
+function gzip_end(zstream)
+function gzip_deflate(zstream, uncompressed_data, expected_compressed_chunk_size)
+```
+
+* gzip_init создает и возвращает контекст gzip потока для последующих вызовов других функций. Значение windowBits см. в документации по zlib (по умолчанию 31). level - уровень сжатия от 1 до 9 (по умолчанию 9), memlevel - допустимый уровень использования памяти от 1 до 8 (по умолчанию 8).
+* gzip_end освобождает контекст gzip. может быть освобожден сборщиком мусора, но лучше вызывать явно.
+* gzip_deflate cжимает очередную часть данных. Данные можно скармливать частями. Cжатые части конкатенируются для получения полных данных. Для финализации потока по окончанию скармливания данных функция должна быть вызвана с uncompressed_data=nil или uncompressed_data="". Возвращается 2 аргумента : сжатые данные и bool признак конца gzip. При ошибках gzip или нехватке памяти возвращается nil.
+* expected_compressed_chunk_size - необязательный параметр для оптимизации выделения памяти под cжимаемые данные. Если буфера не хватает, вызываются realloc, копирующие блоки памяти и влияющие на производительность.  Размер следует выбирать согласно ожидаемой степени сжатия с небольшим запасом. По умолчанию - половина размера uncompressed_data.
+
 ### Системные функции
 
 #### uname
@@ -3025,6 +3058,51 @@ function writeable_file_name(filename)
 - writeable_file_name возвращает filename, если filename содержит абсолютный путь или env `WRITEABLE` отсутствует.
 Иначе берется путь из env `WRITEABLE` и к нему дописывается имя файла filename через append_path.
 
+## Чтение и запись файлов
+
+```
+function readfile(filename)
+```
+
+Читает весь файл. Вызывается error в случае ошибки при открытии или чтении файла.
+
+```
+function z_readfile(filename, expected_ratio)
+```
+
+Автоматически определяет является ли файл gzip. Если да - разжимает, если нет - читает без изменений. Вызывается error в случае ошибки при открытии или чтении файла.
+expected_ratio - ожидаемое соотношение длины разжатых данных к длине сжатых данных (по умолчанию 4).
+
+```
+function writefile(filename, data)
+```
+
+Записывает data в файл. Вызывается error в случае ошибки при открытии файла.
+
+
+## Компрессия данных
+
+```
+function is_gzip_file(filename)
+```
+
+true, если файл является gzip, иначе false. При ошибке открытия файла вызывается error.
+
+```
+function gunzip_file(filename, expected_ratio, read_block_size)
+```
+
+Разжимает файл и возвращает raw string. В случае ошибки открытия или чтения файла вызывается error. При нехватке памяти возвращается nil. read_block_size - частями какого размера читается файл (по умолчанию 16K).
+expected_ratio - ожидаемое соотношение длины разжатых данных к длине сжатых данных (по умолчанию 4).
+
+```
+function gzip_file(filename, data, expected_ratio, level, memlevel, compress_block_size)
+```
+
+Сжимает raw строку в gzip файл. В случае ошибки открытия или чтения файла вызывается error. При испорченных gzip данных или нехватке памяти возвращается nil.
+level - уровень сжатия от 1 до 9 (по умолчанию 9), memlevel - допустимый уровень использования памяти от 1 до 8 (по умолчанию 8). compress_block_size - частями какого размера жмется файл (по умолчанию 16K).
+expected_ratio - ожидаемое соотношение длины разжатых данных к длине сжатых данных (по умолчанию 2).
+
 ## autottl
 
 ```
@@ -4246,8 +4324,10 @@ ip2net фильтрует входные данные, выкидывая неп
 Она берет из stdin список доменов и выводит в stdout результат ресолвинга. Ошибки выводятся в stderr.
 
 ```
---threads=<threads_number>     ; количество потоков. по умолчанию 1.
 --family=<4|6|46>              ; выбор семейства IP адресов : ipv4, ipv6, ipv4+ipv6
+--threads=<threads_number>     ; количество потоков. по умолчанию 1.
+--eagain=<eagain_retries>      ; количество попыток повтора после EAGAIN. по умолчанию 10
+--eagain-delay=<ms>            ; время ожидания в мсек между повторами по EAGAIN. по умолчанию 500.
 --verbose                      ; дебаг-лог на консоль
 --stats=N                      ; выводить статистику каждые N доменов
 --log-resolved=<file>          ; сохранять успешно отресолвленные домены в файл
@@ -4588,6 +4668,8 @@ nfqws2 может работать и самостоятельно без скр
 | IPSET_HOOK                                                                                                                                                                                                                          | скрипт, который получает имя ipset в $1, выдает в stdout список ip, и они добавляются в ipset                                                 |
 | IP2NET_OPT4<br>IP2NET_OPT6                                                                                                                                                                                                          | настройки ip2net для скриптов получения ip листов                                                                                             |
 | MDIG_THREADS                                                                                                                                                                                                                        | количество потоков mdig. используется при ресолвинге хостлистов                                                                               |
+| MDIG_EAGAIN                                                                                                                                                                                                                         | количество попыток при получении EAGAIN |
+| MDIG_EAGAIN_DELAY                                                                                                                                                                                                                   | задержка в мсек между попытками при получении EAGAIN |
 | AUTOHOSTLIST_INCOMING_MAXSEQ<br>AUTOHOSTLIST_RETRANS_MAXSEQ<br>AUTOHOSTLIST_RETRANS_THRESHOLD<br>AUTOHOSTLIST_RETRANS_RESET<br>AUTOHOSTLIST_FAIL_THRESHOLD<br>AUTOHOSTLIST_FAIL_TIME<br>AUTOHOSTLIST_UDP_IN<br>AUTOHOSTLIST_UDP_OUT | параметры [автохостлистов](#детектор-неудач-автохостлистов)                                                                                   |
 | AUTOHOSTLIST_DEBUGLOG                                                                                                                                                                                                               | включение autohostlist debug log. лог пишется в `ipset/zapret-hosts-auto-debug.log`                                                           |
 | GZIP_LISTS                                                                                                                                                                                                                          | применять ли сжатие gzip для генерируемых хост и ip листов                                                                                    |

init.d/custom.d.examples.linux/50-nfqws-ipset

@@ -91,13 +91,13 @@ zapret_custom_firewall_nft()
 	local first_packets_only="$nft_connbytes 1-$NFQWS_MY1_PKT_OUT"
 
 	[ "$DISABLE_IPV4" != 1 ] && {
-	        make_comma_list subnets $NFQWS_MY1_SUBNETS4
+		make_comma_list subnets $NFQWS_MY1_SUBNETS4
 		nft_create_set $NFQWS_MY1_NAME4 "type ipv4_addr; size $NFQWS_MY1_IPSET_SIZE; auto-merge; flags interval;"
 		nft_flush_set $NFQWS_MY1_NAME4
 		nft_add_set_element $NFQWS_MY1_NAME4 "$subnets"
 	}
 	[ "$DISABLE_IPV6" != 1 ] && {
-	        make_comma_list subnets $NFQWS_MY1_SUBNETS6
+		make_comma_list subnets $NFQWS_MY1_SUBNETS6
 		nft_create_set $NFQWS_MY1_NAME6 "type ipv6_addr; size $NFQWS_MY1_IPSET_SIZE; auto-merge; flags interval;"
 		nft_flush_set $NFQWS_MY1_NAME6
 		nft_add_set_element $NFQWS_MY1_NAME6 "$subnets"

init.d/custom.d.examples.linux/80-dns-intercept

@@ -34,19 +34,22 @@ zapret_custom_firewall_nft()
 
 	local rule="udp sport 53 queue num $QNUM bypass"
 
-	# dns client
+	# router
 	nft_print_op "oifname @lanif $rule" "nfqws forward (qnum $QNUM)" "4+6"
 	nft_add_chain forward_dns_feed "type filter hook forward priority mangle;"
+	nft_flush_chain forward_dns_feed
 	nft_add_rule forward_dns_feed oifname @lanif $rule
 
-	# router
+	# dns client
 	nft_print_op "$rule" "nfqws input (qnum $QNUM)" "4+6"
 	nft_add_chain input_dns_feed "type filter hook input priority mangle;"
+	nft_flush_chain input_dns_feed
 	nft_add_rule input_dns_feed $rule
 
 	# dns server
 	nft_print_op "$rule" "nfqws output (qnum $QNUM)" "4+6"
 	nft_add_chain output_dns_feed "type filter hook output priority mangle;"
+	nft_flush_chain output_dns_feed
 	nft_add_rule output_dns_feed $rule
 }
 
@@ -54,6 +57,6 @@ zapret_custom_firewall_nft_flush()
 {
 	local chain
 	for chain in forward_dns_feed input_dns_feed output_dns_feed; do
-		nft_delete_chain $chain 2>/dev/null
+		nft_del_chain $chain 2>/dev/null
 	done
 }

init.d/custom.d.examples.linux/99-lan-filter

@@ -0,0 +1,145 @@
+# this custom script sets FILTER_MARK to specified source ips
+
+# NOTE !!! SCRIPT REQUIRES FILTER_MARK VAR IN CONFIG FILE !!!
+# NOTE !!! WITHOUT FILTER_MARK IT DOES NOTHING !!!
+
+# NOTE !!! ON NON-OPENWRT SYSTEMS SCRIPT REQUIRES IFACE_LAN VAR IN CONFIG FILE !!!
+
+# can override in config :
+# LAN ip/cidr list to be fooled. elements are space separated
+FILTER_LAN_IP="${FILTER_LAN_IP:-192.168.0.0/16}"
+FILTER_LAN_IP6="${FILTER_LAN_IP6:-fc00::/7}"
+# allow fooling from local system (0|1) ?
+FILTER_LAN_ALLOW_OUTPUT="${FILTER_LAN_ALLOW_OUTPUT:-1}"
+
+FILTER_LAN_SET="lanfilter"
+FILTER_LAN_SET6="${FILTER_LAN_SET}6"
+FILTER_LAN_IPSET_SIZE=${FILTER_LAN_IPSET_SIZE:-256}
+FILTER_LAN_IPSET_OPT="${FILTER_LAN_IPSET_OPT:-hash:net hashsize 8192 maxelem $FILTER_LAN_IPSET_SIZE}"
+
+filter_mark_check()
+{
+	[ -n "$FILTER_MARK" ] || {
+		echo "WARNING ! lan filter cannot work without FILTER_MARK set in config"
+		return 1
+	}
+	[ "$DISABLE_IPV4" = 1 -a "$DISABLE_IPV6" = 1 ] && return 1
+	return 0
+}
+
+zapret_custom_firewall()
+{
+	# $1 - 1 - run, 0 - stop
+
+	filter_mark_check || return
+
+	local subnet lanifs rule
+	local setmark="-j MARK --set-mark $FILTER_MARK/$FILTER_MARK"
+	local filt4="-m set --match-set $FILTER_LAN_SET src"
+	local filt6="-m set --match-set $FILTER_LAN_SET6 src"
+
+	get_lanif lanifs
+
+	[ "$DISABLE_IPV4" != 1 ] && {
+		[ "$FILTER_LAN_ALLOW_OUTPUT" = 1 ] && {
+			ipt_print_op $1 "$setmark" "filter output"
+			ipt_add_del $1 OUTPUT -t mangle $setmark
+		}
+		[ -n "$lanifs" ] && {
+			[ "$1" = 1 ] && {
+				ipset create $FILTER_LAN_SET $FILTER_LAN_IPSET_OPT family inet 2>/dev/null
+				ipset flush $FILTER_LAN_SET
+				for subnet in $FILTER_LAN_IP; do
+					echo add $FILTER_LAN_SET $subnet
+				done | ipset -! restore
+			}
+			for lan in $lanifs; do
+				rule="-i $lan $filt4 $setmark"
+				ipt_print_op $1 "$rule" "filter forward"
+				ipt_add_del $1 FORWARD -t mangle $rule
+			done
+		}
+	}
+	[ "$DISABLE_IPV6" != 1 ] && {
+		[ "$FILTER_LAN_ALLOW_OUTPUT" = 1 ] && {
+			ipt_print_op $1 "$setmark" "filter output" 6
+			ipt6_add_del $1 OUTPUT -t mangle $setmark
+		}
+		[ -n "$lanifs" ] && {
+			[ "$1" = 1 ] && {
+				ipset create $FILTER_LAN_SET6 $FILTER_LAN_IPSET_OPT family inet6 2>/dev/null
+				ipset flush $FILTER_LAN_SET6
+				for subnet in $FILTER_LAN_IP6; do
+					echo add $FILTER_LAN_SET6 $subnet
+				done | ipset -! restore
+			}
+			for lan in $lanifs; do
+				rule="-i $lan $filt6 $setmark"
+				ipt_print_op $1 "$rule" "filter forward" 6
+				ipt6_add_del $1 FORWARD -t mangle $rule
+			done
+		}
+	}
+
+	[ "$1" = 1 ] || {
+		ipset destroy $FILTER_LAN_SET 2>/dev/null
+		ipset destroy $FILTER_LAN_SET6 2>/dev/null
+	}
+}
+
+zapret_custom_firewall_nft()
+{
+	filter_mark_check || return
+
+	local subnets rule
+	local setmark="meta mark set meta mark or $FILTER_MARK"
+	local filt4="ip saddr == @$FILTER_LAN_SET"
+	local filt6="ip6 saddr == @$FILTER_LAN_SET6"
+	local lanif="iifname @lanif"
+
+	nft_add_chain forward_lan_filter "type filter hook forward priority mangle;"
+	nft_flush_chain forward_lan_filter
+
+	if [ "$FILTER_LAN_ALLOW_OUTPUT" = 1 ]; then
+		nft_add_chain output_lan_filter "type filter hook output priority mangle;"
+		nft_flush_chain output_lan_filter
+		nft_print_op "$setmark" "filter output" "4+6"
+		nft_add_rule output_lan_filter $setmark
+	else
+		nft_del_chain output_lan_filter 2>/dev/null
+	fi
+
+	[ "$DISABLE_IPV4" != 1 ] && {
+		make_comma_list subnets $FILTER_LAN_IP
+		nft_create_set $FILTER_LAN_SET "type ipv4_addr; size $FILTER_LAN_IPSET_SIZE; auto-merge; flags interval;"
+		nft_flush_set $FILTER_LAN_SET
+		nft_add_set_element $FILTER_LAN_SET "$subnets"
+
+		rule="$lanif $filt4 $setmark"
+		nft_print_op "$rule" "filter forward" "4"
+		nft_add_rule forward_lan_filter $rule
+	}
+	[ "$DISABLE_IPV6" != 1 ] && {
+		make_comma_list subnets $FILTER_LAN_IP6
+		nft_create_set $FILTER_LAN_SET6 "type ipv6_addr; size $FILTER_LAN_IPSET_SIZE; auto-merge; flags interval;"
+		nft_flush_set $FILTER_LAN_SET6
+		nft_add_set_element $FILTER_LAN_SET6 "$subnets"
+
+		rule="$lanif $filt6 $setmark"
+		nft_print_op "$rule" "filter forward" "6"
+		nft_add_rule forward_lan_filter $rule
+	}
+}
+
+
+zapret_custom_firewall_nft_flush()
+{
+	# this function is called after all nft fw rules are deleted
+	# however sets are not deleted. it's desired to clear sets here.
+
+	nft_del_chain forward_lan_filter 2>/dev/null
+	nft_del_chain output_lan_filter 2>/dev/null
+
+	nft_del_set $FILTER_LAN_SET 2>/dev/null
+	nft_del_set $FILTER_LAN_SET6 2>/dev/null
+}

install_easy.sh

@@ -601,7 +601,7 @@ check_dns()
 
 install_systemd()
 {
-	INIT_SCRIPT_SRC="$EXEDIR/init.d/sysv/zapret"
+	INIT_SCRIPT_SRC="$EXEDIR/init.d/sysv/zapret2"
 	CUSTOM_DIR="$ZAPRET_RW/init.d/sysv"
 
 	check_bins

ip2net/ip2net.c

@@ -49,7 +49,7 @@ static int ucmp(const void * a, const void * b, void *arg)
 }
 static uint32_t mask_from_bitcount(uint32_t zct)
 {
-	return zct<32 ? ~((1 << zct) - 1) : 0;
+	return zct<32 ? ~((1u << zct) - 1) : 0;
 }
 // make presorted array unique. return number of unique items.
 // 1,1,2,3,3,0,0,0 (ct=8) => 1,2,3,0 (ct=4)
@@ -138,7 +138,7 @@ static void mask_from_bitcount6_make(uint32_t zct, struct in6_addr *a)
 		int32_t n = (127 - zct) >> 3;
 		memset(a->s6_addr,0xFF,n);
 		memset(a->s6_addr+n,0x00,16-n);
-		a->s6_addr[n] = ~((1 << (zct & 7)) - 1);
+		a->s6_addr[n] = ~((1u << (zct & 7)) - 1);
 	}
 }
 static struct in6_addr ip6_mask[129];

ipset/create_ipset.sh

@@ -68,7 +68,6 @@ ipset_restore()
 {
 	# $1 - ipset name
 	# $2 - filename
-
 	zzexist "$2" || return
 	local fsize=$(zzsize "$2")
 	local svram=0
@@ -77,7 +76,7 @@ ipset_restore()
 
 	local T="Adding to ipset $1 "
 	[ "$svram" = "1" ] && T="$T (saveram)"
-	T="$T : $f"
+	T="$T : $2"
 	echo $T
 
 	if [ "$svram" = "1" ]; then

ipset/def.sh

@@ -44,7 +44,9 @@ ZUSERLIST_EXCLUDE="$IPSET_RW_DIR/zapret-hosts-user-exclude.txt"
 
 [ -n "$IP2NET" ] || IP2NET="$ZAPRET_BASE/ip2net/ip2net"
 [ -n "$MDIG" ] || MDIG="$ZAPRET_BASE/mdig/mdig"
-[ -z "$MDIG_THREADS" ] && MDIG_THREADS=30
+MDIG_THREADS=${MDIG_THREADS:-30}
+MDIG_EAGAIN=${MDIG_EAGAIN:-10}
+MDIG_EAGAIN_DELAY=${MDIG_EAGAIN_DELAY:-500}
 
 
 
@@ -124,7 +126,7 @@ zzcat()
 zz()
 {
  if [ "$GZIP_LISTS" = "1" ]; then
-  gzip -c >"$1.gz"
+  gzip -9c >"$1.gz"
   rm -f "$1"
  else
   cat >"$1"
@@ -161,7 +163,7 @@ digger()
  if [ -x "$MDIG" ]; then
   local cmd
   [ "$2" = "s" ] && cmd=--stats=1000
-  "$MDIG" --family=$1 --threads=$MDIG_THREADS $cmd
+  "$MDIG" --family=$1 --threads=$MDIG_THREADS --eagain=$MDIG_EAGAIN --eagain-delay=$MDIG_EAGAIN_DELAY $cmd
  else
   local A=A
   [ "$1" = "6" ] && A=AAAA
@@ -272,11 +274,10 @@ hup_zapret_daemons()
 {
  echo forcing zapret daemons to reload their hostlist
  if exists killall; then
-  killall -HUP tpws nfqws dvtws 2>/dev/null
+  killall -HUP nfqws2 dvtws2 2>/dev/null
  elif exists pkill; then
-  pkill -HUP ^tpws$
-  pkill -HUP ^nfqws$
-  pkill -HUP ^dvtws$
+  pkill -HUP ^nfqws2$
+  pkill -HUP ^dvtws2$
  else
   echo no mass killer available ! cant HUP zapret daemons
  fi

lua/zapret-lib.lua

@@ -1426,6 +1426,114 @@ function tls_client_hello_mod(tls, options)
 	return tls
 end
 
+-- checks if filename is gzip compressed
+function is_gzip_file(filename)
+	local f, err = io.open(filename, "r")
+	if not f then
+		error("is_gzip_file: "..err)
+	end
+	local hdr = f:read(2)
+	f:close()
+	return hdr and hdr=="\x1F\x8B"
+end
+-- ungzip file to raw string
+-- expected_ratio = uncompressed_size/compressed_size (default 4)
+function gunzip_file(filename, expected_ratio, read_block_size)
+	local f, err = io.open(filename, "r")
+	if not f then
+		error("gunzip_file: "..err)
+	end
+	if not read_block_size then read_block_size=16384 end
+	if not expected_ratio then expected_ratio=4 end
+
+	local decompressed=""
+	gz = gunzip_init()
+	if not gz then
+		error("gunzip_file: stream init error")
+	end
+	repeat
+		local compressed, err = f:read(read_block_size)
+		if not compressed then
+			f:close()
+			gunzip_end(gz)
+			if err then
+				error("gunzip_file: file read error : "..err)
+			else
+				return nil
+			end
+		end
+		local decomp, eof = gunzip_inflate(gz, compressed, #compressed * expected_ratio)
+		if not decomp then
+			f:close()
+			gunzip_end(gz)
+			return nil
+		end
+		decompressed = decompressed .. decomp
+	until eof
+	f:close()
+	gunzip_end(gz)
+	return decompressed
+end
+-- zip file to raw string
+-- expected_ratio = uncompressed_size/compressed_size (default 2)
+-- level : 1..9 (default 9)
+-- memlevel : 1..8 (default 8)
+function gzip_file(filename, data, expected_ratio, level, memlevel, compress_block_size)
+	local f, err = io.open(filename, "w")
+	if not f then
+		error("gzip_file: "..err)
+	end
+	if not write_block_size then compress_block_size=16384 end
+	if not expected_ratio then expected_ratio=2 end
+
+	gz = gzip_init(nil, level, memlevel)
+	if not gz then
+		error("gzip_file: stream init error")
+	end
+	local off=1, block_size
+	repeat
+		block_size = #data-off+1
+		if block_size>compress_block_size then block_size=compress_block_size end
+		local comp, eof = gzip_deflate(gz, string.sub(data,off,off+block_size-1), block_size / expected_ratio)
+		if not comp then
+			f:close()
+			gzip_end(gz)
+			return nil
+		end
+		f:write(comp)
+		off = off + block_size
+	until eof
+	f:close()
+	gzip_end(gz)
+end
+-- reads the whole file
+function readfile(filename)
+	local f, err = io.open(filename, "r")
+	if not f then
+		error("readfile: "..err)
+	end
+	local s,err = f:read("*a")
+	f:close()
+	if err then
+		error("readfile: "..err)
+	end
+	return s
+end
+-- reads plain or gzipped file with transparent decompression
+-- expected_ratio = uncompressed_size/compressed_size (default 4)
+function z_readfile(filename, expected_ratio)
+	return is_gzip_file(filename) and gunzip_file(filename, expected_ratio) or readfile(filename)
+end
+-- write data to filename
+function writefile(filename, data)
+	local f, err = io.open(filename, "w")
+	if not f then
+		error("writefile: "..err)
+	end
+	local s,err = f:write(data)
+	f:close()
+end
+
 -- DISSECTORS
 
 function http_dissect_header(header)

lua/zapret-tests.lua

@@ -13,7 +13,7 @@ end
 
 
 function test_all(...)
-	test_run({test_crypto, test_bin, test_ipstr, test_dissect, test_csum, test_resolve, test_rawsend},...)
+	test_run({test_crypto, test_bin, test_gzip, test_ipstr, test_dissect, test_csum, test_resolve, test_rawsend},...)
 end
 
 
@@ -366,6 +366,35 @@ function test_bin(...)
 	test_run({test_ub, test_bit, test_swap, test_ux},...)
 end
 
+function test_gzip()
+	local s=""
+	for i=1,math.random(2000,3000) do
+		local rnd=brandom(math.random(1,50))
+		s=s..rnd..string.rep(bu8(math.random(0,255)),100-#rnd)
+	end
+	local v=math.random(100001,199999)
+	local level=math.random(1,9)
+	local memlevel=math.random(1,8)
+	print("gzip: original size "..#s)
+	print("gzip: cut point "..(v+1))
+	print("gzip: level "..level)
+	print("gzip: memlevel "..memlevel)
+	local gz = gzip_init(nil, level, memlevel)
+	local zip = gzip_deflate(gz,string.sub(s,1,v))
+	zip = zip..gzip_deflate(gz,string.sub(s,v+1))
+	zip = zip..gzip_deflate(gz,nil) -- finalize
+	gzip_end(gz)
+	print("gzip: deflated size "..#zip)
+	local v=math.random(2,#zip-1)
+	print("gunzip: cut point "..(v+1))
+	gz = gunzip_init()
+	local unzip = gunzip_inflate(gz,string.sub(zip,1,v))
+	unzip = unzip..gunzip_inflate(gz,string.sub(zip,v+1))
+	gunzip_end(gz)
+	print("gunzip: inflated size "..#unzip)
+	print("gzip+gunzip: "..(s==unzip and "OK" or "FAIL"))
+	test_assert(s==unzip)
+end
 
 function test_ipstr()
 	local s_ip, ip, s_ip2
@@ -603,25 +632,50 @@ end
 function test_resolve()
 	local pos
 
-	pos = zero_based_pos(resolve_multi_pos(fake_default_tls,"tls_client_hello","1,extlen,sniext,host,sld,midsld,endsld,endhost,-5"))
+	local tdis = tls_dissect(fake_default_tls)
+	local extlen_pos = 5 + 6 + 32 + 1 + 2 + 1 + #tdis.handshake[TLS_HANDSHAKE_TYPE_CLIENT].dis.session_id + #tdis.handshake[TLS_HANDSHAKE_TYPE_CLIENT].dis.cipher_suites*2 + #tdis.handshake[TLS_HANDSHAKE_TYPE_CLIENT].dis.compression_methods
+	print("fake_default_tls size "..#fake_default_tls.." extlen="..extlen_pos)
+	local m="1,extlen,sniext,host,sld,midsld,endsld,endhost,-5"
+	pos = resolve_multi_pos(fake_default_tls,"tls_client_hello",m,true)
 	test_assert(pos)
-	print("resolve_multi_pos tls : "..table.concat(pos," "))
-	pos = zero_based_pos(resolve_range(fake_default_tls,"tls_client_hello","host,endhost"))
+	print("resolve_multi_pos tls : "..m.." : "..table.concat(pos," "))
+	m = "host,endhost"
+	pos = resolve_range(fake_default_tls,"tls_client_hello",m,false,true)
 	test_assert(pos)
-	print("resolve_range tls : "..table.concat(pos," "))
-	pos = resolve_pos(fake_default_tls,"tls_client_hello","midsld")
+	print("resolve_range tls : "..m.." : "..table.concat(pos," "))
+	m = "1"
+	pos = resolve_pos(fake_default_tls,"tls_client_hello",m,true)
+	test_assert(pos==1)
+	print("resolve_pos tls : "..m.." : "..pos)
+	m = "-1"
+	pos = resolve_pos(fake_default_tls,"tls_client_hello",m,true)
+	test_assert(pos==(#fake_default_tls-1))
+	print("resolve_pos tls : "..m.." : "..pos)
+	m = "extlen"
+	pos = resolve_pos(fake_default_tls,"tls_client_hello",m,true)
+	test_assert(pos==extlen_pos)
+	print("resolve_pos tls : "..m.." : "..pos)
+	m = "midsld"
+	pos = resolve_pos(fake_default_tls,"tls_client_hello",m,true)
 	test_assert(pos)
-	print("resolve_pos tls : "..pos - 1)
-	pos = resolve_pos(fake_default_tls,"tls_client_hello","method")
+	print("resolve_pos tls : "..m.." : "..pos)
+	m = "method"
+	pos = resolve_pos(fake_default_tls,"tls_client_hello",m,true)
 	test_assert(not pos)
-	print("resolve_pos tls non-existent : "..tostring(pos))
+	print("resolve_pos tls non-existent : "..m.." : "..tostring(pos))
 
-	pos = zero_based_pos(resolve_multi_pos(fake_default_http,"http_req","method,host,sld,midsld,endsld,endhost,-5"))
+	local host_pos = string.find(fake_default_http,"Host: ")+6-1
+	print("fake_default_http size "..#fake_default_http.." host="..host_pos)
+	m = "method,host,sld,midsld,endsld,endhost,-5"
+	pos = resolve_multi_pos(fake_default_http,"http_req",m,true)
 	test_assert(pos)
-	print("resolve_multi_pos http : "..table.concat(pos," "))
-	pos = resolve_pos(fake_default_http,"http_req","sniext")
+	test_assert(pos[1]==0)
+	test_assert(pos[2]==host_pos)
+	print("resolve_multi_pos http : "..m.." : "..table.concat(pos," "))
+	m = "sniext"
+	pos = resolve_pos(fake_default_http,"http_req",m,true)
 	test_assert(not pos)
-	print("resolve_pos http non-existent : "..tostring(pos))
+	print("resolve_pos http non-existent : "..m.." : "..tostring(pos))
 end
 
 function test_rawsend(opts)

mdig/mdig.c

@@ -30,7 +30,8 @@
 #endif
 #include <time.h>
 
-#define RESOLVER_EAGAIN_ATTEMPTS 2
+#define RESOLVER_EAGAIN_ATTEMPTS	10
+#define RESOLVER_EAGAIN_DELAY		500
 
 static void trimstr(char *s)
 {
@@ -87,7 +88,7 @@ static bool dom_valid(char *dom)
 static void invalid_domain_beautify(char *dom)
 {
 	for (int i = 0; *dom && i < 64; i++, dom++)
-		if (*dom < 0x20 || *dom>0x7F) *dom = '?';
+		if (*dom < 0x20 || *dom<0) *dom = '?';
 	if (*dom) *dom = 0;
 }
 
@@ -97,7 +98,7 @@ static struct
 {
 	char verbose;
 	char family;
-	int threads;
+	int threads, eagain, eagain_delay;
 	time_t start_time;
 	pthread_mutex_t flock;
 	pthread_mutex_t slock; // stats lock
@@ -193,11 +194,12 @@ static void *t_resolver(void *arg)
 	int i, r;
 	char dom[256];
 	bool is_ok;
-	struct addrinfo hints;
-	struct addrinfo *result;
+	struct addrinfo hints, *result;
+	struct timespec ts_eagain = { .tv_sec = glob.eagain_delay/1000, .tv_nsec=glob.eagain_delay%1000*1000000 };
 
 	VLOG("started");
 
+
 	memset(&hints, 0, sizeof(struct addrinfo));
 	hints.ai_family = (glob.family == FAMILY4) ? AF_INET : (glob.family == FAMILY6) ? AF_INET6 : AF_UNSPEC;
 	hints.ai_socktype = SOCK_DGRAM;
@@ -244,12 +246,16 @@ static void *t_resolver(void *arg)
 			else if (dom_valid(dom))
 			{
 				VLOG("resolving %s", dom);
-				for (i = 0; i < RESOLVER_EAGAIN_ATTEMPTS; i++)
+				for (i = 0; i < glob.eagain; i++)
 				{
 					if ((r = getaddrinfo(dom, NULL, &hints, &result)))
 					{
 						VLOG("failed to resolve %s : result %d (%s)", dom, r, eai_str(r));
-						if (r == EAI_AGAIN) continue; // temporary failure. should retry
+						if (r == EAI_AGAIN)
+						{
+							nanosleep(&ts_eagain, NULL);
+							continue; // temporary failure. should retry
+						}
 					}
 					else
 					{
@@ -447,14 +453,18 @@ int dns_parse_query()
 static void exithelp(void)
 {
 	printf(
-		" --threads=<threads_number>\n"
 		" --family=<4|6|46>\t\t; ipv4, ipv6, ipv4+ipv6\n"
+		" --threads=<threads_number>\n"
+		" --eagain=<eagain_retries>\t; how many times to retry if EAGAIN received. default %u\n"
+		" --eagain-delay=<ms>\t\t; time in msec to wait between EAGAIN attempts. default %u\n"
 		" --verbose\t\t\t; print query progress to stderr\n"
 		" --stats=N\t\t\t; print resolve stats to stderr every N domains\n"
 		" --log-resolved=<file>\t\t; log successfully resolved domains to a file\n"
 		" --log-failed=<file>\t\t; log failed domains to a file\n"
 		" --dns-make-query=<domain>\t; output to stdout binary blob with DNS query. use --family to specify ip version.\n"
-		" --dns-parse-query\t\t; read from stdin binary DNS answer blob and parse it to ipv4/ipv6 addresses\n"
+		" --dns-parse-query\t\t; read from stdin binary DNS answer blob and parse it to ipv4/ipv6 addresses\n",
+		RESOLVER_EAGAIN_ATTEMPTS,
+		RESOLVER_EAGAIN_DELAY
 	);
 	exit(1);
 }
@@ -469,6 +479,8 @@ static void exithelp(void)
 
 enum opt_indices {
 	IDX_HELP,
+	IDX_EAGAIN,
+	IDX_EAGAIN_DELAY,
 	IDX_THREADS,
 	IDX_FAMILY,
 	IDX_VERBOSE,
@@ -483,6 +495,8 @@ enum opt_indices {
 static const struct option long_options[] = {
 	[IDX_HELP] = {"help", no_argument, 0, 0},
 	[IDX_THREADS] = {"threads", required_argument, 0, 0},
+	[IDX_EAGAIN] = {"eagain", required_argument, 0, 0},
+	[IDX_EAGAIN_DELAY] = {"eagain-delay", required_argument, 0, 0},
 	[IDX_FAMILY] = {"family", required_argument, 0, 0},
 	[IDX_VERBOSE] = {"verbose", no_argument, 0, 0},
 	[IDX_STATS] = {"stats", required_argument, 0, 0},
@@ -503,6 +517,8 @@ int main(int argc, char **argv)
 	*fn1 = *fn2 = *dom = 0;
 	glob.family = FAMILY4;
 	glob.threads = 1;
+	glob.eagain = RESOLVER_EAGAIN_ATTEMPTS;
+	glob.eagain_delay = RESOLVER_EAGAIN_DELAY;
 	while ((v = getopt_long_only(argc, argv, "", long_options, &option_index)) != -1)
 	{
 		if (v) exithelp();
@@ -513,13 +529,29 @@ int main(int argc, char **argv)
 			exithelp();
 			break;
 		case IDX_THREADS:
-			glob.threads = optarg ? atoi(optarg) : 0;
+			glob.threads = atoi(optarg);
 			if (glob.threads <= 0 || glob.threads > 100)
 			{
 				fprintf(stderr, "thread number must be within 1..100\n");
 				return 1;
 			}
 			break;
+		case IDX_EAGAIN:
+			glob.eagain = atoi(optarg);
+			if (glob.eagain <= 0 || glob.eagain > 1000)
+			{
+				fprintf(stderr, "eagain must be within 1..1000\n");
+				return 1;
+			}
+			break;
+		case IDX_EAGAIN_DELAY:
+			glob.eagain_delay = atoi(optarg);
+			if (glob.eagain_delay < 0 || glob.eagain_delay > 100000)
+			{
+				fprintf(stderr, "eagain-delay must be within 0..100000\n");
+				return 1;
+			}
+			break;
 		case IDX_FAMILY:
 			if (!strcmp(optarg, "4"))
 				glob.family = FAMILY4;

nfq2/darkmagic.c

@@ -51,11 +51,6 @@ uint32_t net16_add(uint16_t netorder_value, uint16_t cpuorder_increment)
 	return htons(ntohs(netorder_value)+cpuorder_increment);
 }
 
-bool ip_has_df(const struct ip *ip)
-{
-	return ip && !!(ntohs(ip->ip_off) & IP_DF);
-}
-
 uint8_t *tcp_find_option(struct tcphdr *tcp, uint8_t kind)
 {
 	uint8_t *t = (uint8_t*)(tcp+1);
@@ -80,11 +75,6 @@ uint8_t *tcp_find_option(struct tcphdr *tcp, uint8_t kind)
 	}
 	return NULL;
 }
-uint32_t *tcp_find_timestamps(struct tcphdr *tcp)
-{
-	uint8_t *t = tcp_find_option(tcp, TCP_KIND_TS);
-	return (t && t[1]==10) ? (uint32_t*)(t+2) : NULL;
-}
 uint8_t tcp_find_scale_factor(const struct tcphdr *tcp)
 {
 	uint8_t *scale = tcp_find_option((struct tcphdr*)tcp, TCP_KIND_SCALE);

nfq2/darkmagic.h

@@ -80,15 +80,11 @@ void extract_ports(const struct tcphdr *tcphdr, const struct udphdr *udphdr, uin
 void extract_endpoints(const struct ip *ip,const struct ip6_hdr *ip6hdr,const struct tcphdr *tcphdr,const struct udphdr *udphdr, struct sockaddr_storage *src, struct sockaddr_storage *dst);
 bool extract_dst(const uint8_t *data, size_t len, struct sockaddr* dst);
 uint8_t *tcp_find_option(struct tcphdr *tcp, uint8_t kind);
-uint32_t *tcp_find_timestamps(struct tcphdr *tcp);
 uint8_t tcp_find_scale_factor(const struct tcphdr *tcp);
 uint16_t tcp_find_mss(const struct tcphdr *tcp);
 bool tcp_synack_segment(const struct tcphdr *tcphdr);
 bool tcp_syn_segment(const struct tcphdr *tcphdr);
 
-bool ip_has_df(const struct ip *ip);
-
-
 bool make_writeable_dir();
 bool ensure_file_access(const char *filename);
 #ifdef __CYGWIN__

nfq2/desync.c

@@ -274,14 +274,14 @@ static bool auto_hostlist_retrans
 			if (dis->tcp && ctrack->dp->hostlist_auto_retrans_reset && (dis->ip || dis->ip6))
 			{
 				uint8_t pkt[sizeof(struct ip6_hdr)+sizeof(struct tcphdr)];
-				struct ip *ip;
-				struct ip6_hdr *ip6;
+				struct ip *ip=NULL;
+				struct ip6_hdr *ip6=NULL;
 				struct tcphdr *tcp;
 				uint16_t pktlen;
 
 				if (dis->ip)
 				{
-					ip = (struct ip*)pkt; ip6=NULL;
+					ip = (struct ip*)pkt;
 					pktlen = sizeof(struct ip) + sizeof(struct tcphdr);
 					*ip = *dis->ip;
 					ip->ip_hl = sizeof(struct ip)/4; // remove ip options
@@ -292,7 +292,7 @@ static bool auto_hostlist_retrans
 				}
 				else if (dis->ip6)
 				{
-					ip6 = (struct ip6_hdr*)pkt; ip=NULL;
+					ip6 = (struct ip6_hdr*)pkt;
 					pktlen = sizeof(struct ip6_hdr) + sizeof(struct tcphdr);
 					*ip6 = *dis->ip6;
 					ip6->ip6_plen = htons(sizeof(struct tcphdr));
@@ -796,7 +796,7 @@ static uint8_t desync(
 
 			if (b_cutoff_all)
 			{
-				if (lua_instance_cutoff_check(&ctx, bIncoming))
+				if (lua_instance_cutoff_check(params.L, &ctx, bIncoming))
 					DLOG("* lua '%s' : voluntary cutoff\n", instance);
 				else if (check_pos_cutoff(pos, range))
 				{
@@ -827,43 +827,43 @@ static uint8_t desync(
 		{
 			// create arg table that persists across multiple desync function calls
 			lua_newtable(params.L);
-			lua_pushf_dissect(dis);
-			lua_pushf_ctrack(ctrack, tpos, bIncoming);
-			lua_pushf_int("profile_n", dp->n);
-			if (dp->name) lua_pushf_str("profile_name", dp->name);
-			if (dp->n_tpl) lua_pushf_int("template_n", dp->n_tpl);
-			if (dp->name_tpl) lua_pushf_str("template_name", dp->name_tpl);
-			if (dp->cookie) lua_pushf_str("cookie", dp->cookie);
-			lua_pushf_bool("outgoing", !bIncoming);
-			lua_pushf_str("ifin", (ifin && *ifin) ? ifin : NULL);
-			lua_pushf_str("ifout", (ifout && *ifout) ? ifout : NULL);
-			lua_pushf_lint("fwmark", fwmark);
-			lua_pushf_table("target");
+			lua_pushf_dissect(params.L, dis);
+			lua_pushf_ctrack(params.L, ctrack, tpos, bIncoming);
+			lua_pushf_int(params.L, "profile_n", dp->n);
+			if (dp->name) lua_pushf_str(params.L, "profile_name", dp->name);
+			if (dp->n_tpl) lua_pushf_int(params.L, "template_n", dp->n_tpl);
+			if (dp->name_tpl) lua_pushf_str(params.L, "template_name", dp->name_tpl);
+			if (dp->cookie) lua_pushf_str(params.L, "cookie", dp->cookie);
+			lua_pushf_bool(params.L, "outgoing", !bIncoming);
+			lua_pushf_str(params.L, "ifin", (ifin && *ifin) ? ifin : NULL);
+			lua_pushf_str(params.L, "ifout", (ifout && *ifout) ? ifout : NULL);
+			lua_pushf_lint(params.L, "fwmark", fwmark);
+			lua_pushf_table(params.L, "target");
 			lua_getfield(params.L,-1,"target");
-			if (sdport) lua_pushf_int("port",sdport);
-			if (sdp4) lua_pushf_lstr("ip",(const char*)sdp4,sizeof(*sdp4));
-			if (sdp6) lua_pushf_lstr("ip6",(const char*)sdp6,sizeof(*sdp6));
+			if (sdport) lua_pushf_int(params.L, "port",sdport);
+			if (sdp4) lua_pushf_lstr(params.L, "ip",(const char*)sdp4,sizeof(*sdp4));
+			if (sdp6) lua_pushf_lstr(params.L, "ip6",(const char*)sdp6,sizeof(*sdp6));
 			lua_pop(params.L,1);
-			lua_pushf_bool("replay", !!replay_piece_count);
+			lua_pushf_bool(params.L, "replay", !!replay_piece_count);
 			if (replay_piece_count)
 			{
-				lua_pushf_int("replay_piece", replay_piece+1);
-				lua_pushf_int("replay_piece_count", replay_piece_count);
-				lua_pushf_bool("replay_piece_last", (replay_piece+1)>=replay_piece_count);
+				lua_pushf_int(params.L, "replay_piece", replay_piece+1);
+				lua_pushf_int(params.L, "replay_piece_count", replay_piece_count);
+				lua_pushf_bool(params.L, "replay_piece_last", (replay_piece+1)>=replay_piece_count);
 			}
-			lua_pushf_str("l7payload", l7payload_str(l7payload));
-			lua_pushf_str("l7proto", l7proto_str(l7proto));
-			lua_pushf_int("reasm_offset", reasm_offset);
-			lua_pushf_raw("reasm_data", rdata_payload, rlen_payload);
-			lua_pushf_raw("decrypt_data", data_decrypt, len_decrypt);
+			lua_pushf_str(params.L, "l7payload", l7payload_str(l7payload));
+			lua_pushf_str(params.L, "l7proto", l7proto_str(l7proto));
+			lua_pushf_int(params.L, "reasm_offset", reasm_offset);
+			lua_pushf_raw(params.L, "reasm_data", rdata_payload, rlen_payload);
+			lua_pushf_raw(params.L, "decrypt_data", data_decrypt, len_decrypt);
 			//if (ctrack) lua_pushf_reg("instance_cutoff", ctrack->lua_instance_cutoff);
 			if (dis->tcp)
 			{
 				// recommended mss value for generated packets
 				if (rpos && rpos->mss)
-					lua_pushf_int("tcp_mss", rpos->mss);
+					lua_pushf_int(params.L, "tcp_mss", rpos->mss);
 				else
-					lua_pushf_global("tcp_mss", "DEFAULT_MSS");
+					lua_pushf_global(params.L, "tcp_mss", "DEFAULT_MSS");
 			}
 			ref_arg = luaL_ref(params.L, LUA_REGISTRYINDEX);
 
@@ -874,7 +874,7 @@ static uint8_t desync(
 				desync_instance(func->func, dp->n, ctx.func_n, instance, sizeof(instance));
 				ctx.instance = instance;
 
-				if (!lua_instance_cutoff_check(&ctx, bIncoming))
+				if (!lua_instance_cutoff_check(params.L, &ctx, bIncoming))
 				{
 					range = bIncoming ? &func->range_in : &func->range_out;
 					if (check_pos_range(pos, range))
@@ -899,10 +899,10 @@ static uint8_t desync(
 							}
 							lua_pushlightuserdata(params.L, &ctx);
 							lua_rawgeti(params.L, LUA_REGISTRYINDEX, ref_arg);
-							lua_pushf_args(&func->args, -1, true);
-							lua_pushf_str("func", func->func);
-							lua_pushf_int("func_n", ctx.func_n);
-							lua_pushf_str("func_instance", instance);
+							lua_pushf_args(params.L, &func->args, -1, true);
+							lua_pushf_str(params.L, "func", func->func);
+							lua_pushf_int(params.L, "func_n", ctx.func_n);
+							lua_pushf_str(params.L, "func_instance", instance);
 
 							// lua should not store and access ctx outside of this call
 							// if this happens make our best to prevent access to bad memory
@@ -960,7 +960,7 @@ static uint8_t desync(
 			}
 			else
 			{
-				b = lua_reconstruct_dissect(-1, mod_pkt, len_mod_pkt, false, false);
+				b = lua_reconstruct_dissect(params.L, -1, mod_pkt, len_mod_pkt, false, false);
 				lua_pop(params.L, 2);
 				if (!b)
 				{

nfq2/helpers.c

@@ -12,7 +12,7 @@
 
 #define UNIQ_SORT \
 { \
-	int i, j, u; \
+	size_t i, j, u; \
 	for (i = j = 0; j < ct; i++) \
 	{ \
 		u = pu[j++]; \
@@ -642,42 +642,13 @@ bool set_env_exedir(const char *argv0)
 	if ((s = strdup(argv0)))
 	{
 		if ((d = dirname(s)))
-			setenv("EXEDIR",s,1);
+			bOK = !setenv("EXEDIR",d,1);
 		free(s);
 	}
 	return bOK;
 }
 
 
-static void mask_from_preflen6_make(uint8_t plen, struct in6_addr *a)
-{
-	if (plen >= 128)
-		memset(a->s6_addr,0xFF,16);
-	else
-	{
-		uint8_t n = plen >> 3;
-		memset(a->s6_addr,0xFF,n);
-		memset(a->s6_addr+n,0x00,16-n);
-		a->s6_addr[n] = (uint8_t)(0xFF00 >> (plen & 7));
-	}
-}
-struct in6_addr ip6_mask[129];
-void mask_from_preflen6_prepare(void)
-{
-	for (int plen=0;plen<=128;plen++) mask_from_preflen6_make(plen, ip6_mask+plen);
-}
-
-#if defined(__GNUC__) && !defined(__llvm__)
-__attribute__((optimize ("no-strict-aliasing")))
-#endif
-void ip6_and(const struct in6_addr * restrict a, const struct in6_addr * restrict b, struct in6_addr * restrict result)
-{
-	// int128 requires 16-bit alignment. in struct sockaddr_in6.sin6_addr is 8-byte aligned.
-	// it causes segfault on x64 arch with latest compiler. it can cause misalign slowdown on other archs
-	// use 64-bit AND
-	((uint64_t*)result->s6_addr)[0] = ((uint64_t*)a->s6_addr)[0] & ((uint64_t*)b->s6_addr)[0];
-	((uint64_t*)result->s6_addr)[1] = ((uint64_t*)a->s6_addr)[1] & ((uint64_t*)b->s6_addr)[1];
-}
 
 void str_cidr4(char *s, size_t s_len, const struct cidr4 *cidr)
 {

nfq2/helpers.h

@@ -139,15 +139,3 @@ bool parse_cidr4(char *s, struct cidr4 *cidr);
 bool parse_cidr6(char *s, struct cidr6 *cidr);
 
 bool parse_int16(const char *p, int16_t *v);
-
-static inline uint32_t mask_from_preflen(uint32_t preflen)
-{
-	return preflen ? preflen<32 ? ~((1 << (32-preflen)) - 1) : 0xFFFFFFFF : 0;
-}
-void ip6_and(const struct in6_addr * restrict a, const struct in6_addr * restrict b, struct in6_addr * restrict result);
-extern struct in6_addr ip6_mask[129];
-void mask_from_preflen6_prepare(void);
-static inline const struct in6_addr *mask_from_preflen6(uint8_t preflen)
-{
-	return ip6_mask+preflen;
-}

nfq2/hostlist.c

@@ -8,13 +8,8 @@ static bool addpool(hostlist_pool **hostlist, char **s, const char *end, int *ct
 {
 	char *p=*s;
 
-	// comment line
-	if ( *p == '#' || *p == ';' || *p == '/' || *p == '\r' || *p == '\n')
-	{
-		// advance until eol
-		for (; p<end && *p && *p!='\r' && *p != '\n'; p++);
-	}
-	else
+	// comment line ?
+	if ( *p != '#' && *p != ';' && *p != '/' && *p != '\r' && *p != '\n')
 	{
 		// advance until eol lowering all chars
 		uint32_t flags = 0;
@@ -23,7 +18,7 @@ static bool addpool(hostlist_pool **hostlist, char **s, const char *end, int *ct
 			p = ++(*s);
 			flags |= HOSTLIST_POOL_FLAG_STRICT_MATCH;
 		}
-		for (; p<end && *p && *p!='\r' && *p != '\n'; p++) *p=tolower(*p);
+		for (; p<end && *p && *p!=' ' && *p!='\t' && *p!='\r' && *p != '\n'; p++) *p=tolower(*p);
 		if (!HostlistPoolAddStrLen(hostlist, *s, p-*s, flags))
 		{
 			HostlistPoolDestroy(hostlist);
@@ -32,6 +27,8 @@ static bool addpool(hostlist_pool **hostlist, char **s, const char *end, int *ct
 		}
 		if (ct) (*ct)++;
 	}
+	// skip remaining non-eol chars
+	for (; p<end && *p && *p!='\r' && *p != '\n'; p++);
 	// advance to the next line
 	for (; p<end && (!*p || *p=='\r' || *p=='\n') ; p++);
 	*s = p;

nfq2/ipset.c

@@ -12,8 +12,7 @@ static bool addpool(ipset *ips, char **s, const char *end, int *ct)
 	struct cidr4 c4;
 	struct cidr6 c6;
 
-	// advance until eol
-	for (p=*s; p<end && *p && *p!='\r' && *p != '\n'; p++);
+	for (p=*s; p<end && *p && *p!=' ' && *p!='\t' && *p!='\r' && *p != '\n'; p++);
 
 	// comment line
 	if (!(**s == '#' || **s == ';' || **s == '/' || **s == '\r' || **s == '\n' ))
@@ -22,7 +21,6 @@ static bool addpool(ipset *ips, char **s, const char *end, int *ct)
 		if (l>=sizeof(cidr)) l=sizeof(cidr)-1;
 		memcpy(cidr,*s,l);
 		cidr[l]=0;
-		rtrim(cidr);
 
 		if (parse_cidr4(cidr,&c4))
 		{
@@ -46,6 +44,8 @@ static bool addpool(ipset *ips, char **s, const char *end, int *ct)
 			DLOG_ERR("bad ip or subnet : %s\n",cidr);
 	}
 
+	// skip remaining non-eol chars
+	for (; p<end && *p && *p!='\r' && *p != '\n'; p++);
 	// advance to the next line
 	for (; p<end && (!*p || *p=='\r' || *p=='\n') ; p++);
 	*s = p;

nfq2/lua.h

@@ -57,49 +57,49 @@ int lua_absindex(lua_State *L, int idx);
 // push - create object and push to the stack
 // pushf - create object and set it as a named field of a table already present on the stack
 // pushi - create object and set it as a index field of a table already present on the stack
-void lua_pushf_nil(const char *field);
-void lua_pushi_nil(lua_Integer idx);
-void lua_pushf_bool(const char *field, bool b);
-void lua_pushi_bool(lua_Integer idx, bool b);
-void lua_pushf_str(const char *field, const char *str);
-void lua_pushi_str(lua_Integer idx, const char *str);
-void lua_pushf_lstr(const char *field, const char *str, size_t len);
-void lua_pushi_lstr(lua_Integer idx, const char *str, size_t len);
-void lua_pushf_int(const char *field, lua_Integer v);
-void lua_pushi_int(lua_Integer idx, lua_Integer v);
-void lua_pushf_lint(const char *field, int64_t v);
-void lua_pushi_lint(lua_Integer idx, int64_t v);
-void lua_pushf_number(const char *field, lua_Number v);
-void lua_pushi_number(lua_Integer idx, lua_Number v);
-void lua_push_raw(const void *v, size_t l);
-void lua_pushf_raw(const char *field, const void *v, size_t l);
-void lua_pushi_raw(lua_Integer idx, const void *v, size_t l);
-void lua_pushf_reg(const char *field, int ref);
-void lua_pushf_lud(const char *field, void *p);
-void lua_pushf_table(const char *field);
-void lua_pushi_table(lua_Integer idx);
+void lua_pushf_nil(lua_State *L, const char *field);
+void lua_pushi_nil(lua_State *L, lua_Integer idx);
+void lua_pushf_bool(lua_State *L, const char *field, bool b);
+void lua_pushi_bool(lua_State *L, lua_Integer idx, bool b);
+void lua_pushf_str(lua_State *L, const char *field, const char *str);
+void lua_pushi_str(lua_State *L, lua_Integer idx, const char *str);
+void lua_pushf_lstr(lua_State *L, const char *field, const char *str, size_t len);
+void lua_pushi_lstr(lua_State *L, lua_Integer idx, const char *str, size_t len);
+void lua_pushf_int(lua_State *L, const char *field, lua_Integer v);
+void lua_pushi_int(lua_State *L, lua_Integer idx, lua_Integer v);
+void lua_pushf_lint(lua_State *L, const char *field, int64_t v);
+void lua_pushi_lint(lua_State *L, lua_Integer idx, int64_t v);
+void lua_pushf_number(lua_State *L, const char *field, lua_Number v);
+void lua_pushi_number(lua_State *L, lua_Integer idx, lua_Number v);
+void lua_push_raw(lua_State *L, const void *v, size_t l);
+void lua_pushf_raw(lua_State *L, const char *field, const void *v, size_t l);
+void lua_pushi_raw(lua_State *L, lua_Integer idx, const void *v, size_t l);
+void lua_pushf_reg(lua_State *L, const char *field, int ref);
+void lua_pushf_lud(lua_State *L, const char *field, void *p);
+void lua_pushf_table(lua_State *L, const char *field);
+void lua_pushi_table(lua_State *L, lua_Integer idx);
 
-void lua_push_blob(int idx_desync, const char *blob);
-void lua_pushf_blob(int idx_desync, const char *field, const char *blob);
+void lua_push_blob(lua_State *L, int idx_desync, const char *blob);
+void lua_pushf_blob(lua_State *L, int idx_desync, const char *field, const char *blob);
 
-void lua_pushf_tcphdr_options(const struct tcphdr *tcp, size_t len);
-void lua_pushf_tcphdr(const struct tcphdr *tcp, size_t len);
-void lua_pushf_udphdr(const struct udphdr *udp, size_t len);
-void lua_pushf_iphdr(const struct ip *ip, size_t len);
-void lua_pushf_ip6hdr(const struct ip6_hdr *ip6, size_t len);
-void lua_push_dissect(const struct dissect *dis);
-void lua_pushf_dissect(const struct dissect *dis);
-void lua_pushf_ctrack(const t_ctrack *ctrack, const t_ctrack_positions *tpos, bool bIncoming);
-void lua_pushf_args(const struct str2_list_head *args, int idx_desync, bool subst_prefix);
-void lua_pushf_pos(const char *name, const struct packet_pos *pos);
-void lua_pushf_range(const char *name, const struct packet_range *range);
-void lua_pushf_global(const char *field, const char *global);
+void lua_pushf_tcphdr_options(lua_State *L, const struct tcphdr *tcp, size_t len);
+void lua_pushf_tcphdr(lua_State *L, const struct tcphdr *tcp, size_t len);
+void lua_pushf_udphdr(lua_State *L, const struct udphdr *udp, size_t len);
+void lua_pushf_iphdr(lua_State *L, const struct ip *ip, size_t len);
+void lua_pushf_ip6hdr(lua_State *L, const struct ip6_hdr *ip6, size_t len);
+void lua_push_dissect(lua_State *L, const struct dissect *dis);
+void lua_pushf_dissect(lua_State *L, const struct dissect *dis);
+void lua_pushf_ctrack(lua_State *L, const t_ctrack *ctrack, const t_ctrack_positions *tpos, bool bIncoming);
+void lua_pushf_args(lua_State *L, const struct str2_list_head *args, int idx_desync, bool subst_prefix);
+void lua_pushf_pos(lua_State *L, const char *name, const struct packet_pos *pos);
+void lua_pushf_range(lua_State *L, const char *name, const struct packet_range *range);
+void lua_pushf_global(lua_State *L, const char *field, const char *global);
 
-bool lua_reconstruct_ip6hdr(int idx, struct ip6_hdr *ip6, size_t *len, uint8_t last_proto, bool preserve_next);
-bool lua_reconstruct_iphdr(int idx, struct ip *ip, size_t *len);
-bool lua_reconstruct_tcphdr(int idx, struct tcphdr *tcp, size_t *len);
-bool lua_reconstruct_udphdr(int idx, struct udphdr *udp);
-bool lua_reconstruct_dissect(int idx, uint8_t *buf, size_t *len, bool badsum, bool ip6_preserve_next);
+bool lua_reconstruct_ip6hdr(lua_State *L, int idx, struct ip6_hdr *ip6, size_t *len, uint8_t last_proto, bool preserve_next);
+bool lua_reconstruct_iphdr(lua_State *L, int idx, struct ip *ip, size_t *len);
+bool lua_reconstruct_tcphdr(lua_State *L, int idx, struct tcphdr *tcp, size_t *len);
+bool lua_reconstruct_udphdr(lua_State *L, int idx, struct udphdr *udp);
+bool lua_reconstruct_dissect(lua_State *L, int idx, uint8_t *buf, size_t *len, bool badsum, bool ip6_preserve_next);
 
 #define MAGIC_CTX	0xE73DC935
 typedef struct {
@@ -112,4 +112,4 @@ typedef struct {
 	bool incoming,cancel;
 } t_lua_desync_context;
 
-bool lua_instance_cutoff_check(const t_lua_desync_context *ctx, bool bIn);
+bool lua_instance_cutoff_check(lua_State *L, const t_lua_desync_context *ctx, bool bIn);

nfq2/nfqws.c

@@ -377,6 +377,7 @@ ex:
 #ifdef HAS_FILTER_SSID
 	wlan_info_deinit();
 #endif
+	rawsend_cleanup();
 	return res;
 err:
 	if (Fpid) fclose(Fpid);
@@ -576,6 +577,7 @@ exiterr:
 	if (fd[0] != -1) close(fd[0]);
 	if (fd[1] != -1) close(fd[1]);
 	lua_shutdown();
+	rawsend_cleanup();
 	return res;
 }
 
@@ -660,7 +662,6 @@ static int win_main()
 				else if (errno == ENODEV)
 				{
 					DLOG_CONDUP("logical network disappeared. deinitializing windivert.\n");
-					rawsend_cleanup();
 					break;
 				}
 				else if (errno == EINTR)
@@ -711,6 +712,7 @@ static int win_main()
 ex:
 	win_dark_deinit();
 	lua_shutdown();
+	rawsend_cleanup();
 	return res;
 }
 
@@ -1732,7 +1734,6 @@ int main(int argc, char **argv)
 
 	srandom(time(NULL));
 	aes_init_keygen_tables(); // required for aes
-	mask_from_preflen6_prepare();
 	set_env_exedir(argv[0]);
 	set_console_io_buffering();
 #ifdef __CYGWIN__
@@ -2690,7 +2691,6 @@ int main(int argc, char **argv)
 #error unsupported OS
 #endif
 ex:
-	rawsend_cleanup();
 	cleanup_params(&params);
 #ifdef __CYGWIN__
 	if (hMutexArg)

nfq2/params.c

@@ -496,6 +496,8 @@ void cleanup_params(struct params_s *params)
 	hostlist_files_destroy(&params->hostlists);
 	ipset_files_destroy(&params->ipsets);
 	ipcacheDestroy(&params->ipcache);
+	blob_collection_destroy(&params->blobs);
+	strlist_destroy(&params->lua_init_scripts);
 #ifdef __CYGWIN__
 	strlist_destroy(&params->ssid_filter);
 	strlist_destroy(&params->nlm_filter);

nfq2/pools.c

@@ -98,6 +98,7 @@ hostfail_pool *HostFailPoolFind(hostfail_pool *p,const char *s)
 }
 void HostFailPoolDel(hostfail_pool **p, hostfail_pool *elem)
 {
+	free(elem->str);
 	HASH_DEL(*p, elem);
 	free(elem);
 }
@@ -108,11 +109,7 @@ void HostFailPoolPurge(hostfail_pool **pp)
 	HASH_ITER(hh, *pp, elem, tmp)
 	{
 		if (now >= elem->expire)
-		{
-			free(elem->str);
-			HASH_DEL(*pp, elem);
-			free(elem);
-		}
+			HostFailPoolDel(pp, elem);
 	}
 }
 static time_t host_fail_purge_prev=0;