From 87bf78b12cfdde373a0b1463d334e70a2985e979 Mon Sep 17 00:00:00 2001 From: Karen Khachatryan Date: Sat, 26 Sep 2026 13:00:25 +0300 Subject: [PATCH] Use IPinfo Lite for direct country routing - Replace IPverse country feeds with filtered IPinfo Lite IPv4 CIDRs - Reuse update-ipsets.sh during generation to keep initial and runtime direct lists consistent - Validate downloaded IPinfo archives and clean up temporary country data - Keep IPverse as the source for ASN-based direct routing - Make source URLs, filenames, and download settings implementation-owned defaults - Allow local generation to override the ipset path and suppress firewall reloads - Increase the runtime direct-list download timeout to three minutes - Remove Alibaba / AliExpress ASN45102 from the default direct ASN list - Document the new country source, generation flow, and IPinfo attribution --- README.md | 15 ++- .../files/etc/scripts/update-ipsets.sh | 116 ++++++++++++------ servers/example/etc/awg-server.sh | 94 +++++++++----- tools/default.py | 13 +- tools/generate.py | 2 +- tools/generate_runtime.py | 86 +++++++------ tools/generate_server_runtime.py | 6 - tools/validate_runtime_files.py | 14 --- 8 files changed, 215 insertions(+), 131 deletions(-) diff --git a/README.md b/README.md index 263cae4..89e0526 100644 --- a/README.md +++ b/README.md @@ -594,9 +594,9 @@ Hooks при этом все равно запускаются, поэтому ` ./generate_configs.py --skip-awg-download --skip-package-sync --skip-hooks ``` -Dynamic direct-list sources из `tools/default.py` должны быть доступны, если они включены. +Dynamic direct-list sources из `tools/default.py` должны быть доступны, если они включены. При обычной генерации `tools/generate.py` один раз локально запускает настоящий `routers/example/files/etc/scripts/update-ipsets.sh` во временном `IPSETS_DIR`. Полученные `direct-static.txt` и `direct.txt` затем одинаково записываются во все generated router и server каталоги. -Если нужен полностью локальный smoke run без загрузки country/ASN direct-list IP sets, добавьте `--skip-direct-downloads`: +Если нужен полностью локальный smoke run без запуска `update-ipsets.sh` и без загрузки country/ASN direct-list IP sets, добавьте `--skip-direct-downloads`: ```sh ./generate_configs.py --skip-awg-download --skip-package-sync --skip-direct-downloads @@ -815,16 +815,21 @@ Direct lists собираются из нескольких источников - публичные `listen_ip` и `exit_ip` из `config.json` - дополнительные CIDR prefixes из `EXIT_DIRECT_STATIC_IPSETS` -Динамическая часть задается странами и ASN. Для каждой страны объединяются два набора ipverse: `geo-ip-blocks` (более детальные registration/sub-allocation данные) и `country-ip-blocks` (компактные RIR delegation данные). +Динамическая часть задается странами и ASN. Страны берутся из IPinfo Lite через `Alice39s/ipinfo-csv-lite`: daily release содержит готовые CIDR, country code, continent и ASN. Для `DIRECT_COUNTRIES` выбираются только нужные IPv4 CIDR. ASN lists по-прежнему берутся из `ipverse/as-ip-blocks`. На роутерах и exit серверах эти настройки лежат в runtime env: ```sh DIRECT_COUNTRIES='ru cn by' -DIRECT_ASNS='32590 45102' +DIRECT_ASNS='32590' ``` -`update-ipsets.sh` читает `/etc/ipsets/direct-static.txt`, для каждой страны добавляет union `geo-ip-blocks + country-ip-blocks`, затем ASN lists, атомарно обновляет `/etc/ipsets/direct.txt` и перезагружает firewall только если итоговый список изменился. +`update-ipsets.sh` читает `/etc/ipsets/direct-static.txt`, скачивает latest `ipinfo-lite.csv.gz` во временный файл, проверяет gzip и через `grep` + `cut` выбирает готовые IPv4 CIDR для всех `DIRECT_COUNTRIES`, затем добавляет ASN lists. Распакованный CSV в `/tmp` не сохраняется, а gzip удаляется сразу после фильтрации. После этого скрипт атомарно обновляет `/etc/ipsets/direct.txt` и перезагружает firewall только если итоговый список изменился. Python или IPinfo libraries на OpenWrt не нужны. + +Тот же `update-ipsets.sh` запускается один раз локально во время `generate.py` с подмененным `IPSETS_DIR` и `RELOAD_FIREWALL=0`. Поэтому initial `direct-static.txt` и `direct.txt`, которые копируются во все router/server trees, строятся той же runtime реализацией, которая затем работает на OpenWrt. + +В проекте используется IPinfo Lite через `Alice39s/ipinfo-csv-lite` (CC BY-SA 4.0). IP address data powered by [IPinfo](https://ipinfo.io). +URL country source не является произвольным runtime-параметром: `update-ipsets.sh` и `awg-server.sh` ожидают именно формат Alice39s/IPinfo Lite CSV (`cidr,country_code,...`). При смене источника на другой формат URL и parser должны меняться вместе. Трафик к direct destination не получает mark `10000`, поэтому не уходит через exit table. diff --git a/routers/example/files/etc/scripts/update-ipsets.sh b/routers/example/files/etc/scripts/update-ipsets.sh index bdb31b6..1fe0bea 100755 --- a/routers/example/files/etc/scripts/update-ipsets.sh +++ b/routers/example/files/etc/scripts/update-ipsets.sh @@ -1,33 +1,40 @@ #!/bin/sh -if [ -r /etc/router-autoinstall.env ]; then +ENV_FILE="${ENV_FILE:-/etc/router-autoinstall.env}" +if [ -r "$ENV_FILE" ]; then # Generated file, loaded once at startup. Re-run to apply changes. - # shellcheck disable=SC1091 - . /etc/router-autoinstall.env + # shellcheck disable=SC1091 disable=SC1090 + . "$ENV_FILE" fi SCRIPT_NAME="${0##*/}" TAG="${SCRIPT_NAME%.sh}" -URL_GH_RAW="${URL_GH_RAW:-https://raw.githubusercontent.com}" + +# Runtime path/apply knobs may be overridden by the local generator. IPSETS_DIR="${IPSETS_DIR:-/etc/ipsets}" -STATIC_DIRECT_NAME="${STATIC_DIRECT_NAME:-direct-static.txt}" -OUT_DIRECT_NAME="${OUT_DIRECT_NAME:-direct.txt}" +RELOAD_FIREWALL="${RELOAD_FIREWALL:-1}" + +# These defaults are tied to this script's parsers/layouts. A different +# source format requires changing the parser together with the URL. +STATIC_DIRECT_NAME="direct-static.txt" +OUT_DIRECT_NAME="direct.txt" +IPINFO_LITE_CSV_GZ_URL="https://github.com/Alice39s/ipinfo-csv-lite/""\ +releases/latest/download/ipinfo-lite.csv.gz" +URL_IPVERSE_ASN="https://raw.githubusercontent.com/ipverse/as-ip-blocks/master" STATIC_DIRECT="$IPSETS_DIR/$STATIC_DIRECT_NAME" OUT_DIRECT="$IPSETS_DIR/$OUT_DIRECT_NAME" TMP_DIRECT="${OUT_DIRECT}.tmp" TMP_SORTED="${TMP_DIRECT}.sorted" +TMP_IPINFO="/tmp/ipinfo-lite.$$.csv.gz" +TMP_COUNTRIES="/tmp/ipinfo-countries.$$.txt" -URL_IPVERSE_GEO="${URL_IPVERSE_GEO:-$URL_GH_RAW/ipverse/geo-ip-blocks/master}" -URL_IPVERSE_COUNTRY="${URL_IPVERSE_COUNTRY:-$URL_GH_RAW/ipverse/country-ip-blocks/master}" -URL_IPVERSE_ASN="${URL_IPVERSE_ASN:-$URL_GH_RAW/ipverse/as-ip-blocks/master}" +DIRECT_COUNTRIES="${DIRECT_COUNTRIES:-ru cn by}" +DIRECT_ASNS="${DIRECT_ASNS:-32590}" -DIRECT_COUNTRIES="${DIRECT_COUNTRIES:-}" -DIRECT_ASNS="${DIRECT_ASNS:-}" - -CURL_CONNECT_TIMEOUT="${UPDATE_IPSETS_CURL_CONNECT_TIMEOUT:-10}" -CURL_MAX_TIME="${UPDATE_IPSETS_CURL_MAX_TIME:-60}" -CURL_RETRY="${UPDATE_IPSETS_CURL_RETRY:-3}" +UPDATE_IPSETS_CURL_CONNECT_TIMEOUT=10 +UPDATE_IPSETS_CURL_MAX_TIME=180 +UPDATE_IPSETS_CURL_RETRY=3 append_static_direct() { if [ ! -s "$STATIC_DIRECT" ]; then @@ -44,9 +51,9 @@ append_url_list() { data="$( curl -fsSL \ - --connect-timeout "$CURL_CONNECT_TIMEOUT" \ - --max-time "$CURL_MAX_TIME" \ - --retry "$CURL_RETRY" \ + --connect-timeout "$UPDATE_IPSETS_CURL_CONNECT_TIMEOUT" \ + --max-time "$UPDATE_IPSETS_CURL_MAX_TIME" \ + --retry "$UPDATE_IPSETS_CURL_RETRY" \ "$url" )" || { logger -t "$TAG" "ERROR: failed to fetch $label from $url" @@ -69,23 +76,59 @@ append_url_list() { } append_country_lists() { + [ -n "$DIRECT_COUNTRIES" ] || return 0 + + country_regex="" for country in $DIRECT_COUNTRIES; do case "$country" in - [a-z][a-z]) ;; + [A-Za-z][A-Za-z]) ;; *) logger -t "$TAG" "ERROR: bad country code: $country" return 1 ;; esac - append_url_list \ - "$URL_IPVERSE_GEO/country/$country/${country}-ipv4.txt" \ - "geo-country:$country" || return 1 - - append_url_list \ - "$URL_IPVERSE_COUNTRY/country/$country/ipv4-aggregated.txt" \ - "rir-country:$country" || return 1 + # shellcheck disable=SC2018 disable=SC2019 + country="$(printf '%s' "$country" | tr 'a-z' 'A-Z')" + if [ -n "$country_regex" ]; then + country_regex="$country_regex|" + fi + country_regex="$country_regex$country" done + + rm -f "$TMP_IPINFO" "$TMP_COUNTRIES" + curl -fsSL \ + --connect-timeout "$UPDATE_IPSETS_CURL_CONNECT_TIMEOUT" \ + --max-time "$UPDATE_IPSETS_CURL_MAX_TIME" \ + --retry "$UPDATE_IPSETS_CURL_RETRY" \ + -o "$TMP_IPINFO" \ + "$IPINFO_LITE_CSV_GZ_URL" || { + logger -t "$TAG" "ERROR: failed to fetch IPinfo Lite CSV" + rm -f "$TMP_IPINFO" "$TMP_COUNTRIES" + return 1 + } + + if [ ! -s "$TMP_IPINFO" ] || ! gzip -t "$TMP_IPINFO" 2>/dev/null; then + logger -t "$TAG" "ERROR: invalid IPinfo Lite gzip archive" + rm -f "$TMP_IPINFO" "$TMP_COUNTRIES" + return 1 + fi + + gzip -dc "$TMP_IPINFO" \ + | grep -E "^[0-9.]+/[0-9]+,(${country_regex})," \ + | cut -d, -f1 >"$TMP_COUNTRIES" + + if [ ! -s "$TMP_COUNTRIES" ]; then + logger -t "$TAG" \ + "ERROR: no IPinfo CIDRs for countries: $DIRECT_COUNTRIES" + rm -f "$TMP_IPINFO" "$TMP_COUNTRIES" + return 1 + fi + + cat "$TMP_COUNTRIES" >>"$TMP_DIRECT" + rm -f "$TMP_IPINFO" "$TMP_COUNTRIES" + logger -t "$TAG" "OK: added IPinfo countries: $DIRECT_COUNTRIES" + return 0 } append_asn_lists() { @@ -106,6 +149,7 @@ append_asn_lists() { reload_firewall_if_needed() { [ "$1" -eq 1 ] || return 0 + [ "$RELOAD_FIREWALL" = "1" ] || return 0 if [ -x /etc/init.d/firewall ]; then /etc/init.d/firewall reload >/dev/null 2>&1 \ @@ -114,39 +158,39 @@ reload_firewall_if_needed() { } mkdir -p "$IPSETS_DIR" || exit 1 -rm -f "$TMP_DIRECT" "$TMP_SORTED" +rm -f "$TMP_DIRECT" "$TMP_SORTED" "$TMP_IPINFO" "$TMP_COUNTRIES" append_static_direct || { - rm -f "$TMP_DIRECT" "$TMP_SORTED" + rm -f "$TMP_DIRECT" "$TMP_SORTED" "$TMP_IPINFO" "$TMP_COUNTRIES" exit 1 } append_country_lists || { - rm -f "$TMP_DIRECT" "$TMP_SORTED" + rm -f "$TMP_DIRECT" "$TMP_SORTED" "$TMP_IPINFO" "$TMP_COUNTRIES" logger -t "$TAG" "ERROR: failed to build direct ipset" exit 1 } append_asn_lists || { - rm -f "$TMP_DIRECT" "$TMP_SORTED" + rm -f "$TMP_DIRECT" "$TMP_SORTED" "$TMP_IPINFO" "$TMP_COUNTRIES" logger -t "$TAG" "ERROR: failed to build direct ipset" exit 1 } if [ ! -s "$TMP_DIRECT" ]; then - rm -f "$TMP_DIRECT" "$TMP_SORTED" + rm -f "$TMP_DIRECT" "$TMP_SORTED" "$TMP_IPINFO" "$TMP_COUNTRIES" logger -t "$TAG" "ERROR: generated direct ipset is empty" exit 1 fi -sort -u "$TMP_DIRECT" >"$TMP_SORTED" || { - rm -f "$TMP_DIRECT" "$TMP_SORTED" +LC_ALL=C sort -u "$TMP_DIRECT" >"$TMP_SORTED" || { + rm -f "$TMP_DIRECT" "$TMP_SORTED" "$TMP_IPINFO" "$TMP_COUNTRIES" logger -t "$TAG" "ERROR: failed to sort direct list" exit 1 } mv -f "$TMP_SORTED" "$TMP_DIRECT" || { - rm -f "$TMP_DIRECT" "$TMP_SORTED" + rm -f "$TMP_DIRECT" "$TMP_SORTED" "$TMP_IPINFO" "$TMP_COUNTRIES" logger -t "$TAG" "ERROR: failed to replace sorted tmp direct list" exit 1 } @@ -154,13 +198,13 @@ mv -f "$TMP_SORTED" "$TMP_DIRECT" || { changed=0 if [ ! -f "$OUT_DIRECT" ] || ! cmp -s "$TMP_DIRECT" "$OUT_DIRECT"; then mv -f "$TMP_DIRECT" "$OUT_DIRECT" || { - rm -f "$TMP_DIRECT" "$TMP_SORTED" + rm -f "$TMP_DIRECT" "$TMP_SORTED" "$TMP_IPINFO" "$TMP_COUNTRIES" logger -t "$TAG" "ERROR: failed to replace $OUT_DIRECT" exit 1 } changed=1 else - rm -f "$TMP_DIRECT" "$TMP_SORTED" + rm -f "$TMP_DIRECT" "$TMP_SORTED" "$TMP_IPINFO" "$TMP_COUNTRIES" fi reload_firewall_if_needed "$changed" diff --git a/servers/example/etc/awg-server.sh b/servers/example/etc/awg-server.sh index 63e872c..89fa1b2 100755 --- a/servers/example/etc/awg-server.sh +++ b/servers/example/etc/awg-server.sh @@ -41,25 +41,28 @@ BABELD_PIDFILE="/run/${BABELD_CONF_NAME%.conf}.pid" BABELD_START_TIMEOUT="${BABELD_START_TIMEOUT:-10}" BABELD_STOP_TIMEOUT="${BABELD_STOP_TIMEOUT:-3}" -URL_GH_RAW="${URL_GH_RAW:-https://raw.githubusercontent.com}" +# Runtime path may be overridden only when staging/testing the script. IPSETS_DIR="${IPSETS_DIR:-/etc/ipsets}" -STATIC_DIRECT_NAME="${STATIC_DIRECT_NAME:-direct-static.txt}" -OUT_DIRECT_NAME="${OUT_DIRECT_NAME:-direct.txt}" + +# These defaults are tied to the country CSV parser and ipverse ASN layout. +STATIC_DIRECT_NAME="direct-static.txt" +OUT_DIRECT_NAME="direct.txt" +IPINFO_LITE_CSV_GZ_URL="https://github.com/Alice39s/ipinfo-csv-lite/""\ +releases/latest/download/ipinfo-lite.csv.gz" +URL_IPVERSE_ASN="https://raw.githubusercontent.com/ipverse/as-ip-blocks/master" STATIC_DIRECT="$IPSETS_DIR/$STATIC_DIRECT_NAME" OUT_DIRECT="$IPSETS_DIR/$OUT_DIRECT_NAME" TMP_DIRECT="${OUT_DIRECT}.tmp" TMP_SORTED="${TMP_DIRECT}.sorted" +TMP_IPINFO="/tmp/ipinfo-lite.$$.csv.gz" +TMP_COUNTRIES="/tmp/ipinfo-countries.$$.txt" -URL_IPVERSE_GEO="${URL_IPVERSE_GEO:-$URL_GH_RAW/ipverse/geo-ip-blocks/master}" -URL_IPVERSE_COUNTRY="${URL_IPVERSE_COUNTRY:-$URL_GH_RAW/ipverse/country-ip-blocks/master}" -URL_IPVERSE_ASN="${URL_IPVERSE_ASN:-$URL_GH_RAW/ipverse/as-ip-blocks/master}" - -DIRECT_COUNTRIES="${DIRECT_COUNTRIES:-}" -DIRECT_ASNS="${DIRECT_ASNS:-}" -UPDATE_IPSETS_CURL_CONNECT_TIMEOUT="${UPDATE_IPSETS_CURL_CONNECT_TIMEOUT:-10}" -UPDATE_IPSETS_CURL_MAX_TIME="${UPDATE_IPSETS_CURL_MAX_TIME:-60}" -UPDATE_IPSETS_CURL_RETRY="${UPDATE_IPSETS_CURL_RETRY:-3}" +DIRECT_COUNTRIES="${DIRECT_COUNTRIES:-ru cn by}" +DIRECT_ASNS="${DIRECT_ASNS:-32590}" +UPDATE_IPSETS_CURL_CONNECT_TIMEOUT=10 +UPDATE_IPSETS_CURL_MAX_TIME=180 +UPDATE_IPSETS_CURL_RETRY=3 die() { echo "ERROR: $*" >&2 @@ -362,23 +365,58 @@ append_url_list() { } append_country_lists() { + [ -n "$DIRECT_COUNTRIES" ] || return 0 + + country_regex="" for country in $DIRECT_COUNTRIES; do case "$country" in - [a-z][a-z]) ;; + [A-Za-z][A-Za-z]) ;; *) echo "ERROR: bad country code: $country" >&2 return 1 ;; esac - append_url_list \ - "$URL_IPVERSE_GEO/country/$country/${country}-ipv4.txt" \ - "geo-country:$country" || return 1 - - append_url_list \ - "$URL_IPVERSE_COUNTRY/country/$country/ipv4-aggregated.txt" \ - "rir-country:$country" || return 1 + # shellcheck disable=SC2018 disable=SC2019 + country="$(printf '%s' "$country" | tr 'a-z' 'A-Z')" + if [ -n "$country_regex" ]; then + country_regex="$country_regex|" + fi + country_regex="$country_regex$country" done + + rm -f "$TMP_IPINFO" "$TMP_COUNTRIES" + "$CURL_BIN" -fsSL \ + --connect-timeout "${UPDATE_IPSETS_CURL_CONNECT_TIMEOUT}" \ + --max-time "${UPDATE_IPSETS_CURL_MAX_TIME}" \ + --retry "${UPDATE_IPSETS_CURL_RETRY}" \ + -o "$TMP_IPINFO" \ + "$IPINFO_LITE_CSV_GZ_URL" || { + echo "ERROR: failed to fetch IPinfo Lite CSV" >&2 + rm -f "$TMP_IPINFO" "$TMP_COUNTRIES" + return 1 + } + + if [ ! -s "$TMP_IPINFO" ] || ! gzip -t "$TMP_IPINFO" 2>/dev/null; then + echo "ERROR: invalid IPinfo Lite gzip archive" >&2 + rm -f "$TMP_IPINFO" "$TMP_COUNTRIES" + return 1 + fi + + gzip -dc "$TMP_IPINFO" \ + | grep -E "^[0-9.]+/[0-9]+,(${country_regex})," \ + | cut -d, -f1 >"$TMP_COUNTRIES" + + if [ ! -s "$TMP_COUNTRIES" ]; then + echo "ERROR: no IPinfo CIDRs for countries: $DIRECT_COUNTRIES" >&2 + rm -f "$TMP_IPINFO" "$TMP_COUNTRIES" + return 1 + fi + + cat "$TMP_COUNTRIES" >>"$TMP_DIRECT" + rm -f "$TMP_IPINFO" "$TMP_COUNTRIES" + echo "OK: added IPinfo countries: $DIRECT_COUNTRIES" + return 0 } append_asn_lists() { @@ -399,35 +437,35 @@ append_asn_lists() { build_direct_list() { mkdir -p "$IPSETS_DIR" || die "failed to create $IPSETS_DIR" - rm -f "$TMP_DIRECT" "$TMP_SORTED" + rm -f "$TMP_DIRECT" "$TMP_SORTED" "$TMP_IPINFO" "$TMP_COUNTRIES" if ! append_static_direct; then - rm -f "$TMP_DIRECT" "$TMP_SORTED" + rm -f "$TMP_DIRECT" "$TMP_SORTED" "$TMP_IPINFO" "$TMP_COUNTRIES" exit 1 fi if ! append_country_lists; then - rm -f "$TMP_DIRECT" "$TMP_SORTED" + rm -f "$TMP_DIRECT" "$TMP_SORTED" "$TMP_IPINFO" "$TMP_COUNTRIES" exit 1 fi if ! append_asn_lists; then - rm -f "$TMP_DIRECT" "$TMP_SORTED" + rm -f "$TMP_DIRECT" "$TMP_SORTED" "$TMP_IPINFO" "$TMP_COUNTRIES" exit 1 fi if [ ! -s "$TMP_DIRECT" ]; then - rm -f "$TMP_DIRECT" "$TMP_SORTED" + rm -f "$TMP_DIRECT" "$TMP_SORTED" "$TMP_IPINFO" "$TMP_COUNTRIES" die "generated direct list is empty" fi - sort -u "$TMP_DIRECT" >"$TMP_SORTED" || { - rm -f "$TMP_DIRECT" "$TMP_SORTED" + LC_ALL=C sort -u "$TMP_DIRECT" >"$TMP_SORTED" || { + rm -f "$TMP_DIRECT" "$TMP_SORTED" "$TMP_IPINFO" "$TMP_COUNTRIES" die "failed to sort $TMP_DIRECT" } mv -f "$TMP_SORTED" "$TMP_DIRECT" || { - rm -f "$TMP_DIRECT" "$TMP_SORTED" + rm -f "$TMP_DIRECT" "$TMP_SORTED" "$TMP_IPINFO" "$TMP_COUNTRIES" die "failed to replace sorted tmp list" } diff --git a/tools/default.py b/tools/default.py index 9d88b8b..271b2c7 100755 --- a/tools/default.py +++ b/tools/default.py @@ -374,10 +374,15 @@ PACKAGE_REPO_INDEX_FILES = ( "Packages.manifest", ) +# Country source and parser are intentionally coupled to the Alice39s +# IPinfo Lite CSV release format: cidr,country_code,continent_code,asn,name. GITHUB_RAW_BASE_URL = "https://raw.githubusercontent.com" -URL_IPVERSE_GEO = f"{GITHUB_RAW_BASE_URL}/ipverse/geo-ip-blocks/master" -URL_IPVERSE_COUNTRY = f"{GITHUB_RAW_BASE_URL}/ipverse/country-ip-blocks/master" +URL_IPINFO_LITE_CSV_GZ = ( + "https://github.com/Alice39s/ipinfo-csv-lite/releases/latest/download/" + "ipinfo-lite.csv.gz" +) URL_IPVERSE_ASN = f"{GITHUB_RAW_BASE_URL}/ipverse/as-ip-blocks/master" + LOCAL_DIRECT_IPSETS = [ # Bootstrap / local / private / provider-local "0.0.0.0/8", @@ -403,7 +408,6 @@ EXIT_DIRECT_STATIC_IPSETS = [] EXIT_DIRECT_COUNTRIES = ["ru", "cn", "by"] EXIT_DIRECT_ASNS = [ 32590, # Valve Corporation / Steam - 45102, # Alibaba / AliExpress ] # ============================================================ @@ -433,9 +437,10 @@ RUNTIME_IPSETS_DIR = "/etc/ipsets" RUNTIME_DIRECT_STATIC_NAME = REL_DIRECT_STATIC_IPSET.name RUNTIME_DIRECT_OUT_NAME = REL_DIRECT_IPSET.name UPDATE_IPSETS_CURL_CONNECT_TIMEOUT = 10 -UPDATE_IPSETS_CURL_MAX_TIME = 60 +UPDATE_IPSETS_CURL_MAX_TIME = 180 UPDATE_IPSETS_CURL_RETRY = 3 + # ============================================================ # AWG PARAMETER DEFAULTS # ============================================================ diff --git a/tools/generate.py b/tools/generate.py index 1d47454..be5116f 100755 --- a/tools/generate.py +++ b/tools/generate.py @@ -127,7 +127,7 @@ def main(argv: list[str] | None = None) -> None: "--skip-direct-downloads", action="store_true", help=( - "do not download dynamic direct-list country/ASN IP sets; " + "do not run update-ipsets.sh or download dynamic country/ASN IP sets; " "generated direct.txt will contain only static direct entries" ), ) diff --git a/tools/generate_runtime.py b/tools/generate_runtime.py index 8696651..df0a817 100755 --- a/tools/generate_runtime.py +++ b/tools/generate_runtime.py @@ -2,17 +2,18 @@ import sys sys.dont_write_bytecode = True +import tempfile from pathlib import Path try: from .common import * - from .downloads import download_text_lines as curl_download_text_lines from .generate_server_runtime import shell_quote + from .process import run_checked from .tunnel_model import exit_route_env_key except ImportError: from common import * # type: ignore - from downloads import download_text_lines as curl_download_text_lines # type: ignore from generate_server_runtime import shell_quote # type: ignore + from process import run_checked # type: ignore from tunnel_model import exit_route_env_key # type: ignore @@ -35,25 +36,6 @@ def normalize_ipset_lines(lines: list[str]) -> list[str]: return out -def download_text_lines(url: str) -> list[str]: - return [ - line - for line in curl_download_text_lines(url) - if line and not line.startswith("#") - ] - - -def direct_country_urls(country: str) -> tuple[str, str]: - return ( - f"{URL_IPVERSE_GEO}/country/{country}/{country}-ipv4.txt", - f"{URL_IPVERSE_COUNTRY}/country/{country}/ipv4-aggregated.txt", - ) - - -def direct_asn_url(asn: str) -> str: - return f"{URL_IPVERSE_ASN}/as/{asn}/ipv4-aggregated.txt" - - def direct_static_lines(cfg: ConfigData) -> list[str]: lines: list[str] = [] @@ -72,17 +54,32 @@ def direct_static_lines(cfg: ConfigData) -> list[str]: return normalize_ipset_lines(lines) -def direct_dynamic_lines(cfg: ConfigData) -> list[str]: - lines: list[str] = [] +def _read_generated_ipset(path: Path, label: str) -> list[str]: + if not path.is_file(): + die(f"update-ipsets.sh did not create {label}: {path}") - for country in cfg.exit_direct.countries: - for url in direct_country_urls(country): - lines.extend(download_text_lines(url)) + lines = normalize_ipset_lines(path.read_text(encoding="utf-8").splitlines()) + if not lines: + die(f"update-ipsets.sh created empty {label}: {path}") + return lines - for asn in cfg.exit_direct.asns: - lines.extend(download_text_lines(direct_asn_url(asn))) - return normalize_ipset_lines(lines) +def _run_update_ipsets(cfg: ConfigData, ipsets_dir: Path) -> None: + script = ROUTER_EXAMPLE_DIR / "files/etc/scripts/update-ipsets.sh" + if not script.is_file(): + die(f"missing update-ipsets.sh: {script}") + + need("env", "sh", "curl", "gzip", "grep", "cut", "sed", "sort", "cmp", "tr") + + env_args = [ + "ENV_FILE=/dev/null", + f"IPSETS_DIR={ipsets_dir}", + "RELOAD_FIREWALL=0", + f"DIRECT_COUNTRIES={' '.join(cfg.exit_direct.countries)}", + f"DIRECT_ASNS={' '.join(cfg.exit_direct.asns)}", + ] + + run_checked(["env", *env_args, "sh", str(script)]) def build_direct_ipset_lines( @@ -91,8 +88,29 @@ def build_direct_ipset_lines( skip_dynamic_downloads: bool = False, ) -> tuple[list[str], list[str]]: static_lines = direct_static_lines(cfg) - dynamic_lines = [] if skip_dynamic_downloads else direct_dynamic_lines(cfg) - direct_lines = normalize_ipset_lines(static_lines + dynamic_lines) + if skip_dynamic_downloads: + return static_lines, static_lines.copy() + + with tempfile.TemporaryDirectory(prefix="owmb-ipsets-") as tmp_dir: + ipsets_dir = Path(tmp_dir) + static_path = ipsets_dir / RUNTIME_DIRECT_STATIC_NAME + direct_path = ipsets_dir / RUNTIME_DIRECT_OUT_NAME + static_path.write_text("\n".join(static_lines) + "\n", encoding="utf-8") + + _run_update_ipsets(cfg, ipsets_dir) + + generated_static = _read_generated_ipset( + static_path, RUNTIME_DIRECT_STATIC_NAME + ) + direct_lines = _read_generated_ipset(direct_path, RUNTIME_DIRECT_OUT_NAME) + + if generated_static != static_lines: + die("update-ipsets.sh unexpectedly changed direct-static.txt") + + print( + "OK: update-ipsets.sh generated " + f"{len(static_lines)} static and {len(direct_lines)} total direct entries" + ) return static_lines, direct_lines @@ -116,9 +134,6 @@ def build_runtime_env(cfg: ConfigData, router_name: str | None = None) -> str: target_names.append(hub.name) values = { - "IPSETS_DIR": RUNTIME_IPSETS_DIR, - "STATIC_DIRECT_NAME": RUNTIME_DIRECT_STATIC_NAME, - "OUT_DIRECT_NAME": RUNTIME_DIRECT_OUT_NAME, "DIRECT_COUNTRIES": " ".join(cfg.exit_direct.countries), "DIRECT_ASNS": " ".join(cfg.exit_direct.asns), "CHECK_DOH_DOMAIN": CHECK_DOH_DOMAIN, @@ -128,9 +143,6 @@ def build_runtime_env(cfg: ConfigData, router_name: str | None = None) -> str: "CHECK_DOH_PROVIDER_DOMAINS": " ".join(CHECK_DOH_PROVIDER_DOMAINS), "EXIT_ROUTE_TABLE": str(EXIT_POLICY_BASE), "EXIT_ROUTE_INTERVAL": str(EXIT_ROUTE_INTERVAL), - "UPDATE_IPSETS_CURL_CONNECT_TIMEOUT": str(UPDATE_IPSETS_CURL_CONNECT_TIMEOUT), - "UPDATE_IPSETS_CURL_MAX_TIME": str(UPDATE_IPSETS_CURL_MAX_TIME), - "UPDATE_IPSETS_CURL_RETRY": str(UPDATE_IPSETS_CURL_RETRY), "EXIT_ROUTE_TARGETS": " ".join(target_names), } diff --git a/tools/generate_server_runtime.py b/tools/generate_server_runtime.py index 8adc390..df4228a 100755 --- a/tools/generate_server_runtime.py +++ b/tools/generate_server_runtime.py @@ -65,14 +65,8 @@ def build_server_env( **({"IPIP_MTU": str(ipip_mtu_value())} if ipip_mtu_value() is not None else {}), "EXIT_SUBNETS": server_exit_subnets(cfg), "IPSET_NAME": SERVER_ENV_IPSET_NAME, - "IPSETS_DIR": RUNTIME_IPSETS_DIR, - "STATIC_DIRECT_NAME": RUNTIME_DIRECT_STATIC_NAME, - "OUT_DIRECT_NAME": RUNTIME_DIRECT_OUT_NAME, "DIRECT_COUNTRIES": " ".join(cfg.exit_direct.countries), "DIRECT_ASNS": " ".join(cfg.exit_direct.asns), - "UPDATE_IPSETS_CURL_CONNECT_TIMEOUT": str(UPDATE_IPSETS_CURL_CONNECT_TIMEOUT), - "UPDATE_IPSETS_CURL_MAX_TIME": str(UPDATE_IPSETS_CURL_MAX_TIME), - "UPDATE_IPSETS_CURL_RETRY": str(UPDATE_IPSETS_CURL_RETRY), "AWG_SERVICES": awg_services, "BABELD_CONF": server_babeld_conf_remote_path(hub.name), } diff --git a/tools/validate_runtime_files.py b/tools/validate_runtime_files.py index 739484b..6282bc6 100755 --- a/tools/validate_runtime_files.py +++ b/tools/validate_runtime_files.py @@ -337,16 +337,8 @@ def validate_server_env(cfg: ConfigData) -> None: ), "EXIT_SUBNETS": server_exit_subnets(cfg), "IPSET_NAME": SERVER_ENV_IPSET_NAME, - "IPSETS_DIR": RUNTIME_IPSETS_DIR, - "STATIC_DIRECT_NAME": RUNTIME_DIRECT_STATIC_NAME, - "OUT_DIRECT_NAME": RUNTIME_DIRECT_OUT_NAME, "DIRECT_COUNTRIES": " ".join(cfg.exit_direct.countries), "DIRECT_ASNS": " ".join(cfg.exit_direct.asns), - "UPDATE_IPSETS_CURL_CONNECT_TIMEOUT": str( - UPDATE_IPSETS_CURL_CONNECT_TIMEOUT - ), - "UPDATE_IPSETS_CURL_MAX_TIME": str(UPDATE_IPSETS_CURL_MAX_TIME), - "UPDATE_IPSETS_CURL_RETRY": str(UPDATE_IPSETS_CURL_RETRY), "AWG_SERVICES": awg_services, "BABELD_CONF": server_babeld_conf_remote_path(hub.name), } @@ -374,9 +366,6 @@ def expected_runtime_env_values( ) target_names.append(hub.name) expected = { - "IPSETS_DIR": RUNTIME_IPSETS_DIR, - "STATIC_DIRECT_NAME": RUNTIME_DIRECT_STATIC_NAME, - "OUT_DIRECT_NAME": RUNTIME_DIRECT_OUT_NAME, "DIRECT_COUNTRIES": " ".join(cfg.exit_direct.countries), "DIRECT_ASNS": " ".join(cfg.exit_direct.asns), "CHECK_DOH_DOMAIN": CHECK_DOH_DOMAIN, @@ -386,9 +375,6 @@ def expected_runtime_env_values( "CHECK_DOH_PROVIDER_DOMAINS": " ".join(CHECK_DOH_PROVIDER_DOMAINS), "EXIT_ROUTE_TABLE": str(EXIT_POLICY_BASE), "EXIT_ROUTE_INTERVAL": str(EXIT_ROUTE_INTERVAL), - "UPDATE_IPSETS_CURL_CONNECT_TIMEOUT": str(UPDATE_IPSETS_CURL_CONNECT_TIMEOUT), - "UPDATE_IPSETS_CURL_MAX_TIME": str(UPDATE_IPSETS_CURL_MAX_TIME), - "UPDATE_IPSETS_CURL_RETRY": str(UPDATE_IPSETS_CURL_RETRY), "EXIT_ROUTE_TARGETS": " ".join(target_names), }