From e384ae082fa1842a145229ba87fbc3f4e7655ac2 Mon Sep 17 00:00:00 2001 From: zed Date: Sat, 13 Jun 2026 15:06:43 +0300 Subject: [PATCH] Squeeze 537k domains: 5-byte hashes + repartition 40-bit truncated FNV-1a (5 bytes/domain, ~0 collisions to ~500k) plus a custom partition table (drop OTA, ~2.8MB LittleFS). Combined StevenBlack + Hagezi Ultimate = 537,013 domains in 2.56MB flash, still ~40KB RAM. Co-Authored-By: Claude Opus 4.8 (1M context) --- partitions.csv | 6 +++ platformio.ini | 1 + src/main.cpp | 18 ++++--- tools/build_blocklist.py | 105 ++++++++++++++++++++++----------------- 4 files changed, 78 insertions(+), 52 deletions(-) create mode 100644 partitions.csv diff --git a/partitions.csv b/partitions.csv new file mode 100644 index 0000000..eeb7a1b --- /dev/null +++ b/partitions.csv @@ -0,0 +1,6 @@ +# Single app (no OTA) + a fat LittleFS for the ~2.6 MB blocklist. 4 MB flash. +# Name, Type, SubType, Offset, Size +nvs, data, nvs, 0x9000, 0x5000 +phy_init, data, phy, 0xe000, 0x1000 +factory, app, factory, 0x10000, 0x120000 +spiffs, data, spiffs, 0x130000, 0x2D0000 diff --git a/platformio.ini b/platformio.ini index 61dc7da..28e0d58 100644 --- a/platformio.ini +++ b/platformio.ini @@ -3,6 +3,7 @@ platform = espressif32 board = esp32-c3-devkitm-1 framework = arduino board_build.filesystem = littlefs +board_build.partitions = partitions.csv monitor_speed = 115200 upload_port = /dev/ttyACM0 monitor_port = /dev/ttyACM0 diff --git a/src/main.cpp b/src/main.cpp index b7fdc9c..c478336 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -27,21 +27,25 @@ uint32_t numHashes = 0; uint32_t blockedCnt = 0, allowedCnt = 0; uint8_t buf[600]; // DNS packets are small (UDP, <512 typical) -// FNV-1a 64-bit — MUST match tools/build_blocklist.py byte-for-byte. +// 5-byte (40-bit) truncated FNV-1a — MUST match tools/build_blocklist.py. +static const int HASH_BYTES = 5; +static const uint64_t HASH_MASK = (1ULL << (HASH_BYTES * 8)) - 1; static uint64_t fnv1a64(const char* s, size_t n) { uint64_t h = 0xcbf29ce484222325ULL; for (size_t i = 0; i < n; i++) { h ^= (uint8_t)s[i]; h *= 0x100000001b3ULL; } - return h; + return h & HASH_MASK; // truncate to HASH_BYTES } -// Binary-search the flash hash array for h. ~16 cached reads, no RAM copy. +// Binary-search the flash hash array for h. ~20 cached reads, no RAM copy. static bool hashInBlocklist(uint64_t h) { int32_t lo = 0, hi = (int32_t)numHashes - 1; - uint64_t v; + uint8_t b[HASH_BYTES]; while (lo <= hi) { int32_t mid = (lo + hi) >> 1; - blocklist.seek((uint32_t)mid * 8); - blocklist.read((uint8_t*)&v, 8); // little-endian on both host & C3 + blocklist.seek((uint32_t)mid * HASH_BYTES); + blocklist.read(b, HASH_BYTES); + uint64_t v = 0; + for (int k = 0; k < HASH_BYTES; k++) v |= (uint64_t)b[k] << (8 * k); // little-endian if (v < h) lo = mid + 1; else if (v > h) hi = mid - 1; else return true; @@ -122,7 +126,7 @@ void setup() { if (!LittleFS.begin(true)) { Serial.println("LittleFS mount FAILED"); } blocklist = LittleFS.open(BLOCKLIST_PATH, "r"); if (!blocklist) Serial.println("!! blocklist.bin missing — run `pio run -t uploadfs`"); - else { numHashes = blocklist.size() / 8; Serial.printf("blocklist: %u domains (%u bytes)\n", numHashes, (uint32_t)blocklist.size()); } + else { numHashes = blocklist.size() / HASH_BYTES; Serial.printf("blocklist: %u domains (%u bytes)\n", numHashes, (uint32_t)blocklist.size()); } WiFi.mode(WIFI_STA); WiFi.begin(WIFI_SSID, WIFI_PASS); diff --git a/tools/build_blocklist.py b/tools/build_blocklist.py index 212e11a..b7003c3 100644 --- a/tools/build_blocklist.py +++ b/tools/build_blocklist.py @@ -1,65 +1,80 @@ #!/usr/bin/env python3 -"""Preprocess a hosts-format blocklist into a sorted 64-bit FNV-1a hash blob -for the ESP32-C3 ad-blocker. Hashes live in flash and are binary-searched on -the device, so no PSRAM is needed. The same FNV-1a must run on the C3 firmware. +"""Preprocess hosts/domain blocklists into a sorted truncated-FNV-1a hash blob +for the ESP32-C3 ad-blocker. Hashes live in flash and are binary-searched on the +device, so no PSRAM is needed. -Usage: build_blocklist.py [hosts-file|--download] [out.bin] +HASH_BYTES MUST match the firmware (src/main.cpp). 5 bytes (40-bit) keeps +~0 collisions up to ~500k domains while fitting half a million in <3 MB. + +Usage: build_blocklist.py [out.bin] [src ...] + src = local file or URL. With none given, downloads the default big set + (StevenBlack everything + Hagezi Ultimate) ~= 500k+ domains. """ -import sys, struct, os, math, urllib.request +import sys, os, math, urllib.request -FNV64_OFFSET = 0xcbf29ce484222325 -FNV64_PRIME = 0x100000001b3 -MASK = (1 << 64) - 1 +HASH_BYTES = 5 # 40-bit hashes -- must match firmware +MASK = (1 << (HASH_BYTES * 8)) - 1 +FNV_OFFSET = 0xcbf29ce484222325 +FNV_PRIME = 0x100000001b3 +U64 = (1 << 64) - 1 -def fnv1a64(b: bytes) -> int: - h = FNV64_OFFSET +DEFAULT_SOURCES = [ + 'https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/fakenews-gambling-porn-social/hosts', + 'https://raw.githubusercontent.com/hagezi/dns-blocklists/main/domains/ultimate.txt', +] + +def fnv(b: bytes) -> int: + h = FNV_OFFSET for c in b: - h = ((h ^ c) * FNV64_PRIME) & MASK - return h + h = ((h ^ c) * FNV_PRIME) & U64 + return h & MASK # truncate to HASH_BYTES -def normalize(d: str) -> str: - d = d.strip().lower().rstrip('.') +def norm(d: str) -> str: + d = d.strip().lower().lstrip('*').lstrip('.').rstrip('.') return d[4:] if d.startswith('www.') else d +def read_source(src: str) -> str: + if os.path.exists(src): + return open(src, errors='ignore').read() + print(f' downloading {src} ...', file=sys.stderr) + return urllib.request.urlopen(src, timeout=180).read().decode('utf-8', 'ignore') + def main(): - src = sys.argv[1] if len(sys.argv) > 1 else '--download' - out = sys.argv[2] if len(sys.argv) > 2 else 'blocklist.bin' - if src != '--download' and os.path.exists(src): - data = open(src, errors='ignore').read() - else: - url = 'https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts' - print(f'downloading {url} ...', file=sys.stderr) - data = urllib.request.urlopen(url, timeout=90).read().decode('utf-8', 'ignore') + args = sys.argv[1:] + out = args[0] if args else 'blocklist.bin' + sources = args[1:] if len(args) > 1 else DEFAULT_SOURCES domains = set() - for line in data.splitlines(): - line = line.split('#', 1)[0].strip() - if not line: - continue - parts = line.split() - dom = parts[1] if len(parts) >= 2 and parts[0] in ('0.0.0.0', '127.0.0.1', '::1', '::') \ - else parts[0] if len(parts) == 1 else None - if dom: - dom = normalize(dom) - if '.' in dom and ' ' not in dom: - domains.add(dom) + for src in sources: + try: + data = read_source(src) + except Exception as e: + print(f' !! skipped {src}: {e}', file=sys.stderr); continue + for line in data.splitlines(): + line = line.split('#', 1)[0].strip() + if not line or line[0] in '!/': + continue + parts = line.split() + d = parts[1] if len(parts) >= 2 and parts[0] in ('0.0.0.0','127.0.0.1','::1','::') \ + else parts[0] if len(parts) == 1 else None + if d: + d = norm(d) + if '.' in d and ' ' not in d: + domains.add(d) - hashes = sorted(fnv1a64(d.encode()) for d in domains) + hashes = sorted(fnv(d.encode()) for d in domains) collisions = len(hashes) - len(set(hashes)) + uniq = sorted(set(hashes)) # one entry per distinct hash with open(out, 'wb') as f: - for h in hashes: - f.write(struct.pack(' over-block)') print(f'flash blob : {size:,} bytes ({size/1024/1024:.2f} MB) -> {out}') - print(f'C3 flash budget : 4 MB total, ~2.5 MB free after firmware -> fits: {"YES" if size < 2_500_000 else "TIGHT"}') - print(f'lookup : binary search, ~{math.ceil(math.log2(max(n,2)))} cached-flash reads/query') - print(f'optional RAM bloom (1% FP): {m//8:,} bytes (~{m/8/1024:.0f} KB), k={k} [skips flash for ~99% of queries]') + print(f'lookup : ~{math.ceil(math.log2(max(n,2)))} reads/query') if __name__ == '__main__': main()