Squeeze 537k domains: 5-byte hashes + repartition

40-bit truncated FNV-1a (5 bytes/domain, ~0 collisions to ~500k) plus a custom
partition table (drop OTA, ~2.8MB LittleFS). Combined StevenBlack + Hagezi
Ultimate = 537,013 domains in 2.56MB flash, still ~40KB RAM.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

partitions.csv

@@ -0,0 +1,6 @@
+# Single app (no OTA) + a fat LittleFS for the ~2.6 MB blocklist. 4 MB flash.
+# Name,     Type, SubType, Offset,   Size
+nvs,        data, nvs,     0x9000,   0x5000
+phy_init,   data, phy,     0xe000,   0x1000
+factory,    app,  factory, 0x10000,  0x120000
+spiffs,     data, spiffs,  0x130000, 0x2D0000

platformio.ini

@@ -3,6 +3,7 @@ platform = espressif32
 board = esp32-c3-devkitm-1
 framework = arduino
 board_build.filesystem = littlefs
+board_build.partitions = partitions.csv
 monitor_speed = 115200
 upload_port = /dev/ttyACM0
 monitor_port = /dev/ttyACM0

src/main.cpp

@@ -27,21 +27,25 @@ uint32_t numHashes = 0;
 uint32_t blockedCnt = 0, allowedCnt = 0;
 uint8_t buf[600];    // DNS packets are small (UDP, <512 typical)
 
-// FNV-1a 64-bit — MUST match tools/build_blocklist.py byte-for-byte.
+// 5-byte (40-bit) truncated FNV-1a — MUST match tools/build_blocklist.py.
+static const int HASH_BYTES = 5;
+static const uint64_t HASH_MASK = (1ULL << (HASH_BYTES * 8)) - 1;
 static uint64_t fnv1a64(const char* s, size_t n) {
   uint64_t h = 0xcbf29ce484222325ULL;
   for (size_t i = 0; i < n; i++) { h ^= (uint8_t)s[i]; h *= 0x100000001b3ULL; }
-  return h;
+  return h & HASH_MASK;               // truncate to HASH_BYTES
 }
 
-// Binary-search the flash hash array for h. ~16 cached reads, no RAM copy.
+// Binary-search the flash hash array for h. ~20 cached reads, no RAM copy.
 static bool hashInBlocklist(uint64_t h) {
   int32_t lo = 0, hi = (int32_t)numHashes - 1;
-  uint64_t v;
+  uint8_t b[HASH_BYTES];
   while (lo <= hi) {
     int32_t mid = (lo + hi) >> 1;
-    blocklist.seek((uint32_t)mid * 8);
-    blocklist.read((uint8_t*)&v, 8);   // little-endian on both host & C3
+    blocklist.seek((uint32_t)mid * HASH_BYTES);
+    blocklist.read(b, HASH_BYTES);
+    uint64_t v = 0;
+    for (int k = 0; k < HASH_BYTES; k++) v |= (uint64_t)b[k] << (8 * k);   // little-endian
     if (v < h) lo = mid + 1;
     else if (v > h) hi = mid - 1;
     else return true;
@@ -122,7 +126,7 @@ void setup() {
   if (!LittleFS.begin(true)) { Serial.println("LittleFS mount FAILED"); }
   blocklist = LittleFS.open(BLOCKLIST_PATH, "r");
   if (!blocklist) Serial.println("!! blocklist.bin missing — run `pio run -t uploadfs`");
-  else { numHashes = blocklist.size() / 8; Serial.printf("blocklist: %u domains (%u bytes)\n", numHashes, (uint32_t)blocklist.size()); }
+  else { numHashes = blocklist.size() / HASH_BYTES; Serial.printf("blocklist: %u domains (%u bytes)\n", numHashes, (uint32_t)blocklist.size()); }
 
   WiFi.mode(WIFI_STA);
   WiFi.begin(WIFI_SSID, WIFI_PASS);

tools/build_blocklist.py

@@ -1,65 +1,80 @@
 #!/usr/bin/env python3
-"""Preprocess a hosts-format blocklist into a sorted 64-bit FNV-1a hash blob
-for the ESP32-C3 ad-blocker. Hashes live in flash and are binary-searched on
-the device, so no PSRAM is needed. The same FNV-1a must run on the C3 firmware.
+"""Preprocess hosts/domain blocklists into a sorted truncated-FNV-1a hash blob
+for the ESP32-C3 ad-blocker. Hashes live in flash and are binary-searched on the
+device, so no PSRAM is needed.
 
-Usage: build_blocklist.py [hosts-file|--download] [out.bin]
+HASH_BYTES MUST match the firmware (src/main.cpp). 5 bytes (40-bit) keeps
+~0 collisions up to ~500k domains while fitting half a million in <3 MB.
+
+Usage: build_blocklist.py [out.bin] [src ...]
+  src = local file or URL. With none given, downloads the default big set
+  (StevenBlack everything + Hagezi Ultimate) ~= 500k+ domains.
 """
-import sys, struct, os, math, urllib.request
+import sys, os, math, urllib.request
 
-FNV64_OFFSET = 0xcbf29ce484222325
-FNV64_PRIME  = 0x100000001b3
-MASK = (1 << 64) - 1
+HASH_BYTES = 5                          # 40-bit hashes -- must match firmware
+MASK = (1 << (HASH_BYTES * 8)) - 1
+FNV_OFFSET = 0xcbf29ce484222325
+FNV_PRIME  = 0x100000001b3
+U64 = (1 << 64) - 1
 
-def fnv1a64(b: bytes) -> int:
-    h = FNV64_OFFSET
+DEFAULT_SOURCES = [
+    'https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/fakenews-gambling-porn-social/hosts',
+    'https://raw.githubusercontent.com/hagezi/dns-blocklists/main/domains/ultimate.txt',
+]
+
+def fnv(b: bytes) -> int:
+    h = FNV_OFFSET
     for c in b:
-        h = ((h ^ c) * FNV64_PRIME) & MASK
-    return h
+        h = ((h ^ c) * FNV_PRIME) & U64
+    return h & MASK                      # truncate to HASH_BYTES
 
-def normalize(d: str) -> str:
-    d = d.strip().lower().rstrip('.')
+def norm(d: str) -> str:
+    d = d.strip().lower().lstrip('*').lstrip('.').rstrip('.')
     return d[4:] if d.startswith('www.') else d
 
+def read_source(src: str) -> str:
+    if os.path.exists(src):
+        return open(src, errors='ignore').read()
+    print(f'  downloading {src} ...', file=sys.stderr)
+    return urllib.request.urlopen(src, timeout=180).read().decode('utf-8', 'ignore')
+
 def main():
-    src = sys.argv[1] if len(sys.argv) > 1 else '--download'
-    out = sys.argv[2] if len(sys.argv) > 2 else 'blocklist.bin'
-    if src != '--download' and os.path.exists(src):
-        data = open(src, errors='ignore').read()
-    else:
-        url = 'https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts'
-        print(f'downloading {url} ...', file=sys.stderr)
-        data = urllib.request.urlopen(url, timeout=90).read().decode('utf-8', 'ignore')
+    args = sys.argv[1:]
+    out = args[0] if args else 'blocklist.bin'
+    sources = args[1:] if len(args) > 1 else DEFAULT_SOURCES
 
     domains = set()
-    for line in data.splitlines():
-        line = line.split('#', 1)[0].strip()
-        if not line:
-            continue
-        parts = line.split()
-        dom = parts[1] if len(parts) >= 2 and parts[0] in ('0.0.0.0', '127.0.0.1', '::1', '::') \
-              else parts[0] if len(parts) == 1 else None
-        if dom:
-            dom = normalize(dom)
-            if '.' in dom and ' ' not in dom:
-                domains.add(dom)
+    for src in sources:
+        try:
+            data = read_source(src)
+        except Exception as e:
+            print(f'  !! skipped {src}: {e}', file=sys.stderr); continue
+        for line in data.splitlines():
+            line = line.split('#', 1)[0].strip()
+            if not line or line[0] in '!/':
+                continue
+            parts = line.split()
+            d = parts[1] if len(parts) >= 2 and parts[0] in ('0.0.0.0','127.0.0.1','::1','::') \
+                else parts[0] if len(parts) == 1 else None
+            if d:
+                d = norm(d)
+                if '.' in d and ' ' not in d:
+                    domains.add(d)
 
-    hashes = sorted(fnv1a64(d.encode()) for d in domains)
+    hashes = sorted(fnv(d.encode()) for d in domains)
     collisions = len(hashes) - len(set(hashes))
+    uniq = sorted(set(hashes))                       # one entry per distinct hash
     with open(out, 'wb') as f:
-        for h in hashes:
-            f.write(struct.pack('<Q', h))
+        for h in uniq:
+            f.write(h.to_bytes(HASH_BYTES, 'little'))
 
-    n, size = len(hashes), len(hashes) * 8
-    p = 0.01
-    m = int(-n * math.log(p) / (math.log(2) ** 2))
-    k = max(1, round(m / n * math.log(2)))
-    print(f'domains          : {n:,}')
-    print(f'hash collisions  : {collisions}  (each = one over-block; want 0)')
+    n, size = len(uniq), len(uniq) * HASH_BYTES
+    print(f'source domains   : {len(domains):,}')
+    print(f'hash entries     : {n:,}  ({HASH_BYTES}-byte / {HASH_BYTES*8}-bit)')
+    print(f'collisions       : {collisions}  (domains sharing a hash -> over-block)')
     print(f'flash blob       : {size:,} bytes  ({size/1024/1024:.2f} MB)  -> {out}')
-    print(f'C3 flash budget  : 4 MB total, ~2.5 MB free after firmware -> fits: {"YES" if size < 2_500_000 else "TIGHT"}')
-    print(f'lookup           : binary search, ~{math.ceil(math.log2(max(n,2)))} cached-flash reads/query')
-    print(f'optional RAM bloom (1% FP): {m//8:,} bytes (~{m/8/1024:.0f} KB), k={k}  [skips flash for ~99% of queries]')
+    print(f'lookup           : ~{math.ceil(math.log2(max(n,2)))} reads/query')
 
 if __name__ == '__main__':
     main()