From a97dcb379e4b641bfd3a9206556022df055221da Mon Sep 17 00:00:00 2001 From: bluebluesoda <300733005+bluebluesoda@users.noreply.github.com> Date: Mon, 21 Sep 2026 07:53:30 +0000 Subject: [PATCH] feat(ipv6): hand a container a whole /64 out of an extra prefix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - net.ipv6_extra_prefix (prefix mode only, OFF by default and off on every upgrade) lets a container own a whole /64 in addition to its /112 primary. A /112 has 16 host bits, so nothing inside the container can use SLAAC or be delegated further; a /64 is the smallest block that can. - the /64 the host itself uses (the one holding net.ipv6_subnet) and any /64 it holds a global address in are never handed out. That is the only check: the operator may point the key at a prefix that contains the host's own /64. - blocks are stored per account as the CIDR actually given out (users.ipv6_extra_block, migration v20), handed out lowest-first, and belong to the account for life — only deleting it releases the block. An exhausted pool is not an error: the container is simply created without one. - delivery: the container's own eth0 declares `ipv6.routes = , /64` and DROPS ipv6.address. Both halves are load-bearing, found the hard way on a real /60 host: * Incus builds security.ipv6_filtering from the declared addresses and routes, so an undeclared block is dropped by the bridge — the container binds its addresses and answers the host, but everything it sources from the block dies at the veth; * Incus programs a declared route as `via `, and the kernel refuses that form once the gateway is itself covered by a gateway route (the account's own /112 route): "no route to host". Omitting the address makes the routes direct dev routes, which install fine and which Incus restores on every container start. - the guest binds ::1/64, with its real length and NOT as a local route, so the prefix is on-link inside and the customer can carve sub-prefixes out of it; one ndppd rule covers the block like the /112s. - vps add --extra64 / vps extra64 (CLI) mirror the panel entry points; EnsureExtraBlockRoutes (vps install, the boot unit, ipv6-reapply) and a power start re-apply the wiring idempotently. - tests: pool arithmetic, reserved blocks, the /63 membership trap (a /63's two /64s differ in the LAST bit of the fourth hextet), and the guest script. Co-authored-by: CommandCodeBot --- src/internal/admin/expiry_test.go | 2 +- src/internal/cfg/cfg.go | 37 +++ src/internal/cfg/registry.go | 23 ++ src/internal/db/db.go | 13 +- src/internal/db/expiry_test.go | 2 +- src/internal/db/ipv6_test.go | 14 +- src/internal/db/migrate_haproxy_test.go | 7 + src/internal/db/migrate_test.go | 16 +- src/internal/db/user.go | 67 ++++- src/internal/lx/lx.go | 17 ++ src/internal/mgr/batch.go | 8 +- src/internal/mgr/ipv6.go | 61 ++-- src/internal/mgr/ipv6_test.go | 10 +- src/internal/mgr/ipv6extra.go | 359 ++++++++++++++++++++++++ src/internal/mgr/ipv6extra_test.go | 268 ++++++++++++++++++ src/internal/mgr/ipv6pool.go | 2 +- src/internal/mgr/ipv6pool_test.go | 12 +- src/internal/mgr/mgr.go | 98 ++++++- src/internal/mgr/routed_ipv6.go | 75 ++++- src/internal/mgr/share.go | 14 +- src/internal/mgr/transfer.go | 17 +- src/internal/panel/expiry_test.go | 2 +- src/main.go | 42 ++- 23 files changed, 1067 insertions(+), 99 deletions(-) create mode 100644 src/internal/mgr/ipv6extra.go create mode 100644 src/internal/mgr/ipv6extra_test.go diff --git a/src/internal/admin/expiry_test.go b/src/internal/admin/expiry_test.go index dc3917e..7d3bce1 100644 --- a/src/internal/admin/expiry_test.go +++ b/src/internal/admin/expiry_test.go @@ -21,7 +21,7 @@ func TestAdminExpiredUserLocked(t *testing.T) { cookie := adminLogin(t, h, prefix, "correct-horse-battery") u, err := d.CreateUserFull("alice", "h", "10.42.0.2", 1, 30001, 10000, 1, 1024, 10, 0, - db.StatusReady, "", 0, "2000-01-01T00:00:00Z") + db.StatusReady, "", 0, "", "2000-01-01T00:00:00Z") if err != nil { t.Fatal(err) } diff --git a/src/internal/cfg/cfg.go b/src/internal/cfg/cfg.go index 062045e..0847f36 100644 --- a/src/internal/cfg/cfg.go +++ b/src/internal/cfg/cfg.go @@ -223,6 +223,15 @@ type NetCfg struct { // assigned in pool mode (typically the addresses the provider lets the // host bind individually). A user keeps its address until deleted. IPv6Pool []string `yaml:"ipv6_pool,omitempty"` + // IPv6ExtraPrefix is an OPTIONAL extra prefix (normally shorter than /64) + // that whole /64 blocks are carved from and routed to containers, on top of + // the /112 primary address each container already gets from IPv6Subnet. + // Empty — the default, and the state of every upgraded install — disables + // the feature completely: nothing is allocated and no UI appears. + // Deliberately unvalidated beyond "parseable global CIDR": the operator + // may hand us only a slice of what the provider gave them and asserts it is + // theirs. The /64 the host itself uses is never handed to a container. + IPv6ExtraPrefix string `yaml:"ipv6_extra_prefix,omitempty"` } // IPv6 mode values. @@ -806,6 +815,34 @@ func (c *Config) IPv6Network() (*net.IPNet, error) { return n, nil } +// IPv6ExtraPrefixNetwork parses and validates the optional extra prefix whole +// /64 blocks are carved from. Empty means "feature disabled" (nil, no error). +// The value is accepted as-is: the operator asserts the prefix is theirs, so +// there is no ownership check and no length restriction beyond a parseable +// global CIDR carrying an explicit prefix length. A prefix of /64 or longer +// simply yields zero allocatable blocks, which disables the feature silently +// rather than erroring. +func (c *Config) IPv6ExtraPrefixNetwork() (*net.IPNet, error) { + s := c.Net.IPv6ExtraPrefix + if s == "" { + return nil, nil + } + if !strings.Contains(s, "/") { + return nil, fmt.Errorf("invalid ipv6_extra_prefix %q: prefix length required (e.g. /56 or /48)", s) + } + _, n, err := net.ParseCIDR(s) + if err != nil { + return nil, fmt.Errorf("invalid ipv6_extra_prefix %q: %w", s, err) + } + if n.IP.To4() != nil { + return nil, fmt.Errorf("invalid ipv6_extra_prefix %q: not an IPv6 prefix", s) + } + if n.IP.IsPrivate() || n.IP.IsLinkLocalUnicast() || n.IP.IsLoopback() || n.IP.IsUnspecified() { + return nil, fmt.Errorf("invalid ipv6_extra_prefix %q: must be a global (public) prefix", s) + } + return n, nil +} + // IPv6PoolValidated validates and normalizes the configured address pool. // Every entry must be a single global (non-ULA, non-link-local) IPv6 address; // an explicit /128 suffix is accepted and normalized away, any other prefix diff --git a/src/internal/cfg/registry.go b/src/internal/cfg/registry.go index 2a18bda..792c198 100644 --- a/src/internal/cfg/registry.go +++ b/src/internal/cfg/registry.go @@ -374,6 +374,29 @@ var Fields = []Field{ } return nil }}, + {"net.ipv6_extra_prefix", KindOperator, ApplyNextAdd, + "optional extra prefix (prefix mode) whole /64 blocks are carved from and routed to containers; empty = feature off. Accepted as-is — the /64 this host itself uses is never handed to a container", + "2001:1c00:b1b:7f0::/60 (empty for off)", + func(c *Config) string { + if c.Net.IPv6ExtraPrefix == "" { + return "off" + } + return c.Net.IPv6ExtraPrefix + }, + func(c *Config, v string) error { + v = strings.TrimSpace(v) + if v == "" { + c.Net.IPv6ExtraPrefix = "" + return nil + } + old := c.Net.IPv6ExtraPrefix + c.Net.IPv6ExtraPrefix = v + if _, err := c.IPv6ExtraPrefixNetwork(); err != nil { + c.Net.IPv6ExtraPrefix = old + return err + } + return nil + }}, {"incus.image", KindOperator, ApplyNextAdd, "container image alias used on add/reinstall", "vpsmgr/debian-sshd", getStr(func(c *Config) string { return c.Incus.Image }), diff --git a/src/internal/db/db.go b/src/internal/db/db.go index a0a224a..2448668 100644 --- a/src/internal/db/db.go +++ b/src/internal/db/db.go @@ -58,7 +58,7 @@ func (d *DB) Close() error { return d.sql.Close() } // schemaVersion is the current schema version. Every migration in // migrations must be applied in order; Open refuses to start on a database // whose version is newer than this binary understands (downgrade protection). -const schemaVersion = 19 +const schemaVersion = 20 // migrations are applied in order, each inside its own transaction. v1 is the // original schema (baseline); later versions only add/alter, never drop. @@ -332,6 +332,17 @@ var migrations = []struct { `ALTER TABLE users ADD COLUMN ipv6_index INTEGER`, `CREATE UNIQUE INDEX IF NOT EXISTS idx_users_ipv6_index ON users(ipv6_index)`, }}, + // v20 adds the optional whole /64 block a container can be given from + // net.ipv6_extra_prefix (an extra prefix routed alongside the /112 primary + // address). NULL means "this account has none" — the state of every account + // on an install that never set the extra prefix, which is why the unique + // index has to tolerate many NULLs. The CIDR is stored as given rather than + // re-derived from the config, so changing the prefix later never invalidates + // an existing assignment. + {20, []string{ + `ALTER TABLE users ADD COLUMN ipv6_extra_block TEXT`, + `CREATE UNIQUE INDEX IF NOT EXISTS idx_users_ipv6_extra_block ON users(ipv6_extra_block)`, + }}, } // migrationSteps are the Go-side halves of a migration whose data cannot be diff --git a/src/internal/db/expiry_test.go b/src/internal/db/expiry_test.go index 0c1ce02..237d217 100644 --- a/src/internal/db/expiry_test.go +++ b/src/internal/db/expiry_test.go @@ -5,7 +5,7 @@ import "testing" func TestUserExpiryRoundTrip(t *testing.T) { d := openTestDB(t) const exp = "2030-01-02T03:04:05Z" - u, err := d.CreateUserFull("alice", "h", "10.115.0.2", 1, 30001, 10000, 1, 1024, 10, 0, StatusReady, "", 0, exp) + u, err := d.CreateUserFull("alice", "h", "10.115.0.2", 1, 30001, 10000, 1, 1024, 10, 0, StatusReady, "", 0, "", exp) if err != nil { t.Fatal(err) } diff --git a/src/internal/db/ipv6_test.go b/src/internal/db/ipv6_test.go index a5f83af..6ffe26b 100644 --- a/src/internal/db/ipv6_test.go +++ b/src/internal/db/ipv6_test.go @@ -15,7 +15,7 @@ func TestIPv6AddressLifecycle(t *testing.T) { defer d.Close() addr := "2001:db8::9c4" - u, err := d.CreateUserFull("alice", "h", "10.42.0.2", 1, 30001, 10000, 1, 1024, 10, 0, StatusReady, addr, 0, "") + u, err := d.CreateUserFull("alice", "h", "10.42.0.2", 1, 30001, 10000, 1, 1024, 10, 0, StatusReady, addr, 0, "", "") if err != nil { t.Fatal(err) } @@ -24,7 +24,7 @@ func TestIPv6AddressLifecycle(t *testing.T) { } // Same address for a second user must be refused (UNIQUE index). - if _, err := d.CreateUserFull("bob", "h", "10.42.0.3", 2, 30002, 10100, 1, 1024, 10, 0, StatusReady, addr, 0, ""); err == nil { + if _, err := d.CreateUserFull("bob", "h", "10.42.0.3", 2, 30002, 10100, 1, 1024, 10, 0, StatusReady, addr, 0, "", ""); err == nil { t.Fatal("expected UNIQUE violation for duplicate address, got nil") } @@ -47,7 +47,7 @@ func TestIPv6AddressLifecycle(t *testing.T) { } // The address is now assignable again. - if _, err := d.CreateUserFull("carol", "h", "10.42.0.4", 3, 30003, 10200, 1, 1024, 10, 0, StatusReady, addr, 0, ""); err != nil { + if _, err := d.CreateUserFull("carol", "h", "10.42.0.4", 3, 30003, 10200, 1, 1024, 10, 0, StatusReady, addr, 0, "", ""); err != nil { t.Fatalf("reassign after delete: %v", err) } } @@ -61,7 +61,7 @@ func TestIPv6AddressNullShared(t *testing.T) { } defer d.Close() for i, name := range []string{"a", "b", "c"} { - if _, err := d.CreateUserFull(name, "h", "10.42.0."+string(rune('2'+i)), i+1, 30001+i, 10000+i*100, 1, 1024, 10, 0, StatusReady, "", 0, ""); err != nil { + if _, err := d.CreateUserFull(name, "h", "10.42.0."+string(rune('2'+i)), i+1, 30001+i, 10000+i*100, 1, 1024, 10, 0, StatusReady, "", 0, "", ""); err != nil { t.Fatalf("user %s without address: %v", name, err) } } @@ -79,7 +79,7 @@ func TestIPv6IndexLifecycle(t *testing.T) { defer d.Close() index := int64(0x2bd806c9) - u, err := d.CreateUserFull("alice", "h", "10.42.0.2", 1, 30001, 10000, 1, 1024, 10, 0, StatusReady, "", index, "") + u, err := d.CreateUserFull("alice", "h", "10.42.0.2", 1, 30001, 10000, 1, 1024, 10, 0, StatusReady, "", index, "", "") if err != nil { t.Fatal(err) } @@ -95,13 +95,13 @@ func TestIPv6IndexLifecycle(t *testing.T) { } // The same index for a second user must be refused. - if _, err := d.CreateUserFull("bob", "h", "10.42.0.3", 2, 30002, 10100, 1, 1024, 10, 0, StatusReady, "", index, ""); err == nil { + if _, err := d.CreateUserFull("bob", "h", "10.42.0.3", 2, 30002, 10100, 1, 1024, 10, 0, StatusReady, "", index, "", ""); err == nil { t.Fatal("expected UNIQUE violation for a duplicate index, got nil") } // Accounts without one are stored as NULL, so many can coexist. for i, name := range []string{"carol", "dave"} { - if _, err := d.CreateUserFull(name, "h", "10.42.0."+string(rune('4'+i)), 3+i, 30003+i, 10200+i*100, 1, 1024, 10, 0, StatusReady, "", 0, ""); err != nil { + if _, err := d.CreateUserFull(name, "h", "10.42.0."+string(rune('4'+i)), 3+i, 30003+i, 10200+i*100, 1, 1024, 10, 0, StatusReady, "", 0, "", ""); err != nil { t.Fatalf("%s: %v", name, err) } } diff --git a/src/internal/db/migrate_haproxy_test.go b/src/internal/db/migrate_haproxy_test.go index ee3c0a5..98ab484 100644 --- a/src/internal/db/migrate_haproxy_test.go +++ b/src/internal/db/migrate_haproxy_test.go @@ -29,6 +29,13 @@ func rewindToV17(t *testing.T, rows map[string]string) string { if _, err := d.sql.Exec(`ALTER TABLE users DROP COLUMN ipv6_index`); err != nil { t.Fatal(err) } + // Un-apply v20, which added users.ipv6_extra_block (same ALTER caveat). + if _, err := d.sql.Exec(`DROP INDEX IF EXISTS idx_users_ipv6_extra_block`); err != nil { + t.Fatal(err) + } + if _, err := d.sql.Exec(`ALTER TABLE users DROP COLUMN ipv6_extra_block`); err != nil { + t.Fatal(err) + } if _, err := d.sql.Exec(`DELETE FROM settings WHERE key IN ('traefik','haproxy')`); err != nil { t.Fatal(err) } diff --git a/src/internal/db/migrate_test.go b/src/internal/db/migrate_test.go index 596cdf0..69b899d 100644 --- a/src/internal/db/migrate_test.go +++ b/src/internal/db/migrate_test.go @@ -187,7 +187,7 @@ func TestUserStatusRoundTrip(t *testing.T) { t.Fatal(err) } defer d.Close() - u, err := d.CreateUserFull("carol", "h", "10.115.0.4", 3, 30003, 10200, 1, 1024, 10, 0, StatusCreating, "", 0, "") + u, err := d.CreateUserFull("carol", "h", "10.115.0.4", 3, 30003, 10200, 1, 1024, 10, 0, StatusCreating, "", 0, "", "") if err != nil { t.Fatal(err) } @@ -218,6 +218,10 @@ func TestUserStatusRoundTrip(t *testing.T) { // may be left behind — no column, no index, no version row — so the next start // retries the migration from a clean v18 schema instead of finding half of it // applied (which would fail forever with "duplicate column name"). +// +// v20 is unwound along with v19 before the attempt: the applied versions must +// stay an unbroken run (the gap rule), so a schema rewound to v18 has to drop +// every later marker too. func TestMigrateStepFailureRollsBack(t *testing.T) { path := filepath.Join(t.TempDir(), "stepfail.db") d, err := Open(path) @@ -229,9 +233,11 @@ func TestMigrateStepFailureRollsBack(t *testing.T) { } // Back to the v18 state, schema included. for _, s := range []string{ - `DELETE FROM schema_migrations WHERE version = 19`, + `DELETE FROM schema_migrations WHERE version IN (19, 20)`, `DROP INDEX IF EXISTS idx_users_ipv6_index`, `ALTER TABLE users DROP COLUMN ipv6_index`, + `DROP INDEX IF EXISTS idx_users_ipv6_extra_block`, + `ALTER TABLE users DROP COLUMN ipv6_extra_block`, } { if _, err := d.sql.Exec(s); err != nil { t.Fatal(err) @@ -254,17 +260,17 @@ func TestMigrateStepFailureRollsBack(t *testing.T) { t.Fatal(err) } defer raw.Close() - for _, col := range []string{"ipv6_index"} { + for _, col := range []string{"ipv6_index", "ipv6_extra_block"} { if hasColumn(t, raw, "users", col) { t.Errorf("column %s survived the rollback", col) } } var n int - if err := raw.QueryRow(`SELECT count(*) FROM schema_migrations WHERE version = 19`).Scan(&n); err != nil { + if err := raw.QueryRow(`SELECT count(*) FROM schema_migrations WHERE version IN (19, 20)`).Scan(&n); err != nil { t.Fatal(err) } if n != 0 { - t.Errorf("v19 recorded despite the failure") + t.Errorf("v19/v20 recorded despite the failure") } if err := raw.QueryRow(`SELECT count(*) FROM users`).Scan(&n); err != nil { t.Fatal(err) diff --git a/src/internal/db/user.go b/src/internal/db/user.go index d8fac25..faaa427 100644 --- a/src/internal/db/user.go +++ b/src/internal/db/user.go @@ -35,6 +35,13 @@ type User struct { // IPv6Address is the pool-mode IPv6 /128 assigned to this user ("" when // none, e.g. prefix mode or a V4-only container). Released on delete. IPv6Address string + // IPv6ExtraBlock is the whole /64 (a CIDR string, e.g. + // "2001:1c00:b1b:7f1::/64") carved from net.ipv6_extra_prefix and routed to + // this container alongside its /112 primary address ("" = none). Stored as + // the CIDR actually handed out, so a later change of the configured prefix + // cannot invalidate it. It is released only by deleting the account — no + // quota edit can take it back. + IPv6ExtraBlock string // Color is the accent color the operator assigned to this user ("" = // default). A hex string from the admin panel's fixed palette; the user // panel tints its theme with it. @@ -67,9 +74,12 @@ func (d *DB) CreateUser(name, passHash, ip string, idx, sshPort, startPort, cpu, // default otherwise). ipv6Address is the pool-mode /128 assigned to the user // ("" for none) and ipv6Index the prefix-mode /112 block index (0 for none); // both are written in the same transaction so a reservation and the user row -// can never disagree. expiresAt is the quota deadline (RFC3339 UTC, "" = -// permanent). -func (d *DB) CreateUserFull(name, passHash, ip string, idx, sshPort, startPort, cpu, memMB, diskGB, bandwidthGB int, status, ipv6Address string, ipv6Index int64, expiresAt string) (*User, error) { +// can never disagree. ipv6ExtraBlock is the optional whole /64 handed out from +// net.ipv6_extra_prefix ("" for none) — it goes through the same transaction +// for the same reason: a block that is routed to a container but not recorded +// could be handed to a second one. expiresAt is the quota deadline (RFC3339 +// UTC, "" = permanent). +func (d *DB) CreateUserFull(name, passHash, ip string, idx, sshPort, startPort, cpu, memMB, diskGB, bandwidthGB int, status, ipv6Address string, ipv6Index int64, ipv6ExtraBlock, expiresAt string) (*User, error) { if status == "" { status = StatusReady } @@ -80,11 +90,11 @@ func (d *DB) CreateUserFull(name, passHash, ip string, idx, sshPort, startPort, defer tx.Rollback() u := &User{Name: name, PassHash: passHash, Idx: idx, IP: ip, SSHPort: sshPort, StartPort: startPort, - CPU: cpu, MemMB: memMB, DiskGB: diskGB, CreatedAt: now(), Status: status, IPv6Address: ipv6Address, IPv6Index: ipv6Index, ExpiresAt: expiresAt} + CPU: cpu, MemMB: memMB, DiskGB: diskGB, CreatedAt: now(), Status: status, IPv6Address: ipv6Address, IPv6Index: ipv6Index, IPv6ExtraBlock: ipv6ExtraBlock, ExpiresAt: expiresAt} r, err := tx.Exec( - `INSERT INTO users(name, pass_hash, idx, ip, ssh_port, start_port, cpu, mem_mb, disk_gb, created_at, status, ipv6_address, ipv6_index, expires_at) - VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?)`, - u.Name, u.PassHash, u.Idx, u.IP, u.SSHPort, u.StartPort, u.CPU, u.MemMB, u.DiskGB, u.CreatedAt, u.Status, nullIfEmpty(u.IPv6Address), nullIfZero(u.IPv6Index), u.ExpiresAt) + `INSERT INTO users(name, pass_hash, idx, ip, ssh_port, start_port, cpu, mem_mb, disk_gb, created_at, status, ipv6_address, ipv6_index, ipv6_extra_block, expires_at) + VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`, + u.Name, u.PassHash, u.Idx, u.IP, u.SSHPort, u.StartPort, u.CPU, u.MemMB, u.DiskGB, u.CreatedAt, u.Status, nullIfEmpty(u.IPv6Address), nullIfZero(u.IPv6Index), nullIfEmpty(u.IPv6ExtraBlock), u.ExpiresAt) if err != nil { return nil, err } @@ -194,31 +204,33 @@ func scanUser(row *sql.Row) (*User, error) { u := &User{} var ipv6 sql.NullString var ipv6Index sql.NullInt64 + var ipv6Extra sql.NullString err := row.Scan(&u.ID, &u.Name, &u.PassHash, &u.Idx, &u.IP, &u.SSHPort, &u.StartPort, - &u.InitScript, &u.BandwidthQuotaGB, &u.CPU, &u.MemMB, &u.DiskGB, &u.CreatedAt, &u.Status, &ipv6, &ipv6Index, &u.Color, &u.ExpiresAt) + &u.InitScript, &u.BandwidthQuotaGB, &u.CPU, &u.MemMB, &u.DiskGB, &u.CreatedAt, &u.Status, &ipv6, &ipv6Index, &u.Color, &u.ExpiresAt, &ipv6Extra) if err != nil { return nil, err } u.IPv6Address = ipv6.String u.IPv6Index = ipv6Index.Int64 + u.IPv6ExtraBlock = ipv6Extra.String return u, nil } func (d *DB) GetUserByName(name string) (*User, error) { return scanUser(d.sql.QueryRow( - `SELECT id, name, pass_hash, idx, ip, ssh_port, start_port, init_script, bandwidth_quota_gb, cpu, mem_mb, disk_gb, created_at, status, ipv6_address, ipv6_index, color, expires_at + `SELECT id, name, pass_hash, idx, ip, ssh_port, start_port, init_script, bandwidth_quota_gb, cpu, mem_mb, disk_gb, created_at, status, ipv6_address, ipv6_index, color, expires_at, ipv6_extra_block FROM users WHERE name=?`, name)) } func (d *DB) GetUserByID(id int64) (*User, error) { return scanUser(d.sql.QueryRow( - `SELECT id, name, pass_hash, idx, ip, ssh_port, start_port, init_script, bandwidth_quota_gb, cpu, mem_mb, disk_gb, created_at, status, ipv6_address, ipv6_index, color, expires_at + `SELECT id, name, pass_hash, idx, ip, ssh_port, start_port, init_script, bandwidth_quota_gb, cpu, mem_mb, disk_gb, created_at, status, ipv6_address, ipv6_index, color, expires_at, ipv6_extra_block FROM users WHERE id=?`, id)) } func (d *DB) ListUsers() ([]*User, error) { rows, err := d.sql.Query( - `SELECT id, name, pass_hash, idx, ip, ssh_port, start_port, init_script, bandwidth_quota_gb, cpu, mem_mb, disk_gb, created_at, status, ipv6_address, ipv6_index, color, expires_at + `SELECT id, name, pass_hash, idx, ip, ssh_port, start_port, init_script, bandwidth_quota_gb, cpu, mem_mb, disk_gb, created_at, status, ipv6_address, ipv6_index, color, expires_at, ipv6_extra_block FROM users ORDER BY idx`) if err != nil { return nil, err @@ -229,12 +241,14 @@ func (d *DB) ListUsers() ([]*User, error) { u := &User{} var ipv6 sql.NullString var ipv6Index sql.NullInt64 + var ipv6Extra sql.NullString if err := rows.Scan(&u.ID, &u.Name, &u.PassHash, &u.Idx, &u.IP, &u.SSHPort, &u.StartPort, - &u.InitScript, &u.BandwidthQuotaGB, &u.CPU, &u.MemMB, &u.DiskGB, &u.CreatedAt, &u.Status, &ipv6, &ipv6Index, &u.Color, &u.ExpiresAt); err != nil { + &u.InitScript, &u.BandwidthQuotaGB, &u.CPU, &u.MemMB, &u.DiskGB, &u.CreatedAt, &u.Status, &ipv6, &ipv6Index, &u.Color, &u.ExpiresAt, &ipv6Extra); err != nil { return nil, err } u.IPv6Address = ipv6.String u.IPv6Index = ipv6Index.Int64 + u.IPv6ExtraBlock = ipv6Extra.String out = append(out, u) } return out, rows.Err() @@ -325,6 +339,35 @@ func (d *DB) UsedIPv6Addresses() (map[string]bool, error) { return used, rows.Err() } +// UpdateUserIPv6ExtraBlock sets a user's whole-/64 block (carved from +// net.ipv6_extra_prefix), or clears it with "". The column carries a unique +// index, so a block can only ever belong to one user. Nothing in the panel +// calls this with "": a block is released only by deleting the account. +func (d *DB) UpdateUserIPv6ExtraBlock(id int64, block string) error { + _, err := d.sql.Exec(`UPDATE users SET ipv6_extra_block=? WHERE id=?`, nullIfEmpty(block), id) + return err +} + +// UsedIPv6ExtraBlocks returns the set of whole-/64 blocks currently assigned to +// users, so the manager can pick the lowest free one and the UI can count what +// is left. +func (d *DB) UsedIPv6ExtraBlocks() (map[string]bool, error) { + rows, err := d.sql.Query(`SELECT ipv6_extra_block FROM users WHERE ipv6_extra_block IS NOT NULL AND ipv6_extra_block != ''`) + if err != nil { + return nil, err + } + defer rows.Close() + used := map[string]bool{} + for rows.Next() { + var b string + if err := rows.Scan(&b); err != nil { + return nil, err + } + used[b] = true + } + return used, rows.Err() +} + // nullIfEmpty maps an empty string to SQL NULL (the DB stores "no address" as // NULL so the UNIQUE index treats it as not-a-value and many users can share // it — SQLite UNIQUE allows multiple NULLs). diff --git a/src/internal/lx/lx.go b/src/internal/lx/lx.go index dcc1cfd..99f3288 100644 --- a/src/internal/lx/lx.go +++ b/src/internal/lx/lx.go @@ -602,6 +602,17 @@ func (c *Client) SetDisk(name string, gb int) error { // a running container hot-removes the device, which trips an Incus netprio bug // and can leave the option unapplied, so the container is stopped first. func (c *Client) EnsureDeviceOptions(name, devName string, opts map[string]string) (bool, error) { + return c.EnsureDeviceOptionsDrop(name, devName, opts, nil) +} + +// EnsureDeviceOptionsDrop is EnsureDeviceOptions plus a list of option names to +// REMOVE from the device in the same patch. Removing a key is not cosmetic: +// Incus turns a NIC's ipv6.routes entries into `via ` routes, so +// dropping ipv6.address is what makes the routes direct (`dev `) — the +// only form that can carry the whole /64 a container may own, since a +// via-address route is unsupported once the address itself is covered by a +// gateway route (the account's own /112). +func (c *Client) EnsureDeviceOptionsDrop(name, devName string, opts map[string]string, drop []string) (bool, error) { l := c.lockDev(name) l.Lock() defer l.Unlock() @@ -614,6 +625,12 @@ func (c *Client) EnsureDeviceOptions(name, devName string, opts map[string]strin return false, fmt.Errorf("incus: instance %s has no %s device", name, devName) } changed := false + for _, k := range drop { + if _, present := dev[k]; present { + delete(dev, k) + changed = true + } + } for k, v := range opts { if dev[k] != v { dev[k] = v diff --git a/src/internal/mgr/batch.go b/src/internal/mgr/batch.go index 301d85a..790a163 100644 --- a/src/internal/mgr/batch.go +++ b/src/internal/mgr/batch.go @@ -141,7 +141,12 @@ type UserStatus struct { DownGB string BWTotal string // up+down this month, GB ("0.0" when idle) — for sorting IPv6 string - Procs int64 // latest sampled process count (0 when stopped / unavailable) + // IPv6Extra is the whole /64 the account owns from net.ipv6_extra_prefix + // ("" when it has none, and always empty in pool mode). The admin panel + // shows it in the quota dialog, where a block can be handed out once and + // can never be taken back. + IPv6Extra string + Procs int64 // latest sampled process count (0 when stopped / unavailable) } // BatchUsers reads the database-backed resource snapshot. It intentionally @@ -196,6 +201,7 @@ func (m *Manager) BatchUsers() ([]*UserStatus, error) { rs.IPv6 = ipv6 } } + rs.IPv6Extra = u.IPv6ExtraBlock out = append(out, rs) } return out, nil diff --git a/src/internal/mgr/ipv6.go b/src/internal/mgr/ipv6.go index b30a6c0..0ebc22e 100644 --- a/src/internal/mgr/ipv6.go +++ b/src/internal/mgr/ipv6.go @@ -362,16 +362,18 @@ func (m *Manager) enableProxyNDP() error { const ndppdConfPath = "/etc/vpsmgr/ndppd.conf" const ndppdConfLink = "/etc/ndppd.conf" -// ndppdConf renders /etc/ndppd.conf: one `rule ::/112` per container -// under a `proxy ` section, so the in-tree NDP responder knows which -// /112 blocks to answer for on the external link. `extraBlock` adds a /112 -// whose account row does not exist yet (Add wires the container before writing -// its row, so it passes the block it just picked) and `drop` leaves one user -// out by name (its row still exists while Del unwires the container). +// ndppdConf renders /etc/ndppd.conf: one `rule ` per block a container +// owns — its deterministic /112 and, when it has one, its whole /64 from +// net.ipv6_extra_prefix — under a `proxy ` section, so the in-tree NDP +// responder knows which prefixes to answer for on the external link. `fresh` +// adds blocks whose account row does not exist yet (Add wires the container +// before writing its row, so it passes what it just picked, comma-separated) +// and `drop` leaves one user out by name (its row still exists while Del +// unwires the container). // Empty when IPv6 is disabled or no container has a block. The format is kept // ndppd-compatible (each rule line is a bare `rule {`), even though the // daemon is no longer used in prefix mode. -func (m *Manager) ndppdConf(extraBlock, drop string) (string, error) { +func (m *Manager) ndppdConf(fresh, drop string) (string, error) { if !m.cfg.IPv6Enabled() { return "", nil } @@ -395,9 +397,15 @@ func (m *Manager) ndppdConf(extraBlock, drop string) (string, error) { if block != nil { blocks = append(blocks, block.String()) } + // The whole /64 is routed to the container the same way the /112 is, so + // an upstream that resolves the prefix by NDP has to be answered for it + // too. + if u.IPv6ExtraBlock != "" { + blocks = append(blocks, u.IPv6ExtraBlock) + } } - if extraBlock != "" { - blocks = append(blocks, extraBlock) + if fresh != "" { + blocks = append(blocks, strings.Split(fresh, ",")...) } if len(blocks) == 0 { return "", nil @@ -422,8 +430,8 @@ func (m *Manager) ndppdConf(extraBlock, drop string) (string, error) { // root-owned old file (rename checks the parent dir, not the target). When no // container has IPv6 routing the file is removed, so a stale rule can never // misroute. -func (m *Manager) writeNDPPD(extraBlock, drop string) error { - conf, err := m.ndppdConf(extraBlock, drop) +func (m *Manager) writeNDPPD(fresh, drop string) error { + conf, err := m.ndppdConf(fresh, drop) if err != nil { return err } @@ -460,12 +468,13 @@ func (m *Manager) writeNDPPD(extraBlock, drop string) error { return nil } -// WireIPv6 registers a container's /112 with the NDP proxy so its addresses -// are reachable from the internet. block is the container's /112, which Add -// passes explicitly (it wires the container before its account row exists); -// with nil the block is read from the account's stored index. The Incus device -// already routes the /112 to the container. -func (m *Manager) WireIPv6(name string, block *net.IPNet) error { +// WireIPv6 registers a container's blocks with the NDP proxy so its addresses +// are reachable from the internet: its /112, and the whole /64 it may own from +// net.ipv6_extra_prefix. Both are passed explicitly by Add, which wires the +// container before its account row exists; with nil they are read from the +// account (the stored /112 index and the stored /64). The Incus device routes +// the /112, and WireExtraBlock routes the /64. +func (m *Manager) WireIPv6(name string, block, extra *net.IPNet) error { if !m.cfg.IPv6Enabled() { return nil } @@ -476,10 +485,17 @@ func (m *Manager) WireIPv6(name string, block *net.IPNet) error { } block = b } - if block == nil { + var fresh []string + if block != nil { + fresh = append(fresh, block.String()) + } + if extra != nil { + fresh = append(fresh, extra.String()) + } + if len(fresh) == 0 { return nil } - return m.writeNDPPD(block.String(), "") + return m.writeNDPPD(strings.Join(fresh, ","), "") } // UnwireIPv6 removes a container's /112 from the NDP proxy. Returns the error @@ -535,5 +551,10 @@ func (m *Manager) RewireAllIPv6() error { return err } m.cleanLegacyKernelProxy() - return m.writeNDPPD("", "") + if err := m.writeNDPPD("", ""); err != nil { + return err + } + // The whole /64 blocks are installed by the panel, not by Incus, so nothing + // else would bring them back after a reboot. + return m.EnsureExtraBlockRoutes() } diff --git a/src/internal/mgr/ipv6_test.go b/src/internal/mgr/ipv6_test.go index 6720e7a..91b8e57 100644 --- a/src/internal/mgr/ipv6_test.go +++ b/src/internal/mgr/ipv6_test.go @@ -33,7 +33,7 @@ func ipv6TestManager(t *testing.T, subnet string, users map[string]int64) *Manag for name, index := range users { i++ if _, err := d.CreateUserFull(name, "h", fmt.Sprintf("10.42.0.%d", i+1), i, 30000+i, 10000+i*100, - 1, 1024, 10, 0, db.StatusReady, "", index, ""); err != nil { + 1, 1024, 10, 0, db.StatusReady, "", index, "", ""); err != nil { t.Fatal(err) } } @@ -208,7 +208,7 @@ func TestIPv6ContainerScript(t *testing.T) { if err != nil { t.Fatal(err) } - script, err := m.ipv6ContainerScript(addr) + script, err := m.ipv6ContainerScript(addr, "") if err != nil { t.Fatal(err) } @@ -245,7 +245,7 @@ func TestIPv6ContainerScript(t *testing.T) { func TestConfigureContainerIPv6Disabled(t *testing.T) { c := cfg.Default() // IPv6 disabled by default m := &Manager{cfg: c} - if err := m.ConfigureContainerIPv6("alice", ""); err != nil { + if err := m.ConfigureContainerIPv6("alice", "", ""); err != nil { t.Errorf("expected a no-op when IPv6 is disabled, got %v", err) } } @@ -255,7 +255,7 @@ func TestIPv6ContainerScriptNoAddress(t *testing.T) { c := cfg.Default() c.Net.IPv6Subnet = "2602:fada:6::/64" m := &Manager{cfg: c} - if s, err := m.ipv6ContainerScript(""); err != nil || s != "" { + if s, err := m.ipv6ContainerScript("", ""); err != nil || s != "" { t.Errorf(`ipv6ContainerScript("") = %q, %v; want "", nil`, s, err) } } @@ -291,7 +291,7 @@ func TestPoolContainerScript(t *testing.T) { } // An empty address is a no-op (V4-only container) — the empty check lives in // ipv6ContainerScriptFor, the pool script's only caller. - if s, _ := m.ipv6ContainerScriptFor("", ""); s != "" { + if s, _ := m.ipv6ContainerScriptFor("", "", ""); s != "" { t.Errorf("expected empty script for empty address, got %q", s) } } diff --git a/src/internal/mgr/ipv6extra.go b/src/internal/mgr/ipv6extra.go new file mode 100644 index 0000000..8f2a293 --- /dev/null +++ b/src/internal/mgr/ipv6extra.go @@ -0,0 +1,359 @@ +package mgr + +import ( + "errors" + "fmt" + "net" + "os/exec" + "sort" + "strings" + + "vpsmgr/internal/cfg" +) + +// Whole-/64 blocks from an extra prefix (net.ipv6_extra_prefix). +// +// A prefix shorter than /64 can be sliced into whole /64s, one per container. +// That is the only block size that makes the feature worth having: SLAAC needs +// exactly 64 bits, so a container owning a /64 can run its own router, hand +// addresses to nested containers/VMs and generally be a real IPv6 tenant — +// which a /112 (the normal per-container block, cut from the base /64) cannot. +// +// The feature is OFF unless the operator sets net.ipv6_extra_prefix: nothing is +// allocated, no UI appears, and every upgraded install stays exactly as it was. +// +// How a block is delivered: +// +// - The container binds ::1/64 on its eth0 (see +// ipv6ContainerScriptFor), so the whole prefix is on-link inside and any +// address in it is usable. Sub-prefixes the customer carves out internally +// are more specific than that /64 and win over it. +// - The host routes /64 to the container by declaring it on the +// container's NIC: eth0 carries `ipv6.routes = ,/64` +// and NO `ipv6.address` (see applyExtraRoutes for why both parts are +// required). Declaring it there is what makes the bridge's own source +// filter accept packets the container sources from the block, and it makes +// Incus reinstall the route on every container start, so the block survives +// reboots with no panel-side route plumbing. +// - ndppd gets a rule per block, like the /112s, so an upstream that does NDP +// for the prefix is answered. +// +// A block is stored as the CIDR actually handed out (users.ipv6_extra_block), +// not re-derived from the config, so changing the prefix later cannot invalidate +// an existing assignment. It is released only by deleting the account — no +// quota edit can take it back. + +// extraCountCap caps the reported size of an absurdly short prefix (a /24, say) +// so the panel never shows a meaningless number. The allocator itself never +// depends on it: it scans at most used+reserved+1 indices. +const extraCountCap = 1 << 32 + +// ExtraPrefixNetwork returns the configured extra prefix, or nil when the +// feature is off. The registry validates the value on write, so an error here +// means a hand-edited config. +func (m *Manager) ExtraPrefixNetwork() (*net.IPNet, error) { + return m.cfg.IPv6ExtraPrefixNetwork() +} + +// ExtraEnabled reports whether a container can be given a whole /64 right now: +// prefix mode, an extra prefix configured, and at least one free block. +func (m *Manager) ExtraEnabled() bool { + if m.cfg.IPv6ModeEffective() != cfg.IPv6ModePrefix { + return false + } + _, _, _, free, err := m.ExtraCapacity() + return err == nil && free > 0 +} + +// ExtraCapacity reports the whole-/64 pool: how many blocks the configured +// prefix holds, how many of those are reserved for this host, how many are +// assigned to containers, and how many are free. All zeros when the feature is +// off or the prefix is a /64 or longer (which fits no whole /64). +func (m *Manager) ExtraCapacity() (total, reserved, used, free int, err error) { + p, err := m.cfg.IPv6ExtraPrefixNetwork() + if err != nil || p == nil { + return 0, 0, 0, 0, err + } + // Only prefix mode hands out blocks; in pool mode the config value is inert. + if m.cfg.IPv6ModeEffective() != cfg.IPv6ModePrefix { + return 0, 0, 0, 0, nil + } + total = int(extraBlockCount(p)) + reserved = len(m.reservedExtraBlocks(p)) + usedSet, err := m.db.UsedIPv6ExtraBlocks() + if err != nil { + return 0, 0, 0, 0, err + } + // Count only the assignments that still belong to this prefix: the operator + // may have changed it, and an old block outside it must not make the pool + // look fuller — or the free count negative. + for b := range usedSet { + if _, bn, err := net.ParseCIDR(b); err == nil && p.Contains(bn.IP) { + used++ + } + } + free = total - reserved - used + if free < 0 { + free = 0 + } + return total, reserved, used, free, nil +} + +// extraBlockCount returns how many /64s fit in p, or 0 when p is a /64 or +// longer (a /65 holds no whole /64). +func extraBlockCount(p *net.IPNet) uint64 { + ones, _ := p.Mask.Size() + if ones >= 64 || ones < 0 { + return 0 + } + shift := uint(64 - ones) + if shift >= 31 { + // A prefix shorter than /33 — the pool is effectively unlimited. Cap the + // number the caller computes with and reports. + return extraCountCap + } + return uint64(1) << shift +} + +// extraBlockAt returns the idx-th /64 inside p, idx 0 being the lowest. The +// index occupies the bits between the prefix length and the /64 boundary, which +// is exactly what makes the blocks line up with the /64s of the real prefix. +func extraBlockAt(p *net.IPNet, idx uint64) string { + b := make(net.IP, 16) + copy(b, p.IP.To16()) + // p.IP is a network address, so the bits below the prefix are already zero + // and the index can simply be OR-ed into the tail of the /64's first half. + for i := 7; i >= 0 && idx > 0; i-- { + b[i] |= byte(idx & 0xff) + idx >>= 8 + } + return (&net.IPNet{IP: b, Mask: net.CIDRMask(64, 128)}).String() +} + +// reservedExtraBlocks returns the /64s inside extra that must never be handed to +// a container: +// +// - the /64 the host itself uses for pass-through — the one holding +// net.ipv6_subnet, which carries the bridge address, the gateway, every +// container's /112 and, on a provider that delegates a short prefix +// alongside it, the host's own SLAAC address. Routing it to a container +// would both steal the host's own addresses and collide with the kernel's +// existing on-link route for it, breaking the host's upstream connectivity. +// - every /64 the host currently holds a global address in, which catches the +// case the first rule misses: net.ipv6_subnet set to one /64 of the extra +// prefix while the host's own address lives in a different /64 of it. +// +// Both only ever shrink the pool. Blocks outside the extra prefix are not +// returned (they cannot be allocated anyway). +func (m *Manager) reservedExtraBlocks(extra *net.IPNet) map[string]bool { + out := map[string]bool{} + add := func(ip net.IP) { + if ip == nil || ip.To4() != nil || ip.To16() == nil { + return + } + mask := net.CIDRMask(64, 128) + out[(&net.IPNet{IP: ip.To16().Mask(mask), Mask: mask}).String()] = true + } + if n, err := m.cfg.IPv6Network(); err == nil && n != nil { + add(n.IP) + } + // `ip -6 -o addr show scope global` lines look like + // 2: ens18 inet6 2001:db8::1/64 scope global ... + // so the fourth field is the address with its prefix length. + if raw, err := exec.Command("ip", "-6", "-o", "addr", "show", "scope", "global").CombinedOutput(); err == nil { + for _, f := range strings.Fields(string(raw)) { + if ip := net.ParseIP(strings.SplitN(f, "/", 2)[0]); ip != nil { + add(ip) + } + } + } + for b := range out { + if _, bn, err := net.ParseCIDR(b); err != nil || !extra.Contains(bn.IP) { + delete(out, b) + } + } + return out +} + +// pickExtraBlock returns the lowest free whole /64 in the extra prefix, or "" +// when every block is taken or reserved (the caller then creates the container +// without one — allocation is best-effort, never an error). +// +// The scan is bounded by used+reserved+1: by pigeonhole a free block must appear +// within that many consecutive indices, so the cost is O(containers) even for a +// /48, and the pick is deterministic — the admin page can show exactly which +// blocks are in use. +func (m *Manager) pickExtraBlock() (string, error) { + if m.cfg.IPv6ModeEffective() != cfg.IPv6ModePrefix { + return "", nil // only prefix mode hands out blocks + } + p, err := m.cfg.IPv6ExtraPrefixNetwork() + if err != nil || p == nil { + return "", err + } + total := extraBlockCount(p) + if total == 0 { + return "", nil + } + used, err := m.db.UsedIPv6ExtraBlocks() + if err != nil { + return "", err + } + reserved := m.reservedExtraBlocks(p) + limit := uint64(len(used)+len(reserved)) + 1 + if limit > total { + limit = total + } + for idx := uint64(0); idx < limit; idx++ { + b := extraBlockAt(p, idx) + if used[b] || reserved[b] { + continue + } + return b, nil + } + return "", nil +} + +// ExtraEntry is one assigned whole /64: the block and the account holding it. +type ExtraEntry struct { + Block string + User string +} + +// ExtraAssignments lists every assigned block with its owner, for the admin +// page. Sorted by block, so consecutive allocations read in order. +func (m *Manager) ExtraAssignments() []ExtraEntry { + users, err := m.db.ListUsers() + if err != nil { + return nil + } + out := make([]ExtraEntry, 0, len(users)) + for _, u := range users { + if u.IPv6ExtraBlock != "" { + out = append(out, ExtraEntry{Block: u.IPv6ExtraBlock, User: u.Name}) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].Block < out[j].Block }) + return out +} + +// blockRoutes renders an eth0 ipv6.routes value: the account's /112 block, plus +// the whole /64 when it owns one. Incus takes a comma-separated list. +func blockRoutes(block112, extra string) string { + switch { + case extra == "": + return block112 + case block112 == "": + return extra + default: + return block112 + "," + extra + } +} + +// applyExtraRoutes rewrites a container's eth0 IPv6 wiring so its whole /64 is +// routed to it: ipv6.routes carries the /112 AND the block, and ipv6.address is +// REMOVED. Both halves matter, and both were found the hard way: +// +// - Incus builds the NIC's source filter (security.ipv6_filtering) from the +// declared addresses and routes, so the /64 must be declared or the bridge +// drops every packet the container sources from it. +// - Incus programs a declared route as `via `. The kernel +// refuses such a route when the gateway is itself covered by a gateway +// route — which the account's own /112 route is ("no route to host") — so a +// declared /64 can only be installed as a direct `dev` route, which is what +// omitting ipv6.address produces. Nothing is lost: the container binds its +// primary address itself, statically, from the guest script. +// +// The device update restarts the container (Incus device patches do; see +// EnsureDeviceOptionsDrop), so the caller can configure the guest right after. +// Idempotent: an unchanged device is not touched. +func (m *Manager) applyExtraRoutes(name, extra string) error { + block, err := m.IPv6Block(name) + if err != nil { + return err + } + if block == nil { + return fmt.Errorf("%s has no /112 block to route", name) + } + routes := blockRoutes(block.String(), extra) + _, err = m.lx.EnsureDeviceOptionsDrop(name, "eth0", + map[string]string{"ipv6.routes": routes}, []string{"ipv6.address"}) + return err +} + +// EnsureExtraBlockRoutes reapplies the eth0 wiring of every account that owns a +// whole /64. Containers that were created (or upgraded) with only the /112 +// declared would otherwise never get their block back: the kernel-side state is +// part of the instance config, but the declaration is a change made after the +// container existed. Called by EnsureBlockRoutes (`vps install`) and by +// RewireAllIPv6 (the boot unit and `vps ipv6-reapply`), so it also heals a +// container that was recreated out of band. +func (m *Manager) EnsureExtraBlockRoutes() error { + if m.cfg.IPv6ModeEffective() != cfg.IPv6ModePrefix { + return nil + } + users, err := m.db.ListUsers() + if err != nil { + return err + } + var firstErr error + for _, u := range users { + if u.IPv6ExtraBlock == "" { + continue + } + if err := m.applyExtraRoutes(u.Name, u.IPv6ExtraBlock); err != nil && firstErr == nil { + firstErr = err + } + } + return firstErr +} + +// AssignExtraBlock gives an EXISTING container a whole /64 — the admin's +// "quota → assign" path. Idempotent: a container that already has one keeps it +// and no error is raised, so re-submitting the dialog is harmless. +// +// It redeclares the container's NIC (which restarts it: Incus device patches +// do) and then binds the block inside the guest. A guest that refuses the new +// configuration is reported as a warning — the block stays assigned and routed, +// mirroring how a reinstall treats a broken guest. +func (m *Manager) AssignExtraBlock(name string) (string, error) { + m.opMu.Lock() + defer m.opMu.Unlock() + + if m.cfg.IPv6ModeEffective() != cfg.IPv6ModePrefix { + return "", errors.New("this host does not hand out whole /64 blocks (IPv6 prefix mode only)") + } + u, err := m.db.GetUserByName(name) + if err != nil { + return "", err + } + if u.IPv6ExtraBlock != "" { + return u.IPv6ExtraBlock, nil // already assigned + } + block, err := m.pickExtraBlock() + if err != nil { + return "", err + } + if block == "" { + return "", errors.New("no free /64 left in the configured extra prefix") + } + if err := m.db.UpdateUserIPv6ExtraBlock(u.ID, block); err != nil { + return "", err + } + u.IPv6ExtraBlock = block + // Re-declare the NIC so the /64 is routed to the container and allowed by + // the bridge's source filter. This restarts the container. + if err := m.applyExtraRoutes(u.Name, block); err != nil { + // Hand the block back rather than leave it recorded against a container + // that cannot use it. + if rerr := m.db.UpdateUserIPv6ExtraBlock(u.ID, ""); rerr != nil { + fmt.Printf(" ! warn: could not undo the /64 assignment for %s: %v\n", u.Name, rerr) + } + return "", err + } + if err := m.ConfigureContainerIPv6(u.Name, "", block); err != nil { + fmt.Printf(" ! warn: %s was given %s but the container rejected its IPv6 config: %v\n", u.Name, block, err) + } + _ = m.writeNDPPD("", "") + return block, nil +} diff --git a/src/internal/mgr/ipv6extra_test.go b/src/internal/mgr/ipv6extra_test.go new file mode 100644 index 0000000..897790f --- /dev/null +++ b/src/internal/mgr/ipv6extra_test.go @@ -0,0 +1,268 @@ +package mgr + +import ( + "net" + "path/filepath" + "strings" + "testing" + + "vpsmgr/internal/cfg" + "vpsmgr/internal/db" +) + +// extraTestManager is a prefix-mode manager with the standard base /64 and the +// given extra prefix, holding no accounts yet. +func extraTestManager(t *testing.T, extra string) *Manager { + t.Helper() + c := cfg.Default() + c.Net.IPv6Subnet = "2602:fada:6::/64" + c.Net.IPv6ExtraPrefix = extra + d, err := db.Open(filepath.Join(t.TempDir(), "extra.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { d.Close() }) + return &Manager{cfg: c, db: d} +} + +// addExtraUser creates an account carrying the given whole /64. +func addExtraUser(t *testing.T, m *Manager, name, block string, idx int) *db.User { + t.Helper() + u, err := m.db.CreateUserFull(name, "h", "10.42.0."+string(rune('2'+idx)), idx, 30000+idx, 10000+idx*100, + 1, 1024, 10, 0, db.StatusReady, "", int64(idx), block, "") + if err != nil { + t.Fatal(err) + } + return u +} + +// A /60 holds 16 /64s, addressed by an index in the bits between the prefix +// length and the /64 boundary — the mapping every allocation depends on. +func TestExtraBlockLayout(t *testing.T) { + m := extraTestManager(t, "2602:fada:6::/60") + p, err := m.ExtraPrefixNetwork() + if err != nil || p == nil { + t.Fatalf("extra prefix: %v %v", p, err) + } + if got := extraBlockCount(p); got != 16 { + t.Errorf("extraBlockCount(/60) = %d, want 16", got) + } + for idx, want := range map[uint64]string{ + 0: "2602:fada:6::/64", + 1: "2602:fada:6:1::/64", + 15: "2602:fada:6:f::/64", + } { + if got := extraBlockAt(p, idx); got != want { + t.Errorf("extraBlockAt(/60, %d) = %s, want %s", idx, got, want) + } + } + + // The size follows the prefix length; a /64 or longer fits none. + short, _ := (&cfg.Config{Net: cfg.NetCfg{IPv6ExtraPrefix: "2602:fada:6::/56"}}).IPv6ExtraPrefixNetwork() + if got := extraBlockCount(short); got != 256 { + t.Errorf("extraBlockCount(/56) = %d, want 256", got) + } + for _, s := range []string{"2602:fada:6::/64", "2602:fada:6::/80"} { + q, _ := (&cfg.Config{Net: cfg.NetCfg{IPv6ExtraPrefix: s}}).IPv6ExtraPrefixNetwork() + if got := extraBlockCount(q); got != 0 { + t.Errorf("extraBlockCount(%s) = %d, want 0", s, got) + } + } +} + +// Membership and the block walk have to agree for prefixes that are not a whole +// number of hextets, where it is easy to be off by one bit: a /63 holds exactly +// the two /64s that differ in the LAST bit of the fourth hextet, not in the +// third — the mistake that would make the pool look half-used. +func TestExtraBlockMembershipNonByteAligned(t *testing.T) { + m := extraTestManager(t, "2001:db8:5678::/63") + p, err := m.ExtraPrefixNetwork() + if err != nil { + t.Fatal(err) + } + if got := extraBlockAt(p, 1); got != "2001:db8:5678:1::/64" { + t.Errorf("second /64 of a /63 = %s, want 2001:db8:5678:1::/64", got) + } + for _, c := range []struct { + ip string + want bool + }{ + {"2001:db8:5678::", true}, + {"2001:db8:5678:1::", true}, + {"2001:db8:5678:2::", false}, + {"2001:db8:5679::", false}, // a different /63 entirely + {"2602:fada:6::", false}, + } { + if _, bn, err := net.ParseCIDR(c.ip + "/64"); err != nil { + t.Fatal(err) + } else if got := p.Contains(bn.IP); got != c.want { + t.Errorf("contains %s = %v, want %v", c.ip, got, c.want) + } + } +} + +// The /64 this host uses must never be handed out. When the extra prefix +// contains it, it is reserved; when the prefix is somewhere else entirely, +// nothing is. +func TestExtraReservesTheHostsOwnBlock(t *testing.T) { + m := extraTestManager(t, "2602:fada:6::/60") + p, _ := m.ExtraPrefixNetwork() + if res := m.reservedExtraBlocks(p); !res["2602:fada:6::/64"] { + t.Errorf("the host's own /64 is not reserved: %v", res) + } + + // A prefix that does not contain the base /64 reserves nothing of it — the + // pool stays whole. + m2 := extraTestManager(t, "2001:db8:1::/60") + p2, _ := m2.ExtraPrefixNetwork() + if res := m2.reservedExtraBlocks(p2); len(res) != 0 { + t.Errorf("unrelated prefix reserved %v, want none", res) + } + total, reserved, _, free, err := m2.ExtraCapacity() + if err != nil || total != 16 || reserved != 0 || free != 16 { + t.Errorf("capacity = %d/%d/%d (err %v), want 16 total, 0 reserved, 16 free", total, reserved, free, err) + } +} + +// Allocation walks the prefix from the bottom, skipping the host's reserved +// /64, and reports exhaustion with "" (the caller then creates a container +// without a block rather than failing). +func TestPickExtraBlockSkipsReservedAndExhausts(t *testing.T) { + // A /63 holds exactly two /64s; the first is the host's own, so exactly one + // block is allocatable. + m := extraTestManager(t, "2602:fada:6::/63") + if got, _ := m.pickExtraBlock(); got != "2602:fada:6:1::/64" { + t.Fatalf("pickExtraBlock = %q, want the second /64 (the first is the host's)", got) + } + addExtraUser(t, m, "alice", "2602:fada:6:1::/64", 1) + if got, err := m.pickExtraBlock(); got != "" || err != nil { + t.Errorf("pickExtraBlock after exhaustion = %q, %v; want \"\"", got, err) + } + + // Sequential by construction: a /56 hands out the lowest free block each + // time, so the admin page can read the used set off the prefix in order. + m2 := extraTestManager(t, "2001:db8:2::/56") + for i, want := range []string{"2001:db8:2::/64", "2001:db8:2:1::/64", "2001:db8:2:2::/64"} { + got, err := m2.pickExtraBlock() + if err != nil { + t.Fatal(err) + } + if got != want { + t.Fatalf("pick #%d = %q, want %q", i, got, want) + } + addExtraUser(t, m2, "u"+string(rune('a'+i)), got, i+1) + } +} + +// Capacity is what the panel shows and the checkbox greys out on. +func TestExtraCapacityCounts(t *testing.T) { + m := extraTestManager(t, "2602:fada:6::/60") + total, reserved, used, free, err := m.ExtraCapacity() + if err != nil { + t.Fatal(err) + } + if total != 16 || reserved < 1 || used != 0 || free != total-reserved { + t.Errorf("capacity = total %d reserved %d used %d free %d", total, reserved, used, free) + } + addExtraUser(t, m, "alice", "2602:fada:6:1::/64", 1) + if _, _, used, free2, _ := m.ExtraCapacity(); used != 1 || free2 != free-1 { + t.Errorf("after one assignment: used %d free %d, want used 1 free %d", used, free2, free-1) + } + + // Assignments that no longer belong to the configured prefix do not count + // (the operator may have changed it), so the numbers stay consistent. + m2 := extraTestManager(t, "2001:db8:3::/60") + addExtraUser(t, m2, "alice", "2602:ffff::/64", 1) + if _, _, used, _, _ := m2.ExtraCapacity(); used != 0 { + t.Errorf("foreign block counted as used: %d", used) + } +} + +// The feature is off without a prefix, and never runs in pool mode. +func TestExtraDisabledByDefault(t *testing.T) { + m := extraTestManager(t, "") + if m.ExtraEnabled() { + t.Error("ExtraEnabled with no configured prefix") + } + if b, err := m.pickExtraBlock(); b != "" || err != nil { + t.Errorf("pickExtraBlock with no prefix = %q, %v", b, err) + } + if total, _, _, _, _ := m.ExtraCapacity(); total != 0 { + t.Errorf("capacity with no prefix = %d, want 0", total) + } + + c := cfg.Default() + c.Net.IPv6Subnet = "2602:fada:6::/64" + c.Net.IPv6Mode = cfg.IPv6ModePool + c.Net.IPv6ExtraPrefix = "2602:fada:6::/60" + d, err := db.Open(filepath.Join(t.TempDir(), "pool.db")) + if err != nil { + t.Fatal(err) + } + defer d.Close() + mp := &Manager{cfg: c, db: d} + if mp.ExtraEnabled() { + t.Error("ExtraEnabled in pool mode") + } + if b, _ := mp.pickExtraBlock(); b != "" { + t.Errorf("pool mode handed out %q", b) + } +} + +// The guest must bind the whole /64 with its real length (so the prefix is +// on-link inside and the customer can carve sub-prefixes out of it), on both +// guest stacks, and must not let the parent-prefix flush delete it. +func TestIPv6ContainerScriptWithExtraBlock(t *testing.T) { + m := ipv6TestManager(t, "2602:fada:6::/64", map[string]int64{"alice": legacyAlice}) + addr, err := m.IPv6Addr("alice") + if err != nil { + t.Fatal(err) + } + script, err := m.ipv6ContainerScript(addr, "2602:fada:9:1::/64") + if err != nil { + t.Fatal(err) + } + for _, want := range []string{ + "Address=2602:fada:9:1::1/64", // networkd: the /64 address + "ipv6.addresses 2602:fada:6::2bd8:6c9:1/128,2602:fada:9:1::1/64", // RHEL (nmcli) + `[ "$r" = 2602:fada:9:1::/64 ] && continue`, // flush guard + } { + if !strings.Contains(script, want) { + t.Errorf("script missing %q:\n%s", want, script) + } + } + // A local route would claim every address in the block for the container + // itself and make internal re-delegation impossible — the opposite of what + // the block is for. + if strings.Contains(script, "local 2602:fada:9:1::/64") { + t.Errorf("the /64 must not be made a local route:\n%s", script) + } + + // Without a block nothing of the above leaks into the script, and the + // parent-prefix flush keeps its original shape (no skip guard). + plain, err := m.ipv6ContainerScript(addr, "") + if err != nil { + t.Fatal(err) + } + for _, unwanted := range []string{"EXTRAADDR", "2602:fada:9:1", `[ "$r" = `} { + if strings.Contains(plain, unwanted) { + t.Errorf("script without a block contains %q:\n%s", unwanted, plain) + } + } +} + +// Assignments are listed with their owner, in block order. +func TestExtraAssignments(t *testing.T) { + m := extraTestManager(t, "2001:db8:4::/60") + addExtraUser(t, m, "bob", "2001:db8:4:3::/64", 1) + addExtraUser(t, m, "alice", "2001:db8:4:1::/64", 2) + addExtraUser(t, m, "carol", "", 3) // no block: not listed + got := m.ExtraAssignments() + if len(got) != 2 { + t.Fatalf("ExtraAssignments = %v, want 2 entries", got) + } + if got[0].Block != "2001:db8:4:1::/64" || got[0].User != "alice" || + got[1].Block != "2001:db8:4:3::/64" || got[1].User != "bob" { + t.Errorf("ExtraAssignments = %v, want alice's then bob's", got) + } +} diff --git a/src/internal/mgr/ipv6pool.go b/src/internal/mgr/ipv6pool.go index be1522f..9341113 100644 --- a/src/internal/mgr/ipv6pool.go +++ b/src/internal/mgr/ipv6pool.go @@ -325,7 +325,7 @@ func (m *Manager) AssignPoolIPv6(name, addr string) (string, error) { if err := m.lx.WaitReady(u.Name, 180*time.Second); err != nil { return restore(fmt.Errorf("wait for container: %w", err)) } - if err := m.ConfigureContainerIPv6(u.Name, picked); err != nil { + if err := m.ConfigureContainerIPv6(u.Name, picked, ""); err != nil { return restore(fmt.Errorf("configure ipv6 in container: %w", err)) } if err := m.WireIPv6Pool(u.Name, picked); err != nil { diff --git a/src/internal/mgr/ipv6pool_test.go b/src/internal/mgr/ipv6pool_test.go index 205c918..f0a12c3 100644 --- a/src/internal/mgr/ipv6pool_test.go +++ b/src/internal/mgr/ipv6pool_test.go @@ -43,7 +43,7 @@ func TestPickPoolIPv6Auto(t *testing.T) { func TestPickPoolIPv6ExplicitAndUsed(t *testing.T) { m, d := poolTestManager(t, []string{"2001:db8::1", "2001:db8::2"}) // Reserve ::2 by creating a user with it. - if _, err := d.CreateUserFull("alice", "h", "10.42.0.2", 1, 30001, 10000, 1, 1024, 10, 0, db.StatusReady, "2001:db8::2", 0, ""); err != nil { + if _, err := d.CreateUserFull("alice", "h", "10.42.0.2", 1, 30001, 10000, 1, 1024, 10, 0, db.StatusReady, "2001:db8::2", 0, "", ""); err != nil { t.Fatal(err) } // Explicitly picking a used address fails. @@ -66,7 +66,7 @@ func TestPickPoolIPv6ExplicitAndUsed(t *testing.T) { func TestPickPoolIPv6Exhausted(t *testing.T) { m, d := poolTestManager(t, []string{"2001:db8::1"}) - if _, err := d.CreateUserFull("alice", "h", "10.42.0.2", 1, 30001, 10000, 1, 1024, 10, 0, db.StatusReady, "2001:db8::1", 0, ""); err != nil { + if _, err := d.CreateUserFull("alice", "h", "10.42.0.2", 1, 30001, 10000, 1, 1024, 10, 0, db.StatusReady, "2001:db8::1", 0, "", ""); err != nil { t.Fatal(err) } got, err := m.pickPoolIPv6("") @@ -80,7 +80,7 @@ func TestPickPoolIPv6Exhausted(t *testing.T) { func TestFreePoolIPv6List(t *testing.T) { m, d := poolTestManager(t, []string{"2001:db8::1", "2001:db8::2", "2001:db8::3"}) - if _, err := d.CreateUserFull("alice", "h", "10.42.0.2", 1, 30001, 10000, 1, 1024, 10, 0, db.StatusReady, "2001:db8::2", 0, ""); err != nil { + if _, err := d.CreateUserFull("alice", "h", "10.42.0.2", 1, 30001, 10000, 1, 1024, 10, 0, db.StatusReady, "2001:db8::2", 0, "", ""); err != nil { t.Fatal(err) } free := m.FreePoolIPv6List() @@ -101,7 +101,7 @@ func TestIPv6PoolUsage(t *testing.T) { if total != 2 || used != 0 { t.Errorf("empty pool usage = %d/%d, want 2/0", used, total) } - if _, err := d.CreateUserFull("alice", "h", "10.42.0.2", 1, 30001, 10000, 1, 1024, 10, 0, db.StatusReady, "2001:db8::1", 0, ""); err != nil { + if _, err := d.CreateUserFull("alice", "h", "10.42.0.2", 1, 30001, 10000, 1, 1024, 10, 0, db.StatusReady, "2001:db8::1", 0, "", ""); err != nil { t.Fatal(err) } total, used, _ = m.IPv6PoolUsage() @@ -141,7 +141,7 @@ func TestAddPoolIPv6s(t *testing.T) { func TestRemovePoolIPv6(t *testing.T) { m, d := poolTestManager(t, []string{"2001:db8::1", "2001:db8::2"}) // Assigned address cannot be removed. - if _, err := d.CreateUserFull("alice", "h", "10.42.0.2", 1, 30001, 10000, 1, 1024, 10, 0, db.StatusReady, "2001:db8::1", 0, ""); err != nil { + if _, err := d.CreateUserFull("alice", "h", "10.42.0.2", 1, 30001, 10000, 1, 1024, 10, 0, db.StatusReady, "2001:db8::1", 0, "", ""); err != nil { t.Fatal(err) } if err := m.RemovePoolIPv6("2001:db8::1"); err == nil { @@ -163,7 +163,7 @@ func TestRemovePoolIPv6(t *testing.T) { func TestPoolList(t *testing.T) { m, d := poolTestManager(t, []string{"2001:db8::1", "2001:db8::2"}) - if _, err := d.CreateUserFull("alice", "h", "10.42.0.2", 1, 30001, 10000, 1, 1024, 10, 0, db.StatusReady, "2001:db8::1", 0, ""); err != nil { + if _, err := d.CreateUserFull("alice", "h", "10.42.0.2", 1, 30001, 10000, 1, 1024, 10, 0, db.StatusReady, "2001:db8::1", 0, "", ""); err != nil { t.Fatal(err) } entries, err := m.PoolList() diff --git a/src/internal/mgr/mgr.go b/src/internal/mgr/mgr.go index b22d95e..8b83eaa 100644 --- a/src/internal/mgr/mgr.go +++ b/src/internal/mgr/mgr.go @@ -350,6 +350,11 @@ type AddOptions struct { // Ignored unless the config is in pool mode (prefix mode always assigns // the address built from the account's stored block index). IPv6Addr string + // AllocateExtra64 asks for a whole /64 out of net.ipv6_extra_prefix (prefix + // mode only). Best-effort by design: with no extra prefix configured, or + // with every block taken, the container is created without one rather than + // failing. + AllocateExtra64 bool // Days is the initial quota validity in days (0 or negative = permanent). Days int // FromShare is a snapshot share code. When set, the container is cloned @@ -450,10 +455,18 @@ func (m *Manager) Add(name string, opt AddOptions) (*Result, error) { // pool is exhausted / the caller opted out with "none" // - none: no IPv6 at all poolAddr := "" - ipv6 := "" // eth0 ipv6.address handed to Incus (prefix mode only) + // ipv6 is the container's PRIMARY address (bound inside the guest, and what + // the user connects to). The Incus device never carries it: eth0 is given + // routes only, because a declared ipv6.address turns every route into a + // via-address one, and such a route cannot be added once the address is + // itself covered by a gateway route (the account's /112). See + // applyExtraRoutes. + ipv6 := "" var ipv6Index int64 // stored /112 block index (prefix mode only) blockStr := "" var block *net.IPNet // the block those produce (prefix mode only) + extraBlock := "" // optional whole /64 out of net.ipv6_extra_prefix + var extraNet *net.IPNet switch m.cfg.IPv6ModeEffective() { case cfg.IPv6ModePool: if opt.IPv6Addr == "none" { @@ -486,6 +499,23 @@ func (m *Manager) Add(name string, opt AddOptions) (*Result, error) { blockStr = block.String() ipv6 = addHostOffset(block.IP, 1).String() } + // Optional whole /64 (net.ipv6_extra_prefix, off unless the operator + // configured one). Never fatal: an exhausted pool just means no block. + if opt.AllocateExtra64 && m.ExtraEnabled() { + picked64, err := m.pickExtraBlock() + if err != nil { + return nil, err + } + if picked64 != "" { + _, n, err := net.ParseCIDR(picked64) + if err != nil { + return nil, fmt.Errorf("parse extra block %s: %w", picked64, err) + } + extraBlock, extraNet = n.String(), n + } + } + // The routes the NIC declares: the /112, and the /64 when it got one. + blockStr = blockRoutes(blockStr, extraBlock) } // Defend against orphan containers: a crashed create (or an out-of-band // `incus` instance) could already hold this name or the IP NextFreeIdx just @@ -500,7 +530,7 @@ func (m *Manager) Add(name string, opt AddOptions) (*Result, error) { cloned := cloneOwner != "" if cloned { if err := m.lx.CloneFromSnapshot(cloneOwner, cloneSnap, name, - m.cfg.Incus.Pool, m.cfg.Incus.Bridge, ip, ipv6, blockStr, poolAddr, + m.cfg.Incus.Pool, m.cfg.Incus.Bridge, ip, "", blockStr, poolAddr, m.cfg.Net.ExtIF, opt.CPU, opt.MemMB, opt.DiskGB); err != nil { return nil, fmt.Errorf("clone shared checkpoint: %w", err) } @@ -514,7 +544,7 @@ func (m *Manager) Add(name string, opt AddOptions) (*Result, error) { if err := m.lx.EnsureImage(image); err != nil { return nil, fmt.Errorf("ensure image %s: %w", image, err) } - if err := m.lx.Launch(m.cfg.Incus.Pool, m.cfg.Incus.Bridge, name, image, ip, ipv6, blockStr, poolAddr, m.cfg.Net.ExtIF, opt.CPU, opt.MemMB, opt.DiskGB); err != nil { + if err := m.lx.Launch(m.cfg.Incus.Pool, m.cfg.Incus.Bridge, name, image, ip, "", blockStr, poolAddr, m.cfg.Net.ExtIF, opt.CPU, opt.MemMB, opt.DiskGB); err != nil { return nil, fmt.Errorf("launch container: %w", err) } } @@ -584,7 +614,7 @@ func (m *Manager) Add(name string, opt AddOptions) (*Result, error) { // host-routed peer IPv6 container script. No-op when IPv6 is disabled. if m.cfg.IPv6ModeEffective() == cfg.IPv6ModePool { if poolAddr != "" { - if err := m.ConfigureContainerIPv6(name, poolAddr); err != nil { + if err := m.ConfigureContainerIPv6(name, poolAddr, ""); err != nil { cleanup() return nil, fmt.Errorf("config container ipv6: %w", err) } @@ -598,16 +628,16 @@ func (m *Manager) Add(name string, opt AddOptions) (*Result, error) { // bring the guest's /128 + local /112 up FIRST, then publish the NDP // rule. Publishing last means an outside client's first SYN never // races the guest still half-configured during `vps add`. - if err := m.ConfigureContainerIPv6(name, ipv6); err != nil { + if err := m.ConfigureContainerIPv6(name, ipv6, extraBlock); err != nil { cleanup() return nil, fmt.Errorf("config container ipv6: %w", err) } - if err := m.WireIPv6(name, block); err != nil { + if err := m.WireIPv6(name, block, extraNet); err != nil { cleanup() return nil, fmt.Errorf("wire ipv6: %w", err) } } - u, err := m.db.CreateUserFull(name, hash, ip, idx, sshPort, startPort, opt.CPU, opt.MemMB, opt.DiskGB, opt.BandwidthGB, db.StatusCreating, poolAddr, ipv6Index, ExpiryFromDays(opt.Days)) + u, err := m.db.CreateUserFull(name, hash, ip, idx, sshPort, startPort, opt.CPU, opt.MemMB, opt.DiskGB, opt.BandwidthGB, db.StatusCreating, poolAddr, ipv6Index, extraBlock, ExpiryFromDays(opt.Days)) if err != nil { cleanup() return nil, fmt.Errorf("db: %w", err) @@ -672,6 +702,7 @@ type Result struct { DownGB string IPv6 string // primary global address (the one to connect to) IPv6Block string // the /112 block the container owns (informational) + IPv6Extra string // whole /64 handed out from the extra prefix ("" = none) V4Forward bool // whether IPv4 inbound (ssh/ports/domains) is live } @@ -702,6 +733,7 @@ func (m *Manager) resultForState(u *db.User, pass, state string) *Result { return &Result{User: u, Password: pass, PublicIP: m.cfg.DisplayIP(), State: state, Domains: ds, PortsPerUser: cfg.PortsPerUser, UpGB: FormatGB(up), DownGB: FormatGB(down), IPv6: ipv6, IPv6Block: block, + IPv6Extra: u.IPv6ExtraBlock, V4Forward: m.cfg.Net.V4Forward} } @@ -1077,7 +1109,11 @@ func (m *Manager) Power(name, action string) error { if err := m.lx.SetAutostart(u.Name, true); err != nil { return err } - return m.lx.Start(u.Name) + if err := m.lx.Start(u.Name); err != nil { + return err + } + m.reapplyExtraBlockGuest(u) + return nil case "stop": if err := m.lx.SetAutostart(u.Name, false); err != nil { return err @@ -1087,11 +1123,32 @@ func (m *Manager) Power(name, action string) error { if err := m.lx.SetAutostart(u.Name, true); err != nil { return err } - return m.lx.Restart(u.Name) + if err := m.lx.Restart(u.Name); err != nil { + return err + } + m.reapplyExtraBlockGuest(u) + return nil } return errors.New("unknown action") } +// reapplyExtraBlockGuest rebinds a container's whole /64 inside the guest after +// it (re)starts. The host-side route is part of the container's own network +// configuration, so Incus brings it back by itself; the address inside the guest +// is not — it is written by the provider script — so a container that was +// stopped when its block was assigned (or that came back from a host reboot +// without the boot pass reaching it) needs this. Best-effort: a guest that +// rejects its configuration must not turn a successful power operation into an +// error, and the boot pass (`vps ipv6-reapply`) heals it later. +func (m *Manager) reapplyExtraBlockGuest(u *db.User) { + if u == nil || u.IPv6ExtraBlock == "" { + return + } + if err := m.ConfigureContainerIPv6(u.Name, "", ""); err != nil { + fmt.Printf(" ! warn: %s: rebind %s in the guest: %v\n", u.Name, u.IPv6ExtraBlock, err) + } +} + // validSnapName restricts snapshot names to a safe charset so user-supplied // names can never reach the Incus API path with separators, and to Incus's // 63-char name limit. @@ -1451,6 +1508,7 @@ func (m *Manager) Reinstall(name, image string) (string, error) { ipv6 := "" blockStr := "" var block *net.IPNet + var extraNet *net.IPNet if m.cfg.IPv6ModeEffective() == cfg.IPv6ModePool { // NOT set on the Incus eth0 device (rejected: outside bridge subnet). } else { @@ -1459,8 +1517,16 @@ func (m *Manager) Reinstall(name, image string) (string, error) { if block != nil { blockStr = block.String() } + if u.IPv6ExtraBlock != "" { + if _, n, err := net.ParseCIDR(u.IPv6ExtraBlock); err == nil { + extraNet = n + } + } + // The NIC declares the routes: the /112, and the whole /64 when the + // account owns one (it keeps it across a reinstall). + blockStr = blockRoutes(blockStr, u.IPv6ExtraBlock) } - if err := m.lx.Launch(m.cfg.Incus.Pool, m.cfg.Incus.Bridge, u.Name, image, u.IP, ipv6, blockStr, u.IPv6Address, m.cfg.Net.ExtIF, u.CPU, u.MemMB, u.DiskGB); err != nil { + if err := m.lx.Launch(m.cfg.Incus.Pool, m.cfg.Incus.Bridge, u.Name, image, u.IP, "", blockStr, u.IPv6Address, m.cfg.Net.ExtIF, u.CPU, u.MemMB, u.DiskGB); err != nil { rollback() return "", fmt.Errorf("recreate container: %w", err) } @@ -1477,7 +1543,7 @@ func (m *Manager) Reinstall(name, image string) (string, error) { } if m.cfg.IPv6ModeEffective() == cfg.IPv6ModePool { if u.IPv6Address != "" { - if err := m.ConfigureContainerIPv6(u.Name, u.IPv6Address); err != nil { + if err := m.ConfigureContainerIPv6(u.Name, u.IPv6Address, ""); err != nil { rollback() return "", fmt.Errorf("config container ipv6: %w", err) } @@ -1489,11 +1555,11 @@ func (m *Manager) Reinstall(name, image string) (string, error) { } else { // Configure the guest's IPv6 first, then publish the NDP rule, so the // outside world only learns the /112 once the container is ready. - if err := m.ConfigureContainerIPv6(u.Name, ipv6); err != nil { + if err := m.ConfigureContainerIPv6(u.Name, ipv6, u.IPv6ExtraBlock); err != nil { rollback() return "", fmt.Errorf("config container ipv6: %w", err) } - if err := m.WireIPv6(u.Name, block); err != nil { + if err := m.WireIPv6(u.Name, block, extraNet); err != nil { rollback() return "", fmt.Errorf("wire ipv6: %w", err) } @@ -1764,6 +1830,12 @@ func (m *Manager) EnsureBlockRoutes() error { firstErr = err } } + // The whole /64 blocks are routed by the panel rather than by Incus, so they + // need their own pass — both to install them on an upgrade and to bring them + // back after a reboot (that is what RewireAllIPv6 does at boot). + if err := m.EnsureExtraBlockRoutes(); err != nil && firstErr == nil { + firstErr = err + } return firstErr } diff --git a/src/internal/mgr/routed_ipv6.go b/src/internal/mgr/routed_ipv6.go index 75e8628..df124ab 100644 --- a/src/internal/mgr/routed_ipv6.go +++ b/src/internal/mgr/routed_ipv6.go @@ -45,7 +45,7 @@ func (e *RoutedIPv6Error) Unwrap() error { return e.Err } // and re-applies the profile on every boot. No daemons, no waiting. // // Both end by flushing stale on-link / redirect routes for the parent prefix. -func (m *Manager) ipv6ContainerScript(ipv6 string) (string, error) { +func (m *Manager) ipv6ContainerScript(ipv6, extra string) (string, error) { if ipv6 == "" { return "", nil } @@ -56,7 +56,7 @@ func (m *Manager) ipv6ContainerScript(ipv6 string) (string, error) { // Bare prefix (without the trailing ::) used to match routes that belong // to the parent prefix, e.g. 2406:da14:1dd2:a807:753a for a /80. prefix := strings.TrimSuffix(n.IP.String(), "::") - return m.ipv6ContainerScriptFor(ipv6, prefix) + return m.ipv6ContainerScriptFor(ipv6, prefix, extra) } // ipv6ContainerScriptFor renders the same script for an explicit address and @@ -64,7 +64,14 @@ func (m *Manager) ipv6ContainerScript(ipv6 string) (string, error) { // the DB), so prefix is "" — the script then configures the routed-NIC layout: // eth0 statically binds the public /128 with fe80::1 as gateway (Incus's // routed NIC gateway), and eth1 runs DHCPv4 on the private bridge. -func (m *Manager) ipv6ContainerScriptFor(ipv6, prefix string) (string, error) { +// +// extra is the whole /64 the account owns from net.ipv6_extra_prefix ("" when +// it has none). It is bound with its real /64 length, so the prefix is on-link +// inside the container: any address in it is usable and the customer can carve +// sub-prefixes out of it for internal networks. It is deliberately NOT added as +// a local route, which would claim every address in the block for the container +// itself and make exactly that impossible. Pool mode never has one. +func (m *Manager) ipv6ContainerScriptFor(ipv6, prefix, extra string) (string, error) { if ipv6 == "" { return "", nil } @@ -76,12 +83,47 @@ func (m *Manager) ipv6ContainerScriptFor(ipv6, prefix string) (string, error) { if err != nil { return "", err } - flush := fmt.Sprintf(`for r in $(ip -6 route show dev eth0 | awk '{print $1}'); do + // The extra block: its first address is the container's, the rest of the + // /64 is left to it. + extraAddr, extraCIDR := "", "" + if extra != "" { + _, n, err := net.ParseCIDR(extra) + if err != nil { + return "", fmt.Errorf("invalid extra IPv6 block %q: %w", extra, err) + } + extraCIDR = n.String() + extraAddr = addHostOffset(n.IP, 1).String() + } + nmcliExtra := "" + if extraAddr != "" { + nmcliExtra = "," + extraAddr + "/64" + } + // Appended to the networkd config, after the primary /128 binding. Empty + // (the whole block vanishes) when the container has no extra prefix. + networkdExtra := "" + if extraAddr != "" { + networkdExtra = fmt.Sprintf(` if ! grep -qs '^Address=%s/64$' "$CFG" 2>/dev/null; then + cat >> "$CFG" </dev/null || true ;; esac done -ip -6 route flush cache 2>/dev/null || true`, prefix) +ip -6 route flush cache 2>/dev/null || true`, extraSkip, prefix) script := fmt.Sprintf(`set -e # GATEWAY_MAC is the Incus bridge's MAC: the container's default route is # fe80::1 (the bridge gateway), but security.ipv6_filtering drops the guest's @@ -106,7 +148,7 @@ if command -v nmcli >/dev/null 2>&1 && ! systemctl is-active systemd-networkd >/ CONN=$(nmcli -t -f NAME con show 2>/dev/null | grep -i eth0 | head -1) [ -n "$CONN" ] && nmcli con mod "$CONN" \ ipv6.method manual \ - ipv6.addresses %s/128 \ + ipv6.addresses %s/128%s \ ipv6.gateway fe80::1 2>/dev/null || true [ -n "$CONN" ] && nmcli con up "$CONN" >/dev/null 2>&1 || true [ -n "$GATEWAY_MAC" ] && ip -6 neigh replace fe80::1 lladdr "$GATEWAY_MAC" nud permanent dev eth0 2>/dev/null || true @@ -156,6 +198,7 @@ Address=%s/128 ADDR changed=1 fi +%s # Static default route via the bridge's fixed link-local gateway: with RA # off-link/route-prefix off there is no RA-provided default, so without this # the container cannot reach out (only inbound works via the host's proxy). @@ -211,7 +254,7 @@ EOF systemctl daemon-reload || true systemctl enable vpsmgr-ipv6.service >/dev/null 2>&1 || true systemctl restart vpsmgr-ipv6.service >/dev/null 2>&1 || true -%s`, bridgeMAC, localBlock, ipv6, ipv6, ipv6, flush) +%s`, bridgeMAC, localBlock, ipv6, nmcliExtra, ipv6, ipv6, networkdExtra, flush) return script, nil } @@ -316,7 +359,12 @@ fi // primary address. A caller that already knows it passes it — Add launches the // container before creating its account row, so the address comes from the // block index it just picked. An empty addr is read from the account's row. -func (m *Manager) ConfigureContainerIPv6(name, addr string) error { +// +// extra is the whole /64 from net.ipv6_extra_prefix ("" when the account has +// none). Add passes the block it just picked, for the same reason it passes the +// address; everywhere else an empty value is resolved from the account, and an +// account that does not exist yet simply has no block. +func (m *Manager) ConfigureContainerIPv6(name, addr, extra string) error { if !m.cfg.IPv6Enabled() { return nil } @@ -333,12 +381,17 @@ func (m *Manager) ConfigureContainerIPv6(name, addr string) error { return err } } + if extra == "" && !pool { + if u, uerr := m.db.GetUserByName(name); uerr == nil { + extra = u.IPv6ExtraBlock + } + } var script string var err error if pool { - script, err = m.ipv6ContainerScriptFor(addr, "") + script, err = m.ipv6ContainerScriptFor(addr, "", "") } else { - script, err = m.ipv6ContainerScript(addr) + script, err = m.ipv6ContainerScript(addr, extra) } if err != nil || script == "" { return err @@ -369,7 +422,7 @@ func (m *Manager) EnsureRoutedIPv6() error { if st, err := m.lx.State(u.Name); err != nil || st != "Running" { continue // stopped or not created yet } - if err := m.ConfigureContainerIPv6(u.Name, ""); err != nil && firstErr == nil { + if err := m.ConfigureContainerIPv6(u.Name, "", ""); err != nil && firstErr == nil { firstErr = &RoutedIPv6Error{Container: u.Name, Err: err} } } diff --git a/src/internal/mgr/share.go b/src/internal/mgr/share.go index ef98ad2..a4b2279 100644 --- a/src/internal/mgr/share.go +++ b/src/internal/mgr/share.go @@ -252,15 +252,17 @@ func (m *Manager) ReinstallFromShare(name, code string, runInit bool) (string, e m.db.UpdateUserStatus(u.ID, db.StatusFailed) return "", err } - ipv6, blockStr := "", "" + blockStr := "" if m.cfg.IPv6ModeEffective() != cfg.IPv6ModePool { - ipv6, _ = m.IPv6Addr(u.Name) if block, _ := m.IPv6Block(u.Name); block != nil { blockStr = block.String() } + // The account keeps its whole /64 across the reinstall, so the clone + // gets the same declared routes. + blockStr = blockRoutes(blockStr, u.IPv6ExtraBlock) } if err := m.lx.CloneFromSnapshot(owner.Name, snap, tmp, - m.cfg.Incus.Pool, m.cfg.Incus.Bridge, u.IP, ipv6, blockStr, u.IPv6Address, + m.cfg.Incus.Pool, m.cfg.Incus.Bridge, u.IP, "", blockStr, u.IPv6Address, m.cfg.Net.ExtIF, u.CPU, u.MemMB, u.DiskGB); err != nil { m.db.UpdateUserStatus(u.ID, db.StatusFailed) return "", fmt.Errorf("clone shared checkpoint: %w", err) @@ -312,7 +314,7 @@ func (m *Manager) ReinstallFromShare(name, code string, runInit bool) (string, e m.clearAuthorizedKeys(u.Name) if m.cfg.IPv6ModeEffective() == cfg.IPv6ModePool { if u.IPv6Address != "" { - if err := m.ConfigureContainerIPv6(u.Name, u.IPv6Address); err != nil { + if err := m.ConfigureContainerIPv6(u.Name, u.IPv6Address, ""); err != nil { rollback() return "", fmt.Errorf("config container ipv6: %w", err) } @@ -322,11 +324,11 @@ func (m *Manager) ReinstallFromShare(name, code string, runInit bool) (string, e } } } else { - if err := m.ConfigureContainerIPv6(u.Name, ""); err != nil { + if err := m.ConfigureContainerIPv6(u.Name, "", ""); err != nil { rollback() return "", fmt.Errorf("config container ipv6: %w", err) } - if err := m.WireIPv6(u.Name, nil); err != nil { + if err := m.WireIPv6(u.Name, nil, nil); err != nil { rollback() return "", fmt.Errorf("wire ipv6: %w", err) } diff --git a/src/internal/mgr/transfer.go b/src/internal/mgr/transfer.go index 3609635..7b4473e 100644 --- a/src/internal/mgr/transfer.go +++ b/src/internal/mgr/transfer.go @@ -470,7 +470,7 @@ func (m *Manager) TransferImport(ctx context.Context, name string, r io.Reader, } switch { case m.usesTwoNICs(u): - if err := m.ConfigureContainerIPv6(u.Name, u.IPv6Address); err != nil { + if err := m.ConfigureContainerIPv6(u.Name, u.IPv6Address, ""); err != nil { return fail(fmt.Errorf("config container ipv6: %w", err)) } if err := m.WireIPv6Pool(u.Name, u.IPv6Address); err != nil { @@ -480,10 +480,13 @@ func (m *Manager) TransferImport(ctx context.Context, name string, r io.Reader, // Pool mode, no address handed out: the container is pure IPv4 and // there is nothing to route. default: - if err := m.ConfigureContainerIPv6(u.Name, ""); err != nil { + // An imported account starts without a whole /64: the block is + // host-local (like the /112 index it is not carried across machines), + // and the operator can hand one out afterwards from the quota dialog. + if err := m.ConfigureContainerIPv6(u.Name, "", ""); err != nil { return fail(fmt.Errorf("config container ipv6: %w", err)) } - if err := m.WireIPv6(u.Name, nil); err != nil { + if err := m.WireIPv6(u.Name, nil, nil); err != nil { return fail(fmt.Errorf("wire ipv6: %w", err)) } } @@ -832,19 +835,21 @@ func freshNICAddrs() (map[string]string, error) { // following the disk here. func (m *Manager) instanceConfig(u *db.User) (map[string]string, map[string]lx.Device) { poolMode := m.cfg.IPv6ModeEffective() == cfg.IPv6ModePool - ipv6, block, poolAddr := "", "", "" + block, poolAddr := "", "" if poolMode { // Only pool mode keeps the address on the routed NIC, and only when // the user actually has one. poolAddr = u.IPv6Address } else { - ipv6, _ = m.IPv6Addr(u.Name) if b, _ := m.IPv6Block(u.Name); b != nil { block = b.String() } + // The imported account never carries a whole /64 (the block is + // host-local, like the /112 index), so only the /112 is declared. + block = blockRoutes(block, u.IPv6ExtraBlock) } return m.lx.InstanceSpec(m.cfg.Incus.Pool, m.cfg.Incus.Bridge, u.IP, - ipv6, block, poolAddr, m.cfg.Net.ExtIF, u.CPU, u.MemMB, u.DiskGB) + "", block, poolAddr, m.cfg.Net.ExtIF, u.CPU, u.MemMB, u.DiskGB) } // requireStopped refuses to export a container that is not stopped. The check diff --git a/src/internal/panel/expiry_test.go b/src/internal/panel/expiry_test.go index f45fe26..2718bdd 100644 --- a/src/internal/panel/expiry_test.go +++ b/src/internal/panel/expiry_test.go @@ -19,7 +19,7 @@ func TestExpiredAccountLocked(t *testing.T) { prefix := "/" + testSecret hash, _ := pw.Hash("pw") if _, err := d.CreateUserFull("alice", hash, "10.42.0.2", 1, 30001, 10000, 1, 1024, 10, 0, - db.StatusReady, "", 0, "2000-01-01T00:00:00Z"); err != nil { + db.StatusReady, "", 0, "", "2000-01-01T00:00:00Z"); err != nil { t.Fatal(err) } cookie := loginAndCookie(t, h, prefix, "alice", "pw") diff --git a/src/main.go b/src/main.go index 5547a2b..cfbb0c8 100644 --- a/src/main.go +++ b/src/main.go @@ -151,6 +151,12 @@ func main() { } case "quota": err = userQuota(os.Args[2:]) + case "extra64": + if len(os.Args) != 3 { + err = fmt.Errorf("usage: vps extra64 ") + break + } + err = userExtra64(os.Args[2]) case "power": if len(os.Args) != 4 { err = fmt.Errorf("usage: vps power start|stop|restart") @@ -201,8 +207,9 @@ func usage() { fmt.Print(`vps ` + ver.Version + ` usage: vps list [name] all users, or one user's detail - vps add [--cpu 1] [--mem 1G] [--disk 10G] [--bandwidth 100] [--days 30] + vps add [--cpu 1] [--mem 1G] [--disk 10G] [--bandwidth 100] [--days 30] [--extra64] vps quota [--cpu 2] [--mem 2G] [--disk 20G] [--bandwidth 200] [--days 30] [--clear-expiry] + vps extra64 assign a whole /64 block from net.ipv6_extra_prefix vps power start|stop|restart vps passwd reissue user panel password (shown once) vps admin-passwd reset admin panel password (shown once) @@ -1281,11 +1288,13 @@ func userAdd(args []string) error { var cpuS string var memS, diskS, bandwidthS string var days int + var extra64 bool fs.StringVar(&cpuS, "cpu", "", "") fs.StringVar(&memS, "mem", "", "") fs.StringVar(&diskS, "disk", "", "") fs.StringVar(&bandwidthS, "bandwidth", "", "") // GiB/month, 0/empty = unlimited fs.IntVar(&days, "days", 0, "") // validity in days, 0 = permanent + fs.BoolVar(&extra64, "extra64", false, "") // whole /64 from net.ipv6_extra_prefix if err := fs.Parse(args[1:]); err != nil { return err } @@ -1368,7 +1377,7 @@ func userAdd(args []string) error { } defer d.Close() m := mgr.New(c, d) - res, err := m.Add(name, mgr.AddOptions{CPU: cpu, MemMB: mem, DiskGB: disk, BandwidthGB: bandwidth, Days: days}) + res, err := m.Add(name, mgr.AddOptions{CPU: cpu, MemMB: mem, DiskGB: disk, BandwidthGB: bandwidth, Days: days, AllocateExtra64: extra64}) if err != nil { return err } @@ -1394,6 +1403,29 @@ func userDel(name string) error { return nil } +// userExtra64 hands an existing container a whole /64 out of +// net.ipv6_extra_prefix (the same operation as the admin panel's quota dialog). +// Like there, it can only assign: a block stays with the account until the +// account is deleted. +func userExtra64(name string) error { + c, err := cfg.Load() + if err != nil { + return err + } + d, err := db.Open(c.Panel.DB) + if err != nil { + return err + } + defer d.Close() + m := mgr.New(c, d) + block, err := m.AssignExtraBlock(name) + if err != nil { + return err + } + fmt.Printf("user %s: whole /64 %s\n", name, block) + return nil +} + func userQuota(args []string) error { if len(args) < 1 { return fmt.Errorf("usage: vps quota [--cpu 2] [--mem 2G] [--disk 20G] [--bandwidth 100] [--days 30] [--clear-expiry]") @@ -1646,6 +1678,9 @@ func printAdded(r *mgr.Result) { if r.IPv6 != "" { fmt.Printf("ipv6: %s\n", r.IPv6) } + if r.IPv6Extra != "" { + fmt.Printf("ipv6 /64: %s (routed to the container; claimed and released with the account)\n", r.IPv6Extra) + } if r.Password != "" { fmt.Printf("password: %s (panel + root)\n", r.Password) } else { @@ -1671,6 +1706,9 @@ func printResult(r *mgr.Result) { if r.IPv6 != "" { fmt.Printf("ipv6: %s\n", r.IPv6) } + if r.IPv6Extra != "" { + fmt.Printf("ipv6 /64: %s\n", r.IPv6Extra) + } if r.V4Forward { fmt.Printf("ssh: %d\n", u.SSHPort) fmt.Printf("ports: %s\n", mgr.UserPorts(u.StartPort, r.PortsPerUser))