diff --git a/README.md b/README.md index 07c332d4..3b02931c 100644 --- a/README.md +++ b/README.md @@ -325,6 +325,25 @@ Keys in the terminal version: Agents read their config at startup, so a running session keeps its model until you start a new one. +### Moving to another machine + +```sh +magpie backup # writes magpie.magpie-backup, asks for a passphrase twice +magpie backup --no-keys ~/b.magpie-backup # the same with no API keys in it +magpie restore magpie.magpie-backup # on the other machine +magpie restore --no-agents b.magpie-backup # providers, settings, profiles; agents left as they are +``` + +A backup holds your providers (with their keys, unless `--no-keys`), the +pictures picked for them, the settings, the profiles and every agent's model. +It is encrypted on your machine (AES-256-GCM, the key derived from the +passphrase with PBKDF2-SHA256); nothing in it can be read without the +passphrase. Restoring replaces providers with the same id and adds the rest; +one that came without a key keeps the key already there. Agent models are set +only for agents installed on that machine. Subscriptions are not in it: sign +in to them on each machine. Piped in, the passphrase is the first line of +stdin. + ## Files - `~/.config/magpie/profiles.json` — saved profiles diff --git a/backup_cmd.go b/backup_cmd.go new file mode 100644 index 00000000..91a7d0f3 --- /dev/null +++ b/backup_cmd.go @@ -0,0 +1,147 @@ +package main + +import ( + "bufio" + "errors" + "fmt" + "os" + "strings" + + "golang.org/x/term" + + "github.com/yetone/magpie/internal/backup" + "github.com/yetone/magpie/internal/edit" +) + +// backupCmd writes every provider, setting, profile and agent model to one +// file sealed with a passphrase: magpie backup [--no-keys] [file]. +func backupCmd(args []string) error { + keys, file := true, "" + for _, a := range args { + switch { + case a == "--no-keys": + keys = false + case strings.HasPrefix(a, "-"): + return fmt.Errorf("unknown flag %s (magpie backup [--no-keys] [file])", a) + case file == "": + file = a + default: + return errors.New("usage: magpie backup [--no-keys] [file]") + } + } + if file == "" { + file = "magpie" + backup.Ext + } + b, err := backup.Collect(keys, version) + if err != nil { + return err + } + pass, err := passphrase("Passphrase for the backup: ", true) + if err != nil { + return err + } + data, err := backup.Seal(b, pass) + if err != nil { + return err + } + if err := edit.WriteAtomic(file, data); err != nil { + return err + } + os.Chmod(file, 0o600) + what := "with their keys" + if !keys { + what = "without keys" + } + fmt.Printf("%s %s: %d providers %s, %d profiles, %d agent settings\n", + green.Render("saved"), file, len(b.Providers), what, len(b.Profiles), len(b.Agents)) + fmt.Println(muted.Render("Subscriptions are not in it: sign in to them on the other machine. Keep the passphrase: without it the file can't be opened.")) + return nil +} + +// restoreCmd puts a backup in: magpie restore [--no-agents] . +func restoreCmd(args []string) error { + parts, file := backup.All, "" + for _, a := range args { + switch { + case a == "--no-agents": + parts.Agents = false + case strings.HasPrefix(a, "-"): + return fmt.Errorf("unknown flag %s (magpie restore [--no-agents] )", a) + case file == "": + file = a + default: + return errors.New("usage: magpie restore [--no-agents] ") + } + } + if file == "" { + return errors.New("usage: magpie restore [--no-agents] ") + } + data, err := os.ReadFile(file) + if err != nil { + return err + } + pass, err := passphrase("Passphrase: ", false) + if err != nil { + return err + } + b, err := backup.Open(data, pass) + if err != nil { + return err + } + r, err := backup.Restore(b, parts) + if err != nil { + return err + } + fmt.Printf("%s providers: %d added, %d replaced", green.Render("restored"), r.Added, r.Replaced) + if r.Settings { + fmt.Print("; settings") + } + fmt.Printf("; %d profiles; %d agent settings changed\n", r.Profiles, r.Agents) + if len(r.NeedKey) > 0 { + fmt.Println(muted.Render("Needs a key (magpie provider key ): " + strings.Join(r.NeedKey, ", "))) + } + if len(r.Skipped) > 0 { + fmt.Println(muted.Render("Left as they are (agent not here, or its model can't be reached yet): " + strings.Join(r.Skipped, ", "))) + } + return nil +} + +// passphrase reads one from the terminal without echo, asked twice when +// it is being set; piped in, it is the first line of stdin. +func passphrase(prompt string, confirm bool) (string, error) { + fd := int(os.Stdin.Fd()) + if !term.IsTerminal(fd) { + line, err := bufio.NewReader(os.Stdin).ReadString('\n') + line = strings.TrimRight(line, "\r\n") + if line == "" { + if err != nil { + return "", fmt.Errorf("no passphrase on stdin: %w", err) + } + return "", errors.New("the passphrase is empty") + } + return line, nil + } + read := func(p string) (string, error) { + fmt.Fprint(os.Stderr, p) + b, err := term.ReadPassword(fd) + fmt.Fprintln(os.Stderr) + return string(b), err + } + pass, err := read(prompt) + if err != nil { + return "", err + } + if pass == "" { + return "", errors.New("the passphrase is empty") + } + if confirm { + again, err := read("Again: ") + if err != nil { + return "", err + } + if again != pass { + return "", errors.New("the passphrases differ") + } + } + return pass, nil +} diff --git a/go.mod b/go.mod index 89fc5f74..8c7987de 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/tidwall/jsonc v0.3.3 github.com/wailsapp/wails/v3 v3.0.0-beta.24 golang.org/x/sys v0.46.0 + golang.org/x/term v0.44.0 modernc.org/sqlite v1.44.3 ) diff --git a/go.sum b/go.sum index d855248f..0df65532 100644 --- a/go.sum +++ b/go.sum @@ -89,6 +89,8 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc= +golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y= golang.org/x/text v0.39.0 h1:UbZz4pLOvn600D6Oh6GGEI6VAmndrEBLv8/6BEXzyus= golang.org/x/text v0.39.0/go.mod h1:3UwRclnC2g0TU9x8PZiyfOajCd1zaUNHF9cvqcQZ+ZM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/internal/backup/backup.go b/internal/backup/backup.go new file mode 100644 index 00000000..529e3dbb --- /dev/null +++ b/internal/backup/backup.go @@ -0,0 +1,290 @@ +// Package backup packs what the user set up in magpie into one file, sealed +// with a passphrase, to carry to another machine: the providers (their keys +// too, unless left out), the pictures picked for them, the settings, the +// profiles and every agent's model. Subscriptions are not in it: each is +// the sign-in of an agent on this machine, so each machine signs in on its +// own. +// +// The file is JSON: a header naming how it is sealed, and the bundle +// encrypted with AES-256-GCM under a key derived from the passphrase with +// PBKDF2-SHA256. Nothing in it can be read without the passphrase. +package backup + +import ( + "crypto/aes" + "crypto/cipher" + "crypto/pbkdf2" + "crypto/rand" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "maps" + "os" + "path/filepath" + "regexp" + "slices" + "strings" + "time" + + "github.com/yetone/magpie/internal/agent" + "github.com/yetone/magpie/internal/edit" + "github.com/yetone/magpie/internal/profile" + "github.com/yetone/magpie/internal/provider" + "github.com/yetone/magpie/internal/settings" +) + +// Ext is the extension backups are saved with. +const Ext = ".magpie-backup" + +const ( + format = "magpie-backup" + iterations = 600_000 +) + +// Bundle is what a backup holds. +type Bundle struct { + Version int `json:"version"` + Created time.Time `json:"created"` + App string `json:"app,omitempty"` // the magpie that made it + Keys bool `json:"keys"` // whether the providers carry their keys + Providers []provider.Provider `json:"providers"` + Icons map[string][]byte `json:"icons,omitempty"` // pictures picked for providers, by file name + Settings *settings.Settings `json:"settings,omitempty"` + Profiles map[string]profile.Profile `json:"profiles,omitempty"` + Agents profile.Profile `json:"agents,omitempty"` // every agent's fields as they are now +} + +type envelope struct { + Format string `json:"format"` + Version int `json:"version"` + KDF string `json:"kdf"` + Iterations int `json:"iterations"` + Salt []byte `json:"salt"` + Nonce []byte `json:"nonce"` + Data []byte `json:"data"` +} + +// ErrPassphrase is a passphrase that does not open the backup. +var ErrPassphrase = errors.New("wrong passphrase, or the file was changed") + +// Collect gathers the bundle; without keys, providers carry none, nor any +// header that looks like one. +func Collect(keys bool, app string) (Bundle, error) { + b := Bundle{Version: 1, Created: time.Now().UTC(), App: app, Keys: keys} + for _, p := range provider.Stored() { + if !keys { + p = withoutKeys(p) + } + b.Providers = append(b.Providers, p) + if name, ok := strings.CutPrefix(p.Icon, "file:"); ok { + if f := provider.IconFile(name); f != "" { + if data, err := os.ReadFile(f); err == nil { + if b.Icons == nil { + b.Icons = map[string][]byte{} + } + b.Icons[name] = data + } + } + } + } + if _, err := os.Stat(settings.Path()); err == nil { + s := settings.Load() + b.Settings = &s + } + ps, err := profile.Load() + if err != nil { + return b, err + } + if len(ps) > 0 { + b.Profiles = ps + } + if snap := profile.Snapshot(); len(snap) > 0 { + b.Agents = snap + } + return b, nil +} + +var secretHeader = regexp.MustCompile(`(?i)auth|key|token|secret|cookie|session|password`) + +func withoutKeys(p provider.Provider) provider.Provider { + p.Key, p.KeyName, p.Keys, p.KeyProtocol = "", "", nil, "" + if len(p.Headers) > 0 { + h := map[string]string{} + for k, v := range p.Headers { + if !secretHeader.MatchString(k) { + h[k] = v + } + } + p.Headers = h + } + return p +} + +// Seal encrypts the bundle under the passphrase. +func Seal(b Bundle, pass string) ([]byte, error) { + if pass == "" { + return nil, errors.New("a backup needs a passphrase") + } + plain, err := json.Marshal(b) + if err != nil { + return nil, err + } + e := envelope{Format: format, Version: 1, KDF: "pbkdf2-sha256", Iterations: iterations, + Salt: make([]byte, 16), Nonce: make([]byte, 12)} + rand.Read(e.Salt) + rand.Read(e.Nonce) + gcm, err := aead(pass, e) + if err != nil { + return nil, err + } + e.Data = gcm.Seal(nil, e.Nonce, plain, header(e)) + return json.MarshalIndent(e, "", " ") +} + +// Open decrypts a backup. +func Open(data []byte, pass string) (Bundle, error) { + var e envelope + if json.Unmarshal(data, &e) != nil || e.Format != format { + return Bundle{}, errors.New("not a magpie backup") + } + if e.Version != 1 || e.KDF != "pbkdf2-sha256" { + return Bundle{}, errors.New("this backup was made by a newer magpie; update magpie to open it") + } + if e.Iterations < 100_000 || e.Iterations > 10_000_000 || len(e.Nonce) != 12 || len(e.Salt) < 16 { + return Bundle{}, errors.New("not a magpie backup") + } + gcm, err := aead(pass, e) + if err != nil { + return Bundle{}, err + } + plain, err := gcm.Open(nil, e.Nonce, e.Data, header(e)) + if err != nil { + return Bundle{}, ErrPassphrase + } + var b Bundle + if err := json.Unmarshal(plain, &b); err != nil { + return Bundle{}, err + } + return b, nil +} + +// header is what the sealing covers besides the bundle: the envelope's +// own fields, so none of them can be changed unnoticed. +func header(e envelope) []byte { + return fmt.Appendf(nil, "%s/%d/%s/%d/%x", e.Format, e.Version, e.KDF, e.Iterations, e.Salt) +} + +func aead(pass string, e envelope) (cipher.AEAD, error) { + key, err := pbkdf2.Key(sha256.New, pass, e.Salt, e.Iterations, 32) + if err != nil { + return nil, err + } + block, err := aes.NewCipher(key) + if err != nil { + return nil, err + } + return cipher.NewGCM(block) +} + +// Parts picks what a restore puts back. +type Parts struct { + Providers, Settings, Profiles, Agents bool +} + +// All is every part. +var All = Parts{true, true, true, true} + +// Result says what a restore did. +type Result struct { + Added, Replaced int // providers + NeedKey []string // providers that came without a key and have none here + Settings bool + Profiles int + Agents int // agent fields changed + Skipped []string // agent fields left alone: the agent is not on this machine, or the value failed +} + +// Restore puts the chosen parts of the bundle in. Providers come first, so +// the agents' models that go through them resolve. Only agents on this +// machine are set. +func Restore(b Bundle, parts Parts) (Result, error) { + var r Result + if parts.Providers { + for name, data := range b.Icons { + if f := provider.IconFile(name); f != "" && len(data) <= provider.MaxIcon { + if _, err := os.Stat(f); err != nil { + os.MkdirAll(filepath.Dir(f), 0o755) + edit.WriteAtomic(f, data) + } + } + } + var err error + if r.Added, r.Replaced, err = provider.Restore(b.Providers); err != nil { + return r, err + } + for _, p := range provider.Stored() { + if !p.Ready() && slices.ContainsFunc(b.Providers, func(q provider.Provider) bool { return q.ID == p.ID }) { + r.NeedKey = append(r.NeedKey, p.Name) + } + } + } + if parts.Settings && b.Settings != nil { + if err := settings.Save(*b.Settings); err != nil { + return r, err + } + r.Settings = true + } + if parts.Profiles && len(b.Profiles) > 0 { + for name, p := range b.Profiles { + if err := profile.Save(name, p); err != nil { + return r, err + } + r.Profiles++ + } + } + if parts.Agents && len(b.Agents) > 0 { + here := map[string]bool{} + for _, a := range agent.Detected() { + here[a.ID] = true + } + // one value at a time, so one that fails (a model of a subscription + // not signed in here) leaves the rest to go in + for _, k := range profileKeys(b.Agents) { + id, _, _ := strings.Cut(k, ".") + if !here[id] { + r.Skipped = append(r.Skipped, k) + continue + } + n, err := profile.Apply(profile.Profile{k: b.Agents[k]}) + if err != nil { + r.Skipped = append(r.Skipped, k) + continue + } + r.Agents += n + } + } + return r, nil +} + +// profileKeys orders a profile's keys the way Apply does: providers, then +// models, then the rest. +func profileKeys(p profile.Profile) []string { + rank := func(k string) int { + switch { + case strings.HasSuffix(k, ".provider"): + return 0 + case strings.HasSuffix(k, ".model"): + return 1 + } + return 2 + } + keys := slices.Collect(maps.Keys(p)) + slices.SortFunc(keys, func(a, b string) int { + if ra, rb := rank(a), rank(b); ra != rb { + return ra - rb + } + return strings.Compare(a, b) + }) + return keys +} diff --git a/internal/backup/backup_test.go b/internal/backup/backup_test.go new file mode 100644 index 00000000..2b973b09 --- /dev/null +++ b/internal/backup/backup_test.go @@ -0,0 +1,169 @@ +package backup + +import ( + "errors" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/yetone/magpie/internal/profile" + "github.com/yetone/magpie/internal/provider" + "github.com/yetone/magpie/internal/settings" +) + +// home gives the test a machine of its own: no agents, no magpie files. +func home(t *testing.T) { + t.Helper() + h := t.TempDir() + t.Setenv("HOME", h) + t.Setenv("XDG_CONFIG_HOME", filepath.Join(h, ".config")) + t.Setenv("XDG_CACHE_HOME", filepath.Join(h, ".cache")) + t.Setenv("PATH", h) + for _, v := range []string{"CLAUDE_CONFIG_DIR", "CODEX_HOME", "DSH_HOME", "GEMINI_CLI_HOME", "OPENCODE_CONFIG"} { + t.Setenv(v, "") + } +} + +func setUp(t *testing.T) { + t.Helper() + icon, err := provider.StoreIcon([]byte("\x89PNG\r\n\x1a\n0000000000000000")) + if err != nil { + t.Fatal(err) + } + for _, p := range []provider.Provider{ + {ID: "acme", Name: "Acme", Chat: "https://acme.example.com/v1", Key: "sk-acme", Icon: icon, + Keys: []provider.KeyAccount{{Name: "second", Key: "sk-acme-2"}}, + Headers: map[string]string{"X-Team": "a", "X-Api-Key": "hdr-secret"}}, + {ID: "beta", Name: "Beta", Chat: "https://beta.example.com/v1", Key: "sk-beta"}, + } { + if err := provider.Save(p); err != nil { + t.Fatal(err) + } + } + if err := settings.Save(settings.Settings{Theme: "dark", Lang: "zh"}); err != nil { + t.Fatal(err) + } + if err := profile.Save("work", profile.Profile{"claude.model": "acme/m1"}); err != nil { + t.Fatal(err) + } +} + +func TestRoundTrip(t *testing.T) { + home(t) + setUp(t) + b, err := Collect(true, "test") + if err != nil { + t.Fatal(err) + } + data, err := Seal(b, "correct horse") + if err != nil { + t.Fatal(err) + } + for _, secret := range []string{"sk-acme", "hdr-secret", "acme.example.com", "work"} { + if strings.Contains(string(data), secret) { + t.Fatalf("%q readable in the file", secret) + } + } + if _, err := Open(data, "wrong"); !errors.Is(err, ErrPassphrase) { + t.Fatalf("wrong passphrase: %v", err) + } + + home(t) // another machine + got, err := Open(data, "correct horse") + if err != nil { + t.Fatal(err) + } + r, err := Restore(got, All) + if err != nil { + t.Fatal(err) + } + if r.Added != 2 || !r.Settings || r.Profiles != 1 || len(r.NeedKey) != 0 { + t.Fatalf("result: %+v", r) + } + p, err := provider.Find("acme") + if err != nil || p.Key != "sk-acme" || len(p.Keys) != 1 || p.Headers["X-Api-Key"] != "hdr-secret" { + t.Fatalf("acme: %+v %v", p, err) + } + name, _ := strings.CutPrefix(p.Icon, "file:") + if _, err := os.Stat(provider.IconFile(name)); err != nil { + t.Fatalf("icon: %v", err) + } + if s := settings.Load(); s.Theme != "dark" || s.Lang != "zh" { + t.Fatalf("settings: %+v", s) + } + if ps, _ := profile.Load(); ps["work"]["claude.model"] != "acme/m1" { + t.Fatalf("profiles: %+v", ps) + } +} + +// Without keys, nothing secret leaves; restored over a machine that has the +// provider, its keys there stay, and one new there is named as needing a key. +func TestNoKeys(t *testing.T) { + home(t) + setUp(t) + b, err := Collect(false, "test") + if err != nil { + t.Fatal(err) + } + for _, p := range b.Providers { + if p.Key != "" || len(p.Keys) != 0 { + t.Fatalf("key in a keyless backup: %+v", p) + } + if _, ok := p.Headers["X-Api-Key"]; ok { + t.Fatalf("auth header in a keyless backup: %+v", p.Headers) + } + } + data, _ := Seal(b, "pw") + + home(t) + if err := provider.Save(provider.Provider{ID: "acme", Name: "Acme old", Chat: "https://old.example.com/v1", Key: "sk-here"}); err != nil { + t.Fatal(err) + } + got, err := Open(data, "pw") + if err != nil { + t.Fatal(err) + } + r, err := Restore(got, Parts{Providers: true}) + if err != nil { + t.Fatal(err) + } + if r.Added != 1 || r.Replaced != 1 || !slices.Equal(r.NeedKey, []string{"Beta"}) || r.Settings || r.Profiles != 0 { + t.Fatalf("result: %+v", r) + } + if p, _ := provider.Find("acme"); p.Key != "sk-here" || p.Chat != "https://acme.example.com/v1" || p.Headers["X-Team"] != "a" { + t.Fatalf("acme: %+v", p) + } +} + +func TestNotABackup(t *testing.T) { + for _, data := range []string{"", "{}", `{"format":"magpie-backup","version":9,"kdf":"x"}`} { + if _, err := Open([]byte(data), "pw"); err == nil || errors.Is(err, ErrPassphrase) { + t.Fatalf("%q: %v", data, err) + } + } + if _, err := Seal(Bundle{}, ""); err == nil { + t.Fatal("sealed with no passphrase") + } +} + +func TestTampered(t *testing.T) { + data, err := Seal(Bundle{Version: 1}, "pw") + if err != nil { + t.Fatal(err) + } + // lowering the work of the key derivation is noticed + bad := strings.Replace(string(data), `"iterations": 600000`, `"iterations": 100000`, 1) + if _, err := Open([]byte(bad), "pw"); !errors.Is(err, ErrPassphrase) { + t.Fatalf("tampered header: %v", err) + } +} + +func TestProfileKeys(t *testing.T) { + got := profileKeys(profile.Profile{"codex.effort": "", "claude.model": "", "codex.provider": "", "codex.model": ""}) + want := []string{"codex.provider", "claude.model", "codex.model", "codex.effort"} + if !slices.Equal(got, want) { + t.Fatalf("%v", got) + } +} diff --git a/internal/provider/backup.go b/internal/provider/backup.go new file mode 100644 index 00000000..4f42a611 --- /dev/null +++ b/internal/provider/backup.go @@ -0,0 +1,42 @@ +package provider + +// A backup carries providers.json's entries as they are stored, and puts +// them back on another machine (see internal/backup). + +// Stored is the providers as providers.json keeps them: keys included, +// signed-in accounts only as the model picks the user made for them. +func Stored() []Provider { return load().Providers } + +// Restore puts providers from a backup in. Each replaces the one here with +// its id; one that came without keys keeps the keys already here. It +// returns how many were new and how many replaced one here. +func Restore(ps []Provider) (added, replaced int, err error) { + f := load() + for _, p := range ps { + p.IconURL = "" + if p.ID == "" || p.ID != Slug(p.ID) || p.ID == "magpie" { + continue + } + i := -1 + for j := range f.Providers { + if f.Providers[j].ID == p.ID { + i = j + break + } + } + if i < 0 { + f.Providers = append(f.Providers, p) + added++ + continue + } + if p.Key == "" && len(p.Keys) == 0 { + p.Key, p.KeyName, p.Keys, p.KeyProtocol = f.Providers[i].Key, f.Providers[i].KeyName, f.Providers[i].Keys, f.Providers[i].KeyProtocol + } + f.Providers[i] = p + replaced++ + } + if added+replaced == 0 { + return 0, 0, nil + } + return added, replaced, store(f) +} diff --git a/main.go b/main.go index a99c9458..6ce4561c 100644 --- a/main.go +++ b/main.go @@ -40,6 +40,9 @@ const usage = `magpie — one place to pick every agent's model magpie profiles list profiles magpie rm delete a profile + magpie backup [--no-keys] [file] providers, keys, settings, profiles and agent models in one file, sealed with a passphrase + magpie restore [--no-agents] put a backup in on this machine + magpie providers list your providers: host, key, models, who uses them magpie presets the vendors magpie knows: add one with just a key magpie provider add e.g. magpie provider add deepseek sk-… @@ -146,6 +149,10 @@ func run(args []string) error { return usageCmd(args) case "update": return updateCmd(args) + case "backup": + return backupCmd(args[1:]) + case "restore": + return restoreCmd(args[1:]) case "claude-mcp-helper": // internal: stdio MCP subprocess spawned by Claude Code return claudebridge.RunMCP(args[1:]) }