diff --git a/README.md b/README.md index 4ff591bd..f86eb10d 100644 --- a/README.md +++ b/README.md @@ -688,6 +688,21 @@ otherwise): curl -fsSL https://usemagpie.ai/install.sh | sh ``` +Behind a firewall, or without access to GitHub, both the installer and +`magpie update` take `--proxy ` (http, https, socks5, socks5h; or +`MAGPIE_PROXY`, and `HTTPS_PROXY` / `ALL_PROXY` as usual) and +`--mirror `, a GitHub download mirror of your choosing put before +the release's github.com URL (or `MAGPIE_MIRROR`; `magpie update mirror +|off` keeps one for every update, the app's too): + +```sh +curl -fsSL https://usemagpie.ai/install.sh | sh -s -- --proxy http://127.0.0.1:7890 +magpie update --mirror https://mirror.example/ +``` + +No mirror is used unless you give one, and the SHA-256 every download is +checked against still comes from usemagpie.ai, never from the mirror. + Mac releases are signed and notarised; the Windows and Linux builds are not signed yet (Windows SmartScreen may ask before the first run). Every build keeps itself current: the app diff --git a/internal/gui/api.go b/internal/gui/api.go index b0013b91..d518e359 100644 --- a/internal/gui/api.go +++ b/internal/gui/api.go @@ -769,6 +769,8 @@ func Handler(w Windows, gw *gateway.Server) http.Handler { in.TextSize = cur.TextSize // the version the Update pill was hidden for, set from the pill in.UpdateSkip = cur.UpdateSkip + // the GitHub mirror updates come through, set by magpie update mirror + in.UpdateMirror = cur.UpdateMirror if v := strings.TrimSpace(in.Vision); v != "" && v != "off" && v != cur.Vision { if _, _, ok := provider.Resolve(v); !ok { fail(rw, fmt.Errorf("no model %s to describe images", v)) diff --git a/internal/gui/settings_keep_test.go b/internal/gui/settings_keep_test.go index 778f9e20..1ff87222 100644 --- a/internal/gui/settings_keep_test.go +++ b/internal/gui/settings_keep_test.go @@ -136,6 +136,7 @@ func TestSettingsSaveKeepsWhatItDoesNotSend(t *testing.T) { CodexAutoReset: []string{"me@example.com"}, TextSize: 125, UpdateSkip: "0.1.500", + UpdateMirror: "https://mirror.example/", Window: []int{900, 700}, } if err := settings.Save(was); err != nil { diff --git a/internal/netproxy/allproxy_test.go b/internal/netproxy/allproxy_test.go new file mode 100644 index 00000000..6e6d9316 --- /dev/null +++ b/internal/netproxy/allproxy_test.go @@ -0,0 +1,59 @@ +package netproxy + +import ( + "net/url" + "testing" +) + +// ALL_PROXY, which curl reads and Go doesn't, is magpie's too when neither +// HTTPS_PROXY nor HTTP_PROXY is set: a shell with only +// ALL_PROXY=socks5://… gets magpie update through it (akic404 on Discord), +// and NO_PROXY still keeps hosts off it. +func TestAllProxyFromTheEnvironment(t *testing.T) { + t.Setenv("XDG_CONFIG_HOME", t.TempDir()) + for _, k := range []string{"HTTPS_PROXY", "https_proxy", "HTTP_PROXY", "http_proxy", "ALL_PROXY", "all_proxy", "NO_PROXY", "no_proxy"} { + t.Setenv(k, "") + } + sysCache.at, sysCache.p = farFuture, Proxy{} + t.Cleanup(func() { sysCache.at = zero }) + + t.Setenv("all_proxy", "socks5h://127.0.0.1:1080") + t.Setenv("NO_PROXY", "internal.example, .corp.example:443") + gh, _ := url.Parse("https://github.com/yetone/magpie-releases/releases/download/v1/magpie") + if p, err := For(gh); err != nil || p == nil || p.String() != "socks5h://127.0.0.1:1080" { + t.Fatalf("ALL_PROXY: %v %v", p, err) + } + if proxy, source := Describe(); proxy != "socks5h://127.0.0.1:1080" || source != "environment" { + t.Fatalf("described as %q (%s)", proxy, source) + } + for _, raw := range []string{"https://internal.example/x", "https://api.internal.example/x", "https://git.corp.example/x", "http://127.0.0.1:3/x"} { + u, _ := url.Parse(raw) + if p, _ := For(u); p != nil { + t.Errorf("%s went through %v despite NO_PROXY", raw, p) + } + } + t.Setenv("NO_PROXY", "*") + if p, _ := For(gh); p != nil { + t.Fatalf("NO_PROXY=* still proxied: %v", p) + } +} + +func TestNoProxyList(t *testing.T) { + for _, c := range []struct { + host, list string + want bool + }{ + {"github.com", "", false}, + {"github.com", "github.com", true}, + {"objects.github.com", "github.com", true}, + {"notgithub.com", "github.com", false}, + {"api.example.com", "*.example.com", true}, + {"example.com", ".example.com", true}, + {"example.com", "example.com:443", true}, + {"localhost", "", true}, + } { + if got := noProxy(c.host, c.list); got != c.want { + t.Errorf("noProxy(%q, %q) = %v, want %v", c.host, c.list, got, c.want) + } + } +} diff --git a/internal/netproxy/netproxy.go b/internal/netproxy/netproxy.go index 9d447cb0..10eacf98 100644 --- a/internal/netproxy/netproxy.go +++ b/internal/netproxy/netproxy.go @@ -264,7 +264,55 @@ func Describe() (proxy, source string) { return "", "none" } -func fromEnv(u *url.URL) (*url.URL, error) { return http.ProxyFromEnvironment(&http.Request{URL: u}) } +// fromEnv is the proxy the environment names for u: HTTPS_PROXY or +// HTTP_PROXY as Go reads them, else ALL_PROXY (curl's, which Go doesn't +// read: a shell with only ALL_PROXY=socks5://… set has curl go through it, +// and magpie, its updates among its requests, too). NO_PROXY is kept for +// it as well. +func fromEnv(u *url.URL) (*url.URL, error) { + if p, err := http.ProxyFromEnvironment(&http.Request{URL: u}); p != nil || err != nil { + return p, err + } + all := envOf("ALL_PROXY", "all_proxy") + if all == "" || noProxy(u.Hostname(), envOf("NO_PROXY", "no_proxy")) { + return nil, nil + } + return Parse(all) +} + +func envOf(keys ...string) string { + for _, k := range keys { + if v := strings.TrimSpace(os.Getenv(k)); v != "" { + return v + } + } + return "" +} + +// noProxy matches host against NO_PROXY as curl does: "*" for every host, +// else entries apart by commas, each the host or a domain it is under +// (example.com and .example.com both take api.example.com); a port on an +// entry is ignored. Loopback is never proxied anyway. +func noProxy(host, list string) bool { + host = strings.ToLower(strings.TrimSuffix(host, ".")) + if host == "" || loopback(host) { + return true + } + for _, e := range strings.Split(list, ",") { + e = strings.ToLower(strings.TrimSpace(e)) + if e == "*" { + return true + } + if h, _, err := net.SplitHostPort(e); err == nil { + e = h + } + e = strings.TrimPrefix(strings.TrimPrefix(e, "*"), ".") + if e != "" && (host == e || strings.HasSuffix(host, "."+e)) { + return true + } + } + return false +} // Proxy is the system's proxy: one address, and the hosts that skip it. type Proxy struct { diff --git a/internal/netproxy/netproxy_test.go b/internal/netproxy/netproxy_test.go index 0cfd4617..d4e86ae0 100644 --- a/internal/netproxy/netproxy_test.go +++ b/internal/netproxy/netproxy_test.go @@ -60,7 +60,7 @@ func TestParseWindows(t *testing.T) { func TestFor(t *testing.T) { dir := t.TempDir() t.Setenv("XDG_CONFIG_HOME", dir) - for _, k := range []string{"HTTPS_PROXY", "https_proxy", "HTTP_PROXY", "http_proxy", "ALL_PROXY", "NO_PROXY", "no_proxy"} { + for _, k := range []string{"HTTPS_PROXY", "https_proxy", "HTTP_PROXY", "http_proxy", "ALL_PROXY", "all_proxy", "NO_PROXY", "no_proxy"} { t.Setenv(k, "") } sysCache.at, sysCache.p = farFuture, Proxy{URL: "http://127.0.0.1:7890", Bypass: []string{"*.corp"}} diff --git a/internal/settings/settings.go b/internal/settings/settings.go index 73d27dbe..9990b735 100644 --- a/internal/settings/settings.go +++ b/internal/settings/settings.go @@ -129,6 +129,13 @@ type Settings struct { // one of UpdateEveries, 0 for every six hours. NoAutoUpdate bool `json:"noAutoUpdate,omitempty"` UpdateEvery int `json:"updateEvery,omitempty"` + // UpdateMirror is a GitHub download mirror an update's file is fetched + // through, its prefix put before the release's github.com URL + // (https://mirror.example/https://github.com/…): "" for none, which is + // the default — magpie names no mirror of its own. The file is still + // checked against the SHA-256 the update feed at usemagpie.ai gives, + // never one from the mirror. magpie update mirror sets it. + UpdateMirror string `json:"updateMirror,omitempty"` // Vision is the model that describes an image to a model that can't see // it: a model's id (provider/model, group/), "off" to turn such an // image away, or empty for one magpie picks (see gateway.seer). @@ -607,6 +614,11 @@ func Save(s Settings) error { if !slices.Contains(UpdateEveries, s.UpdateEvery) { return fmt.Errorf("magpie checks for updates every %v minutes, not %d", UpdateEveries, s.UpdateEvery) } + if m := s.UpdateMirror; m != "" { + if u, err := url.Parse(m); err != nil || (u.Scheme != "https" && u.Scheme != "http") || u.Host == "" { + return fmt.Errorf("an update mirror is an http(s) address put before the github.com URL, like https://mirror.example/, not %q", m) + } + } if s.UsageAlert < 0 || s.UsageAlert > 100 { return fmt.Errorf("a usage alert is at a percentage from 1 to 100, or 0 for off, not %d", s.UsageAlert) } diff --git a/internal/update/mirror.go b/internal/update/mirror.go new file mode 100644 index 00000000..9645b1c4 --- /dev/null +++ b/internal/update/mirror.go @@ -0,0 +1,105 @@ +package update + +import ( + "context" + "fmt" + "net/url" + "strings" + + "github.com/yetone/magpie/internal/netproxy" + "github.com/yetone/magpie/internal/settings" +) + +// A network that can't reach GitHub can have a release's file come +// through a GitHub download mirror: its prefix goes before the file's +// github.com URL (https://mirror.example/https://github.com/…). magpie +// names no mirror of its own; one is used only when given (magpie update +// --mirror, or Settings' UpdateMirror, which magpie update mirror sets). +// +// A mirror is a host magpie doesn't trust: the update feed, with each +// file's SHA-256, is still asked at usemagpie.ai and never through the +// mirror, the file is checked against that hash, and the macOS app must +// still be signed by the team that signed the one installed. A mirror +// that changes a byte gets its download refused. + +type mirrorKey struct{} + +// WithMirror has the downloads made under ctx go through the mirror +// prefix; "off" goes through none, whatever Settings say, and "" leaves +// ctx as it is. +func WithMirror(ctx context.Context, prefix string) context.Context { + if prefix = strings.TrimSpace(prefix); prefix == "" { + return ctx + } + return context.WithValue(ctx, mirrorKey{}, prefix) +} + +// WithProxy has the update feed and the downloads asked under ctx go +// through proxy (http://, https://, socks5://, socks5h://; host:port is +// http; "direct" for none), over Settings' Proxy and the environment's. +func WithProxy(ctx context.Context, proxy string) context.Context { + return netproxy.With(ctx, proxy) +} + +// CheckMirror says what is wrong with a mirror prefix, nil when it is one. +func CheckMirror(prefix string) error { + u, err := url.Parse(strings.TrimSpace(prefix)) + if err != nil || (u.Scheme != "https" && u.Scheme != "http") || u.Host == "" { + return fmt.Errorf("a mirror is an http(s) address put before the github.com URL, like https://mirror.example/, not %q", prefix) + } + return nil +} + +// CheckProxy says what is wrong with a proxy address, nil when it is one. +func CheckProxy(proxy string) error { + if proxy == "direct" { + return nil + } + u, err := netproxy.Parse(proxy) + if err != nil || u.Host == "" { + return fmt.Errorf("a proxy is an address like http://127.0.0.1:7890 or socks5://127.0.0.1:1080, not %q", proxy) + } + switch u.Scheme { + case "http", "https", "socks5", "socks5h": + return nil + } + return fmt.Errorf("a proxy is http://, https://, socks5:// or socks5h://, not %q", proxy) +} + +// mirrorOf is the mirror downloads under ctx go through: WithMirror's, +// else Settings', "" for none. +func mirrorOf(ctx context.Context) string { + m, _ := ctx.Value(mirrorKey{}).(string) + if m == "" { + m = strings.TrimSpace(settings.Load().UpdateMirror) + } + if m == "off" || m == "direct" { + return "" + } + return m +} + +// mirrored is raw fetched through the mirror prefix: only a file on +// GitHub is; anything else (a test's server, a feed pointed elsewhere) +// goes as it is. +func mirrored(prefix, raw string) string { + if prefix == "" { + return raw + } + u, err := url.Parse(raw) + if err != nil || !githubHost(u.Hostname()) { + return raw + } + if !strings.HasSuffix(prefix, "/") { + prefix += "/" + } + return prefix + raw +} + +func githubHost(h string) bool { + h = strings.ToLower(h) + return h == "github.com" || strings.HasSuffix(h, ".github.com") || strings.HasSuffix(h, ".githubusercontent.com") +} + +// Mirror is the mirror downloads under ctx go through, "" for none. +func Mirror(ctx context.Context) string { return mirrorOf(ctx) } diff --git a/internal/update/mirror_test.go b/internal/update/mirror_test.go new file mode 100644 index 00000000..28aaf0b0 --- /dev/null +++ b/internal/update/mirror_test.go @@ -0,0 +1,169 @@ +package update + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "slices" + "strings" + "sync" + "testing" + + "github.com/yetone/magpie/internal/settings" +) + +func updateHome(t *testing.T) { + t.Helper() + h := t.TempDir() + t.Setenv("HOME", h) + t.Setenv("USERPROFILE", h) + t.Setenv("XDG_CONFIG_HOME", filepath.Join(h, ".config")) + for _, k := range []string{"HTTPS_PROXY", "https_proxy", "HTTP_PROXY", "http_proxy", "ALL_PROXY", "all_proxy", "NO_PROXY", "no_proxy"} { + t.Setenv(k, "") + } +} + +// A release's file comes through a GitHub download mirror only when one is +// given (magpie update --mirror, or Settings' UpdateMirror): its prefix +// before the github.com URL. The feed, with the file's SHA-256, is never +// asked through it, and a file the mirror changed is refused (akic404 on +// Discord: install and update without a proxy). +func TestUpdateThroughAMirror(t *testing.T) { + updateHome(t) + body := []byte("new magpie") + sum := sha256.Sum256(body) + asset := Asset{URL: "https://github.com/yetone/magpie-releases/releases/download/v9.9.9/magpie-cli-linux-amd64", SHA256: hex.EncodeToString(sum[:])} + + var mu sync.Mutex + var seen []string + tamper := false + mirror := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + seen = append(seen, r.URL.Path) + bad := tamper + mu.Unlock() + if bad { + w.Write([]byte("not magpie")) + return + } + w.Write(body) + })) + defer mirror.Close() + feed := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + json.NewEncoder(w).Encode(Release{Version: "9.9.9", Assets: map[string]Asset{BinaryAsset(): asset}}) + })) + defer feed.Close() + t.Setenv("MAGPIE_UPDATE_FEED", feed.URL+"/api/latest") + + // none unless given: no mirror is magpie's own + if m := Mirror(context.Background()); m != "" { + t.Fatalf("a mirror by default: %q", m) + } + ctx := WithMirror(context.Background(), mirror.URL) + rel, err := Latest(ctx) + if err != nil { + t.Fatal(err) + } + path := filepath.Join(t.TempDir(), "magpie.new") + if err := download(ctx, rel.Assets[BinaryAsset()], path); err != nil { + t.Fatal(err) + } + if b, _ := os.ReadFile(path); string(b) != string(body) { + t.Fatalf("downloaded %q", b) + } + mu.Lock() + if want := []string{"/" + asset.URL}; !slices.Equal(seen, want) { + t.Fatalf("the mirror was asked %v, want %v (the feed never)", seen, want) + } + seen, tamper = nil, true + mu.Unlock() + + // what the mirror changed doesn't match the feed's checksum + os.Remove(path) + err = download(ctx, asset, path) + if err == nil || !strings.Contains(err.Error(), "does not match its checksum") || !strings.Contains(err.Error(), "mirror") { + t.Fatalf("a tampered file: %v", err) + } + if _, serr := os.Stat(path); !os.IsNotExist(serr) { + t.Fatal("the tampered file was kept") + } + + // Settings' mirror is the app's too; off turns it off for one run + mu.Lock() + tamper, seen = false, nil + mu.Unlock() + if err := settings.Save(settings.Settings{UpdateMirror: mirror.URL + "/"}); err != nil { + t.Fatal(err) + } + if m := Mirror(context.Background()); m != mirror.URL+"/" { + t.Fatalf("Settings' mirror: %q", m) + } + if err := download(context.Background(), asset, path); err != nil { + t.Fatal(err) + } + if m := Mirror(WithMirror(context.Background(), "off")); m != "" { + t.Fatalf("off: %q", m) + } + // a file not on GitHub goes as it is + if got := mirrored(mirror.URL, feed.URL+"/x"); got != feed.URL+"/x" { + t.Fatalf("a non-GitHub URL mirrored: %q", got) + } + if err := settings.Save(settings.Settings{UpdateMirror: "ftp://nope"}); err == nil { + t.Fatal("an ftp mirror was saved") + } +} + +// magpie update --proxy takes the feed and the download through the proxy +// given, over Settings' and the environment's. +func TestUpdateThroughAProxyGiven(t *testing.T) { + updateHome(t) + body := []byte("new magpie") + sum := sha256.Sum256(body) + var mu sync.Mutex + var seen []string + px := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + seen = append(seen, r.URL.String()) + mu.Unlock() + if r.URL.Path == "/asset" { + w.Write(body) + return + } + json.NewEncoder(w).Encode(Release{Version: "9.9.9", Assets: map[string]Asset{ + BinaryAsset(): {URL: "http://releases.magpie.invalid/asset", SHA256: hex.EncodeToString(sum[:])}, + }}) + })) + defer px.Close() + if err := settings.Save(settings.Settings{Proxy: "direct"}); err != nil { + t.Fatal(err) + } + t.Setenv("MAGPIE_UPDATE_FEED", "http://feed.magpie.invalid/api/latest") + ctx := WithProxy(context.Background(), px.URL) + rel, err := Latest(ctx) + if err != nil { + t.Fatal(err) + } + if err := download(ctx, rel.Assets[BinaryAsset()], filepath.Join(t.TempDir(), "magpie.new")); err != nil { + t.Fatal(err) + } + mu.Lock() + defer mu.Unlock() + if want := []string{"http://feed.magpie.invalid/api/latest", "http://releases.magpie.invalid/asset"}; !slices.Equal(seen, want) { + t.Fatalf("through the proxy: %v, want %v", seen, want) + } + for _, bad := range []string{"ftp://127.0.0.1:21", "socks4://127.0.0.1:1080"} { + if CheckProxy(bad) == nil { + t.Errorf("%s taken as a proxy", bad) + } + } + for _, good := range []string{"127.0.0.1:7890", "http://127.0.0.1:7890", "socks5://127.0.0.1:1080", "socks5h://u:p@127.0.0.1:1080", "direct"} { + if err := CheckProxy(good); err != nil { + t.Errorf("%s: %v", good, err) + } + } +} diff --git a/internal/update/update.go b/internal/update/update.go index dace7981..82d2229d 100644 --- a/internal/update/update.go +++ b/internal/update/update.go @@ -591,7 +591,10 @@ func download(ctx context.Context, a Asset, path string) error { } func fetch(ctx context.Context, a Asset, path string) error { - req, err := http.NewRequestWithContext(ctx, "GET", a.URL, nil) + // through a mirror when one is given (mirror.go): the hash checked + // below is still the feed's + u := mirrored(mirrorOf(ctx), a.URL) + req, err := http.NewRequestWithContext(ctx, "GET", u, nil) if err != nil { return err } @@ -601,6 +604,9 @@ func fetch(ctx context.Context, a Asset, path string) error { } defer res.Body.Close() if res.StatusCode != 200 { + if u != a.URL { + return fmt.Errorf("download %s through the mirror %s: %s", filepath.Base(path), strings.TrimSuffix(u, a.URL), res.Status) + } return fmt.Errorf("download %s: %s", filepath.Base(path), res.Status) } f, err := os.Create(path) @@ -623,6 +629,9 @@ func fetch(ctx context.Context, a Asset, path string) error { } if err == nil && !strings.EqualFold(hex.EncodeToString(h.Sum(nil)), a.SHA256) { err = fmt.Errorf("%s does not match its checksum", filepath.Base(path)) + if u != a.URL { + err = fmt.Errorf("%s from the mirror %s does not match its checksum from %s; not installed", filepath.Base(path), strings.TrimSuffix(u, a.URL), Site) + } } if err != nil { os.Remove(path) diff --git a/main.go b/main.go index d13ea8bf..8d2d9494 100644 --- a/main.go +++ b/main.go @@ -96,7 +96,11 @@ const usage = `magpie — one place to pick every agent's model magpie quota [] [--json] what is left of every subscription, plan and key balance magpie sync refresh the model catalog and vendor model lists magpie agents list every supported agent - magpie update [check] install the newest release (check: only say if there is one) + magpie update [check] [--proxy ] [--mirror ] + install the newest release (check: only say if there is one); --proxy: an + http(s):// or socks5:// proxy for it; --mirror: a GitHub download mirror put + before the github.com URL (none unless given; still checked against usemagpie.ai's SHA-256) + magpie update mirror [|off] the mirror every update, the app's own too, is downloaded through magpie update auto [on|off] [30m|1h|6h|24h] whether the app looks for updates by itself, and how often (6h) agents: claude (cc), codex, gemini, opencode (oc), mimocode, pi, goose, cursor, zed, copilot, crush diff --git a/site/install.test.mjs b/site/install.test.mjs index 74a175ff..32c75166 100644 --- a/site/install.test.mjs +++ b/site/install.test.mjs @@ -9,7 +9,7 @@ import { fileURLToPath } from "node:url"; const installer = fileURLToPath(new URL("./public/install.sh", import.meta.url)); -function install(t, { termux = true, arch = "aarch64", override = false, prefixOnly = false, missing = false, corrupt = false, desktop = false } = {}) { +function install(t, { termux = true, arch = "aarch64", override = false, prefixOnly = false, missing = false, corrupt = false, desktop = false, github = false, args = [], env: extra = {}, tamper = "" } = {}) { const dir = mkdtempSync(join(tmpdir(), "magpie-install-")); t.after(() => rmSync(dir, { recursive: true, force: true })); const tools = join(dir, "tools"); @@ -26,7 +26,7 @@ function install(t, { termux = true, arch = "aarch64", override = false, prefixO // A Linux asset is present even when Android's is missing: never use it. for (const name of [file, `magpie-cli-linux-${cpu}`]) { if (missing && name === file) continue; - assets[name] = { url: `https://download.invalid/${name}`, sha256: corrupt ? "0".repeat(64) : createHash("sha256").update(payload).digest("hex") }; + assets[name] = { url: github ? `https://github.com/yetone/magpie-releases/releases/download/vtest/${name}` : `https://download.invalid/${name}`, sha256: corrupt ? "0".repeat(64) : createHash("sha256").update(payload).digest("hex") }; } const requests = join(dir, "requests"); writeFileSync(join(tools, "uname"), `#!/bin/sh\ncase "$1" in -m) echo '${arch}';; -s) echo Linux;; esac\n`, { mode: 0o755 }); @@ -36,8 +36,10 @@ const fs = require("node:fs"); const args = process.argv.slice(2); const url = args.at(-1); fs.appendFileSync(process.env.INSTALL_REQUESTS, url + String.fromCharCode(10)); +fs.appendFileSync(process.env.INSTALL_ARGS, JSON.stringify(args) + String.fromCharCode(10)); +const tamper = process.env.INSTALL_TAMPER; if (url.endsWith("/api/latest")) process.stdout.write(process.env.INSTALL_FEED); -else fs.writeFileSync(args[args.indexOf("-o") + 1], process.env.INSTALL_PAYLOAD); +else fs.writeFileSync(args[args.indexOf("-o") + 1], tamper && url.startsWith(tamper) ? "changed by the mirror" : process.env.INSTALL_PAYLOAD); `, { mode: 0o755 }); const env = { ...process.env, @@ -52,9 +54,16 @@ else fs.writeFileSync(args[args.indexOf("-o") + 1], process.env.INSTALL_PAYLOAD) INSTALL_REQUESTS: requests, INSTALL_FEED: JSON.stringify({ version: "test", assets }), INSTALL_PAYLOAD: payload, + INSTALL_ARGS: join(dir, "args"), + INSTALL_TAMPER: tamper, + MAGPIE_PROXY: "", + MAGPIE_MIRROR: "", + ...extra, }; - const result = spawnSync("sh", [installer], { env, encoding: "utf8" }); - return { ...result, bin, file, payload, share: env.XDG_DATA_HOME, requests: readFileSync(requests, "utf8") }; + const result = spawnSync("sh", [installer, ...args], { env, encoding: "utf8" }); + const read = (p) => (existsSync(p) ? readFileSync(p, "utf8") : ""); + const calls = read(env.INSTALL_ARGS).trim().split("\n").filter(Boolean).map((l) => JSON.parse(l)); + return { ...result, bin, file, payload, share: env.XDG_DATA_HOME, requests: read(requests), calls }; } test("Termux downloads Android arm64 into PREFIX/bin even with WebKit installed", (t) => { @@ -104,3 +113,53 @@ test("ordinary Linux still selects its CLI or desktop build", (t) => { assert.equal(existsSync(join(r.share, "applications/magpie.desktop")), desktop); } }); + +// akic404 on Discord: installing without a way round the firewall. A proxy +// and a GitHub mirror can be given; neither is used unless given. +test("no proxy and no mirror unless given", (t) => { + const r = install(t, { termux: false, github: true }); + assert.equal(r.status, 0, r.stderr); + for (const a of r.calls) assert.ok(!a.includes("--proxy"), JSON.stringify(a)); + assert.match(r.requests, /^https:\/\/github\.com\/yetone\/magpie-releases\//m); +}); + +test("--proxy takes every download through it, as does MAGPIE_PROXY", (t) => { + for (const [opts, want] of [ + [{ args: ["--proxy", "socks5h://127.0.0.1:1080"] }, "socks5h://127.0.0.1:1080"], + [{ args: ["--proxy=http://127.0.0.1:7890"] }, "http://127.0.0.1:7890"], + [{ env: { MAGPIE_PROXY: "http://10.0.0.1:3128" } }, "http://10.0.0.1:3128"], + ]) { + const r = install(t, { termux: false, desktop: true, github: true, ...opts }); + assert.equal(r.status, 0, r.stderr); + assert.equal(r.calls.length, 3); // the feed, the file, the icon + for (const a of r.calls) assert.equal(a[a.indexOf("--proxy") + 1], want, JSON.stringify(a)); + } + const bad = install(t, { termux: false, args: ["--proxy", "ftp://x"] }); + assert.notEqual(bad.status, 0); + assert.match(bad.stderr, /--proxy is an address/); + assert.equal(bad.calls.length, 0); +}); + +test("--mirror fetches the file through the mirror, the feed and its checksum from the site", (t) => { + const r = install(t, { termux: false, github: true, args: ["--mirror", "https://gh.mirror.example"] }); + assert.equal(r.status, 0, r.stderr); + const [feed, file] = r.requests.trim().split("\n"); + assert.equal(feed, "https://usemagpie.ai/api/latest"); + assert.equal(file, "https://gh.mirror.example/https://github.com/yetone/magpie-releases/releases/download/vtest/magpie-cli-linux-arm64"); + assert.equal(readFileSync(join(r.bin, "magpie"), "utf8"), r.payload); + assert.match(r.stdout, /through https:\/\/gh\.mirror\.example\//); + + const env = install(t, { termux: false, github: true, env: { MAGPIE_MIRROR: "https://gh.mirror.example/" } }); + assert.equal(env.status, 0, env.stderr); + assert.match(env.requests, /^https:\/\/gh\.mirror\.example\/https:\/\/github\.com\//m); +}); + +test("a file the mirror changed is refused and the installed magpie kept", (t) => { + const r = install(t, { termux: false, github: true, args: ["--mirror", "https://gh.mirror.example/"], tamper: "https://gh.mirror.example/" }); + assert.notEqual(r.status, 0); + assert.match(r.stderr, /from https:\/\/gh\.mirror\.example\/ does not match its checksum from https:\/\/usemagpie\.ai; not installed/); + assert.equal(readFileSync(join(r.bin, "magpie"), "utf8"), "old installation"); + const bad = install(t, { termux: false, args: ["--mirror", "gh.mirror.example"] }); + assert.notEqual(bad.status, 0); + assert.match(bad.stderr, /--mirror is an http\(s\) address/); +}); diff --git a/site/public/docs/ja/start.html b/site/public/docs/ja/start.html index 8314537d..7f7fdf94 100644 --- a/site/public/docs/ja/start.html +++ b/site/public/docs/ja/start.html @@ -256,6 +256,8 @@ magpie claude group/opus-anywhere # これを使う

実行中のセッションは開始時の設定を使い続けます。新しいセッションを始めてください。Codex の場合は Codex アプリも再起動してください。

エージェントの設定を手で編集すると、何か壊れますか？

いいえ。magpie は毎回ファイルを読み直し、自分が設定するキーだけに触れるので、他の設定やコメントはそのまま残ります。Import… で取り込んだプロバイダはコピーなので、その後エージェント側の設定を変えても再度コピーはされません。

+

GitHub に届かない環境で、インストールや更新をするには？

+

インストーラと magpie update にプロキシを渡します：curl -fsSL https://usemagpie.ai/install.sh | sh -s -- --proxy http://127.0.0.1:7890、または magpie update --proxy socks5://127.0.0.1:1080（http・https・socks5・socks5h に対応。HTTPS_PROXY と ALL_PROXY も使われます）。--mirror <prefix> を付けると、指定した GitHub ダウンロードミラーから <prefix>https://github.com/… としてリリースファイルを取得します。magpie update mirror <prefix> でアプリ自身の更新を含め毎回使い、magpie update mirror off で解除します。指定しない限りミラーは使われず、ファイルの SHA-256 はミラーではなく常に usemagpie.ai から取得するので、ミラーが書き換えたファイルはインストールされません。アプリ自身の更新は Settings → Proxy を通ります。

困ったときや、共有したい構成があるときは Discord でどうぞ。ユーザーにすぐ使えるプロバイダを配りたいベンダーの方は magpie に追加 をご覧ください。

diff --git a/site/public/docs/start.html b/site/public/docs/start.html index b3678b70..9e8fc497 100644 --- a/site/public/docs/start.html +++ b/site/public/docs/start.html @@ -256,6 +256,8 @@ magpie claude group/opus-anywhere # use it

A running session keeps what it started with. Start a new one; for Codex, restart the Codex app too.

Does editing the agent’s config by hand break anything?

No. magpie reads the files each time and touches only the keys it sets, so your other settings and comments stay. Providers brought over with Import… are copies: later changes in the agent’s settings aren’t copied again.

+

Installing or updating behind a firewall, or without access to GitHub?

+

Give the installer and magpie update a proxy: curl -fsSL https://usemagpie.ai/install.sh | sh -s -- --proxy http://127.0.0.1:7890, or magpie update --proxy socks5://127.0.0.1:1080 (http, https, socks5 and socks5h; HTTPS_PROXY and ALL_PROXY are honoured too). --mirror <prefix> fetches the release file from a GitHub download mirror you choose, as <prefix>https://github.com/…; magpie update mirror <prefix> keeps it for every update, the app’s too, and magpie update mirror off removes it. No mirror is used unless you give one, and the file’s SHA-256 still comes from usemagpie.ai, never from the mirror: a file the mirror changed is refused. The app’s own updates go through Settings → Proxy.

Stuck, or have a setup worth sharing? Ask on Discord. Vendors who want to hand their users a ready-made provider: see Add to magpie.

diff --git a/site/public/docs/zh/start.html b/site/public/docs/zh/start.html index c7f0214a..d0a00fca 100644 --- a/site/public/docs/zh/start.html +++ b/site/public/docs/zh/start.html @@ -256,6 +256,8 @@ magpie claude group/opus-anywhere # 使用它

已在运行的会话保持启动时的模型。开一个新会话；如果是 Codex，也请重启 Codex 应用。

手动改 Agent 的配置会冲突吗？

不会。magpie 每次都重新读取文件，只动它设置的那几个键，你的其他设置和注释都会保留。用 导入… 带进来的供应商是一份拷贝：之后在 Agent 里的改动不会再同步过来。

+

没有梯子，安装或更新连不上 GitHub 怎么办？

+

给安装脚本和 magpie update 加代理：curl -fsSL https://usemagpie.ai/install.sh | sh -s -- --proxy http://127.0.0.1:7890，或 magpie update --proxy socks5://127.0.0.1:1080（支持 http、https、socks5、socks5h；环境变量 HTTPS_PROXY、ALL_PROXY 同样生效）。--mirror <前缀> 从你指定的 GitHub 下载镜像取发布文件，即 <前缀>https://github.com/…；magpie update mirror <前缀> 让之后每次更新（包括应用自己的更新）都用它，magpie update mirror off 取消。不指定就不用任何镜像；文件的 SHA-256 始终来自 usemagpie.ai，而不是镜像，镜像改过的文件会被拒绝安装。应用自己的更新走 设置 → 代理。

遇到问题，或者有值得分享的配置？来 Discord 聊。想给自己的用户一键添加供应商的厂商，请看 Add to magpie。

diff --git a/site/public/install.sh b/site/public/install.sh index ba5259ec..9731de44 100644 --- a/site/public/install.sh +++ b/site/public/install.sh @@ -8,14 +8,58 @@ # otherwise or with MAGPIE_CLI=1. # Termux: installs the Android terminal build into $PREFIX/bin. # Every download is checked against the release's SHA-256. +# +# Options (curl -fsSL https://usemagpie.ai/install.sh | sh -s -- --proxy …): +# --proxy a proxy for every download: http://, https://, +# socks5:// or socks5h:// (MAGPIE_PROXY= does the same; +# without either, curl's own https_proxy / HTTPS_PROXY / +# ALL_PROXY are used as usual) +# --mirror a GitHub download mirror: the release file is fetched +# from (MAGPIE_MIRROR= does +# the same). None is used unless given. The release +# list and its SHA-256 still come from usemagpie.ai, +# never from the mirror, so a file the mirror changed is +# refused. set -eu site=https://usemagpie.ai bin="${MAGPIE_BIN_DIR:-$HOME/.local/bin}" +proxy="${MAGPIE_PROXY:-}" +mirror="${MAGPIE_MIRROR:-}" say() { printf ' %s\n' "$*"; } die() { printf 'magpie: %s\n' "$*" >&2; exit 1; } +while [ $# -gt 0 ]; do + case "$1" in + --proxy) [ $# -ge 2 ] || die "--proxy needs a value"; proxy=$2; shift 2 ;; + --proxy=*) proxy=${1#--proxy=}; shift ;; + --mirror) [ $# -ge 2 ] || die "--mirror needs a value"; mirror=$2; shift 2 ;; + --mirror=*) mirror=${1#--mirror=}; shift ;; + -h|--help) + printf '%s\n' "usage: curl -fsSL $site/install.sh | sh -s -- [--proxy ] [--mirror ]" \ + " --proxy http://, https://, socks5:// or socks5h:// proxy for the downloads (or MAGPIE_PROXY=)" \ + " --mirror GitHub download mirror put before the release's github.com URL (or MAGPIE_MIRROR=);" \ + " the SHA-256 is still checked against $site's" + exit 0 ;; + *) die "unknown option $1 (--proxy , --mirror )" ;; + esac +done +case "$proxy" in + ''|http://?*|https://?*|socks5://?*|socks5h://?*) ;; + *) die "--proxy is an address like http://127.0.0.1:7890 or socks5://127.0.0.1:1080, not $proxy" ;; +esac +case "$mirror" in + '') ;; + http://?*|https://?*) case "$mirror" in */) ;; *) mirror="$mirror/" ;; esac ;; + *) die "--mirror is an http(s) address put before the github.com URL, like https://mirror.example/, not $mirror" ;; +esac + +# get: curl, through the proxy when one was given +get() { + if [ -n "$proxy" ]; then curl -fsSL --proxy "$proxy" "$@"; else curl -fsSL "$@"; fi +} + case "$(uname -m)" in arm64|aarch64) arch=arm64 ;; x86_64|amd64) arch=amd64 ;; @@ -42,7 +86,8 @@ esac tmp=$(mktemp -d) trap 'rm -rf "$tmp"' EXIT -feed=$(curl -fsSL "$site/api/latest") || die "could not reach $site" +# the release list and its checksums: always from the site, never a mirror +feed=$(get "$site/api/latest") || die "could not reach $site${proxy:+ through $proxy}" # The feed is one line of JSON; pull this file's url and hash out of it. entry=$(printf '%s' "$feed" | sed -n "s/.*\"$file\":{\([^}]*\)}.*/\1/p") url=$(printf '%s' "$entry" | sed -n 's/.*"url":"\([^"]*\)".*/\1/p') @@ -53,11 +98,19 @@ if [ "$os" = android ] && { [ -z "$url" ] || [ -z "$sum" ]; }; then fi [ -n "$url" ] && [ -n "$sum" ] || die "the newest release has no $file" -say "downloading magpie $version ($file)" -curl -fsSL -o "$tmp/$file" "$url" +from=$url +case "$url" in + https://github.com/*) [ -z "$mirror" ] || from="$mirror$url" ;; +esac +if [ "$from" != "$url" ]; then say "downloading magpie $version ($file) through $mirror" +else say "downloading magpie $version ($file)"; fi +get -o "$tmp/$file" "$from" || die "could not download $from" if command -v sha256sum >/dev/null; then got=$(sha256sum "$tmp/$file" | cut -d' ' -f1) else got=$(shasum -a 256 "$tmp/$file" | cut -d' ' -f1); fi -[ "$got" = "$sum" ] || die "$file does not match its checksum" +if [ "$got" != "$sum" ]; then + [ "$from" = "$url" ] || die "$file from $mirror does not match its checksum from $site; not installed" + die "$file does not match its checksum" +fi mkdir -p "$bin" if [ "$os" = darwin ]; then @@ -73,7 +126,7 @@ else if [ "$file" = "magpie-linux-$arch" ]; then share="${XDG_DATA_HOME:-$HOME/.local/share}" mkdir -p "$share/applications" "$share/icons/hicolor/256x256/apps" - curl -fsSL -o "$share/icons/hicolor/256x256/apps/magpie.png" "$site/img/icon-256.png" || true + get -o "$share/icons/hicolor/256x256/apps/magpie.png" "$site/img/icon-256.png" || true cat > "$share/applications/magpie.desktop" <|off]`: the GitHub +// download mirror every update goes through, the app's own among them +// (settings.UpdateMirror). With nothing, it says which. +func updateMirrorCmd(args []string) error { + s := settings.Load() + if len(args) > 1 { + return fmt.Errorf("magpie update mirror takes one prefix, or off") + } + if len(args) == 1 { + switch m := strings.TrimSpace(args[0]); m { + case "off", "none", "": + s.UpdateMirror = "" + default: + if err := update.CheckMirror(m); err != nil { + return err + } + s.UpdateMirror = m + } + if err := settings.Save(s); err != nil { + return err + } + } + if s.UpdateMirror == "" { + fmt.Println("updates are downloaded from GitHub", muted.Render("· magpie update mirror has them come through a mirror; the checksum is still usemagpie.ai's")) + return nil + } + fmt.Println("updates are downloaded through", bold.Render(s.UpdateMirror), muted.Render("· each is checked against usemagpie.ai's checksum · magpie update mirror off goes back to GitHub")) + return nil +} + +// updateFlags takes --proxy and --mirror (each as --flag value or +// --flag=value) out of args. +func updateFlags(args []string) (rest []string, proxy, mirror string, err error) { + for i := 0; i < len(args); i++ { + a := args[i] + name, val, eq := strings.Cut(a, "=") + if name != "--proxy" && name != "--mirror" { + rest = append(rest, a) + continue + } + if !eq { + if i+1 >= len(args) { + return nil, "", "", fmt.Errorf("%s needs a value", name) + } + i++ + val = args[i] + } + if name == "--proxy" { + if err := update.CheckProxy(val); err != nil { + return nil, "", "", err + } + proxy = val + continue + } + if val != "off" { + if err := update.CheckMirror(val); err != nil { + return nil, "", "", err + } + } + mirror = val + } + return rest, proxy, mirror, nil +} + +// updateCmd is `magpie update [check] [--proxy ] [--mirror ]`: +// the app replaces its bundle, the terminal build its binary. func updateCmd(args []string) error { + args, proxy, mirror, err := updateFlags(args) + if err != nil { + return err + } if len(args) > 1 && args[1] == "auto" { return updateAutoCmd(args[2:]) } + if len(args) > 1 && args[1] == "mirror" { + return updateMirrorCmd(args[2:]) + } + if len(args) > 2 || len(args) == 2 && args[1] != "check" { + return fmt.Errorf("usage: magpie update [check] [--proxy ] [--mirror ]") + } ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) defer cancel() + ctx = update.WithMirror(update.WithProxy(ctx, proxy), mirror) rel, err := update.Latest(ctx) if err != nil { return err @@ -81,7 +157,7 @@ func updateCmd(args []string) error { if update.Stuck(app) != "" { return fmt.Errorf("%s can't be replaced where it is; move magpie to Applications, or download the new version from %s", tilde(app), update.Site) } - fmt.Println(muted.Render(" downloading " + update.AppAsset() + " …")) + fmt.Println(muted.Render(" downloading " + update.AppAsset() + via(ctx) + " …")) staged, err := update.Stage(ctx, rel, app) if err != nil { return err @@ -106,10 +182,18 @@ func updateCmd(args []string) error { if exe, err := update.Executable(); err == nil && update.Homebrew(exe) { return fmt.Errorf("this magpie was installed with Homebrew; update it with: brew upgrade magpie") } - fmt.Println(muted.Render(" downloading " + update.BinaryAsset() + " …")) + fmt.Println(muted.Render(" downloading " + update.BinaryAsset() + via(ctx) + " …")) if err := update.ReplaceBinary(ctx, rel); err != nil { return err } fmt.Println(green.Render("✓"), "updated to", rel.Version) return nil } + +// via says the mirror a download goes through, if any. +func via(ctx context.Context) string { + if m := update.Mirror(ctx); m != "" { + return " through " + m + } + return "" +} diff --git a/update_flags_test.go b/update_flags_test.go new file mode 100644 index 00000000..52a3823a --- /dev/null +++ b/update_flags_test.go @@ -0,0 +1,44 @@ +package main + +import ( + "slices" + "testing" + + "github.com/yetone/magpie/internal/settings" +) + +// magpie update takes --proxy and --mirror, as --flag value or +// --flag=value, before or after check; magpie update mirror keeps one for +// every update, the app's too, and off clears it. +func TestUpdateFlags(t *testing.T) { + rest, proxy, mirror, err := updateFlags([]string{"update", "--proxy", "socks5://127.0.0.1:1080", "check", "--mirror=https://mirror.example/"}) + if err != nil || !slices.Equal(rest, []string{"update", "check"}) || proxy != "socks5://127.0.0.1:1080" || mirror != "https://mirror.example/" { + t.Fatalf("%v %q %q %v", rest, proxy, mirror, err) + } + for _, bad := range [][]string{ + {"update", "--proxy"}, + {"update", "--proxy", "ftp://x:1"}, + {"update", "--mirror", "mirror.example"}, + } { + if _, _, _, err := updateFlags(bad); err == nil { + t.Errorf("%v taken", bad) + } + } + if err := updateCmd([]string{"update", "chek"}); err == nil { + t.Error("magpie update chek ran an update") + } + + groupsHome(t) + if err := updateCmd([]string{"update", "mirror", "https://mirror.example/"}); err != nil { + t.Fatal(err) + } + if got := settings.Load().UpdateMirror; got != "https://mirror.example/" { + t.Fatalf("saved %q", got) + } + if err := updateCmd([]string{"update", "mirror", "nope"}); err == nil || settings.Load().UpdateMirror != "https://mirror.example/" { + t.Fatalf("a bad mirror: %v, kept %q", err, settings.Load().UpdateMirror) + } + if err := updateCmd([]string{"update", "mirror", "off"}); err != nil || settings.Load().UpdateMirror != "" { + t.Fatalf("off: %v, kept %q", err, settings.Load().UpdateMirror) + } +}