From e5f8ff6026139f49b5d4c40ea1ff80e7d120be9a Mon Sep 17 00:00:00 2001 From: yetone Date: Sat, 3 Oct 2026 17:12:45 +0800 Subject: [PATCH] =?UTF-8?q?site,=20gui:=20release=20notes=20survive=20GitH?= =?UTF-8?q?ub=20limiting=20usemagpie.ai,=20and=20a=20failure=20to=20get=20?= =?UTF-8?q?them=20says=20so,=20apart=20from=20a=20release=20without=20any?= =?UTF-8?q?=20(#661)=20=E2=80=94=20/api/notes=20answered=20503=20{"error":?= =?UTF-8?q?"no=20releases"}=20with=20Cache-Control=20public,=20max-age=3D3?= =?UTF-8?q?00=20whenever=20GitHub's=20API=20limited=20the=20worker's=20sha?= =?UTF-8?q?red=20IP,=20and=20/api/latest=20fell=20back=20to=20the=20releas?= =?UTF-8?q?e=20page's=20redirect=20with=20notes=20""=20(kept=20300=20s=20t?= =?UTF-8?q?oo),=20so=20the=20update=20window=20had=20nothing=20to=20show.?= =?UTF-8?q?=20The=20worker=20now:=20never=20keeps=20an=20error=20(503=20is?= =?UTF-8?q?=20no-store);=20takes=20the=20notes=20from=20GitHub's=20release?= =?UTF-8?q?s=20feed=20(releases.atom,=20not=20API-limited)=20when=20the=20?= =?UTF-8?q?API=20says=20no,=20turning=20its=20HTML=20back=20into=20the=20m?= =?UTF-8?q?arkdown=20the=20app=20draws=20and=20putting=20back=20the=20=20marker=20GitHub=20strips=20(before=20the=20f?= =?UTF-8?q?irst=20Chinese=20heading=20after=20an=20English=20one);=20keeps?= =?UTF-8?q?=20the=20last=20whole=20answer=20for=2030=20days=20in=20the=20e?= =?UTF-8?q?dge=20cache=20and=20serves=20it=20(merged=20with=20the=20feed's?= =?UTF-8?q?=20newer=20releases)=20when=20GitHub=20fails;=20keeps=20a=20deg?= =?UTF-8?q?raded=20answer=2060=20s=20instead=20of=20300;=20and=20sends=20a?= =?UTF-8?q?=20GITHUB=5FTOKEN=20secret=20to=20the=20API=20where=20one=20is?= =?UTF-8?q?=20set=20(none=20is=20yet).=20In=20the=20app,=20/api/whatsnew?= =?UTF-8?q?=20says=20error=20(and=20the=20release=20page's=20url)=20when?= =?UTF-8?q?=20the=20notes=20couldn't=20be=20had,=20and=20Settings'=20What'?= =?UTF-8?q?s=20new=20opens=20a=20dialog=20saying=20"Couldn't=20load=20the?= =?UTF-8?q?=20release=20notes"=20with=20Try=20again=20and=20"Open=20the=20?= =?UTF-8?q?release=20page"=20(through=20the=20app),=20or=20"No=20release?= =?UTF-8?q?=20notes=20provided."=20with=20the=20release=20page=20and=20no?= =?UTF-8?q?=20retry=20for=20a=20version=20with=20none,=20instead=20of=20a?= =?UTF-8?q?=20passing=20toast;=20en/zh.=20Tests=20that=20fail=20on=20the?= =?UTF-8?q?=20old=20code=20and=20pass:=20site/worker.test.mjs=20(API=20lim?= =?UTF-8?q?ited:=20notes=20from=20the=20feed=20in=20either=20language,=20b?= =?UTF-8?q?riefly=20kept;=20all=20of=20GitHub=20down:=20503=20no-store,=20?= =?UTF-8?q?then=20the=20last=20whole=20answer;=20the=20token=20sent),=20Te?= =?UTF-8?q?stWhatsNewTellsFailureFromNone,=20and=20whats-new-failed.test.c?= =?UTF-8?q?js=20(WebKit,=20en/zh:=20failure=20with=20retry=20and=20link,?= =?UTF-8?q?=20retry=20showing=20the=20notes,=20none=20without=20retry,=20n?= =?UTF-8?q?o=20click=20moves=20the=20page).=20The=20feed=20parser=20was=20?= =?UTF-8?q?tried=20on=20the=20real=20releases.atom=20(v0.1.730=E2=80=93739?= =?UTF-8?q?:=20English=20and=20Chinese=20split=20as=20the=20API's=20bodies?= =?UTF-8?q?=20are).=20go=20vet,=20builds=20(darwin,=20GOOS=3Dlinux,=20GOOS?= =?UTF-8?q?=3Dwindows),=20go=20test=20-tags=20nogui=20-count=3D1=20./...?= =?UTF-8?q?=20under=20a=20temp=20HOME,=20node=20--test=20site/=20and=20wha?= =?UTF-8?q?ts-new.test.cjs=20pass.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/gui/assets/app.css | 3 + internal/gui/assets/app.js | 34 ++- internal/gui/assets/i18n.js | 2 + internal/gui/tests/whats-new-failed.test.cjs | 147 +++++++++++ internal/gui/whatsnew.go | 36 ++- internal/gui/whatsnew_fail_test.go | 49 ++++ internal/update/notes.go | 6 + site/worker.js | 253 +++++++++++++++---- site/worker.test.mjs | 119 ++++++++- 9 files changed, 590 insertions(+), 59 deletions(-) create mode 100644 internal/gui/tests/whats-new-failed.test.cjs create mode 100644 internal/gui/whatsnew_fail_test.go diff --git a/internal/gui/assets/app.css b/internal/gui/assets/app.css index 6e321a02..1ad0a8e7 100644 --- a/internal/gui/assets/app.css +++ b/internal/gui/assets/app.css @@ -2261,6 +2261,9 @@ html.theming, html.theming *, html.theming *::before, html.theming *::after { .whatsnew .wn-notes code { font: 11.5px/1.4 var(--mono, ui-monospace, monospace); padding: 0 4px; border-radius: 4px; background: var(--pill); } .whatsnew .wn-link { color: var(--accent); text-decoration: none; } .whatsnew .wn-link:hover { text-decoration: underline; } +.whatsnew .wn-msg { margin: 4px 0 6px; font-size: 12.5px; color: var(--fg-2); } +.whatsnew .wn-failed .wn-msg { color: var(--red); } +.whatsnew .wn-acts { display: flex; align-items: center; gap: 12px; font-size: 12.5px; } /* Keep native window chrome outside the scrim and dialog. In particular, the macOS traffic lights must never sit on top of modal content. */ body.window.mac .modal { top: calc(50px / var(--zoom, 1)); padding: 16px 24px 24px; } diff --git a/internal/gui/assets/app.js b/internal/gui/assets/app.js index 213a785c..2522f12a 100644 --- a/internal/gui/assets/app.js +++ b/internal/gui/assets/app.js @@ -1983,6 +1983,8 @@ async function whatsNewOnce() { // openWhatsNew is Settings' way back to the notes: the current version's (or // those since the last update), after the waiting update's when u has one. +// The notes that couldn't be had (the site failing, #661) say so, with a +// retry and the release page, apart from a version that has none. async function openWhatsNew(u, b) { if (b) { b.disabled = true; b.classList.add("busy"); } const w = await api(updatePath("whatsnew?all=1")).catch(() => null); @@ -1991,16 +1993,20 @@ async function openWhatsNew(u, b) { if (u?.notes && u.latest && ["ready", "available", "downloading"].includes(u.state) && !list.some((r) => r.version === u.latest)) { list.unshift({ version: u.latest, notes: u.notes, url: u.url, pending: true }); } - if (!list.length) return status(t("Couldn't load the release notes"), "err"); - showWhatsNew(list); + const failed = !w || !!w.error; + if (!list.length && !w?.current) return status(t("Couldn't load the release notes"), "err"); + showWhatsNew(list, false, { failed, current: w?.current, url: w?.url, retry: () => openWhatsNew(u) }); } // auto: shown by itself after an update, with "Don't show again today" // (#525: releases come many a day); Settings' row opens it without one. -function showWhatsNew(releases, auto) { +// more (from Settings): failed, the current version's notes couldn't be +// had, retry asks again; current and url, the version and its release +// page, for a dialog with no notes. +function showWhatsNew(releases, auto, more = {}) { const ed = el("div", "editor whatsnew"); const head = el("div", "ehead"); - head.append(el("b", "", t("What's new in {v}", { v: "v" + releases[0].version }))); + head.append(el("b", "", t("What's new in {v}", { v: "v" + (releases[0]?.version || more.current) }))); ed.append(head); for (const r of releases) { const sec = el("section", "wn-rel"); @@ -2010,6 +2016,26 @@ function showWhatsNew(releases, auto) { sec.append(h, noteBlocks(r.notes)); ed.append(sec); } + if (more.failed || !releases.length) { + const sec = el("section", "wn-rel wn-none" + (more.failed ? " wn-failed" : "")); + const h = el("div", "wn-ver"); + if (more.current) h.append(el("b", "", "v" + more.current)); + const p = el("p", "wn-msg", more.failed ? t("Couldn't load the release notes") : t("No release notes provided.")); + const acts = el("div", "wn-acts"); + if (more.failed && more.retry) { + const again = el("button", "text", t("Try again")); + again.onclick = async (e) => { + e.stopPropagation(); + again.disabled = true; + again.classList.add("busy"); + await more.retry(); + }; + acts.append(again); + } + if (more.url) acts.append(noteLink(t("Open the release page"), more.url)); + sec.append(h, p, acts); + ed.append(sec); + } const bar = el("div", "bar"); const ok = el("button", "text primary", t("Close")); ok.onclick = (e) => { e.stopPropagation(); closeConfirmAsk(); }; diff --git a/internal/gui/assets/i18n.js b/internal/gui/assets/i18n.js index f1197eff..222c8bdb 100644 --- a/internal/gui/assets/i18n.js +++ b/internal/gui/assets/i18n.js @@ -1656,6 +1656,8 @@ const I18N = { "Not installed yet": "尚未安装", "Don't show again today": "今天不再弹出", "Couldn't load the release notes": "无法获取更新说明", + "No release notes provided.": "这个版本没有更新说明。", + "Open the release page": "打开发布页面", "Check": "检查", "Couldn't check for updates": "检查更新失败", "Couldn't download {v}": "{v} 下载失败", diff --git a/internal/gui/tests/whats-new-failed.test.cjs b/internal/gui/tests/whats-new-failed.test.cjs new file mode 100644 index 00000000..c434039a --- /dev/null +++ b/internal/gui/tests/whats-new-failed.test.cjs @@ -0,0 +1,147 @@ +// Run with Node's test runner and Playwright on the module path; see README.md. +// #661: the site failing to give the release notes (a 503 while GitHub +// limited it) left Settings' What's new with nothing but a passing "Couldn't +// load the release notes". Now the dialog opens and says so, with Try again +// and the release page; a version whose notes are really empty says "No +// release notes provided." instead, with the release page and no retry. Try +// again shows the notes once they come; no click moves the page; the release +// page opens in the browser through the app. In English and Chinese; the API +// is faked. +const assert = require("node:assert/strict"); +const fs = require("node:fs/promises"); +const path = require("node:path"); +const { test } = require("node:test"); +const { chromium, webkit } = require("playwright"); + +const assets = path.resolve(__dirname, "../assets"); +const PAGE = "https://github.com/yetone/magpie-releases/releases/tag/v0.1.737"; + +function settingsPayload(lang) { + return { + theme: "light", lang, tray: "panel", quotaLeft: false, currency: "usd", + dock: false, dockWindow: false, proxy: "", redact: false, redactPersonal: false, redactWords: [], + codexWarmup: "", claudeWarmup: "", codexWarmAt: "", claudeWarmAt: "", workbuddyCheckin: false, noStats: false, + trayUsage: "", trayUsageEvery: 3, vision: "", imageGen: "", + version: "0.1.737", dir: "~/.config/magpie", gateway: "http://127.0.0.1:3425", + proxyNow: "none", proxySource: "none", login: false, + visionModels: [], imageGenModels: [], workbuddyCheckins: [], lanURLs: [], + fx: { rate: 7.2, at: new Date().toISOString(), stale: false }, + }; +} + +// ctl.mode: "fail" (the site failing), "none" (no notes), "ok" +function serve(lang, ctl) { + return async (route) => { + const req = route.request(), url = new URL(req.url()); + const json = (data) => route.fulfill({ json: data }); + if (url.pathname === "/boot.js") return route.fulfill({ contentType: "text/javascript", body: `window.bootPrefs = {lang:"${lang}",theme:"light",web:false};` }); + if (url.pathname === "/wails/runtime.js") return route.fulfill({ contentType: "text/javascript", body: "export const Window = {};" }); + if (url.pathname === "/api/state") return json({ agents: [], profiles: [], settings: { lang, theme: "light" } }); + if (url.pathname === "/api/settings") return json(settingsPayload(lang)); + if (url.pathname === "/api/plugins") return json({ plugins: [] }); + if (url.pathname === "/api/groups") return json({ groups: [], models: [] }); + if (url.pathname === "/api/usage/quotas") return json([]); + if (url.pathname === "/api/update") return json({ state: "latest", current: "0.1.737" }); + if (url.pathname === "/api/whatsnew") { + const all = url.searchParams.has("all"); + if (!all) return json({ show: false, current: "0.1.737", releases: [] }); + ctl.asked++; + const base = { show: false, current: "0.1.737", url: PAGE }; + if (ctl.mode === "fail") return json({ ...base, releases: [], error: "release notes: 503 Service Unavailable" }); + if (ctl.mode === "none") return json({ ...base, releases: [] }); + return json({ ...base, releases: [{ version: "0.1.737", notes: "## Bug Fixes\n\n- Notes came back. (#661)" }] }); + } + if (url.pathname === "/api/open") { ctl.opened.push(req.postDataJSON().url); return route.fulfill({ status: 204 }); } + if (url.pathname.startsWith("/api/")) return json({}); + const file = path.join(assets, url.pathname === "/" ? "index.html" : url.pathname); + const contentType = { ".html": "text/html", ".js": "text/javascript", ".css": "text/css", ".svg": "image/svg+xml", ".png": "image/png" }[path.extname(file)]; + await route.fulfill({ body: await fs.readFile(file), contentType }); + }; +} + +const words = { + en: { open: "Open", failed: "Couldn't load the release notes", none: "No release notes provided.", retry: "Try again", page: "Open the release page", title: "What's new in v0.1.737" }, + zh: { open: "打开", failed: "无法获取更新说明", none: "这个版本没有更新说明。", retry: "重试", page: "打开发布页面", title: "v0.1.737 更新内容" }, +}; + +for (const engine of (process.env.BROWSER ? [process.env.BROWSER] : ["chromium", "webkit"])) { + for (const lang of ["en", "zh"]) { + const w = words[lang]; + test(`${engine} ${lang}: release notes that couldn't be had, apart from none (#661)`, async (t) => { + const browser = await (engine === "webkit" ? webkit.launch() : chromium.launch({ channel: "chromium" })); + const context = await browser.newContext({ viewport: { width: 900, height: 700 }, reducedMotion: "reduce" }); + t.after(async () => browser.close()); + const ctl = { mode: "fail", asked: 0, opened: [] }; + const page = await context.newPage(); + page.setDefaultTimeout(5000); + const errors = []; + page.on("pageerror", (e) => errors.push(e.message)); + page.on("dialog", (d) => { errors.push("dialog: " + d.message()); d.dismiss(); }); + await page.route("**/*", serve(lang, ctl)); + await page.goto("http://magpie.test/?view=settings&tab=about"); + const dialog = page.locator("#modal .editor.whatsnew"); + const view = page.locator("#view-settings"); + const again = page.locator("#about .row.pref.whatsnew-row").getByRole("button", { name: w.open, exact: true }); + await again.waitFor(); + await page.mouse.move(450, 400); + for (let i = 0; i < 60; i++) { + const b = await again.boundingBox(); + if (b && b.y > 80 && b.y + b.height < 640) break; + await page.mouse.wheel(0, !b || b.y > 80 ? 120 : -120); + await page.waitForTimeout(30); + } + await page.waitForTimeout(200); + const at = await view.evaluate((v) => v.scrollTop); + const still = async (what) => assert.equal(await view.evaluate((v) => v.scrollTop), at, what + " moved the page"); + + // the site failing: said so, with Try again and the release page + await again.click(); + await dialog.waitFor(); + await still("Open"); + assert.equal(await dialog.locator(".ehead b").textContent(), w.title); + assert.equal(await dialog.locator(".wn-msg").textContent(), w.failed); + assert.equal(await dialog.getByText(w.none).count(), 0); + const retry = dialog.getByRole("button", { name: w.retry }); + const link = dialog.getByRole("link", { name: w.page }); + assert.equal(await link.getAttribute("href"), PAGE); + if (process.env.ARTIFACT_DIR) { + await fs.mkdir(process.env.ARTIFACT_DIR, { recursive: true }); + await page.waitForTimeout(400); + await page.screenshot({ path: path.join(process.env.ARTIFACT_DIR, `${engine}-${lang}-notes-failed.png`) }); + } + const border = await page.evaluate(() => [...document.querySelectorAll("#modal .whatsnew, #modal .whatsnew *")].map((e) => getComputedStyle(e).borderLeftWidth).filter((b) => parseFloat(b) > 1)); + assert.deepEqual(border, [], "no left-border accent"); + // the release page opens in the browser, through the app + await link.click(); + for (let i = 0; i < 40 && !ctl.opened.length; i++) await page.waitForTimeout(25); + assert.deepEqual(ctl.opened, [PAGE]); + assert.equal(page.url(), "http://magpie.test/?view=settings&tab=about", "the page didn't navigate"); + // Try again while the site still fails: the same, asked again + await retry.click(); + for (let i = 0; i < 40 && ctl.asked < 2; i++) await page.waitForTimeout(25); + await dialog.getByRole("button", { name: w.retry }).waitFor(); + assert.equal(ctl.asked, 2); + // Try again once the site answers: the notes + ctl.mode = "ok"; + await dialog.getByRole("button", { name: w.retry }).click(); + await dialog.locator(".wn-notes li").waitFor(); + assert.equal(await dialog.locator(".wn-msg").count(), 0); + assert.equal(await dialog.locator(".wn-notes li").textContent(), "Notes came back. (#661)"); + await still("Try again"); + await page.keyboard.press("Escape"); + await page.locator("#modal").waitFor({ state: "hidden" }); + + // a version with no notes: said so, the release page, no retry + ctl.mode = "none"; + await again.click(); + await dialog.waitFor(); + await still("Open"); + assert.equal(await dialog.locator(".wn-msg").textContent(), w.none); + assert.equal(await dialog.getByRole("button", { name: w.retry }).count(), 0); + assert.equal(await dialog.getByRole("link", { name: w.page }).getAttribute("href"), PAGE); + await page.keyboard.press("Escape"); + await page.locator("#modal").waitFor({ state: "hidden" }); + assert.deepEqual(errors, []); + }); + } +} diff --git a/internal/gui/whatsnew.go b/internal/gui/whatsnew.go index 07c3d30a..789f3c4b 100644 --- a/internal/gui/whatsnew.go +++ b/internal/gui/whatsnew.go @@ -136,6 +136,11 @@ type whatsNewJSON struct { Show bool `json:"show"` Current string `json:"current"` Releases []update.Note `json:"releases"` + // Error: the notes couldn't be had (the site down, or GitHub failing + // it), as opposed to releases that have none (#661); URL is the + // current version's release page, where they can be read instead + Error string `json:"error,omitempty"` + URL string `json:"url,omitempty"` } // get is what a page shows: after an upgrade, the notes since, once (a @@ -156,8 +161,13 @@ func (n *whatsNew) get(ctx context.Context, all bool, lang string) whatsNewJSON if (!popup && !all) || !update.Released(Version) { return j } + j.URL = update.ReleasePage(Version) if notes == nil { - notes = fetchNotes(ctx, after, Version, lang) + var err error + notes, err = fetchNotes(ctx, after, Version, lang) + if err != nil { + j.Error = err.Error() + } if len(notes) > 0 { // kept: they don't change while magpie runs n.mu.Lock() if n.notes == nil { @@ -182,22 +192,28 @@ func (n *whatsNew) seen() { // fetchNotes asks the site for the notes since after; a site without the // list yet, or one that can't be reached, leaves the update feed, whose -// newest release has its notes, for when that is the current one. -func fetchNotes(ctx context.Context, after, upto, lang string) []update.Note { +// newest release has its notes, for when that is the current one. The +// error says the notes couldn't be had, which the page tells apart from +// releases without notes (#661): the list failed and the feed had none. +func fetchNotes(ctx context.Context, after, upto, lang string) ([]update.Note, error) { ctx, cancel := context.WithTimeout(ctx, 15*time.Second) defer cancel() - notes, err := update.NotesBetween(ctx, after, upto, lang) - if err == nil && len(notes) > 0 { - return notes + notes, failed := update.NotesBetween(ctx, after, upto, lang) + if failed == nil && len(notes) > 0 { + return notes, nil } - if err != nil { - log.Println("release notes:", err) + if failed != nil { + log.Println("release notes:", failed) } rel, err := update.LatestIn(ctx, lang) if err != nil { - return nil + return nil, failed } - return update.Between([]update.Note{{Version: rel.Version, Notes: rel.Notes, URL: rel.URL}}, after, upto) + notes = update.Between([]update.Note{{Version: rel.Version, Notes: rel.Notes, URL: rel.URL}}, after, upto) + if len(notes) == 0 { + return nil, failed + } + return notes, nil } // pageLang is the language a page asks in, its own (?lang=, zh or en), or diff --git a/internal/gui/whatsnew_fail_test.go b/internal/gui/whatsnew_fail_test.go new file mode 100644 index 00000000..9fa5ef01 --- /dev/null +++ b/internal/gui/whatsnew_fail_test.go @@ -0,0 +1,49 @@ +package gui + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/yetone/magpie/internal/update" +) + +// #661: the site failing to give the notes (a 503 while GitHub limits it, +// and the update feed's newest without notes) is told apart from releases +// that have none, and the page gets the release page to read them on. +func TestWhatsNewTellsFailureFromNone(t *testing.T) { + t.Setenv("XDG_CONFIG_HOME", t.TempDir()) + down := true + notes := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if down { + w.Header().Set("Cache-Control", "public, max-age=300") + http.Error(w, `{"error":"no releases"}`, http.StatusServiceUnavailable) + return + } + json.NewEncoder(w).Encode(map[string]any{"releases": []update.Note{}}) + })) + defer notes.Close() + feed := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + json.NewEncoder(w).Encode(update.Release{Version: "0.1.737"}) + })) + defer feed.Close() + t.Setenv("MAGPIE_NOTES_FEED", notes.URL) + t.Setenv("MAGPIE_UPDATE_FEED", feed.URL) + old := Version + defer func() { Version = old }() + Version = "0.1.737" + n := &whatsNew{} + + j := n.get(context.Background(), true, "en") + if j.Error == "" || len(j.Releases) != 0 || j.URL != "https://github.com/yetone/magpie-releases/releases/tag/v0.1.737" { + t.Fatalf("site failing: %+v", j) + } + // asked again once the site answers: no notes, and no error + down = false + j = n.get(context.Background(), true, "en") + if j.Error != "" || len(j.Releases) != 0 || j.URL == "" { + t.Fatalf("no notes: %+v", j) + } +} diff --git a/internal/update/notes.go b/internal/update/notes.go index 90e7416d..f77c8e7b 100644 --- a/internal/update/notes.go +++ b/internal/update/notes.go @@ -29,6 +29,12 @@ func NotesFeed() string { return Site + "/api/notes" } +// ReleasePage is version's page on GitHub, where its notes are when the +// site can't give them (#661). +func ReleasePage(version string) string { + return "https://github.com/yetone/magpie-releases/releases/tag/v" + strings.TrimPrefix(strings.TrimSpace(version), "v") +} + // NotesBetween asks the site for the notes of every release after after // (none when "") up to and including upto, newest first, in lang (see // InLang). diff --git a/site/worker.js b/site/worker.js index 5be15834..a8aeab75 100644 --- a/site/worker.js +++ b/site/worker.js @@ -48,20 +48,20 @@ export default { return Response.redirect(url.toString(), 301); } if (url.pathname === "/api/latest") { - const rel = await latest(ctx); - if (!rel) return json({ error: "no release yet" }, 503); + const got = await latest(ctx, env); + if (!got) return json({ error: "no release yet" }, 503, NO_STORE); const lang = url.searchParams.get("lang"); - return json({ ...rel, notes: inLang(rel.notes, lang) }, 200, { "Cache-Control": `public, max-age=${TTL}` }); + return json({ ...got.rel, notes: inLang(got.rel.notes, lang) }, 200, cacheFor(got)); } if (url.pathname === "/api/notes") { - const list = await releases(ctx); - if (!list) return json({ error: "no releases" }, 503); + const got = await releases(ctx, env); + if (!got) return json({ error: "no releases" }, 503, NO_STORE); const after = url.searchParams.get("after"), upto = url.searchParams.get("upto"); const lang = url.searchParams.get("lang"); - const pick = list + const pick = got.list .filter((r) => (!after || newer(r.version, after)) && (!upto || !newer(r.version, upto))) .map((r) => ({ ...r, notes: inLang(r.notes, lang) })); - return json({ releases: pick }, 200, { "Cache-Control": `public, max-age=${TTL}` }); + return json({ releases: pick }, 200, cacheFor(got)); } if (url.pathname === "/download" || url.pathname.startsWith("/download/")) { const want = url.pathname.split("/")[2] || "mac-arm64"; @@ -160,29 +160,75 @@ function beacon(res) { return new HTMLRewriter().on("body", { element: (el) => el.append(BEACON, { html: true }) }).transform(res); } -// latest is the newest release, condensed, with each file's SHA-256 taken -// from the release's SHA256SUMS. GitHub's API gives the notes and sizes but -// limits anonymous callers by IP, and a worker shares its IP with many -// others; when the API says no, the release page's redirect gives the -// version and SHA256SUMS the files, which is all an update needs. -async function latest(ctx) { - const cache = caches.default; - const key = new Request("https://usemagpie.ai/__latest"); - const hit = await cache.match(key); - if (hit) return hit.json(); +// How long answers are kept. A whole answer (from GitHub's API) for TTL; a +// degraded one (the API said no, so the release page and the releases feed +// stood in, or the last whole answer did) for BRIEF, so that one bad minute +// at GitHub doesn't leave apps without notes for long (#661). An error is +// never kept: the next ask tries again. +const BRIEF = 60; +const KEEP = 30 * 86400; // the last whole answer, for when GitHub fails +const NO_STORE = { "Cache-Control": "no-store" }; - const rel = (await fromAPI()) || (await fromPages()); +function cacheFor(got) { + return { "Cache-Control": `public, max-age=${got.degraded ? BRIEF : TTL}` }; +} + +// cached and remember are the edge's cache (caches.default) by name. +async function cached(name) { + const hit = await caches.default.match(new Request(`https://usemagpie.ai/__${name}`)); + return hit ? hit.json() : null; +} + +function remember(ctx, name, v, ttl) { + ctx.waitUntil(caches.default.put(new Request(`https://usemagpie.ai/__${name}`), json(v, 200, { "Cache-Control": `max-age=${ttl}` }))); +} + +// latest is the newest release, condensed, with each file's SHA-256 taken +// from the release's SHA256SUMS: {rel, degraded}, or null when GitHub can't +// say. GitHub's API gives the notes and sizes but limits anonymous callers +// by IP, and a worker shares its IP with many others (a GITHUB_TOKEN secret +// lifts that: `npx wrangler secret put GITHUB_TOKEN`); when the API says +// no, the release page's redirect gives the version, SHA256SUMS the files +// and the releases feed the notes, which is all an update needs. The last +// whole answer is kept, and stands in for a degraded one of its version. +async function latest(ctx, env) { + const hit = await cached("latest"); + if (hit) return hit; + const full = await fromAPI(env); + if (full) { + const got = { rel: full, degraded: false }; + remember(ctx, "latest", got, TTL); + remember(ctx, "latest-good", full, KEEP); + return got; + } + const good = await cached("latest-good"); + const pages = await fromPages(); + let rel = pages; + if (good && (!pages || !newer(pages.version, good.version))) rel = good; + else if (pages && !pages.notes && good && good.version === pages.version) rel = good; if (!rel) return null; - ctx.waitUntil(cache.put(key, json(rel, 200, { "Cache-Control": `max-age=${TTL}` }))); - return rel; + const got = { rel, degraded: true }; + remember(ctx, "latest", got, BRIEF); + return got; } const UA = { "User-Agent": "usemagpie.ai" }; -async function fromAPI() { - const res = await fetch(`https://api.github.com/repos/${REPO}/releases/latest`, { - headers: { ...UA, Accept: "application/vnd.github+json" }, - }); +// api asks GitHub's API, with the GITHUB_TOKEN secret where one is set. +function api(path, env) { + const headers = { ...UA, Accept: "application/vnd.github+json" }; + if (env && env.GITHUB_TOKEN) headers.Authorization = `Bearer ${env.GITHUB_TOKEN}`; + return fetch(`https://api.github.com/repos/${REPO}/${path}`, { headers }); +} + +async function fromAPI(env) { + let res; + try { + res = await api("releases/latest", env); + } catch (e) { + console.log("github api", e); + return null; + } if (!res.ok) { console.log("github api", res.status, await res.text()); return null; @@ -200,16 +246,23 @@ async function fromAPI() { } async function fromPages() { - const res = await fetch(`https://github.com/${REPO}/releases/latest`, { headers: UA, redirect: "manual" }); + let res; + try { + res = await fetch(`https://github.com/${REPO}/releases/latest`, { headers: UA, redirect: "manual" }); + } catch (e) { + console.log("github latest redirect", e); + return null; + } const tag = (res.headers.get("Location") || "").split("/tag/")[1]; if (!tag) { console.log("github latest redirect", res.status); return null; } - const sums = await checksums(tag); + const [sums, feed] = await Promise.all([checksums(tag), fromFeed()]); if (!sums) return null; const url = `https://github.com/${REPO}/releases/tag/${tag}`; - const rel = { version: tag.replace(/^v/, ""), notes: "", url, published: null, assets: {} }; + const entry = (feed || []).find((r) => r.version === tag.replace(/^v/, "")); + const rel = { version: tag.replace(/^v/, ""), notes: entry ? entry.notes : "", url, published: entry ? entry.published : null, assets: {} }; for (const [name, sha256] of Object.entries(sums)) { rel.assets[name] = { url: `https://github.com/${REPO}/releases/download/${tag}/${name}`, size: 0, sha256 }; } @@ -217,24 +270,130 @@ async function fromPages() { } // releases is the newest hundred releases' notes, newest first, kept as -// long as the newest release is. Drafts and pre-releases are left out. -async function releases(ctx) { - const cache = caches.default; - const key = new Request("https://usemagpie.ai/__releases"); - const hit = await cache.match(key); - if (hit) return hit.json(); - const res = await fetch(`https://api.github.com/repos/${REPO}/releases?per_page=100`, { - headers: { ...UA, Accept: "application/vnd.github+json" }, - }); - if (!res.ok) { - console.log("github api releases", res.status, await res.text()); +// long as the newest release is: {list, degraded}, or null when GitHub +// can't say. Drafts and pre-releases are left out. When the API says no, +// the releases feed (the newest ten, not rate-limited) and the last whole +// list stand in. +async function releases(ctx, env) { + const hit = await cached("releases"); + if (hit) return hit; + let res = null; + try { + res = await api("releases?per_page=100", env); + } catch (e) { + console.log("github api releases", e); + } + if (res && res.ok) { + const list = (await res.json()) + .filter((r) => !r.draft && !r.prerelease) + .map((r) => ({ version: r.tag_name.replace(/^v/, ""), notes: r.body || "", url: r.html_url, published: r.published_at })); + const got = { list, degraded: false }; + remember(ctx, "releases", got, TTL); + remember(ctx, "releases-good", list, KEEP); + return got; + } + if (res) console.log("github api releases", res.status, await res.text()); + const [good, feed] = await Promise.all([cached("releases-good"), fromFeed()]); + if (!good && !feed) return null; + // the last whole list's notes (markdown, both languages) where it has the + // release, the feed's for the newer ones + const have = new Set((good || []).map((r) => r.version)); + const list = [...(feed || []).filter((r) => !have.has(r.version)), ...(good || [])]; + list.sort((a, b) => (newer(a.version, b.version) ? -1 : newer(b.version, a.version) ? 1 : 0)); + const got = { list, degraded: true }; + remember(ctx, "releases", got, BRIEF); + return got; +} + +// fromFeed is the newest releases from GitHub's releases feed (Atom), which +// isn't the API and so isn't limited by it: [{version, notes, url, +// published}], newest first, or null. The feed has each release's notes as +// HTML, which notes() turns back into the markdown the app draws. +async function fromFeed() { + let res; + try { + res = await fetch(`https://github.com/${REPO}/releases.atom`, { headers: UA }); + } catch (e) { + console.log("github feed", e); return null; } - const list = (await res.json()) - .filter((r) => !r.draft && !r.prerelease) - .map((r) => ({ version: r.tag_name.replace(/^v/, ""), notes: r.body || "", url: r.html_url, published: r.published_at })); - ctx.waitUntil(cache.put(key, json(list, 200, { "Cache-Control": `max-age=${TTL}` }))); - return list; + if (!res.ok) { + console.log("github feed", res.status); + return null; + } + return parseFeed(await res.text()); +} + +export function parseFeed(xml) { + const out = []; + for (const [, entry] of xml.matchAll(/([\s\S]*?)<\/entry>/g)) { + const href = (entry.match(/]*href="([^"]*\/releases\/tag\/[^"]+)"/) || [])[1]; + if (!href) continue; + const url = unxml(href); + const tag = decodeURIComponent(url.split("/tag/")[1]); + if (/-/.test(tag)) continue; // a pre-release + const content = (entry.match(/]*>([\s\S]*?)<\/content>/) || [])[1] || ""; + const updated = (entry.match(/([^<]*)<\/updated>/) || [])[1] || null; + out.push({ version: tag.replace(/^v/, ""), notes: notes(unxml(content)), url, published: updated }); + } + return out; +} + +function unxml(s) { + return s.replace(/</g, "<").replace(/>/g, ">").replace(/"/g, '"').replace(/'/g, "'").replace(/&/g, "&"); +} + +function unhtml(s) { + return s + .replace(/&#x([0-9a-f]+);/gi, (_, h) => String.fromCodePoint(parseInt(h, 16))) + .replace(/&#(\d+);/g, (_, d) => String.fromCodePoint(+d)) + .replace(/ /g, " ") + .replace(/</g, "<").replace(/>/g, ">").replace(/"/g, '"').replace(/'|'/g, "'") + .replace(/&/g, "&"); +} + +// inline is a piece of HTML as markdown text: bold, code and links kept, +// every other tag dropped. +function inline(html) { + return unhtml( + html + .replace(/<(strong|b)>([\s\S]*?)<\/\1>/g, "**$2**") + .replace(/([\s\S]*?)<\/code>/g, "`$1`") + .replace(/]*href="([^"]*)"[^>]*>([\s\S]*?)<\/a>/g, (_, h, t) => `[${t}](${h})`) + .replace(//g, " ") + .replace(/<[^>]+>/g, ""), + ).replace(/\s+/g, " ").trim(); +} + +const HAN = /\p{Script=Han}/u; + +// notes turns a release's HTML (as the feed has it) back into markdown: +// headings, bullets and paragraphs. GitHub drops the HTML comment the +// Chinese notes start under, so it goes back before the first heading in +// Chinese that follows an English one. +export function notes(html) { + const lines = []; + let english = false, zh = false, prev = ""; + for (const [, tag, body] of html.matchAll(/<(h[1-6]|li|p|pre)\b[^>]*>([\s\S]*?)<\/\1>/g)) { + const text = tag === "pre" ? unhtml(body.replace(/<[^>]+>/g, "")).trim() : inline(body); + if (!text) continue; + if (tag[0] === "h") { + const isZh = HAN.test(text); + if (isZh && english && !zh) { + lines.push("", ZH); + zh = true; + } + if (!isZh) english = true; + lines.push("", "#".repeat(+tag[1]) + " " + text); + } else if (tag === "li") { + if (prev !== "li") lines.push(""); + lines.push("- " + text); + } else { + lines.push("", text); + } + prev = tag; + } + return lines.join("\n").replace(/\n{3,}/g, "\n\n").trim(); } // A release's notes are in English, then (since the release workflow @@ -276,7 +435,13 @@ function newer(a, b) { // checksums reads a release's SHA256SUMS: {file name: hash}. async function checksums(tag) { - const r = await fetch(`https://github.com/${REPO}/releases/download/${tag}/SHA256SUMS`, { headers: UA }); + let r; + try { + r = await fetch(`https://github.com/${REPO}/releases/download/${tag}/SHA256SUMS`, { headers: UA }); + } catch (e) { + console.log("SHA256SUMS", e); + return null; + } if (!r.ok) return null; const sums = {}; for (const line of (await r.text()).split("\n")) { diff --git a/site/worker.test.mjs b/site/worker.test.mjs index 93108933..81bd4f3e 100644 --- a/site/worker.test.mjs +++ b/site/worker.test.mjs @@ -19,8 +19,48 @@ globalThis.caches = { put: async (req, res) => void store.set(req.url, await res.text()), }, }; -globalThis.fetch = async (u) => { +// apiDown stands in for GitHub's API limiting the worker's shared IP (#661); +// pagesDown for github.com itself failing too. seen keeps each API ask's +// headers. +let apiDown = false, pagesDown = false; +const seen = []; +const FEED = ` + + + tag:github.com,2008:Repository/1/v0.1.3 + 2026-10-03T08:54:51Z + + magpie v0.1.3 + <h3>Features</h3> +<ul> +<li>One thing (#1)</li> +</ul> +<h3>Install</h3> +<p>Download it.</p> +<h3>新功能</h3> +<ul> +<li>一件事 (#1)</li> +</ul> + + + tag:github.com,2008:Repository/1/v0.1.2 + 2026-10-02T08:54:51Z + + <h3>Fixes</h3> +<ul> +<li>Older, <strong>English</strong> alone, <code>a &amp;lt; b</code> <a href="https://x.dev/">link</a></li> +</ul> + +`; +globalThis.fetch = async (u, init = {}) => { u = String(u); + if (u.startsWith("https://api.github.com/")) { + seen.push(new Headers(init.headers)); + if (apiDown) return new Response('{"message":"API rate limit exceeded"}', { status: 403 }); + } else if (pagesDown) return new Response("unicorn", { status: 503 }); + if (u === "https://github.com/yetone/magpie-releases/releases/latest") + return new Response(null, { status: 302, headers: { Location: "https://github.com/yetone/magpie-releases/releases/tag/v0.1.3" } }); + if (u.endsWith("/releases.atom")) return new Response(FEED); const gh = (tag) => ({ tag_name: tag, body: BODIES[tag], html_url: "https://github.com/r/" + tag, published_at: "2026-10-01T00:00:00Z", draft: false, prerelease: false, assets: [{ name: "magpie-linux-amd64", size: 1, browser_download_url: "https://dl/x" }] }); if (u.endsWith("/releases/latest")) return Response.json(gh("v0.1.3")); if (u.includes("/releases?per_page=")) return Response.json(Object.keys(BODIES).map(gh)); @@ -103,3 +143,80 @@ test("docs: the bare docs paths open each language's guide", async () => { assert.equal(new URL(res.headers.get("Location"), "https://usemagpie.ai").pathname, to, from); } }); + +// #661: the update window showed no notes while GitHub's API limited the +// worker, and the failure was kept for five minutes. +const ask = (path, env = {}) => worker.fetch(new Request("https://usemagpie.ai" + path), env, ctx); + +test("latest: with the API limited, the notes come from the releases feed, kept briefly", async () => { + store.clear(); + apiDown = true; + try { + const res = await ask("/api/latest"); + assert.equal(res.status, 200); + const body = await res.json(); + assert.equal(body.version, "0.1.3"); + assert.match(body.notes, /One thing \(#1\)/); + assert.doesNotMatch(body.notes, /一件事/); + assert.equal(body.assets["magpie-linux-amd64"].sha256, "abc"); + assert.equal(res.headers.get("Cache-Control"), "public, max-age=60"); + const zh = await (await ask("/api/latest?lang=zh")).json(); + assert.match(zh.notes, /^### 新功能\n\n- 一件事 \(#1\)$/); + } finally { + apiDown = false; + } +}); + +test("notes: with the API limited, the releases feed's notes, in either language", async () => { + store.clear(); + apiDown = true; + try { + const res = await ask("/api/notes?after=0.1.1&upto=0.1.3&lang=zh"); + assert.equal(res.status, 200); + assert.equal(res.headers.get("Cache-Control"), "public, max-age=60"); + const list = (await res.json()).releases; + assert.deepEqual(list.map((r) => r.version), ["0.1.3", "0.1.2"]); + assert.equal(list[0].notes, "### 新功能\n\n- 一件事 (#1)"); + assert.equal(list[1].notes, "### Fixes\n\n- Older, **English** alone, `a < b` [link](https://x.dev/)"); + assert.equal(list[0].url, "https://github.com/yetone/magpie-releases/releases/tag/v0.1.3"); + } finally { + apiDown = false; + } +}); + +test("notes and latest: GitHub failing altogether is an error never kept, then the last whole answer", async () => { + store.clear(); + apiDown = pagesDown = true; + try { + for (const path of ["/api/notes?upto=0.1.3", "/api/latest"]) { + const res = await ask(path); + assert.equal(res.status, 503, path); + assert.equal(res.headers.get("Cache-Control"), "no-store", path); + } + // a whole answer once, then GitHub fails again: that one, kept briefly + apiDown = pagesDown = false; + assert.equal((await ask("/api/notes?upto=0.1.3")).headers.get("Cache-Control"), "public, max-age=300"); + assert.equal((await ask("/api/latest")).status, 200); + for (const k of ["https://usemagpie.ai/__releases", "https://usemagpie.ai/__latest"]) store.delete(k); + apiDown = pagesDown = true; + const res = await ask("/api/notes?upto=0.1.3"); + assert.equal(res.status, 200); + assert.equal(res.headers.get("Cache-Control"), "public, max-age=60"); + assert.equal((await res.json()).releases[0].notes, EN); + const lat = await ask("/api/latest"); + assert.equal(lat.status, 200); + assert.equal((await lat.json()).notes, EN); + } finally { + apiDown = pagesDown = false; + } +}); + +test("api: a GITHUB_TOKEN secret is sent to GitHub's API, and none without it", async () => { + store.clear(); + seen.length = 0; + await ask("/api/notes?upto=0.1.3", { GITHUB_TOKEN: "ghp_x" }); + assert.equal(seen.at(-1).get("Authorization"), "Bearer ghp_x"); + store.clear(); + await ask("/api/notes?upto=0.1.3"); + assert.equal(seen.at(-1).get("Authorization"), null); +});