- {[1, 2, 3, 4, 5, 6].map((item) =>
)}
+
+ {Array.from({ length: itemCount }, (_, index) => )}
);
}
diff --git a/frontend/src/ToolbarPopover.tsx b/frontend/src/ToolbarPopover.tsx
index b60a322..b55aeca 100644
--- a/frontend/src/ToolbarPopover.tsx
+++ b/frontend/src/ToolbarPopover.tsx
@@ -75,6 +75,7 @@ export function ToolbarPopover({
aria-label={triggerLabel}
aria-expanded={open}
aria-controls={id}
+ data-state={open ? "open" : "closed"}
title={triggerLabel}
>
diff --git a/frontend/src/api.ts b/frontend/src/api.ts
index 18b637e..6310949 100644
--- a/frontend/src/api.ts
+++ b/frontend/src/api.ts
@@ -4,6 +4,7 @@ import type {
GalleryImage,
GallerySummary,
ImagesPage,
+ PhotoDetails,
SortMode,
ThumbnailStatus,
} from "./types";
@@ -11,6 +12,7 @@ import type {
const REQUEST_TIMEOUT_MS = 15_000;
const IMAGE_SEARCH_REQUEST_TIMEOUT_MS = 60_000;
const SIMILAR_REQUEST_TIMEOUT_MS = 60_000;
+const PHOTO_DETAILS_REQUEST_TIMEOUT_MS = 30_000;
type JsonObject = Record;
type JsonValidator = (value: unknown) => value is T;
@@ -60,6 +62,27 @@ function isImagesPage(value: unknown): value is ImagesPage {
&& (value.nextOffset === null || isNonNegativeInteger(value.nextOffset));
}
+function isHistogramChannel(value: unknown): value is number[] {
+ return Array.isArray(value)
+ && value.length === 256
+ && value.every(isNonNegativeInteger);
+}
+
+function isPhotoDetails(value: unknown): value is PhotoDetails {
+ return isObject(value)
+ && isNonNegativeInteger(value.fileSize)
+ && isNonNegativeInteger(value.modifiedMs)
+ && Array.isArray(value.exif)
+ && value.exif.every((field) => isObject(field)
+ && typeof field.label === "string"
+ && typeof field.value === "string")
+ && isObject(value.histogram)
+ && isHistogramChannel(value.histogram.red)
+ && isHistogramChannel(value.histogram.green)
+ && isHistogramChannel(value.histogram.blue)
+ && isHistogramChannel(value.histogram.luminance);
+}
+
function isThumbnailStatus(value: unknown): value is ThumbnailStatus {
return isObject(value)
&& isNonNegativeInteger(value.total)
@@ -203,3 +226,15 @@ export function getSimilarImages(
SIMILAR_REQUEST_TIMEOUT_MS,
);
}
+
+export function getPhotoDetails(
+ imageId: string,
+ signal?: AbortSignal,
+): Promise {
+ return getJson(
+ `/api/images/${encodeURIComponent(imageId)}/details`,
+ isPhotoDetails,
+ signal,
+ PHOTO_DETAILS_REQUEST_TIMEOUT_MS,
+ );
+}
diff --git a/frontend/src/format.ts b/frontend/src/format.ts
index 5fdf089..db2aceb 100644
--- a/frontend/src/format.ts
+++ b/frontend/src/format.ts
@@ -1,5 +1,24 @@
const COUNT_FORMATTER = new Intl.NumberFormat("zh-CN");
+const FILE_SIZE_UNITS = ["KB", "MB", "GB", "TB"] as const;
export function formatCount(value: number): string {
return COUNT_FORMATTER.format(value);
}
+
+export function formatFileSize(bytes: number): string {
+ if (bytes < 1024) return `${COUNT_FORMATTER.format(bytes)} B`;
+ let value = bytes / 1024;
+ let unit: string = FILE_SIZE_UNITS[0];
+ for (let index = 1; index < FILE_SIZE_UNITS.length && value >= 1024; index += 1) {
+ value /= 1024;
+ unit = FILE_SIZE_UNITS[index];
+ }
+ const maximumFractionDigits = value >= 100 ? 0 : value >= 10 ? 1 : 2;
+ return `${value.toLocaleString(undefined, { maximumFractionDigits })} ${unit}`;
+}
+
+export function formatPixelCount(megapixels: number): string {
+ return megapixels >= 1
+ ? `${megapixels.toFixed(1)} MP`
+ : `${Math.round(megapixels * 1000)} KP`;
+}
diff --git a/frontend/src/main.tsx b/frontend/src/main.tsx
index d485e35..b3710f7 100644
--- a/frontend/src/main.tsx
+++ b/frontend/src/main.tsx
@@ -1,6 +1,7 @@
import { render } from "preact";
import App from "./App";
import "./styles.css";
+import "./PhotoInformation.css";
const root = document.getElementById("root");
if (!root) throw new Error("Missing #root application container");
diff --git a/frontend/src/styles.css b/frontend/src/styles.css
index 24cb5cc..c1c7cd1 100644
--- a/frontend/src/styles.css
+++ b/frontend/src/styles.css
@@ -237,6 +237,16 @@ select:focus-visible,
background: rgba(231, 241, 238, .74);
}
+.gallery-sidebar-toggle[aria-expanded="true"],
+.gallery-sidebar-toggle[aria-expanded="true"]:hover,
+.gallery-sidebar-toggle[aria-expanded="true"]:focus-visible {
+ width: calc(var(--sidebar-width) - 20px);
+ justify-items: center;
+ padding: 0;
+ color: #216b63;
+ background: rgba(231, 241, 238, .74);
+}
+
.sidebar-toggle-button {
color: #59615e;
}
@@ -525,21 +535,31 @@ select:focus-visible,
transition: transform var(--motion-medium) var(--ease-emphasized);
}
-.toolbar-popover-trigger:hover,
.toolbar-popover.is-open .toolbar-popover-trigger,
-.toolbar-action-button:hover {
+.toolbar-popover-trigger[aria-expanded="true"] {
color: #0f6d64;
background: #eaf2f0;
border-color: #c5ddd8;
}
-.explore-toggle.is-active {
+.explore-toggle.is-active,
+.explore-toggle[aria-pressed="true"],
+.explore-toggle[aria-pressed="true"]:hover,
+.explore-toggle[aria-pressed="true"]:focus-visible {
color: #0f6d64;
background: #e8f2f0;
border-color: #bfdad5;
box-shadow: 0 3px 10px rgba(30, 100, 91, .1);
}
+.toolbar-popover-trigger[aria-expanded="true"],
+.toolbar-popover-trigger[aria-expanded="true"]:hover,
+.toolbar-popover-trigger[aria-expanded="true"]:focus-visible {
+ color: #0f6d64;
+ background: #eaf2f0;
+ border-color: #c5ddd8;
+}
+
.toolbar-state-dot {
position: absolute;
width: 5px;
@@ -564,6 +584,7 @@ select:focus-visible,
}
.explore-toggle.is-active .toolbar-state-dot,
+.explore-toggle[aria-pressed="true"] .toolbar-state-dot,
.topbar-search.has-query .toolbar-state-dot {
opacity: 1;
transform: scale(1);
@@ -1059,7 +1080,8 @@ select:focus-visible,
pointer-events: auto;
}
-.viewer-header .viewer-control {
+.viewer-header .viewer-control,
+.viewer-details-toolbar .viewer-control {
background: transparent;
border-color: transparent;
box-shadow: none;
@@ -1067,8 +1089,8 @@ select:focus-visible,
-webkit-backdrop-filter: none;
}
-.viewer-header .viewer-control:hover,
-.viewer-header .viewer-control:focus-visible {
+.viewer-header .viewer-control:focus-visible,
+.viewer-details-toolbar .viewer-control:focus-visible {
background: rgba(255, 255, 255, .105);
border-color: transparent;
}
@@ -1106,9 +1128,6 @@ select:focus-visible,
text-decoration: none;
}
-.viewer-control:hover,
-.viewer-nav:hover:not(:disabled),
-.viewer-scroll-cue:hover,
.viewer-control:focus-visible,
.viewer-nav:focus-visible,
.viewer-scroll-cue:focus-visible {
@@ -1239,14 +1258,26 @@ select:focus-visible,
top: 50%;
width: 48px;
height: 48px;
- opacity: .64;
+ color: rgba(249, 252, 251, .88);
+ background: linear-gradient(145deg, rgba(70, 88, 82, .42), rgba(23, 29, 27, .34));
+ border: 0;
+ box-shadow: 0 10px 28px rgba(0, 0, 0, .24);
+ backdrop-filter: blur(20px) saturate(1.35);
+ -webkit-backdrop-filter: blur(20px) saturate(1.35);
+ opacity: .78;
transform: translateY(-50%);
}
-.viewer-nav:hover:not(:disabled),
-.viewer-nav:focus-visible { opacity: 1; }
+.viewer-nav:focus-visible {
+ color: #fff;
+ background: linear-gradient(145deg, rgba(89, 112, 104, .52), rgba(31, 40, 37, .43));
+ border: 0;
+ box-shadow: 0 13px 34px rgba(0, 0, 0, .3);
+ opacity: 1;
+ transform: translateY(-50%) scale(1.035);
+}
-.viewer-nav:active:not(:disabled) { transform: translateY(-50%) scale(.93); }
+.viewer-nav:active:not(:disabled) { transform: translateY(-50%) scale(.95); }
.viewer-nav:disabled {
opacity: 0;
@@ -1274,15 +1305,13 @@ select:focus-visible,
width: 50px;
height: 50px;
padding: 0;
- color: rgba(248, 250, 249, .76);
- background: rgba(18, 22, 20, .6);
- border: 1px solid rgba(255, 255, 255, .13);
+ color: rgba(249, 252, 251, .86);
+ background: linear-gradient(145deg, rgba(70, 88, 82, .44), rgba(23, 29, 27, .36));
+ border: 0;
border-radius: 50%;
- box-shadow:
- 0 10px 32px rgba(0, 0, 0, .3),
- inset 0 1px rgba(255, 255, 255, .04);
- backdrop-filter: blur(16px) saturate(1.12);
- -webkit-backdrop-filter: blur(16px) saturate(1.12);
+ box-shadow: 0 12px 34px rgba(0, 0, 0, .28);
+ backdrop-filter: blur(22px) saturate(1.4);
+ -webkit-backdrop-filter: blur(22px) saturate(1.4);
cursor: pointer;
pointer-events: auto;
transition:
@@ -1291,13 +1320,17 @@ select:focus-visible,
transform 150ms var(--ease-standard);
}
-.viewer-scroll-cue:hover,
.viewer-scroll-cue:focus-visible {
color: #fff;
- background: rgba(48, 55, 52, .8);
- border-color: rgba(255, 255, 255, .23);
- outline: none;
- transform: translateY(2px);
+ background: linear-gradient(145deg, rgba(91, 115, 106, .54), rgba(31, 41, 37, .45));
+ border: 0;
+ box-shadow: 0 15px 40px rgba(0, 0, 0, .34);
+ transform: translateY(2px) scale(1.035);
+}
+
+.viewer-scroll-cue:focus-visible {
+ outline: 2px solid #8fd2c9;
+ outline-offset: 3px;
}
.viewer-scroll-cue:active {
@@ -1305,12 +1338,12 @@ select:focus-visible,
}
.viewer-scroll-cue svg {
- filter: drop-shadow(0 2px 5px rgba(0, 0, 0, .64));
+ filter: drop-shadow(0 2px 5px rgba(0, 0, 0, .48));
animation: viewer-details-chevron 1.8s var(--ease-standard) infinite;
}
.viewer-details-return:focus-visible,
-.viewer-details-download:focus-visible {
+.viewer-details-toolbar .viewer-control:focus-visible {
outline: 2px solid #8fd2c9;
outline-offset: 3px;
}
@@ -1334,7 +1367,7 @@ select:focus-visible,
}
.viewer-details-inner {
- width: min(960px, calc(100% - 40px));
+ width: min(1640px, calc(100% - 48px));
min-height: 100vh;
min-height: 100dvh;
margin: 0 auto;
@@ -1345,12 +1378,12 @@ select:focus-visible,
position: sticky;
z-index: 3;
top: 0;
- min-height: 74px;
+ min-height: 62px;
display: flex;
align-items: center;
justify-content: space-between;
gap: 18px;
- padding: 13px 0;
+ padding: 8px 0;
background: linear-gradient(to bottom, rgba(18, 21, 20, .98), rgba(18, 21, 20, .9));
border-bottom: 1px solid rgba(255, 255, 255, .08);
backdrop-filter: blur(16px) saturate(1.08);
@@ -1367,7 +1400,7 @@ select:focus-visible,
.viewer-details-heading h2 {
margin: 0;
color: #fff;
- font-size: 16px;
+ font-size: 15px;
font-weight: 680;
letter-spacing: -.01em;
}
@@ -1378,160 +1411,17 @@ select:focus-visible,
font-variant-numeric: tabular-nums;
}
-.viewer-details-actions {
- display: flex;
- align-items: center;
- gap: 8px;
+.viewer-details-toolbar {
+ flex: 0 0 auto;
}
.viewer-details-content {
- width: min(840px, 100%);
+ width: min(1520px, 100%);
display: flex;
flex-direction: column;
- gap: 28px;
+ gap: 18px;
margin: 0 auto;
- padding: clamp(30px, 5vh, 54px) 0 max(112px, calc(84px + env(safe-area-inset-bottom)));
-}
-
-.viewer-details-summary {
- min-width: 0;
- display: grid;
- grid-template-columns: 104px minmax(0, 1fr);
- align-items: center;
- gap: 20px;
- margin: 0;
- padding: 18px;
- background: rgba(255, 255, 255, .045);
- border: 1px solid rgba(255, 255, 255, .075);
- border-radius: 18px;
- box-shadow: 0 18px 54px rgba(0, 0, 0, .13);
-}
-
-.viewer-details-summary img {
- width: 104px;
- height: 104px;
- display: block;
- object-fit: cover;
- background: #0b0d0c;
- border: 1px solid rgba(255, 255, 255, .08);
- border-radius: 13px;
-}
-
-.viewer-details-summary > div {
- min-width: 0;
- display: flex;
- flex-direction: column;
- align-items: flex-start;
- gap: 7px;
-}
-
-.viewer-details-kind {
- color: rgba(143, 210, 201, .72);
- font-size: 10px;
- font-weight: 650;
- letter-spacing: .05em;
- text-transform: uppercase;
-}
-
-.viewer-details-summary h3 {
- max-width: 100%;
- margin: 0;
- overflow: hidden;
- color: #fff;
- font-size: clamp(16px, 2.4vw, 21px);
- font-weight: 650;
- line-height: 1.35;
- text-overflow: ellipsis;
- white-space: nowrap;
-}
-
-.viewer-details-summary p {
- margin: 0;
- color: rgba(239, 243, 241, .5);
- font-size: 11px;
- font-variant-numeric: tabular-nums;
-}
-
-.viewer-details-section {
- display: flex;
- flex-direction: column;
- gap: 10px;
-}
-
-.viewer-details-section > h3 {
- margin: 0 2px;
- color: rgba(248, 250, 249, .82);
- font-size: 12px;
- font-weight: 650;
-}
-
-.viewer-details-grid {
- display: grid;
- grid-template-columns: repeat(4, minmax(0, 1fr));
- gap: 1px;
- margin: 0;
- overflow: hidden;
- background: rgba(255, 255, 255, .07);
- border: 1px solid rgba(255, 255, 255, .07);
- border-radius: 15px;
-}
-
-.viewer-details-grid > div {
- min-width: 0;
- display: flex;
- flex-direction: column;
- gap: 6px;
- padding: 15px;
- background: rgba(29, 34, 32, .98);
-}
-
-.viewer-details-grid dt {
- color: rgba(239, 243, 241, .45);
- font-size: 10px;
-}
-
-.viewer-details-grid dd {
- margin: 0;
- overflow: hidden;
- color: rgba(250, 251, 250, .9);
- font-size: 12px;
- font-variant-numeric: tabular-nums;
- text-overflow: ellipsis;
- white-space: nowrap;
-}
-
-.viewer-details-download {
- min-height: 62px;
- display: grid;
- grid-template-columns: 24px minmax(0, 1fr) 20px;
- align-items: center;
- gap: 13px;
- padding: 11px 16px;
- color: rgba(246, 248, 247, .84);
- background: rgba(255, 255, 255, .045);
- border: 1px solid rgba(255, 255, 255, .075);
- border-radius: 15px;
- text-decoration: none;
- transition: color var(--motion-fast) ease, background var(--motion-fast) ease;
-}
-
-.viewer-details-download:hover {
- color: #fff;
- background: rgba(255, 255, 255, .085);
-}
-
-.viewer-details-download > span {
- min-width: 0;
- display: flex;
- flex-direction: column;
- gap: 3px;
-}
-
-.viewer-details-download strong { font-size: 12px; }
-
-.viewer-details-download small {
- color: rgba(239, 243, 241, .46);
- font-size: 10px;
+ padding: 22px 0 max(80px, calc(64px + env(safe-area-inset-bottom)));
}
.viewer-similar-trigger.is-active {
@@ -1540,14 +1430,32 @@ select:focus-visible,
border-color: rgba(143, 210, 201, .38);
}
+.viewer-header .viewer-control[aria-pressed="true"],
+.viewer-details-toolbar .viewer-control[aria-pressed="true"],
+.viewer-header .viewer-control[aria-pressed="true"]:hover,
+.viewer-header .viewer-control[aria-pressed="true"]:focus-visible,
+.viewer-details-toolbar .viewer-control[aria-pressed="true"]:hover,
+.viewer-details-toolbar .viewer-control[aria-pressed="true"]:focus-visible {
+ color: #b8eee7;
+ background: rgba(40, 122, 114, .48);
+ border-color: rgba(143, 210, 201, .38);
+}
+
.viewer-similar-section {
scroll-margin-top: 90px;
display: flex;
flex-direction: column;
- gap: 16px;
+ gap: 10px;
padding-top: 2px;
}
+.viewer-similar-section.is-idle {
+ height: 0;
+ gap: 0;
+ overflow: hidden;
+ padding: 0;
+}
+
.viewer-similar-heading {
min-width: 0;
display: flex;
@@ -1561,68 +1469,23 @@ select:focus-visible,
min-width: 0;
display: flex;
flex-direction: column;
- gap: 4px;
-}
-
-.viewer-similar-heading h3,
-.viewer-similar-heading p {
- margin: 0;
+ gap: 0;
}
.viewer-similar-heading h3 {
+ margin: 0;
color: rgba(250, 252, 251, .94);
font-size: 14px;
font-weight: 680;
}
-.viewer-similar-heading p {
- color: rgba(239, 243, 241, .46);
- font-size: 10px;
-}
-
.viewer-similar-heading > svg {
flex: 0 0 auto;
color: rgba(143, 210, 201, .64);
}
-.viewer-similar-search-button,
-.viewer-similar-load-more {
+.viewer-similar-auto-load {
color: rgba(247, 250, 249, .86);
- background: rgba(255, 255, 255, .05);
- border: 1px solid rgba(255, 255, 255, .085);
- transition:
- color var(--motion-fast) ease,
- background var(--motion-fast) ease,
- border-color var(--motion-fast) ease,
- transform var(--motion-fast) var(--ease-standard);
-}
-
-.viewer-similar-search-button {
- min-height: 62px;
- display: grid;
- grid-template-columns: 24px minmax(0, 1fr) 20px;
- align-items: center;
- gap: 13px;
- padding: 11px 16px;
- text-align: left;
- border-radius: 15px;
-}
-
-.viewer-similar-search-button:hover,
-.viewer-similar-load-more:hover:not(:disabled) {
- color: #fff;
- background: rgba(74, 135, 126, .16);
- border-color: rgba(143, 210, 201, .2);
-}
-
-.viewer-similar-search-button:active,
-.viewer-similar-load-more:active:not(:disabled) {
- transform: scale(.988);
-}
-
-.viewer-similar-search-button > span {
- font-size: 12px;
- font-weight: 650;
}
.viewer-similar-masonry {
@@ -1721,7 +1584,7 @@ select:focus-visible,
justify-content: center;
}
-.viewer-similar-load-more {
+.viewer-similar-auto-load {
min-width: 210px;
min-height: 40px;
display: flex;
@@ -1729,23 +1592,16 @@ select:focus-visible,
justify-content: center;
gap: 8px;
padding: 9px 15px;
- border-radius: 999px;
font-size: 11px;
font-weight: 620;
}
-.viewer-similar-load-more:disabled {
- opacity: .56;
- cursor: wait;
+.viewer-similar-pagination[data-state="ready"] .viewer-similar-auto-load {
+ color: rgba(239, 243, 241, .36);
}
-.viewer-similar-load-more small {
- padding-left: 8px;
- color: rgba(239, 243, 241, .42);
- border-left: 1px solid rgba(255, 255, 255, .12);
- font-size: 9px;
- font-variant-numeric: tabular-nums;
- font-weight: 500;
+.viewer-similar-pagination[data-state="loading"] .viewer-similar-auto-load {
+ color: rgba(184, 238, 231, .72);
}
.viewer-similar-count {
@@ -1755,7 +1611,7 @@ select:focus-visible,
.viewer-similar-skeleton {
display: grid;
- grid-template-columns: repeat(3, minmax(0, 1fr));
+ grid-template-columns: repeat(var(--similar-columns, 3), minmax(0, 1fr));
align-items: start;
gap: 10px;
}
@@ -1884,6 +1740,44 @@ select:focus-visible,
}
@media (hover: hover) and (pointer: fine) {
+ .toolbar-popover-trigger:hover,
+ .toolbar-action-button:hover {
+ color: #0f6d64;
+ background: #eaf2f0;
+ border-color: #c5ddd8;
+ }
+
+ .viewer-header .viewer-control:hover,
+ .viewer-details-toolbar .viewer-control:hover {
+ background: rgba(255, 255, 255, .105);
+ border-color: transparent;
+ }
+
+ .viewer-control:hover,
+ .viewer-nav:hover:not(:disabled),
+ .viewer-scroll-cue:hover {
+ color: #fff;
+ background: rgba(48, 55, 52, .76);
+ border-color: rgba(255, 255, 255, .22);
+ }
+
+ .viewer-nav:hover:not(:disabled) {
+ color: #fff;
+ background: linear-gradient(145deg, rgba(89, 112, 104, .52), rgba(31, 40, 37, .43));
+ border: 0;
+ box-shadow: 0 13px 34px rgba(0, 0, 0, .3);
+ opacity: 1;
+ transform: translateY(-50%) scale(1.035);
+ }
+
+ .viewer-scroll-cue:hover {
+ color: #fff;
+ background: linear-gradient(145deg, rgba(91, 115, 106, .54), rgba(31, 41, 37, .45));
+ border: 0;
+ box-shadow: 0 15px 40px rgba(0, 0, 0, .34);
+ transform: translateY(2px) scale(1.035);
+ }
+
.brand:hover .brand-mark {
background: #216e66;
box-shadow: 0 5px 13px rgba(32, 100, 92, .2);
@@ -2028,29 +1922,14 @@ select:focus-visible,
}
.viewer-details-inner { width: calc(100% - 24px); }
.viewer-details-header {
- min-height: 66px;
- padding: 10px 0;
+ min-height: 58px;
+ padding: 6px 0;
}
.viewer-details-content {
- gap: 22px;
- padding: 24px 0 max(88px, calc(64px + env(safe-area-inset-bottom)));
+ gap: 16px;
+ padding: 16px 0 max(72px, calc(56px + env(safe-area-inset-bottom)));
}
- .viewer-details-summary {
- grid-template-columns: 80px minmax(0, 1fr);
- gap: 14px;
- padding: 14px;
- border-radius: 15px;
- }
- .viewer-details-summary img {
- width: 80px;
- height: 80px;
- border-radius: 11px;
- }
- .viewer-details-summary h3 { font-size: 15px; }
- .viewer-details-grid { grid-template-columns: repeat(2, minmax(0, 1fr)); }
- .viewer-details-grid > div { padding: 13px 14px; }
- .viewer-similar-section { scroll-margin-top: 76px; }
- .viewer-similar-heading p { max-width: 240px; }
+ .viewer-similar-section { scroll-margin-top: 68px; }
.viewer-similar-masonry {
gap: 6px;
}
@@ -2060,11 +1939,9 @@ select:focus-visible,
}
.viewer-similar-card figcaption { padding: 8px; font-size: 9px; }
.viewer-similar-skeleton {
- grid-template-columns: repeat(2, minmax(0, 1fr));
gap: 6px;
}
- .viewer-similar-skeleton span:nth-child(n + 5) { display: none; }
- .viewer-similar-load-more { width: 100%; }
+ .viewer-similar-auto-load { width: 100%; }
.viewer-load-error {
bottom: max(64px, calc(52px + env(safe-area-inset-bottom)));
}
@@ -2124,6 +2001,19 @@ select:focus-visible,
.viewer-load-error { bottom: max(58px, calc(48px + env(safe-area-inset-bottom))); }
}
+@media (max-width: 380px) {
+ .viewer-details-header { gap: 10px; }
+ .viewer-details-heading span { display: none; }
+ .viewer-details-toolbar {
+ gap: 1px;
+ padding: 3px;
+ }
+ .viewer-details-toolbar .viewer-control {
+ width: 36px;
+ height: 36px;
+ }
+}
+
@keyframes viewer-media-swipe-in-next {
from { opacity: .24; transform: translate3d(min(22vw, 96px), 0, 0) scale(.985); }
to { opacity: 1; transform: translate3d(0, 0, 0) scale(1); }
diff --git a/frontend/src/types.ts b/frontend/src/types.ts
index 6e1bb6a..902cac8 100644
--- a/frontend/src/types.ts
+++ b/frontend/src/types.ts
@@ -25,6 +25,25 @@ export interface ImagesPage {
nextOffset: number | null;
}
+export interface PhotoExifField {
+ label: string;
+ value: string;
+}
+
+export interface PhotoHistogram {
+ red: number[];
+ green: number[];
+ blue: number[];
+ luminance: number[];
+}
+
+export interface PhotoDetails {
+ fileSize: number;
+ modifiedMs: number;
+ exif: PhotoExifField[];
+ histogram: PhotoHistogram;
+}
+
export interface ThumbnailStatus {
total: number;
ready: number;
diff --git a/src/config.rs b/src/config.rs
index dbc724e..17428e5 100644
--- a/src/config.rs
+++ b/src/config.rs
@@ -1,4 +1,10 @@
-use std::{env, ffi::OsString, net::SocketAddr, path::PathBuf, time::Duration};
+use std::{
+ env,
+ ffi::{OsStr, OsString},
+ net::SocketAddr,
+ path::PathBuf,
+ time::Duration,
+};
use anyhow::{Context, Result, anyhow, bail};
@@ -6,7 +12,7 @@ use anyhow::{Context, Result, anyhow, bail};
pub struct Config {
pub gallery_dir: PathBuf,
pub cache_dir: PathBuf,
- pub text_search: Option,
+ pub text_search: Option,
pub listen: SocketAddr,
pub initial_batch: usize,
pub workers: usize,
@@ -19,6 +25,26 @@ pub struct TextSearchFiles {
pub vocabulary: PathBuf,
}
+#[derive(Clone, Debug)]
+pub enum TextSearchSource {
+ Local(TextSearchFiles),
+ AutoDownload(TextSearchFiles),
+}
+
+#[derive(Clone, Debug)]
+enum TextSearchModelInput {
+ Local(PathBuf),
+ AutoDownload,
+ Disabled,
+}
+
+const TEXT_SEARCH_MODEL_FILENAME: &str = "model.safetensors";
+const TEXT_SEARCH_VOCABULARY_FILENAME: &str = "vocab.txt";
+const TEXT_SEARCH_ENABLED_VALUE: &str = "true";
+const TEXT_SEARCH_DISABLED_VALUE: &str = "false";
+const TEXT_SEARCH_LEGACY_AUTO_VALUE: &str = "auto";
+const TEXT_SEARCH_AUTO_CACHE_DIRECTORY: &str = "models/chinese-clip-vit-base-patch16-f4a64596";
+
impl Config {
pub fn from_env_and_args() -> Result {
let cwd = env::current_dir().context("cannot determine the working directory")?;
@@ -46,7 +72,7 @@ impl Config {
let mut text_search_model = if is_overridden("--text-search-model") {
None
} else {
- env_path(&get_env, "PIXHELF_TEXT_SEARCH_MODEL", &cwd)?
+ env_text_search_model(&get_env, &cwd)?.or(Some(TextSearchModelInput::AutoDownload))
};
let mut text_search_vocabulary = if is_overridden("--text-search-vocab") {
None
@@ -85,7 +111,7 @@ impl Config {
"--gallery-dir" => gallery_dir = path_value(&mut args, &arg, &cwd)?,
"--cache-dir" => cache_dir = path_value(&mut args, &arg, &cwd)?,
"--text-search-model" => {
- text_search_model = Some(path_value(&mut args, &arg, &cwd)?);
+ text_search_model = Some(text_search_model_value(&mut args, &arg, &cwd)?);
}
"--text-search-vocab" => {
text_search_vocabulary = Some(path_value(&mut args, &arg, &cwd)?);
@@ -117,13 +143,23 @@ impl Config {
cache_dir = canonical_directory(&cache_dir, "cache")?;
ensure_cache_is_external(&gallery_dir, &cache_dir)?;
let text_search = match text_search_model {
- Some(model) => {
- let model = canonical_file(&model, "text-search model")?;
- let vocabulary = text_search_vocabulary
- .unwrap_or_else(|| model.parent().unwrap_or(&cwd).join("vocab.txt"));
- let vocabulary = canonical_file(&vocabulary, "text-search vocabulary")?;
- Some(TextSearchFiles { model, vocabulary })
+ Some(TextSearchModelInput::Local(model)) => Some(TextSearchSource::Local(
+ resolve_text_search_files(&model, text_search_vocabulary.as_deref(), &cwd)?,
+ )),
+ Some(TextSearchModelInput::AutoDownload) if text_search_vocabulary.is_some() => {
+ bail!("text-search vocabulary cannot be used with automatic model download");
}
+ Some(TextSearchModelInput::AutoDownload) => {
+ let directory = cache_dir.join(TEXT_SEARCH_AUTO_CACHE_DIRECTORY);
+ Some(TextSearchSource::AutoDownload(TextSearchFiles {
+ model: directory.join(TEXT_SEARCH_MODEL_FILENAME),
+ vocabulary: directory.join(TEXT_SEARCH_VOCABULARY_FILENAME),
+ }))
+ }
+ Some(TextSearchModelInput::Disabled) if text_search_vocabulary.is_some() => {
+ bail!("text-search vocabulary requires text search to be enabled");
+ }
+ Some(TextSearchModelInput::Disabled) => None,
None if text_search_vocabulary.is_some() => {
bail!("--text-search-vocab requires --text-search-model");
}
@@ -158,6 +194,31 @@ fn canonical_file(path: &std::path::Path, label: &str) -> Result {
.with_context(|| format!("cannot resolve {label} file: {}", path.display()))
}
+fn resolve_text_search_files(
+ model_or_directory: &std::path::Path,
+ vocabulary: Option<&std::path::Path>,
+ cwd: &std::path::Path,
+) -> Result {
+ let model = if model_or_directory.is_dir() {
+ model_or_directory.join(TEXT_SEARCH_MODEL_FILENAME)
+ } else {
+ model_or_directory.to_path_buf()
+ };
+ let model = canonical_file(&model, "text-search model")?;
+ let vocabulary = vocabulary.map_or_else(
+ || {
+ model
+ .parent()
+ .unwrap_or(cwd)
+ .join(TEXT_SEARCH_VOCABULARY_FILENAME)
+ },
+ std::path::Path::to_path_buf,
+ );
+ let vocabulary = canonical_file(&vocabulary, "text-search vocabulary")?;
+
+ Ok(TextSearchFiles { model, vocabulary })
+}
+
fn ensure_cache_is_external(
gallery_dir: &std::path::Path,
cache_dir: &std::path::Path,
@@ -204,6 +265,14 @@ fn env_path(
Ok(env_value(get_env, name)?.map(|value| resolve_path(PathBuf::from(value), cwd)))
}
+fn env_text_search_model(
+ get_env: &impl Fn(&str) -> Option,
+ cwd: &std::path::Path,
+) -> Result> {
+ Ok(env_value(get_env, "PIXHELF_TEXT_SEARCH_MODEL")?
+ .map(|value| parse_text_search_model(value, cwd)))
+}
+
fn env_number(
get_env: &impl Fn(&str) -> Option,
name: &str,
@@ -223,6 +292,29 @@ fn path_value(
Ok(resolve_path(PathBuf::from(value(args, flag)?), cwd))
}
+fn text_search_model_value(
+ args: &mut impl Iterator- ,
+ flag: &str,
+ cwd: &std::path::Path,
+) -> Result {
+ Ok(parse_text_search_model(
+ OsString::from(value(args, flag)?),
+ cwd,
+ ))
+}
+
+fn parse_text_search_model(value: OsString, cwd: &std::path::Path) -> TextSearchModelInput {
+ if value == OsStr::new(TEXT_SEARCH_ENABLED_VALUE)
+ || value == OsStr::new(TEXT_SEARCH_LEGACY_AUTO_VALUE)
+ {
+ TextSearchModelInput::AutoDownload
+ } else if value == OsStr::new(TEXT_SEARCH_DISABLED_VALUE) {
+ TextSearchModelInput::Disabled
+ } else {
+ TextSearchModelInput::Local(resolve_path(PathBuf::from(value), cwd))
+ }
+}
+
fn resolve_path(path: PathBuf, cwd: &std::path::Path) -> PathBuf {
if path.is_absolute() {
path
@@ -264,8 +356,8 @@ Usage: pixhelf [OPTIONS]\n\n\
Options:\n \
--gallery-dir PATH Gallery root (default: ./pic)\n \
--cache-dir PATH Thumbnail cache (default: ./.pixhelf-cache/thumbnails)\n \
- --text-search-model PATH Optional OFA Chinese-CLIP ViT-B/16 safetensors model\n \
- --text-search-vocab PATH Chinese-CLIP vocab.txt (default: beside model)\n \
+ --text-search-model VALUE 'true', 'false', Chinese-CLIP directory, or model file (default: true)\n \
+ --text-search-vocab PATH Optional vocab.txt override (default: beside model)\n \
--listen HOST:PORT Listen address (default: 0.0.0.0:3002)\n \
--initial-batch N Priority thumbnail batch (default: 60)\n \
--workers N Background thumbnail workers (default: 2-4, based on CPU)\n \
@@ -349,7 +441,7 @@ mod tests {
#[test]
fn resolves_only_existing_model_files() {
let temp = tempfile::tempdir().unwrap();
- let model = temp.path().join("model.safetensors");
+ let model = temp.path().join(TEXT_SEARCH_MODEL_FILENAME);
std::fs::write(&model, b"model").unwrap();
assert_eq!(
@@ -360,6 +452,98 @@ mod tests {
assert!(canonical_file(&temp.path().join("missing"), "model").is_err());
}
+ #[test]
+ fn resolves_text_search_directory_and_legacy_file_path() {
+ let temp = tempfile::tempdir().unwrap();
+ let directory = temp.path().join("chinese-clip");
+ std::fs::create_dir(&directory).unwrap();
+ let model = directory.join(TEXT_SEARCH_MODEL_FILENAME);
+ let vocabulary = directory.join(TEXT_SEARCH_VOCABULARY_FILENAME);
+ std::fs::write(&model, b"model").unwrap();
+ std::fs::write(&vocabulary, b"vocabulary").unwrap();
+
+ let from_directory = resolve_text_search_files(&directory, None, temp.path()).unwrap();
+ let from_file = resolve_text_search_files(&model, None, temp.path()).unwrap();
+
+ assert_eq!(from_directory.model, model.canonicalize().unwrap());
+ assert_eq!(
+ from_directory.vocabulary,
+ vocabulary.canonicalize().unwrap()
+ );
+ assert_eq!(from_file.model, from_directory.model);
+ assert_eq!(from_file.vocabulary, from_directory.vocabulary);
+ }
+
+ #[test]
+ fn semantic_search_is_enabled_by_default_and_uses_the_persistent_cache() {
+ let temp = tempfile::tempdir().unwrap();
+ let gallery = temp.path().join("gallery");
+ let cache = temp.path().join("cache");
+ std::fs::create_dir(&gallery).unwrap();
+ let environment = HashMap::from([
+ ("PIXHELF_GALLERY_DIR", gallery.as_os_str().to_os_string()),
+ ("PIXHELF_CACHE_DIR", cache.as_os_str().to_os_string()),
+ ]);
+
+ let config = Config::from_sources(temp.path().to_path_buf(), [], |name| {
+ environment.get(name).cloned()
+ })
+ .unwrap();
+ let expected = config.cache_dir.join(TEXT_SEARCH_AUTO_CACHE_DIRECTORY);
+
+ let Some(TextSearchSource::AutoDownload(files)) = config.text_search else {
+ panic!("expected automatic model download");
+ };
+ assert_eq!(files.model, expected.join(TEXT_SEARCH_MODEL_FILENAME));
+ assert_eq!(
+ files.vocabulary,
+ expected.join(TEXT_SEARCH_VOCABULARY_FILENAME)
+ );
+ }
+
+ #[test]
+ fn automatic_model_download_rejects_a_vocabulary_override() {
+ let temp = tempfile::tempdir().unwrap();
+ std::fs::create_dir(temp.path().join("pic")).unwrap();
+ let args = [
+ "--text-search-model",
+ "true",
+ "--text-search-vocab",
+ "vocab.txt",
+ ]
+ .map(str::to_owned);
+
+ let error = Config::from_sources(temp.path().to_path_buf(), args, |_| None).unwrap_err();
+ assert!(error.to_string().contains("automatic model download"));
+ }
+
+ #[test]
+ fn boolean_false_disables_text_search_and_auto_remains_compatible() {
+ let temp = tempfile::tempdir().unwrap();
+ assert!(matches!(
+ parse_text_search_model(OsString::from("false"), temp.path()),
+ TextSearchModelInput::Disabled
+ ));
+ assert!(matches!(
+ parse_text_search_model(OsString::from("auto"), temp.path()),
+ TextSearchModelInput::AutoDownload
+ ));
+
+ let gallery = temp.path().join("gallery");
+ let cache = temp.path().join("cache");
+ std::fs::create_dir(&gallery).unwrap();
+ let environment = HashMap::from([
+ ("PIXHELF_GALLERY_DIR", gallery.as_os_str().to_os_string()),
+ ("PIXHELF_CACHE_DIR", cache.as_os_str().to_os_string()),
+ ("PIXHELF_TEXT_SEARCH_MODEL", OsString::from("false")),
+ ]);
+ let config = Config::from_sources(temp.path().to_path_buf(), [], |name| {
+ environment.get(name).cloned()
+ })
+ .unwrap();
+ assert!(config.text_search.is_none());
+ }
+
#[test]
fn rejects_cache_directories_inside_the_gallery() {
let temp = tempfile::tempdir().unwrap();
diff --git a/src/main.rs b/src/main.rs
index 6d60b87..ea3bf86 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -1,15 +1,16 @@
mod config;
mod gallery;
+mod photo_details;
mod similarity;
mod support;
mod text_search;
mod thumbs;
mod web;
-use std::{path::PathBuf, sync::Arc};
+use std::{path::PathBuf, sync::Arc, time::Duration};
use anyhow::{Context, Result};
-use config::Config;
+use config::{Config, TextSearchFiles, TextSearchSource};
use gallery::{GalleryIndex, scan_gallery};
use text_search::TextSearchIndex;
use thumbs::ThumbnailManager;
@@ -18,6 +19,8 @@ use tracing::{error, info, warn};
use tracing_subscriber::EnvFilter;
use web::AppState;
+const AUTOMATIC_MODEL_RETRY_INTERVAL: Duration = Duration::from_secs(5 * 60);
+
#[tokio::main]
async fn main() -> Result<()> {
tracing_subscriber::fmt()
@@ -50,20 +53,10 @@ async fn main() -> Result<()> {
.map(|record| record.id.clone())
.collect();
let index = Arc::new(RwLock::new(Arc::new(index)));
- let text_search = match config.text_search.as_ref() {
- Some(files) => match TextSearchIndex::load(&files.model, &files.vocabulary) {
- Ok(index) => Some(index),
- Err(error) => {
- warn!(
- model = %files.model.display(),
- vocabulary = %files.vocabulary.display(),
- %error,
- "natural-language search model unavailable; using filename search"
- );
- None
- }
- },
- None => None,
+ let (text_search, automatic_text_search) = match config.text_search.as_ref() {
+ Some(TextSearchSource::Local(files)) => (load_text_search(files), None),
+ Some(TextSearchSource::AutoDownload(files)) => (None, Some(files.clone())),
+ None => (None, None),
};
let thumbnails = ThumbnailManager::new_with_text_search(
config.cache_dir.clone(),
@@ -99,6 +92,9 @@ async fn main() -> Result<()> {
.await
.with_context(|| format!("cannot listen on {}", config.listen))?;
info!(listen = %config.listen, "Pixhelf is ready");
+ if let Some(files) = automatic_text_search {
+ start_automatic_text_search(Arc::clone(&thumbnails), files);
+ }
axum::serve(listener, web::router(state))
.with_graceful_shutdown(shutdown_signal())
@@ -106,6 +102,55 @@ async fn main() -> Result<()> {
Ok(())
}
+fn load_text_search(files: &TextSearchFiles) -> Option> {
+ match TextSearchIndex::load(&files.model, &files.vocabulary) {
+ Ok(index) => Some(index),
+ Err(error) => {
+ warn!(
+ model = %files.model.display(),
+ vocabulary = %files.vocabulary.display(),
+ %error,
+ "natural-language search model unavailable; using filename search"
+ );
+ None
+ }
+ }
+}
+
+fn start_automatic_text_search(thumbnails: Arc, files: TextSearchFiles) {
+ tokio::spawn(async move {
+ loop {
+ match prepare_automatic_text_search(&files).await {
+ Ok(index) => {
+ if thumbnails.enable_text_search(index) {
+ info!(
+ "background model preparation complete; natural-language search enabled"
+ );
+ }
+ return;
+ }
+ Err(error) => {
+ warn!(
+ retry_seconds = AUTOMATIC_MODEL_RETRY_INTERVAL.as_secs(),
+ error = %format!("{error:#}"),
+ "natural-language search model is not ready; filename search remains available"
+ );
+ tokio::time::sleep(AUTOMATIC_MODEL_RETRY_INTERVAL).await;
+ }
+ }
+ }
+ });
+}
+
+async fn prepare_automatic_text_search(files: &TextSearchFiles) -> Result> {
+ text_search::ensure_automatic_model(files).await?;
+ let model = files.model.clone();
+ let vocabulary = files.vocabulary.clone();
+ tokio::task::spawn_blocking(move || TextSearchIndex::load(&model, &vocabulary))
+ .await
+ .context("natural-language search setup task stopped")?
+}
+
async fn rescan_gallery(state: AppState, config: Config) {
loop {
tokio::time::sleep(config.scan_interval).await;
diff --git a/src/photo_details.rs b/src/photo_details.rs
new file mode 100644
index 0000000..ba47d27
--- /dev/null
+++ b/src/photo_details.rs
@@ -0,0 +1,448 @@
+use std::{fs::File, io::BufReader, path::Path};
+
+use anyhow::{Context, Result};
+use exif::{Exif, In, Tag, Value};
+use serde::Serialize;
+
+use crate::gallery::ImageRecord;
+
+#[derive(Clone, Debug, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct PhotoDetails {
+ pub(crate) file_size: u64,
+ pub(crate) modified_ms: u64,
+ pub(crate) exif: Vec,
+ pub(crate) histogram: PhotoHistogram,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
+pub(crate) struct ExifField {
+ pub(crate) label: &'static str,
+ pub(crate) value: String,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
+pub(crate) struct PhotoHistogram {
+ pub(crate) red: Vec,
+ pub(crate) green: Vec,
+ pub(crate) blue: Vec,
+ pub(crate) luminance: Vec,
+}
+
+impl PhotoDetails {
+ pub(crate) fn read(record: &ImageRecord, thumbnail: &Path) -> Result {
+ record.ensure_source_is_current()?;
+ let exif = read_exif(&record.path)?;
+ let histogram = read_histogram(thumbnail)?;
+ record.ensure_source_is_current()?;
+ Ok(Self {
+ file_size: record.size,
+ modified_ms: record.modified_ms,
+ exif,
+ histogram,
+ })
+ }
+}
+
+fn read_histogram(path: &Path) -> Result {
+ let image = image::open(path)
+ .with_context(|| format!("cannot decode histogram source {}", path.display()))?
+ .to_rgb8();
+ let mut red = vec![0_u32; 256];
+ let mut green = vec![0_u32; 256];
+ let mut blue = vec![0_u32; 256];
+ let mut luminance = vec![0_u32; 256];
+
+ for pixel in image.pixels() {
+ let [r, g, b] = pixel.0;
+ red[usize::from(r)] += 1;
+ green[usize::from(g)] += 1;
+ blue[usize::from(b)] += 1;
+ // Integer Rec. 709 coefficients, rounded to the nearest 8-bit value.
+ let luma =
+ (54_u16 * u16::from(r) + 183_u16 * u16::from(g) + 19_u16 * u16::from(b) + 128) >> 8;
+ luminance[usize::from(luma)] += 1;
+ }
+
+ Ok(PhotoHistogram {
+ red,
+ green,
+ blue,
+ luminance,
+ })
+}
+
+fn read_exif(path: &Path) -> Result> {
+ let file =
+ File::open(path).with_context(|| format!("cannot open EXIF source {}", path.display()))?;
+ let mut reader = BufReader::new(file);
+ let mut parser = exif::Reader::new();
+ parser.continue_on_error(true);
+ let Some(exif) = parser
+ .read_from_container(&mut reader)
+ .or_else(|error| error.distill_partial_result(|_| {}))
+ .ok()
+ else {
+ return Ok(Vec::new());
+ };
+
+ Ok(exif_fields(&exif))
+}
+
+fn exif_fields(exif: &Exif) -> Vec {
+ let mut fields = Vec::with_capacity(14);
+
+ if let Some(mut captured_at) = ascii_value(exif, Tag::DateTimeOriginal)
+ .or_else(|| ascii_value(exif, Tag::DateTimeDigitized))
+ .or_else(|| ascii_value(exif, Tag::DateTime))
+ {
+ captured_at = normalize_exif_datetime(&captured_at);
+ if let Some(offset) = ascii_value(exif, Tag::OffsetTimeOriginal)
+ .or_else(|| ascii_value(exif, Tag::OffsetTimeDigitized))
+ .or_else(|| ascii_value(exif, Tag::OffsetTime))
+ {
+ captured_at.push(' ');
+ captured_at.push_str(&offset);
+ }
+ push_field(&mut fields, "拍摄时间", Some(captured_at));
+ }
+
+ push_field(
+ &mut fields,
+ "色彩空间",
+ unsigned_value(exif, Tag::ColorSpace).map(|value| match value {
+ 1 => "sRGB".to_owned(),
+ 0xffff => "未校准".to_owned(),
+ _ => value.to_string(),
+ }),
+ );
+
+ let make = ascii_value(exif, Tag::Make);
+ let model = ascii_value(exif, Tag::Model);
+ push_field(&mut fields, "相机", combine_make_and_model(make, model));
+ push_field(&mut fields, "镜头", ascii_value(exif, Tag::LensModel));
+ push_field(
+ &mut fields,
+ "快门",
+ rational_value(exif, Tag::ExposureTime).map(format_exposure_time),
+ );
+ push_field(
+ &mut fields,
+ "光圈",
+ rational_value(exif, Tag::FNumber).map(|value| format!("f/{}", compact_decimal(value))),
+ );
+ push_field(
+ &mut fields,
+ "ISO",
+ unsigned_value(exif, Tag::PhotographicSensitivity)
+ .or_else(|| unsigned_value(exif, Tag::ISOSpeed))
+ .or_else(|| unsigned_value(exif, Tag::RecommendedExposureIndex))
+ .map(|value| value.to_string()),
+ );
+ push_field(
+ &mut fields,
+ "焦距",
+ rational_value(exif, Tag::FocalLength)
+ .map(|value| format!("{} mm", compact_decimal(value))),
+ );
+ push_field(
+ &mut fields,
+ "35mm 等效",
+ unsigned_value(exif, Tag::FocalLengthIn35mmFilm)
+ .filter(|value| *value > 0)
+ .map(|value| format!("{value} mm")),
+ );
+ push_field(
+ &mut fields,
+ "曝光补偿",
+ signed_rational_value(exif, Tag::ExposureBiasValue).map(format_exposure_bias),
+ );
+ push_field(
+ &mut fields,
+ "曝光程序",
+ unsigned_value(exif, Tag::ExposureProgram).map(exposure_program),
+ );
+ push_field(
+ &mut fields,
+ "测光模式",
+ unsigned_value(exif, Tag::MeteringMode).map(metering_mode),
+ );
+ push_field(
+ &mut fields,
+ "闪光灯",
+ unsigned_value(exif, Tag::Flash).map(|value| {
+ if value & 1 == 1 {
+ "已闪光".to_owned()
+ } else {
+ "未闪光".to_owned()
+ }
+ }),
+ );
+ push_field(
+ &mut fields,
+ "白平衡",
+ unsigned_value(exif, Tag::WhiteBalance).map(|value| match value {
+ 0 => "自动".to_owned(),
+ 1 => "手动".to_owned(),
+ _ => value.to_string(),
+ }),
+ );
+ push_field(&mut fields, "处理软件", ascii_value(exif, Tag::Software));
+
+ fields
+}
+
+fn push_field(fields: &mut Vec, label: &'static str, value: Option) {
+ let Some(value) = value.map(|value| value.trim().to_owned()) else {
+ return;
+ };
+ if !value.is_empty() {
+ fields.push(ExifField { label, value });
+ }
+}
+
+fn field_value(exif: &Exif, tag: Tag) -> Option<&Value> {
+ exif.get_field(tag, In::PRIMARY).map(|field| &field.value)
+}
+
+fn ascii_value(exif: &Exif, tag: Tag) -> Option {
+ let Value::Ascii(values) = field_value(exif, tag)? else {
+ return None;
+ };
+ let value = values.first()?;
+ let value = String::from_utf8_lossy(value)
+ .trim_matches(|character: char| character == '\0' || character.is_whitespace())
+ .to_owned();
+ (!value.is_empty()).then_some(value)
+}
+
+fn unsigned_value(exif: &Exif, tag: Tag) -> Option {
+ field_value(exif, tag)?.get_uint(0)
+}
+
+fn rational_value(exif: &Exif, tag: Tag) -> Option {
+ match field_value(exif, tag)? {
+ Value::Rational(values) => values.first().map(|value| value.to_f64()),
+ _ => None,
+ }
+ .filter(|value| value.is_finite() && *value >= 0.0)
+}
+
+fn signed_rational_value(exif: &Exif, tag: Tag) -> Option {
+ match field_value(exif, tag)? {
+ Value::SRational(values) => values.first().map(|value| value.to_f64()),
+ Value::Rational(values) => values.first().map(|value| value.to_f64()),
+ _ => None,
+ }
+ .filter(|value| value.is_finite())
+}
+
+fn normalize_exif_datetime(value: &str) -> String {
+ let bytes = value.as_bytes();
+ if bytes.len() >= 10 && bytes.get(4) == Some(&b':') && bytes.get(7) == Some(&b':') {
+ if let (Some(year), Some(month), Some(day), Some(rest)) = (
+ value.get(..4),
+ value.get(5..7),
+ value.get(8..10),
+ value.get(10..),
+ ) {
+ return format!("{year}-{month}-{day}{rest}");
+ }
+ value.to_owned()
+ } else {
+ value.to_owned()
+ }
+}
+
+fn combine_make_and_model(make: Option, model: Option) -> Option {
+ match (make, model) {
+ (Some(make), Some(model)) if model.to_lowercase().starts_with(&make.to_lowercase()) => {
+ Some(model)
+ }
+ (Some(make), Some(model)) => Some(format!("{make} {model}")),
+ (Some(make), None) => Some(make),
+ (None, Some(model)) => Some(model),
+ (None, None) => None,
+ }
+}
+
+fn format_exposure_time(seconds: f64) -> String {
+ if seconds <= 0.0 {
+ return "0 秒".to_owned();
+ }
+ if seconds < 1.0 {
+ let denominator = (1.0 / seconds).round();
+ let reciprocal = 1.0 / seconds;
+ if denominator.is_finite()
+ && denominator >= 1.0
+ && (reciprocal - denominator).abs() <= (reciprocal * 0.005).max(0.01)
+ {
+ return format!("1/{} 秒", denominator as u64);
+ }
+ }
+ format!("{} 秒", compact_decimal(seconds))
+}
+
+fn format_exposure_bias(value: f64) -> String {
+ if value.abs() < 0.005 {
+ "0 EV".to_owned()
+ } else {
+ format!("{value:+.1} EV")
+ }
+}
+
+fn compact_decimal(value: f64) -> String {
+ if (value - value.round()).abs() < 0.005 {
+ format!("{value:.0}")
+ } else if value.abs() < 10.0 {
+ format!("{value:.1}")
+ } else {
+ format!("{value:.0}")
+ }
+}
+
+fn exposure_program(value: u32) -> String {
+ match value {
+ 0 => "未定义".to_owned(),
+ 1 => "手动".to_owned(),
+ 2 => "程序自动".to_owned(),
+ 3 => "光圈优先".to_owned(),
+ 4 => "快门优先".to_owned(),
+ 5 => "创意程序".to_owned(),
+ 6 => "动作程序".to_owned(),
+ 7 => "人像模式".to_owned(),
+ 8 => "风景模式".to_owned(),
+ _ => value.to_string(),
+ }
+}
+
+fn metering_mode(value: u32) -> String {
+ match value {
+ 0 => "未知".to_owned(),
+ 1 => "平均".to_owned(),
+ 2 => "中央重点平均".to_owned(),
+ 3 => "点测光".to_owned(),
+ 4 => "多点测光".to_owned(),
+ 5 => "多区测光".to_owned(),
+ 6 => "局部测光".to_owned(),
+ 255 => "其他".to_owned(),
+ _ => value.to_string(),
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use exif::{Field, Rational};
+ use image::ImageEncoder;
+ use std::io::Cursor;
+
+ #[test]
+ fn histogram_counts_rgb_and_luminance_bins() {
+ let temp = tempfile::tempdir().unwrap();
+ let path = temp.path().join("histogram.png");
+ let mut image = image::RgbImage::new(2, 1);
+ image.put_pixel(0, 0, image::Rgb([255, 0, 0]));
+ image.put_pixel(1, 0, image::Rgb([0, 255, 0]));
+ image.save(&path).unwrap();
+
+ let histogram = read_histogram(&path).unwrap();
+ assert_eq!(histogram.red[0], 1);
+ assert_eq!(histogram.red[255], 1);
+ assert_eq!(histogram.green[0], 1);
+ assert_eq!(histogram.green[255], 1);
+ assert_eq!(histogram.blue[0], 2);
+ assert_eq!(histogram.luminance.iter().sum::(), 2);
+ }
+
+ #[test]
+ fn formats_common_photography_values() {
+ assert_eq!(
+ normalize_exif_datetime("2026:08:29 14:03:02"),
+ "2026-08-29 14:03:02"
+ );
+ assert_eq!(
+ combine_make_and_model(Some("Canon".into()), Some("Canon EOS R6".into())),
+ Some("Canon EOS R6".into())
+ );
+ assert_eq!(format_exposure_time(1.0 / 125.0), "1/125 秒");
+ assert_eq!(format_exposure_time(1.3), "1.3 秒");
+ assert_eq!(format_exposure_time(0.4), "0.4 秒");
+ assert_eq!(format_exposure_bias(-2.0 / 3.0), "-0.7 EV");
+ }
+
+ #[test]
+ fn reads_curated_fields_from_a_jpeg_container() {
+ let temp = tempfile::tempdir().unwrap();
+ let path = temp.path().join("camera.jpg");
+ let fields = [
+ Field {
+ tag: Tag::Make,
+ ifd_num: In::PRIMARY,
+ value: Value::Ascii(vec![b"Canon".to_vec()]),
+ },
+ Field {
+ tag: Tag::Model,
+ ifd_num: In::PRIMARY,
+ value: Value::Ascii(vec![b"Canon EOS R6".to_vec()]),
+ },
+ Field {
+ tag: Tag::DateTimeOriginal,
+ ifd_num: In::PRIMARY,
+ value: Value::Ascii(vec![b"2026:08:29 14:03:02".to_vec()]),
+ },
+ Field {
+ tag: Tag::ExposureTime,
+ ifd_num: In::PRIMARY,
+ value: Value::Rational(vec![Rational { num: 1, denom: 125 }]),
+ },
+ Field {
+ tag: Tag::FNumber,
+ ifd_num: In::PRIMARY,
+ value: Value::Rational(vec![Rational { num: 28, denom: 10 }]),
+ },
+ Field {
+ tag: Tag::PhotographicSensitivity,
+ ifd_num: In::PRIMARY,
+ value: Value::Short(vec![400]),
+ },
+ Field {
+ tag: Tag::ColorSpace,
+ ifd_num: In::PRIMARY,
+ value: Value::Short(vec![1]),
+ },
+ ];
+ let mut metadata = Cursor::new(Vec::new());
+ let mut writer = exif::experimental::Writer::new();
+ for field in &fields {
+ writer.push_field(field);
+ }
+ writer.write(&mut metadata, false).unwrap();
+
+ let file = File::create(&path).unwrap();
+ let mut encoder = image::codecs::jpeg::JpegEncoder::new_with_quality(file, 90);
+ encoder.set_exif_metadata(metadata.into_inner()).unwrap();
+ encoder
+ .write_image(
+ &[180, 90, 30, 30, 90, 180],
+ 2,
+ 1,
+ image::ExtendedColorType::Rgb8,
+ )
+ .unwrap();
+
+ let fields = read_exif(&path).unwrap();
+ let value = |label| {
+ fields
+ .iter()
+ .find(|field| field.label == label)
+ .map(|field| field.value.as_str())
+ };
+ assert_eq!(value("拍摄时间"), Some("2026-08-29 14:03:02"));
+ assert_eq!(value("相机"), Some("Canon EOS R6"));
+ assert_eq!(value("快门"), Some("1/125 秒"));
+ assert_eq!(value("光圈"), Some("f/2.8"));
+ assert_eq!(value("ISO"), Some("400"));
+ assert_eq!(value("色彩空间"), Some("sRGB"));
+ }
+}
diff --git a/src/text_search.rs b/src/text_search.rs
index 1e46419..dd8068b 100644
--- a/src/text_search.rs
+++ b/src/text_search.rs
@@ -11,21 +11,11 @@ use std::{
Arc, Mutex, RwLock,
atomic::{AtomicBool, AtomicU64, Ordering},
},
+ time::Duration,
};
-#[cfg(test)]
-use std::fs;
-
-use anyhow::{Context, Result, anyhow, bail};
-use candle_core::{DType, Device, IndexOp, Tensor};
-use candle_nn::VarBuilder;
-use candle_transformers::models::chinese_clip::{ChineseClipConfig, ChineseClipModel};
-use image::{DynamicImage, GenericImageView, ImageReader, imageops::FilterType};
+use anyhow::{Result, anyhow};
use serde::Serialize;
-use tokenizers::{
- Tokenizer, TruncationParams, models::wordpiece::WordPiece, normalizers::bert::BertNormalizer,
- pre_tokenizers::bert::BertPreTokenizer, processors::bert::BertProcessing,
-};
use tokio::sync::Notify;
use tracing::{info, warn};
@@ -38,14 +28,22 @@ use crate::support::{
mutex_lock, read_lock, write_lock,
};
-const MODEL_NAME: &str = "Chinese-CLIP ViT-B/16";
-const MODEL_EDGE: u32 = 224;
-const TEXT_CONTEXT_LENGTH: usize = 52;
+mod download;
+mod model;
+
+pub(crate) use download::ensure_automatic_model;
+
+#[cfg(test)]
+pub(crate) use model::TextImageModel;
+use model::{LazyModel, MODEL_NAME};
+
const EMBEDDING_SIZE: usize = 512;
const EMBEDDING_MAGIC: &[u8; 8] = b"PXHFCN1\0";
const EMBEDDING_EXTENSION: &str = "cnclip";
const MODEL_FINGERPRINT_BYTES: usize = FINGERPRINT_BYTES;
const QUERY_CACHE_LIMIT: usize = 16;
+const MODEL_IDLE_TIMEOUT: Duration = Duration::from_secs(60);
+const MODEL_IDLE_CHECK_INTERVAL: Duration = Duration::from_secs(10);
pub(crate) type TextSearchEmbedding = QuantizedEmbedding;
@@ -56,13 +54,8 @@ pub(crate) fn text_image_similarity(
query.cosine_similarity(image)
}
-pub(crate) trait TextImageModel: Send {
- fn embed_thumbnail(&mut self, thumbnail: &Path) -> Result;
- fn embed_text(&mut self, text: &str) -> Result;
-}
-
pub(crate) struct TextSearchIndex {
- model: Arc>>,
+ model: Arc>,
model_fingerprint: Fingerprint,
model_token: String,
entries: RwLock>>,
@@ -116,17 +109,13 @@ impl TextSearchIndex {
pub(crate) fn load(model_path: &Path, vocabulary_path: &Path) -> Result> {
let fingerprint =
fingerprint_files(&[model_path, vocabulary_path], "text-search model file")?;
- let model = ChineseClip::load(model_path, vocabulary_path).with_context(|| {
- format!(
- "cannot load natural-language search model: {}",
- model_path.display()
- )
- })?;
- let index = Self::with_model(Box::new(model), fingerprint);
+ let model = LazyModel::chinese_clip(model_path.to_owned(), vocabulary_path.to_owned());
+ let index = Self::with_model(model, fingerprint);
info!(
model = MODEL_NAME,
+ path = %model_path.display(),
fingerprint = %index.model_token,
- "local natural-language image search enabled"
+ "local natural-language image search enabled; model will load on demand"
);
Ok(index)
}
@@ -136,13 +125,18 @@ impl TextSearchIndex {
model: Box,
model_fingerprint: [u8; MODEL_FINGERPRINT_BYTES],
) -> Arc {
- Self::with_model(model, model_fingerprint)
+ Self::with_model(LazyModel::preloaded(model), model_fingerprint)
}
- fn with_model(
- model: Box,
+ #[cfg(test)]
+ fn with_test_loader(
+ loader: impl Fn() -> Result> + Send + Sync + 'static,
model_fingerprint: [u8; MODEL_FINGERPRINT_BYTES],
) -> Arc {
+ Self::with_model(LazyModel::with_loader(loader), model_fingerprint)
+ }
+
+ fn with_model(model: LazyModel, model_fingerprint: [u8; MODEL_FINGERPRINT_BYTES]) -> Arc {
Arc::new(Self {
model: Arc::new(Mutex::new(model)),
model_fingerprint,
@@ -163,6 +157,10 @@ impl TextSearchIndex {
tokio::spawn(async move {
index.worker_loop().await;
});
+ let index = Arc::clone(self);
+ tokio::spawn(async move {
+ index.model_reaper_loop().await;
+ });
}
pub(crate) fn reconcile(&self, valid_ids: &HashSet<&str>) {
@@ -275,6 +273,24 @@ impl TextSearchIndex {
}
}
+ async fn model_reaper_loop(self: Arc) {
+ loop {
+ tokio::time::sleep(MODEL_IDLE_CHECK_INTERVAL).await;
+ if mutex_lock(&self.model).release_if_idle(MODEL_IDLE_TIMEOUT) {
+ info!(
+ model = MODEL_NAME,
+ idle_seconds = MODEL_IDLE_TIMEOUT.as_secs(),
+ "released idle natural-language search model"
+ );
+ }
+ }
+ }
+
+ #[cfg(test)]
+ fn release_model_for_test(&self) -> bool {
+ mutex_lock(&self.model).release_if_idle(Duration::ZERO)
+ }
+
async fn worker_loop(self: Arc) {
loop {
let id = self.queue.pop().await;
@@ -376,129 +392,6 @@ fn write_embedding(
)
}
-struct ChineseClip {
- model: ChineseClipModel,
- tokenizer: Tokenizer,
- device: Device,
-}
-
-impl ChineseClip {
- fn load(model_path: &Path, vocabulary_path: &Path) -> Result {
- let device = Device::Cpu;
- // SAFETY: the mapped model is immutable for the process lifetime. The
- // documentation requires stopping Pixhelf before replacing it.
- let variables =
- unsafe { VarBuilder::from_mmaped_safetensors(&[model_path], DType::F32, &device)? };
- let config = ChineseClipConfig::clip_vit_base_patch16();
- let model = ChineseClipModel::new(variables, &config)?;
- let tokenizer = load_tokenizer(vocabulary_path)?;
- Ok(Self {
- model,
- tokenizer,
- device,
- })
- }
-}
-
-impl TextImageModel for ChineseClip {
- fn embed_thumbnail(&mut self, thumbnail: &Path) -> Result {
- let input = preprocess_thumbnail(thumbnail)?;
- let values = self
- .model
- .get_image_features(&input)?
- .i(0)?
- .to_vec1::()?;
- TextSearchEmbedding::quantize(&values, "text-search image model")
- }
-
- fn embed_text(&mut self, text: &str) -> Result {
- let encoding = self
- .tokenizer
- .encode(text, true)
- .map_err(|error| anyhow!("cannot tokenize text query: {error}"))?;
- if encoding.is_empty() {
- bail!("text query did not produce any tokens");
- }
- let input_ids = Tensor::new(encoding.get_ids(), &self.device)?.unsqueeze(0)?;
- let token_type_ids = Tensor::new(encoding.get_type_ids(), &self.device)?.unsqueeze(0)?;
- let attention_mask =
- Tensor::new(encoding.get_attention_mask(), &self.device)?.unsqueeze(0)?;
- let values = self
- .model
- .get_text_features(&input_ids, Some(&token_type_ids), Some(&attention_mask))?
- .i(0)?
- .to_vec1::()?;
- TextSearchEmbedding::quantize(&values, "text-search text model")
- }
-}
-
-fn load_tokenizer(vocabulary_path: &Path) -> Result {
- let vocabulary = vocabulary_path.to_str().with_context(|| {
- format!(
- "vocabulary path is not UTF-8: {}",
- vocabulary_path.display()
- )
- })?;
- let model = WordPiece::from_file(vocabulary)
- .unk_token("[UNK]".to_owned())
- .build()
- .map_err(|error| anyhow!("cannot load Chinese-CLIP vocabulary: {error}"))?;
- let mut tokenizer = Tokenizer::new(model);
- let cls_id = tokenizer
- .token_to_id("[CLS]")
- .context("Chinese-CLIP vocabulary is missing [CLS]")?;
- let sep_id = tokenizer
- .token_to_id("[SEP]")
- .context("Chinese-CLIP vocabulary is missing [SEP]")?;
- tokenizer.with_normalizer(Some(BertNormalizer::default()));
- tokenizer.with_pre_tokenizer(Some(BertPreTokenizer));
- tokenizer.with_post_processor(Some(BertProcessing::new(
- ("[SEP]".to_owned(), sep_id),
- ("[CLS]".to_owned(), cls_id),
- )));
- tokenizer
- .with_truncation(Some(TruncationParams {
- max_length: TEXT_CONTEXT_LENGTH,
- ..Default::default()
- }))
- .map_err(|error| anyhow!("cannot configure Chinese-CLIP tokenizer: {error}"))?;
- Ok(tokenizer)
-}
-
-fn preprocess_thumbnail(path: &Path) -> Result {
- let image = ImageReader::open(path)
- .with_context(|| format!("cannot open thumbnail {}", path.display()))?
- .with_guessed_format()?
- .decode()
- .with_context(|| format!("cannot decode thumbnail {}", path.display()))?;
- preprocess_image(&image)
-}
-
-fn preprocess_image(image: &DynamicImage) -> Result {
- let (width, height) = image.dimensions();
- if width == 0 || height == 0 {
- bail!("cannot embed an empty image");
- }
- // The official ChineseCLIPFeatureExtractor resizes directly to 224x224
- // and does not centre-crop this checkpoint.
- let pixels = image
- .resize_exact(MODEL_EDGE, MODEL_EDGE, FilterType::CatmullRom)
- .to_rgb8();
- const MEAN: [f32; 3] = [0.481_454_66, 0.457_827_5, 0.408_210_73];
- const STANDARD_DEVIATION: [f32; 3] = [0.268_629_54, 0.261_302_6, 0.275_777_1];
- let mut values = Vec::with_capacity((MODEL_EDGE * MODEL_EDGE * 3) as usize);
- for channel in 0..3 {
- values.extend(pixels.pixels().map(|pixel| {
- (f32::from(pixel.0[channel]) / 255.0 - MEAN[channel]) / STANDARD_DEVIATION[channel]
- }));
- }
- Ok(Tensor::from_vec(
- values,
- (1, 3, MODEL_EDGE as usize, MODEL_EDGE as usize),
- &Device::Cpu,
- )?)
-}
-
#[cfg(test)]
mod tests {
use super::*;
@@ -589,59 +482,75 @@ mod tests {
assert!(embedding_path(&thumbnail).is_file());
}
- #[test]
- fn tokenizer_uses_chinese_bert_special_tokens() {
- let temp = tempfile::tempdir().unwrap();
- let path = temp.path().join("vocab.txt");
- let mut vocabulary = vec!["[PAD]".to_owned()];
- vocabulary.extend((0..99).map(|index| format!("[unused{index}]")));
- vocabulary.extend(
- ["[UNK]", "[CLS]", "[SEP]", "[MASK]", "海", "边", "日", "落"].map(str::to_owned),
+ #[tokio::test]
+ async fn reloadable_model_loads_on_demand_and_can_be_released() {
+ let loads = Arc::new(AtomicUsize::new(0));
+ let image_calls = Arc::new(AtomicUsize::new(0));
+ let text_calls = Arc::new(AtomicUsize::new(0));
+ let index = TextSearchIndex::with_test_loader(
+ {
+ let loads = Arc::clone(&loads);
+ let image_calls = Arc::clone(&image_calls);
+ let text_calls = Arc::clone(&text_calls);
+ move || {
+ loads.fetch_add(1, Ordering::Relaxed);
+ Ok(Box::new(ColourAndWordModel {
+ image_calls: Arc::clone(&image_calls),
+ text_calls: Arc::clone(&text_calls),
+ }))
+ }
+ },
+ [10; MODEL_FINGERPRINT_BYTES],
);
- fs::write(&path, format!("{}\n", vocabulary.join("\n"))).unwrap();
- let tokenizer = load_tokenizer(&path).unwrap();
- let encoding = tokenizer.encode("海边日落", true).unwrap();
- assert_eq!(encoding.get_ids().first(), Some(&101));
- assert_eq!(encoding.get_ids().last(), Some(&102));
- assert!(encoding.len() >= 6);
+
+ assert_eq!(loads.load(Ordering::Relaxed), 0);
+ index.embed_query("红色图片").await.unwrap();
+ assert_eq!(loads.load(Ordering::Relaxed), 1);
+ assert!(index.release_model_for_test());
+
+ index.embed_query("blue image").await.unwrap();
+ assert_eq!(loads.load(Ordering::Relaxed), 2);
+ assert_eq!(text_calls.load(Ordering::Relaxed), 2);
}
- #[test]
- fn preprocessing_matches_official_chinese_clip_shape() {
- let image = DynamicImage::ImageRgb8(image::RgbImage::from_fn(400, 200, |x, y| {
- image::Rgb([(x % 255) as u8, (y % 255) as u8, 128])
- }));
- let tensor = preprocess_image(&image).unwrap();
- assert_eq!(tensor.dims(), &[1, 3, 224, 224]);
- assert!(tensor.flatten_all().unwrap().to_vec1::().unwrap()[0].is_finite());
- }
-
- #[test]
- fn official_model_runs_when_supplied_by_the_test_environment() {
- let (Some(model_path), Some(vocabulary_path)) = (
- std::env::var_os("PIXHELF_CHINESE_CLIP_MODEL"),
- std::env::var_os("PIXHELF_CHINESE_CLIP_VOCAB"),
- ) else {
- return;
- };
+ #[tokio::test]
+ async fn cached_image_embedding_does_not_load_model() {
let temp = tempfile::tempdir().unwrap();
- let red = temp.path().join("red.png");
- let blue = temp.path().join("blue.png");
- image::RgbImage::from_pixel(224, 224, image::Rgb([235, 30, 25]))
- .save(&red)
- .unwrap();
- image::RgbImage::from_pixel(224, 224, image::Rgb([25, 40, 235]))
- .save(&blue)
- .unwrap();
+ let thumbnail = temp.path().join("cached.webp");
+ let fingerprint = [11; MODEL_FINGERPRINT_BYTES];
+ write_embedding(
+ &embedding_path(&thumbnail),
+ &fingerprint,
+ &TextSearchEmbedding::for_test([(0, 100)]),
+ )
+ .unwrap();
+ let loads = Arc::new(AtomicUsize::new(0));
+ let index = TextSearchIndex::with_test_loader(
+ {
+ let loads = Arc::clone(&loads);
+ move || {
+ loads.fetch_add(1, Ordering::Relaxed);
+ Ok(Box::new(ColourAndWordModel {
+ image_calls: Arc::new(AtomicUsize::new(0)),
+ text_calls: Arc::new(AtomicUsize::new(0)),
+ }))
+ }
+ },
+ fingerprint,
+ );
+ index.reconcile(&HashSet::from(["cached"]));
+ index.enqueue("cached", &thumbnail, false);
+ index.start_worker();
- let mut model =
- ChineseClip::load(Path::new(&model_path), Path::new(&vocabulary_path)).unwrap();
- let query = model.embed_text("一张红色的图片").unwrap();
- let red = model.embed_thumbnail(&red).unwrap();
- let blue = model.embed_thumbnail(&blue).unwrap();
- let red_score = text_image_similarity(&query, &red);
- let blue_score = text_image_similarity(&query, &blue);
- eprintln!("official Chinese-CLIP scores: red={red_score:.4}, blue={blue_score:.4}");
- assert!(red_score > blue_score);
+ tokio::time::timeout(Duration::from_secs(2), async {
+ while !index.status().background_complete {
+ tokio::time::sleep(Duration::from_millis(10)).await;
+ }
+ })
+ .await
+ .expect("cached text-search index timed out");
+
+ assert!(index.embedding("cached").is_some());
+ assert_eq!(loads.load(Ordering::Relaxed), 0);
}
}
diff --git a/src/text_search/download.rs b/src/text_search/download.rs
new file mode 100644
index 0000000..62f573c
--- /dev/null
+++ b/src/text_search/download.rs
@@ -0,0 +1,496 @@
+use std::{
+ ffi::OsString,
+ io::{ErrorKind, Read},
+ path::{Path, PathBuf},
+ time::Duration,
+};
+
+use anyhow::{Context, Result, bail};
+use reqwest::{Client, StatusCode, header::RANGE};
+use sha2::{Digest, Sha256};
+use tokio::{fs, io::AsyncWriteExt};
+use tracing::{info, warn};
+
+use crate::config::TextSearchFiles;
+
+const MODEL_REVISION: &str = "f4a64596bbcf9a2a94591b74b9dc39b2e4e77e3e";
+const MODEL_BASE_URL: &str = "https://huggingface.co/OFA-Sys/chinese-clip-vit-base-patch16/resolve";
+const PROGRESS_INTERVAL: u64 = 64 * 1024 * 1024;
+const DOWNLOAD_ATTEMPTS: usize = 5;
+
+const ARTIFACTS: [Artifact; 2] = [
+ Artifact {
+ kind: ArtifactKind::Model,
+ filename: "model.safetensors",
+ sha256: "29cc0b2bcf6ff777f2e15742be92b110e4acbdb2068356e862c4637a4b15fe4f",
+ size: 753_106_020,
+ },
+ Artifact {
+ kind: ArtifactKind::Vocabulary,
+ filename: "vocab.txt",
+ sha256: "45bbac6b341c319adc98a532532882e91a9cefc0329aa57bac9ae761c27b291c",
+ size: 109_540,
+ },
+];
+
+#[derive(Clone, Copy)]
+struct Artifact {
+ kind: ArtifactKind,
+ filename: &'static str,
+ sha256: &'static str,
+ size: u64,
+}
+
+#[derive(Clone, Copy)]
+enum ArtifactKind {
+ Model,
+ Vocabulary,
+}
+
+impl Artifact {
+ fn destination<'a>(&self, files: &'a TextSearchFiles) -> &'a Path {
+ match self.kind {
+ ArtifactKind::Model => &files.model,
+ ArtifactKind::Vocabulary => &files.vocabulary,
+ }
+ }
+
+ fn url(&self) -> String {
+ format!("{MODEL_BASE_URL}/{MODEL_REVISION}/{}", self.filename)
+ }
+}
+
+pub(crate) async fn ensure_automatic_model(files: &TextSearchFiles) -> Result<()> {
+ let mut missing = Vec::new();
+ for artifact in ARTIFACTS {
+ let destination = artifact.destination(files);
+ if cached_artifact_is_valid(destination, artifact).await? {
+ info!(file = %destination.display(), "using cached model file");
+ } else {
+ missing.push(artifact);
+ }
+ }
+
+ if missing.is_empty() {
+ return Ok(());
+ }
+
+ for destination in [&files.model, &files.vocabulary] {
+ let parent = destination.parent().with_context(|| {
+ format!("model cache path has no parent: {}", destination.display())
+ })?;
+ fs::create_dir_all(parent)
+ .await
+ .with_context(|| format!("cannot create model cache: {}", parent.display()))?;
+ }
+
+ let client = Client::builder()
+ .connect_timeout(Duration::from_secs(30))
+ .user_agent(concat!("pixhelf/", env!("CARGO_PKG_VERSION")))
+ .build()
+ .context("cannot initialise the model downloader")?;
+ for artifact in missing {
+ let url = artifact.url();
+ download_artifact(&client, artifact.destination(files), artifact, &url).await?;
+ }
+
+ info!(
+ directory = %files.model.parent().unwrap_or(&files.model).display(),
+ "natural-language search model downloaded"
+ );
+ Ok(())
+}
+
+async fn download_artifact(
+ client: &Client,
+ destination: &Path,
+ artifact: Artifact,
+ url: &str,
+) -> Result<()> {
+ for attempt in 1..=DOWNLOAD_ATTEMPTS {
+ let result = download_artifact_once(client, destination, artifact, url).await;
+ if result.is_ok() || attempt == DOWNLOAD_ATTEMPTS {
+ return result.with_context(|| {
+ format!(
+ "failed to download {} after {attempt} attempts",
+ artifact.filename
+ )
+ });
+ }
+
+ let error = result.expect_err("failed download must contain an error");
+ let retry_seconds = 1_u64 << (attempt - 1);
+ warn!(
+ file = artifact.filename,
+ attempt,
+ retry_seconds,
+ error = %format!("{error:#}"),
+ "model download failed; retrying"
+ );
+ tokio::time::sleep(Duration::from_secs(retry_seconds)).await;
+ }
+ unreachable!("download attempt loop always returns")
+}
+
+async fn download_artifact_once(
+ client: &Client,
+ destination: &Path,
+ artifact: Artifact,
+ url: &str,
+) -> Result<()> {
+ let partial = suffixed_path(destination, ".part");
+ let mut downloaded = file_size_or_zero(&partial).await?;
+
+ if downloaded == artifact.size {
+ if sha256_file(partial.clone()).await? == artifact.sha256 {
+ install_artifact(&partial, destination, artifact.sha256).await?;
+ return Ok(());
+ }
+ fs::remove_file(&partial)
+ .await
+ .with_context(|| format!("cannot remove invalid download: {}", partial.display()))?;
+ downloaded = 0;
+ } else if downloaded > artifact.size {
+ fs::remove_file(&partial)
+ .await
+ .with_context(|| format!("cannot remove invalid download: {}", partial.display()))?;
+ downloaded = 0;
+ }
+
+ let resumed_from = downloaded;
+ let mut request = client.get(url);
+ if resumed_from > 0 {
+ request = request.header(RANGE, format!("bytes={resumed_from}-"));
+ }
+
+ info!(
+ file = artifact.filename,
+ downloaded_bytes = resumed_from,
+ total_bytes = artifact.size,
+ "downloading natural-language search model"
+ );
+ let mut response = request
+ .send()
+ .await
+ .with_context(|| format!("cannot download {url}"))?;
+ if resumed_from > 0 && response.status() != StatusCode::PARTIAL_CONTENT {
+ info!(
+ file = artifact.filename,
+ "download source does not support this range request; restarting the file"
+ );
+ response = client
+ .get(url)
+ .send()
+ .await
+ .with_context(|| format!("cannot restart download {url}"))?;
+ }
+ let status = response.status();
+ if !status.is_success() {
+ bail!("model download returned HTTP {status}: {url}");
+ }
+
+ let resuming = resumed_from > 0 && status == StatusCode::PARTIAL_CONTENT;
+ if !resuming {
+ downloaded = 0;
+ }
+ let mut options = fs::OpenOptions::new();
+ options.create(true).write(true);
+ if resuming {
+ options.append(true);
+ } else {
+ options.truncate(true);
+ }
+ let mut output = options
+ .open(&partial)
+ .await
+ .with_context(|| format!("cannot open model download: {}", partial.display()))?;
+ let mut next_progress = downloaded.saturating_add(PROGRESS_INTERVAL);
+
+ while let Some(chunk) = response
+ .chunk()
+ .await
+ .with_context(|| format!("model download interrupted: {}", artifact.filename))?
+ {
+ output
+ .write_all(&chunk)
+ .await
+ .with_context(|| format!("cannot write model download: {}", partial.display()))?;
+ downloaded = downloaded.saturating_add(chunk.len() as u64);
+ if downloaded > artifact.size {
+ bail!(
+ "downloaded {} is larger than expected ({} > {} bytes)",
+ artifact.filename,
+ downloaded,
+ artifact.size
+ );
+ }
+ if downloaded >= next_progress || downloaded == artifact.size {
+ info!(
+ file = artifact.filename,
+ percent = downloaded.saturating_mul(100) / artifact.size,
+ downloaded_bytes = downloaded,
+ total_bytes = artifact.size,
+ "model download progress"
+ );
+ next_progress = downloaded.saturating_add(PROGRESS_INTERVAL);
+ }
+ }
+ output
+ .sync_all()
+ .await
+ .with_context(|| format!("cannot save model download: {}", partial.display()))?;
+ drop(output);
+
+ if downloaded != artifact.size {
+ bail!(
+ "downloaded {} has the wrong size ({} of {} bytes); restart Pixhelf to resume",
+ artifact.filename,
+ downloaded,
+ artifact.size
+ );
+ }
+
+ let checksum = sha256_file(partial.clone()).await?;
+ if checksum != artifact.sha256 {
+ fs::remove_file(&partial).await.with_context(|| {
+ format!(
+ "cannot remove checksum-mismatched file: {}",
+ partial.display()
+ )
+ })?;
+ bail!(
+ "downloaded {} failed SHA-256 verification",
+ artifact.filename
+ );
+ }
+
+ install_artifact(&partial, destination, artifact.sha256).await
+}
+
+async fn cached_artifact_is_valid(path: &Path, artifact: Artifact) -> Result {
+ let metadata = match fs::metadata(path).await {
+ Ok(metadata) => metadata,
+ Err(error) if error.kind() == ErrorKind::NotFound => return Ok(false),
+ Err(error) => {
+ return Err(error)
+ .with_context(|| format!("cannot inspect cached model: {}", path.display()));
+ }
+ };
+ if !metadata.is_file() || metadata.len() != artifact.size {
+ return Ok(false);
+ }
+
+ let marker = checksum_marker(path);
+ if let Ok(recorded) = fs::read_to_string(&marker).await
+ && recorded.trim() == artifact.sha256
+ && marker_is_current(&marker, &metadata).await
+ {
+ return Ok(true);
+ }
+
+ if sha256_file(path.to_path_buf()).await? != artifact.sha256 {
+ return Ok(false);
+ }
+ write_checksum_marker(path, artifact.sha256).await?;
+ Ok(true)
+}
+
+async fn marker_is_current(marker: &Path, artifact: &std::fs::Metadata) -> bool {
+ let Ok(marker) = fs::metadata(marker).await else {
+ return false;
+ };
+ match (marker.modified(), artifact.modified()) {
+ (Ok(marker_time), Ok(artifact_time)) => marker_time >= artifact_time,
+ _ => true,
+ }
+}
+
+async fn file_size_or_zero(path: &Path) -> Result {
+ match fs::metadata(path).await {
+ Ok(metadata) if metadata.is_file() => Ok(metadata.len()),
+ Ok(_) => bail!("model download path is not a file: {}", path.display()),
+ Err(error) if error.kind() == ErrorKind::NotFound => Ok(0),
+ Err(error) => {
+ Err(error).with_context(|| format!("cannot inspect download: {}", path.display()))
+ }
+ }
+}
+
+async fn sha256_file(path: PathBuf) -> Result {
+ let display = path.display().to_string();
+ tokio::task::spawn_blocking(move || {
+ let mut file = std::fs::File::open(&path)
+ .with_context(|| format!("cannot open file for verification: {display}"))?;
+ let mut hasher = Sha256::new();
+ let mut buffer = vec![0_u8; 1024 * 1024];
+ loop {
+ let read = file
+ .read(&mut buffer)
+ .with_context(|| format!("cannot verify file: {display}"))?;
+ if read == 0 {
+ break;
+ }
+ hasher.update(&buffer[..read]);
+ }
+ Ok(format!("{:x}", hasher.finalize()))
+ })
+ .await
+ .context("model verification task stopped")?
+}
+
+async fn install_artifact(partial: &Path, destination: &Path, checksum: &str) -> Result<()> {
+ if fs::try_exists(destination).await.with_context(|| {
+ format!(
+ "cannot inspect model destination: {}",
+ destination.display()
+ )
+ })? {
+ fs::remove_file(destination)
+ .await
+ .with_context(|| format!("cannot replace cached model: {}", destination.display()))?;
+ }
+ fs::rename(partial, destination).await.with_context(|| {
+ format!(
+ "cannot install downloaded model: {} -> {}",
+ partial.display(),
+ destination.display()
+ )
+ })?;
+ write_checksum_marker(destination, checksum).await
+}
+
+async fn write_checksum_marker(path: &Path, checksum: &str) -> Result<()> {
+ let marker = checksum_marker(path);
+ let temporary = suffixed_path(&marker, ".tmp");
+ fs::write(&temporary, format!("{checksum}\n"))
+ .await
+ .with_context(|| format!("cannot write model checksum: {}", temporary.display()))?;
+ if fs::try_exists(&marker)
+ .await
+ .with_context(|| format!("cannot inspect model checksum: {}", marker.display()))?
+ {
+ fs::remove_file(&marker)
+ .await
+ .with_context(|| format!("cannot replace model checksum: {}", marker.display()))?;
+ }
+ fs::rename(&temporary, &marker)
+ .await
+ .with_context(|| format!("cannot save model checksum: {}", marker.display()))
+}
+
+fn checksum_marker(path: &Path) -> PathBuf {
+ suffixed_path(path, ".sha256")
+}
+
+fn suffixed_path(path: &Path, suffix: &str) -> PathBuf {
+ let mut value = OsString::from(path.as_os_str());
+ value.push(suffix);
+ PathBuf::from(value)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ const TEST_ARTIFACT: Artifact = Artifact {
+ kind: ArtifactKind::Model,
+ filename: "model.safetensors",
+ sha256: "9372c470eeadd5ecd9c3c74c2b3cb633f8e2f2fad799250a0f70d652b6b825e4",
+ size: 5,
+ };
+
+ #[tokio::test]
+ async fn verifies_and_marks_an_existing_cached_file() {
+ let temp = tempfile::tempdir().unwrap();
+ let path = temp.path().join(TEST_ARTIFACT.filename);
+ fs::write(&path, b"model").await.unwrap();
+
+ assert!(
+ cached_artifact_is_valid(&path, TEST_ARTIFACT)
+ .await
+ .unwrap()
+ );
+ assert_eq!(
+ fs::read_to_string(checksum_marker(&path))
+ .await
+ .unwrap()
+ .trim(),
+ TEST_ARTIFACT.sha256
+ );
+ }
+
+ #[tokio::test]
+ async fn rejects_same_sized_corrupt_cached_file() {
+ let temp = tempfile::tempdir().unwrap();
+ let path = temp.path().join(TEST_ARTIFACT.filename);
+ fs::write(&path, b"wrong").await.unwrap();
+
+ assert!(
+ !cached_artifact_is_valid(&path, TEST_ARTIFACT)
+ .await
+ .unwrap()
+ );
+ assert!(!checksum_marker(&path).exists());
+ }
+
+ #[tokio::test]
+ #[ignore = "requires access to the official model repository"]
+ async fn downloads_and_verifies_the_official_vocabulary() {
+ let temp = tempfile::tempdir().unwrap();
+ let artifact = ARTIFACTS[1];
+ let path = temp.path().join(artifact.filename);
+ let client = Client::builder()
+ .connect_timeout(Duration::from_secs(30))
+ .build()
+ .unwrap();
+
+ let url = artifact.url();
+ download_artifact(&client, &path, artifact, &url)
+ .await
+ .unwrap();
+
+ assert!(cached_artifact_is_valid(&path, artifact).await.unwrap());
+
+ let content = fs::read(&path).await.unwrap();
+ fs::remove_file(&path).await.unwrap();
+ fs::remove_file(checksum_marker(&path)).await.unwrap();
+ fs::write(suffixed_path(&path, ".part"), &content[..content.len() / 2])
+ .await
+ .unwrap();
+
+ download_artifact(&client, &path, artifact, &url)
+ .await
+ .unwrap();
+ assert_eq!(fs::read(&path).await.unwrap(), content);
+ }
+
+ #[tokio::test]
+ #[ignore = "downloads and loads the complete 753 MB Hugging Face checkpoint"]
+ async fn downloads_and_loads_the_complete_huggingface_checkpoint() {
+ let temp = tempfile::tempdir().unwrap();
+ let files = TextSearchFiles {
+ model: temp.path().join(ARTIFACTS[0].filename),
+ vocabulary: temp.path().join(ARTIFACTS[1].filename),
+ };
+
+ ensure_automatic_model(&files).await.unwrap();
+
+ let model = files.model.clone();
+ let vocabulary = files.vocabulary.clone();
+ tokio::task::spawn_blocking(move || {
+ crate::text_search::model::validate_checkpoint(&model, &vocabulary)
+ })
+ .await
+ .unwrap()
+ .unwrap();
+ }
+
+ #[test]
+ fn uses_the_pinned_huggingface_checkpoint() {
+ let artifact = ARTIFACTS[0];
+ let url = artifact.url();
+ assert!(url.starts_with("https://huggingface.co/OFA-Sys/"));
+ assert!(url.contains(MODEL_REVISION));
+ assert!(url.ends_with("/model.safetensors"));
+ }
+}
diff --git a/src/text_search/model.rs b/src/text_search/model.rs
new file mode 100644
index 0000000..d949231
--- /dev/null
+++ b/src/text_search/model.rs
@@ -0,0 +1,334 @@
+//! Chinese-CLIP loading, preprocessing, and idle model lifecycle.
+
+use std::{
+ path::{Path, PathBuf},
+ sync::Arc,
+ time::{Duration, Instant},
+};
+
+use anyhow::{Context, Result, anyhow, bail};
+use candle_core::{DType, Device, IndexOp, Tensor};
+use candle_nn::VarBuilder;
+use candle_transformers::models::chinese_clip::{ChineseClipConfig, ChineseClipModel};
+use image::{DynamicImage, GenericImageView, ImageReader, imageops::FilterType};
+use tokenizers::{
+ Tokenizer, TruncationParams, models::wordpiece::WordPiece, normalizers::bert::BertNormalizer,
+ pre_tokenizers::bert::BertPreTokenizer, processors::bert::BertProcessing,
+};
+use tracing::info;
+
+use super::TextSearchEmbedding;
+
+pub(super) const MODEL_NAME: &str = "Chinese-CLIP ViT-B/16";
+const MODEL_EDGE: u32 = 224;
+const TEXT_CONTEXT_LENGTH: usize = 52;
+
+pub(crate) trait TextImageModel: Send {
+ fn embed_thumbnail(&mut self, thumbnail: &Path) -> Result;
+ fn embed_text(&mut self, text: &str) -> Result;
+}
+
+type ModelLoader = dyn Fn() -> Result> + Send + Sync + 'static;
+
+pub(super) struct LazyModel {
+ current: Option>,
+ loader: Option>,
+ load_error: Option,
+ last_used: Option,
+}
+
+impl LazyModel {
+ pub(super) fn chinese_clip(model_path: PathBuf, vocabulary_path: PathBuf) -> Self {
+ Self::reloadable(Arc::new(move || {
+ let model = ChineseClip::load(&model_path, &vocabulary_path).with_context(|| {
+ format!(
+ "cannot load natural-language search model: {}",
+ model_path.display()
+ )
+ })?;
+ Ok(Box::new(model))
+ }))
+ }
+
+ #[cfg(test)]
+ pub(super) fn preloaded(model: Box) -> Self {
+ Self {
+ current: Some(model),
+ loader: None,
+ load_error: None,
+ last_used: Some(Instant::now()),
+ }
+ }
+
+ #[cfg(test)]
+ pub(super) fn with_loader(
+ loader: impl Fn() -> Result> + Send + Sync + 'static,
+ ) -> Self {
+ Self::reloadable(Arc::new(loader))
+ }
+
+ fn reloadable(loader: Arc) -> Self {
+ Self {
+ current: None,
+ loader: Some(loader),
+ load_error: None,
+ last_used: None,
+ }
+ }
+
+ pub(super) fn embed_thumbnail(&mut self, path: &Path) -> Result {
+ self.run(|model| model.embed_thumbnail(path))
+ }
+
+ pub(super) fn embed_text(&mut self, text: &str) -> Result {
+ self.run(|model| model.embed_text(text))
+ }
+
+ fn run(
+ &mut self,
+ operation: impl FnOnce(&mut dyn TextImageModel) -> Result,
+ ) -> Result {
+ self.ensure_loaded()?;
+ let result = operation(self.current.as_deref_mut().expect("model must be loaded"));
+ self.last_used = Some(Instant::now());
+ result
+ }
+
+ fn ensure_loaded(&mut self) -> Result<()> {
+ if self.current.is_some() {
+ return Ok(());
+ }
+ if let Some(error) = &self.load_error {
+ bail!("natural-language search model is unavailable: {error}");
+ }
+ let loader = self
+ .loader
+ .as_ref()
+ .context("natural-language search model cannot be reloaded")?
+ .clone();
+ info!(
+ model = MODEL_NAME,
+ "loading natural-language search model on demand"
+ );
+ match loader() {
+ Ok(model) => {
+ self.current = Some(model);
+ self.last_used = Some(Instant::now());
+ info!(model = MODEL_NAME, "natural-language search model loaded");
+ Ok(())
+ }
+ Err(error) => {
+ let message = format!("{error:#}");
+ self.load_error = Some(message.clone());
+ Err(anyhow!(message))
+ }
+ }
+ }
+
+ pub(super) fn release_if_idle(&mut self, timeout: Duration) -> bool {
+ if self.loader.is_none() || self.current.is_none() {
+ return false;
+ }
+ if !self
+ .last_used
+ .is_some_and(|last_used| last_used.elapsed() >= timeout)
+ {
+ return false;
+ }
+ self.current = None;
+ self.last_used = None;
+ true
+ }
+}
+
+struct ChineseClip {
+ model: ChineseClipModel,
+ tokenizer: Tokenizer,
+ device: Device,
+}
+
+impl ChineseClip {
+ fn load(model_path: &Path, vocabulary_path: &Path) -> Result {
+ let device = Device::Cpu;
+ // SAFETY: Pixhelf treats the configured model as immutable while the
+ // process is running. Reloads use the same canonical file.
+ let variables =
+ unsafe { VarBuilder::from_mmaped_safetensors(&[model_path], DType::F32, &device)? };
+ let config = ChineseClipConfig::clip_vit_base_patch16();
+ Ok(Self {
+ model: ChineseClipModel::new(variables, &config)?,
+ tokenizer: load_tokenizer(vocabulary_path)?,
+ device,
+ })
+ }
+}
+
+#[cfg(test)]
+pub(super) fn validate_checkpoint(model_path: &Path, vocabulary_path: &Path) -> Result<()> {
+ ChineseClip::load(model_path, vocabulary_path).map(|_| ())
+}
+
+impl TextImageModel for ChineseClip {
+ fn embed_thumbnail(&mut self, thumbnail: &Path) -> Result {
+ let input = preprocess_thumbnail(thumbnail)?;
+ let values = self
+ .model
+ .get_image_features(&input)?
+ .i(0)?
+ .to_vec1::()?;
+ TextSearchEmbedding::quantize(&values, "text-search image model")
+ }
+
+ fn embed_text(&mut self, text: &str) -> Result {
+ let encoding = self
+ .tokenizer
+ .encode(text, true)
+ .map_err(|error| anyhow!("cannot tokenize text query: {error}"))?;
+ if encoding.is_empty() {
+ bail!("text query did not produce any tokens");
+ }
+ let input_ids = Tensor::new(encoding.get_ids(), &self.device)?.unsqueeze(0)?;
+ let token_type_ids = Tensor::new(encoding.get_type_ids(), &self.device)?.unsqueeze(0)?;
+ let attention_mask =
+ Tensor::new(encoding.get_attention_mask(), &self.device)?.unsqueeze(0)?;
+ let values = self
+ .model
+ .get_text_features(&input_ids, Some(&token_type_ids), Some(&attention_mask))?
+ .i(0)?
+ .to_vec1::()?;
+ TextSearchEmbedding::quantize(&values, "text-search text model")
+ }
+}
+
+fn load_tokenizer(vocabulary_path: &Path) -> Result {
+ let vocabulary = vocabulary_path.to_str().with_context(|| {
+ format!(
+ "vocabulary path is not UTF-8: {}",
+ vocabulary_path.display()
+ )
+ })?;
+ let model = WordPiece::from_file(vocabulary)
+ .unk_token("[UNK]".to_owned())
+ .build()
+ .map_err(|error| anyhow!("cannot load Chinese-CLIP vocabulary: {error}"))?;
+ let mut tokenizer = Tokenizer::new(model);
+ let cls_id = tokenizer
+ .token_to_id("[CLS]")
+ .context("Chinese-CLIP vocabulary is missing [CLS]")?;
+ let sep_id = tokenizer
+ .token_to_id("[SEP]")
+ .context("Chinese-CLIP vocabulary is missing [SEP]")?;
+ tokenizer.with_normalizer(Some(BertNormalizer::default()));
+ tokenizer.with_pre_tokenizer(Some(BertPreTokenizer));
+ tokenizer.with_post_processor(Some(BertProcessing::new(
+ ("[SEP]".to_owned(), sep_id),
+ ("[CLS]".to_owned(), cls_id),
+ )));
+ tokenizer
+ .with_truncation(Some(TruncationParams {
+ max_length: TEXT_CONTEXT_LENGTH,
+ ..Default::default()
+ }))
+ .map_err(|error| anyhow!("cannot configure Chinese-CLIP tokenizer: {error}"))?;
+ Ok(tokenizer)
+}
+
+fn preprocess_thumbnail(path: &Path) -> Result {
+ let image = ImageReader::open(path)
+ .with_context(|| format!("cannot open thumbnail {}", path.display()))?
+ .with_guessed_format()?
+ .decode()
+ .with_context(|| format!("cannot decode thumbnail {}", path.display()))?;
+ preprocess_image(&image)
+}
+
+fn preprocess_image(image: &DynamicImage) -> Result {
+ let (width, height) = image.dimensions();
+ if width == 0 || height == 0 {
+ bail!("cannot embed an empty image");
+ }
+ // This checkpoint resizes directly to 224x224 without a centre crop.
+ let pixels = image
+ .resize_exact(MODEL_EDGE, MODEL_EDGE, FilterType::CatmullRom)
+ .to_rgb8();
+ const MEAN: [f32; 3] = [0.481_454_66, 0.457_827_5, 0.408_210_73];
+ const STANDARD_DEVIATION: [f32; 3] = [0.268_629_54, 0.261_302_6, 0.275_777_1];
+ let mut values = Vec::with_capacity((MODEL_EDGE * MODEL_EDGE * 3) as usize);
+ for channel in 0..3 {
+ values.extend(pixels.pixels().map(|pixel| {
+ (f32::from(pixel.0[channel]) / 255.0 - MEAN[channel]) / STANDARD_DEVIATION[channel]
+ }));
+ }
+ Ok(Tensor::from_vec(
+ values,
+ (1, 3, MODEL_EDGE as usize, MODEL_EDGE as usize),
+ &Device::Cpu,
+ )?)
+}
+
+#[cfg(test)]
+mod tests {
+ use std::fs;
+
+ use super::*;
+ use crate::text_search::text_image_similarity;
+
+ #[test]
+ fn tokenizer_uses_chinese_bert_special_tokens() {
+ let temp = tempfile::tempdir().unwrap();
+ let path = temp.path().join("vocab.txt");
+ let mut vocabulary = vec!["[PAD]".to_owned()];
+ vocabulary.extend((0..99).map(|index| format!("[unused{index}]")));
+ vocabulary.extend(
+ ["[UNK]", "[CLS]", "[SEP]", "[MASK]", "海", "边", "日", "落"].map(str::to_owned),
+ );
+ fs::write(&path, format!("{}\n", vocabulary.join("\n"))).unwrap();
+
+ let encoding = load_tokenizer(&path)
+ .unwrap()
+ .encode("海边日落", true)
+ .unwrap();
+ assert_eq!(encoding.get_ids().first(), Some(&101));
+ assert_eq!(encoding.get_ids().last(), Some(&102));
+ assert!(encoding.len() >= 6);
+ }
+
+ #[test]
+ fn preprocessing_matches_official_chinese_clip_shape() {
+ let image = DynamicImage::ImageRgb8(image::RgbImage::from_fn(400, 200, |x, y| {
+ image::Rgb([(x % 255) as u8, (y % 255) as u8, 128])
+ }));
+ let tensor = preprocess_image(&image).unwrap();
+ assert_eq!(tensor.dims(), &[1, 3, 224, 224]);
+ assert!(tensor.flatten_all().unwrap().to_vec1::().unwrap()[0].is_finite());
+ }
+
+ #[test]
+ fn official_model_runs_when_supplied_by_the_test_environment() {
+ let (Some(model_path), Some(vocabulary_path)) = (
+ std::env::var_os("PIXHELF_CHINESE_CLIP_MODEL"),
+ std::env::var_os("PIXHELF_CHINESE_CLIP_VOCAB"),
+ ) else {
+ return;
+ };
+ let temp = tempfile::tempdir().unwrap();
+ let red = temp.path().join("red.png");
+ let blue = temp.path().join("blue.png");
+ image::RgbImage::from_pixel(224, 224, image::Rgb([235, 30, 25]))
+ .save(&red)
+ .unwrap();
+ image::RgbImage::from_pixel(224, 224, image::Rgb([25, 40, 235]))
+ .save(&blue)
+ .unwrap();
+
+ let mut model =
+ ChineseClip::load(Path::new(&model_path), Path::new(&vocabulary_path)).unwrap();
+ let query = model.embed_text("一张红色的图片").unwrap();
+ let red = model.embed_thumbnail(&red).unwrap();
+ let blue = model.embed_thumbnail(&blue).unwrap();
+ assert!(
+ text_image_similarity(&query, &red) > text_image_similarity(&query, &blue),
+ "red query should rank the red image first"
+ );
+ }
+}
diff --git a/src/thumbs.rs b/src/thumbs.rs
index bf9fd8b..1ba4a5e 100644
--- a/src/thumbs.rs
+++ b/src/thumbs.rs
@@ -54,7 +54,7 @@ pub struct ThumbnailManager {
queue: JobQueue,
initial_ready: AtomicBool,
similarity_warmup_running: AtomicBool,
- text_search: Option>,
+ text_search: RwLock>>,
}
#[derive(Debug)]
@@ -132,10 +132,48 @@ impl ThumbnailManager {
queue: JobQueue::default(),
initial_ready: AtomicBool::new(false),
similarity_warmup_running: AtomicBool::new(false),
- text_search,
+ text_search: RwLock::new(text_search),
}))
}
+ fn text_search(&self) -> Option> {
+ read_lock(&self.text_search).clone()
+ }
+
+ /// Attach an automatically downloaded search model without restarting the
+ /// web service, then backfill every thumbnail that is already available.
+ pub fn enable_text_search(&self, text_search: Arc) -> bool {
+ {
+ let mut current = write_lock(&self.text_search);
+ if current.is_some() {
+ return false;
+ }
+ *current = Some(Arc::clone(&text_search));
+ }
+
+ // The model is visible before this snapshot. A thumbnail completed
+ // after that point enqueues itself in the regular worker path, while a
+ // thumbnail completed before it is included below.
+ let (ids, ready) = {
+ let entries = read_lock(&self.entries);
+ let ids = entries.keys().cloned().collect::>();
+ let ready = entries
+ .values()
+ .filter(|entry| matches!(*mutex_lock(&entry.state), ThumbState::Ready))
+ .filter(|entry| valid_cache_file(&entry.cache_path))
+ .map(|entry| (entry.record.id.clone(), entry.cache_path.clone()))
+ .collect::>();
+ (ids, ready)
+ };
+ let current_ids = ids.iter().map(String::as_str).collect::>();
+ text_search.reconcile(¤t_ids);
+ for (id, thumbnail) in ready {
+ text_search.enqueue(&id, &thumbnail, false);
+ }
+ text_search.start_worker();
+ true
+ }
+
pub fn reconcile(&self, records: &[Arc]) {
let current_ids: HashSet<&str> = records.iter().map(|record| record.id.as_str()).collect();
let mut missing = Vec::new();
@@ -180,7 +218,7 @@ impl ThumbnailManager {
for id in missing {
self.queue.push_background(id);
}
- if let Some(text_search) = &self.text_search {
+ if let Some(text_search) = self.text_search() {
text_search.reconcile(¤t_ids);
let ready = {
let entries = read_lock(&self.entries);
@@ -304,8 +342,8 @@ impl ThumbnailManager {
initial_batch_ready: self.initial_ready.load(Ordering::Acquire),
background_complete: ready + failed == total && processing == 0,
text_search: self
- .text_search
- .as_ref()
+ .text_search()
+ .as_deref()
.map_or_else(TextSearchStatus::disabled, |index| index.status()),
}
}
@@ -314,19 +352,19 @@ impl ThumbnailManager {
&self,
query: &str,
) -> Result>> {
- let Some(index) = &self.text_search else {
+ let Some(index) = self.text_search() else {
return Ok(None);
};
index.embed_query(query).await.map(Some)
}
pub fn text_search_embedding(&self, id: &str) -> Option> {
- self.text_search.as_ref()?.embedding(id)
+ self.text_search()?.embedding(id)
}
pub fn text_search_cache_token(&self) -> String {
- self.text_search
- .as_ref()
+ self.text_search()
+ .as_deref()
.map_or_else(|| "off".to_owned(), |index| index.cache_token())
}
@@ -447,7 +485,7 @@ impl ThumbnailManager {
}
};
if ready {
- if let Some(text_search) = &self.text_search {
+ if let Some(text_search) = self.text_search() {
text_search.enqueue(&id, &entry.cache_path, false);
}
entry.notify.notify_waiters();
@@ -656,6 +694,18 @@ mod tests {
use super::*;
use crate::gallery::scan_gallery;
+ struct ConstantTextSearchModel;
+
+ impl crate::text_search::TextImageModel for ConstantTextSearchModel {
+ fn embed_thumbnail(&mut self, _thumbnail: &Path) -> Result {
+ Ok(TextSearchEmbedding::for_test([(0, 100)]))
+ }
+
+ fn embed_text(&mut self, _text: &str) -> Result {
+ Ok(TextSearchEmbedding::for_test([(0, 100)]))
+ }
+ }
+
#[test]
fn generates_one_bounded_webp_thumbnail() {
let temp = tempfile::tempdir().unwrap();
@@ -760,6 +810,42 @@ mod tests {
.expect("similarity descriptor warmup timed out");
}
+ #[tokio::test]
+ async fn downloaded_text_search_attaches_to_existing_thumbnails() {
+ let temp = tempfile::tempdir().unwrap();
+ let gallery = temp.path().join("gallery");
+ fs::create_dir(&gallery).unwrap();
+ image::RgbImage::new(20, 10)
+ .save(gallery.join("image.png"))
+ .unwrap();
+ let gallery = scan_gallery(&gallery, None).unwrap();
+ let manager = ThumbnailManager::new(temp.path().join("cache")).unwrap();
+ manager.reconcile(&gallery.images);
+ manager.start_workers(1);
+ let id = gallery.images[0].id.clone();
+ manager.ensure_ready(&id).await.unwrap();
+ assert!(!manager.status().text_search.enabled);
+
+ let text_search = TextSearchIndex::with_test_model(
+ Box::new(ConstantTextSearchModel),
+ [17; crate::support::embedding::FINGERPRINT_BYTES],
+ );
+ assert!(manager.enable_text_search(text_search));
+
+ tokio::time::timeout(Duration::from_secs(2), async {
+ loop {
+ let status = manager.status().text_search;
+ if status.background_complete && status.ready == 1 {
+ break;
+ }
+ tokio::time::sleep(Duration::from_millis(10)).await;
+ }
+ })
+ .await
+ .expect("dynamically attached text-search index timed out");
+ assert!(manager.text_search_embedding(&id).is_some());
+ }
+
#[tokio::test]
async fn changed_source_requires_a_fresh_gallery_record() {
let temp = tempfile::tempdir().unwrap();
diff --git a/src/web.rs b/src/web.rs
index 575aebd..b504b78 100644
--- a/src/web.rs
+++ b/src/web.rs
@@ -24,6 +24,7 @@ use tracing::error;
use crate::{
gallery::{Album, GalleryIndex, ImageRecord},
+ photo_details::PhotoDetails,
similarity::{
DiversityFingerprint, MIN_SIMILARITY_SCORE, redundancy_score, signature_for_thumbnail,
similarity_score,
@@ -56,6 +57,7 @@ const TEXT_SEARCH_RANKING_CACHE_LIMIT: usize = 16;
const TEXT_SEARCH_RESULT_LIMIT: usize = 600;
const FILENAME_MATCH_BOOST: f32 = 3.0;
const MAX_QUERY_VALUE_BYTES: usize = 4096;
+const PHOTO_DETAILS_CACHE_LIMIT: usize = 64;
static SIMILARITY_LIMIT: OnceLock> = OnceLock::new();
type SimilarityRanking = Arc>>;
@@ -64,6 +66,8 @@ static SIMILARITY_RANKING_CACHE: OnceLock = OnceLock::ne
type TextSearchRanking = Arc>>;
type TextSearchRankingCache = Mutex>;
static TEXT_SEARCH_RANKING_CACHE: OnceLock = OnceLock::new();
+type PhotoDetailsCache = Mutex>>;
+static PHOTO_DETAILS_CACHE: OnceLock = OnceLock::new();
fn similarity_limit() -> Arc {
Arc::clone(SIMILARITY_LIMIT.get_or_init(|| Arc::new(Semaphore::new(1))))
@@ -95,6 +99,18 @@ fn cache_text_search_ranking(key: String, ranking: TextSearchRanking) {
mutex_lock(text_search_ranking_cache()).insert(key, ranking);
}
+fn photo_details_cache() -> &'static PhotoDetailsCache {
+ PHOTO_DETAILS_CACHE.get_or_init(|| Mutex::new(BoundedCache::new(PHOTO_DETAILS_CACHE_LIMIT)))
+}
+
+fn cached_photo_details(key: &str) -> Option> {
+ mutex_lock(photo_details_cache()).get_cloned(key)
+}
+
+fn cache_photo_details(key: String, details: Arc) {
+ mutex_lock(photo_details_cache()).insert(key, details);
+}
+
#[derive(Clone)]
pub struct AppState {
pub index: Arc>>,
@@ -250,6 +266,7 @@ pub fn router(state: AppState) -> Router {
.route("/api/health", get(health))
.route("/api/gallery", get(gallery_summary))
.route("/api/images", get(images))
+ .route("/api/images/{id}/details", get(image_details))
.route("/api/images/{id}/similar", get(similar_images))
.route("/api/status", get(thumbnail_status))
.route("/api/images/{id}/thumbnail", get(thumbnail))
@@ -833,6 +850,81 @@ async fn thumbnail_status(State(state): State) -> Response {
response
}
+async fn image_details(
+ State(state): State,
+ Path(id): Path,
+ headers: HeaderMap,
+) -> Response {
+ let image = {
+ let index = state.index.read().await;
+ let Some(image) = index.image(&id).cloned() else {
+ return StatusCode::NOT_FOUND.into_response();
+ };
+ image
+ };
+ let metadata = match tokio::fs::metadata(&image.path).await {
+ Ok(metadata) => metadata,
+ Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
+ return StatusCode::NOT_FOUND.into_response();
+ }
+ Err(error) => {
+ error!(path = %image.path.display(), %error, "cannot inspect image details source");
+ return StatusCode::INTERNAL_SERVER_ERROR.into_response();
+ }
+ };
+ if !image.matches_metadata(&metadata) {
+ return StatusCode::NOT_FOUND.into_response();
+ }
+
+ let etag = format!("\"details-v1-{id}\"");
+ if is_not_modified(&headers, &etag) {
+ return not_modified(&etag, "private, max-age=31536000, immutable");
+ }
+
+ let cache_key = format!("{}:{}", image.path.display(), image.id);
+ if let Some(details) = cached_photo_details(&cache_key) {
+ return with_cache_headers(
+ Json(details.as_ref()).into_response(),
+ &etag,
+ "private, max-age=31536000, immutable",
+ );
+ }
+
+ let thumbnail = match state.thumbnails.ensure_ready(&id).await {
+ Ok(path) => path,
+ Err(ThumbnailError::NotFound) => return StatusCode::NOT_FOUND.into_response(),
+ Err(error) => {
+ error!(%id, %error, "cannot prepare histogram source");
+ return StatusCode::INTERNAL_SERVER_ERROR.into_response();
+ }
+ };
+ let details = match tokio::task::spawn_blocking({
+ let image = Arc::clone(&image);
+ move || PhotoDetails::read(&image, &thumbnail)
+ })
+ .await
+ {
+ Ok(Ok(details)) => Arc::new(details),
+ Ok(Err(error)) => {
+ if image.ensure_source_is_current().is_err() {
+ return StatusCode::NOT_FOUND.into_response();
+ }
+ error!(%id, %error, "cannot read image details");
+ return StatusCode::INTERNAL_SERVER_ERROR.into_response();
+ }
+ Err(error) => {
+ error!(%id, %error, "image details task stopped");
+ return StatusCode::INTERNAL_SERVER_ERROR.into_response();
+ }
+ };
+ cache_photo_details(cache_key, Arc::clone(&details));
+ with_cache_headers(
+ Json(details.as_ref()).into_response(),
+ &etag,
+ "private, max-age=31536000, immutable",
+ )
+}
+
async fn thumbnail(
State(state): State,
Path(id): Path,
@@ -1310,6 +1402,10 @@ mod tests {
request("/api/images/missing/original").await.status(),
StatusCode::NOT_FOUND
);
+ assert_eq!(
+ request("/api/images/missing/details").await.status(),
+ StatusCode::NOT_FOUND
+ );
assert_eq!(
request("/api/images/missing/similar").await.status(),
StatusCode::NOT_FOUND
@@ -1551,6 +1647,70 @@ mod tests {
assert!(!body.is_empty());
}
+ #[tokio::test]
+ async fn image_details_expose_file_data_and_histogram() {
+ let temp = tempfile::tempdir().unwrap();
+ let gallery = temp.path().join("gallery");
+ std::fs::create_dir(&gallery).unwrap();
+ image::RgbImage::from_pixel(20, 10, image::Rgb([180, 90, 30]))
+ .save(gallery.join("image.png"))
+ .unwrap();
+ let index = crate::gallery::scan_gallery(&gallery, None).unwrap();
+ let id = index.images[0].id.clone();
+ let expected_size = index.images[0].size;
+ let thumbnails = ThumbnailManager::new(temp.path().join("cache")).unwrap();
+ thumbnails.reconcile(&index.images);
+ thumbnails.start_workers(1);
+ let app = router(AppState {
+ index: Arc::new(RwLock::new(Arc::new(index))),
+ thumbnails,
+ });
+
+ let response = app
+ .clone()
+ .oneshot(
+ Request::builder()
+ .uri(format!("/api/images/{id}/details"))
+ .body(Body::empty())
+ .unwrap(),
+ )
+ .await
+ .unwrap();
+ assert_eq!(response.status(), StatusCode::OK);
+ assert_eq!(
+ response.headers()[header::CACHE_CONTROL],
+ "private, max-age=31536000, immutable"
+ );
+ let etag = response.headers()[header::ETAG].clone();
+ let body = axum::body::to_bytes(response.into_body(), usize::MAX)
+ .await
+ .unwrap();
+ let details: serde_json::Value = serde_json::from_slice(&body).unwrap();
+ assert_eq!(details["fileSize"], expected_size);
+ assert!(details["modifiedMs"].as_u64().unwrap() > 0);
+ assert!(details["exif"].as_array().unwrap().is_empty());
+ for channel in ["red", "green", "blue", "luminance"] {
+ let bins = details["histogram"][channel].as_array().unwrap();
+ assert_eq!(bins.len(), 256);
+ assert_eq!(
+ bins.iter().map(|bin| bin.as_u64().unwrap()).sum::(),
+ 200
+ );
+ }
+
+ let cached = app
+ .oneshot(
+ Request::builder()
+ .uri(format!("/api/images/{id}/details"))
+ .header(header::IF_NONE_MATCH, etag)
+ .body(Body::empty())
+ .unwrap(),
+ )
+ .await
+ .unwrap();
+ assert_eq!(cached.status(), StatusCode::NOT_MODIFIED);
+ }
+
#[tokio::test]
async fn stale_image_ids_do_not_serve_replaced_files() {
let temp = tempfile::tempdir().unwrap();