refactor(markdown): preserve source ranges for math

This commit is contained in:
kuddev committed 2026-07-19 16:50:22 +08:00
1 parent 50ebf31903
commit 51ebaa5bf9
3 files changed
+321 -59

No files matched your search

+15 -3
View File
@@ -262,7 +262,7 @@ fn wrap_inline(inline: &FormattedTextInline, max_cols: usize) -> Vec<Vec<Span>>
.iter()
.enumerate()
.flat_map(|(frag, fragment)| {
fragment.text.chars().map(move |ch| WrapChar {
fragment.text().chars().map(move |ch| WrapChar {
ch,
frag,
cols: ch.width().unwrap_or(0).max(1),
@@ -439,6 +439,14 @@ impl LayoutCtx<'_> {
}
self.y += ch * 0.5;
},
// 原生数学布局接入前保留确定的源码回退,解析失败也不会吞掉文档内容。
FormattedTextLine::DisplayMath(math) => {
self.y += ch * 0.35;
let mut span = Span::label(math.as_str(), false);
span.code = true;
self.push(ch * BODY_LINE, indent, 1.0, vec![span], decor);
self.y += ch * 0.5;
},
FormattedTextLine::UnorderedList(list) => {
self.list_item(&list.text, list.indent_level, "• ", quote_depth, extra_indent);
},
@@ -546,7 +554,11 @@ impl LayoutCtx<'_> {
fn table(&mut self, table: &crate::markdown::FormattedTable, quote_depth: usize, indent: f32) {
let ch = self.cell_h;
let inline_cols = |inline: &FormattedTextInline| -> usize {
inline.iter().flat_map(|f| f.text.chars()).map(|c| c.width().unwrap_or(0).max(1)).sum()
inline
.iter()
.flat_map(|f| f.text().chars())
.map(|c| c.width().unwrap_or(0).max(1))
.sum()
};
let col_count =
table.headers.len().max(table.rows.iter().map(Vec::len).max().unwrap_or(0)).max(1);
@@ -603,7 +615,7 @@ fn cell_spans(
let mut used = 0usize;
if let Some(inline) = cell {
'outer: for (frag, fragment) in inline.iter().enumerate() {
for ch in fragment.text.chars() {
for ch in fragment.text().chars() {
let cols = ch.width().unwrap_or(0).max(1);
if used + cols > width {
// No room left: truncate with an ellipsis if one fits.
+168 -17
View File
@@ -12,6 +12,7 @@
use std::collections::VecDeque;
use std::fmt;
use std::ops::Range;
use std::sync::Arc;
pub mod parser;
@@ -42,12 +43,14 @@ impl CustomWeight {
/// A parsed Markdown document: a flat sequence of block-level lines.
#[derive(Debug, Clone, PartialEq, Eq, Default)]
pub struct FormattedText {
/// Markdown 源码只保留这一份共享字节;fragment 用范围引用它,避免逐层复制。
pub source: Arc<str>,
pub lines: VecDeque<FormattedTextLine>,
}
impl FormattedText {
pub fn new(lines: impl Into<VecDeque<FormattedTextLine>>) -> Self {
Self { lines: lines.into() }
Self { source: Arc::from(""), lines: lines.into() }
}
/// The document's raw text, without any of the markdown markers.
@@ -56,6 +59,108 @@ impl FormattedText {
}
}
/// UTF-8 源码中的半开字节范围。文档输入上限在解析入口保证可用 `u32` 表示。
#[derive(Clone, Copy, Debug, Default, Eq, Hash, PartialEq)]
pub struct SourceRange {
pub start: u32,
pub end: u32,
}
impl SourceRange {
pub fn new(range: Range<usize>) -> Option<Self> {
Some(Self { start: range.start.try_into().ok()?, end: range.end.try_into().ok()? })
}
pub fn as_usize(self) -> Range<usize> {
self.start as usize..self.end as usize
}
}
/// 绝大多数内容引用 Markdown 源码;只有实体解码、软换行等合成文本单独分配。
#[derive(Clone, Debug)]
pub enum TextRef {
Source { source: Arc<str>, range: SourceRange },
Generated(Box<str>),
}
impl Default for TextRef {
fn default() -> Self {
Self::Generated(Box::default())
}
}
impl PartialEq for TextRef {
fn eq(&self, other: &Self) -> bool {
self.as_str() == other.as_str()
}
}
impl Eq for TextRef {}
impl TextRef {
pub fn source(source: Arc<str>, range: Range<usize>) -> Option<Self> {
let range = SourceRange::new(range)?;
source.get(range.as_usize())?;
Some(Self::Source { source, range })
}
pub fn generated(text: impl Into<Box<str>>) -> Self {
Self::Generated(text.into())
}
pub fn as_str(&self) -> &str {
match self {
Self::Source { source, range } => &source[range.as_usize()],
Self::Generated(text) => text,
}
}
/// 只合并同一源码中的连续范围;跨 Markdown 标记的内容保持独立,避免错误扩大范围。
fn append(&mut self, next: Self) -> Result<(), Self> {
match (self, next) {
(
Self::Source { source, range },
Self::Source { source: next_source, range: next_range },
) if Arc::ptr_eq(source, &next_source) && range.end == next_range.start => {
range.end = next_range.end;
Ok(())
},
(slot @ Self::Generated(_), Self::Generated(next)) => {
let mut combined = String::from(slot.as_str());
combined.push_str(&next);
*slot = Self::Generated(combined.into_boxed_str());
Ok(())
},
// 实体解码或反斜杠转义会在同样式文本中留下源码间隙;此时只合成这一小段,
// 既保持旧的 fragment 合并语义,也不扩大 range 到 Markdown 标记字节。
(slot, next) => {
let mut combined = String::from(slot.as_str());
combined.push_str(next.as_str());
*slot = Self::Generated(combined.into_boxed_str());
Ok(())
},
}
}
}
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
pub enum MathMode {
Inline,
Display,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct MathSource {
pub source: TextRef,
pub mode: MathMode,
}
impl MathSource {
pub fn as_str(&self) -> &str {
self.source.as_str()
}
}
/// One block-level element of the document.
#[derive(Debug, Clone, Eq, PartialEq)]
pub enum FormattedTextLine {
@@ -69,16 +174,20 @@ pub enum FormattedTextLine {
HorizontalRule,
Image(FormattedImage),
Table(FormattedTable),
DisplayMath(MathSource),
/// A block nested inside `depth` levels of `>` quoting. Quoting wraps the
/// inner block instead of forking every variant, so the flat list (and
/// the virtual scroller's block indexing) survives.
Quote { depth: usize, line: Box<FormattedTextLine> },
Quote {
depth: usize,
line: Box<FormattedTextLine>,
},
}
impl FormattedTextLine {
pub fn raw_text(&self) -> String {
let join = |inline: &FormattedTextInline| -> String {
inline.iter().map(|fragment| fragment.text.as_str()).collect()
inline.iter().map(FormattedTextFragment::text).collect()
};
let mut text = match self {
Self::CodeBlock(text) => text.code.clone(),
@@ -90,6 +199,7 @@ impl FormattedTextLine {
Self::LineBreak | Self::HorizontalRule => "\n".to_string(),
Self::Image(image) => format!("{}\n", image.alt_text),
Self::Table(table) => table.to_plain_text(),
Self::DisplayMath(math) => math.as_str().to_owned(),
Self::Quote { line, .. } => line.raw_text(),
};
// Each `FormattedTextLine` unit represents a complete line.
@@ -111,7 +221,8 @@ impl FormattedTextLine {
| FormattedTextLine::LineBreak
| FormattedTextLine::HorizontalRule
| FormattedTextLine::Image(_)
| FormattedTextLine::Table(_) => None,
| FormattedTextLine::Table(_)
| FormattedTextLine::DisplayMath(_) => None,
}
}
@@ -123,7 +234,7 @@ impl FormattedTextLine {
let mut char_count = 0;
for fragment in inline_fragments {
let range_start = char_count;
char_count += fragment.text.chars().count();
char_count += fragment.text().chars().count();
if let Some(Hyperlink::Url(url)) = &fragment.styles.hyperlink {
hyperlinks.push((range_start..char_count, url.clone()));
}
@@ -196,7 +307,7 @@ impl FormattedTable {
/// Serialize to GFM pipe-table markdown (used for raw-text extraction).
pub fn to_plain_text(&self) -> String {
fn inline_to_text(inline: &FormattedTextInline) -> String {
inline.iter().map(|f| f.text.as_str()).collect()
inline.iter().map(FormattedTextFragment::text).collect()
}
let mut lines = Vec::new();
@@ -222,10 +333,22 @@ impl FormattedTable {
pub type FormattedTextInline = Vec<FormattedTextFragment>;
/// A fragment of formatted text: the text itself plus formatting flags.
#[derive(Debug, Clone, Eq, PartialEq)]
pub enum FragmentContent {
Text(TextRef),
Math(MathSource),
}
impl Default for FragmentContent {
fn default() -> Self {
Self::Text(TextRef::default())
}
}
/// A fragment of formatted text or an inline mathematical object plus formatting flags.
#[derive(Debug, Clone, Eq, PartialEq, Default)]
pub struct FormattedTextFragment {
pub text: String,
pub content: FragmentContent,
pub styles: FormattedTextStyles,
}
@@ -249,29 +372,26 @@ pub struct FormattedTextStyles {
impl FormattedTextFragment {
pub fn plain_text(text: impl Into<String>) -> Self {
Self { text: text.into(), styles: Default::default() }
Self::from_generated(text, Default::default())
}
pub fn bold(text: impl Into<String>) -> Self {
Self {
text: text.into(),
styles: FormattedTextStyles {
weight: Some(CustomWeight::Bold),
..Default::default()
},
content: FragmentContent::Text(TextRef::generated(text.into().into_boxed_str())),
styles: FormattedTextStyles { weight: Some(CustomWeight::Bold), ..Default::default() },
}
}
pub fn italic(text: impl Into<String>) -> Self {
Self {
text: text.into(),
content: FragmentContent::Text(TextRef::generated(text.into().into_boxed_str())),
styles: FormattedTextStyles { italic: true, ..Default::default() },
}
}
pub fn hyperlink(tag: impl Into<String>, url: impl Into<String>) -> Self {
Self {
text: tag.into(),
content: FragmentContent::Text(TextRef::generated(tag.into().into_boxed_str())),
styles: FormattedTextStyles {
hyperlink: Some(Hyperlink::Url(url.into())),
..Default::default()
@@ -281,10 +401,41 @@ impl FormattedTextFragment {
pub fn inline_code(text: impl Into<String>) -> Self {
Self {
text: text.into(),
content: FragmentContent::Text(TextRef::generated(text.into().into_boxed_str())),
styles: FormattedTextStyles { inline_code: true, ..Default::default() },
}
}
pub fn from_text(text: TextRef, styles: FormattedTextStyles) -> Self {
Self { content: FragmentContent::Text(text), styles }
}
pub fn math(source: MathSource, styles: FormattedTextStyles) -> Self {
Self { content: FragmentContent::Math(source), styles }
}
pub fn text(&self) -> &str {
match &self.content {
FragmentContent::Text(text) => text.as_str(),
FragmentContent::Math(math) => math.as_str(),
}
}
pub fn is_math(&self) -> bool {
matches!(self.content, FragmentContent::Math(_))
}
pub(crate) fn append_text(&mut self, text: TextRef) -> Result<(), TextRef> {
match &mut self.content {
FragmentContent::Text(current) => current.append(text),
FragmentContent::Math(_) => Err(text),
}
}
fn from_generated(text: impl Into<String>, styles: FormattedTextStyles) -> Self {
let text = text.into().into_boxed_str();
Self { content: FragmentContent::Text(TextRef::generated(text)), styles }
}
}
impl fmt::Debug for FormattedTextStyles {
+138 -39
View File
@@ -6,15 +6,16 @@
//! This file owns ONLY the event folding. The AST lives in `super`, the
//! on-screen layout (wrapping, virtual scrolling) in `display::markdown_view`.
use pulldown_cmark::{
Alignment, CodeBlockKind, Event, Options, Parser, Tag, TagEnd,
};
use std::ops::Range;
use std::sync::Arc;
use pulldown_cmark::{Alignment, CodeBlockKind, Event, Options, Parser, Tag, TagEnd};
use super::{
CodeBlockText, CustomWeight, FormattedImage, FormattedIndentTextInline, FormattedTable,
FormattedTaskList, FormattedText, FormattedTextFragment, FormattedTextHeader,
FormattedTextInline, FormattedTextLine, FormattedTextStyles, Hyperlink,
OrderedFormattedIndentTextInline, TableAlignment,
FormattedTextInline, FormattedTextLine, FormattedTextStyles, Hyperlink, MathMode, MathSource,
OrderedFormattedIndentTextInline, TableAlignment, TextRef,
};
/// Parse a Markdown document. GFM tables, task lists and strikethrough are
@@ -25,10 +26,11 @@ pub fn parse_markdown(text: &str) -> FormattedText {
options.insert(Options::ENABLE_STRIKETHROUGH);
options.insert(Options::ENABLE_TASKLISTS);
options.insert(Options::ENABLE_YAML_STYLE_METADATA_BLOCKS);
options.insert(Options::ENABLE_MATH);
let mut fold = Fold::default();
for event in Parser::new_ext(text, options) {
fold.event(event);
let mut fold = Fold::new(Arc::from(text));
for (event, range) in Parser::new_ext(text, options).into_offset_iter() {
fold.event(event, range);
}
fold.finish()
}
@@ -55,8 +57,8 @@ struct TableFold {
}
/// Event-stream folding state. One instance per document.
#[derive(Default)]
struct Fold {
source: Arc<str>,
out: Vec<FormattedTextLine>,
/// Inline run being collected for the current paragraph / heading / list
/// item / table cell.
@@ -83,7 +85,27 @@ struct Fold {
}
impl Fold {
fn event(&mut self, event: Event<'_>) {
fn new(source: Arc<str>) -> Self {
Self {
source,
out: Vec::new(),
inline: Vec::new(),
strong: 0,
emphasis: 0,
strike: 0,
links: Vec::new(),
list_stack: Vec::new(),
pending_item: None,
quote_depth: 0,
heading: None,
code: None,
table: None,
image: None,
in_metadata: false,
}
}
fn event(&mut self, event: Event<'_>, range: Range<usize>) {
match event {
Event::Start(tag) => self.start(tag),
Event::End(tag) => self.end(tag),
@@ -96,22 +118,24 @@ impl Fold {
} else if let Some(image) = &mut self.image {
image.alt_text.push_str(&text);
} else {
self.push_text(&text, false);
self.push_source_text(range, &text, false);
}
},
Event::Code(text) => {
if let Some(image) = &mut self.image {
image.alt_text.push_str(&text);
} else {
self.push_text(&text, true);
self.push_source_text(range, &text, true);
}
},
// The viewer renders HTML as the literal text the author wrote.
Event::Html(html) | Event::InlineHtml(html) => self.push_text(&html, false),
Event::SoftBreak => self.push_text(" ", false),
Event::Html(html) | Event::InlineHtml(html) => {
self.push_source_text(range, &html, false)
},
Event::SoftBreak => self.push_generated_text(" ", false),
// A hard break stays inside the paragraph block: the layout pass
// turns embedded '\n' into a forced visual line break.
Event::HardBreak => self.push_text("\n", false),
Event::HardBreak => self.push_generated_text("\n", false),
Event::Rule => {
self.flush_inline_as_paragraph();
self.push_block(FormattedTextLine::HorizontalRule);
@@ -122,10 +146,23 @@ impl Fold {
}
},
Event::FootnoteReference(name) => {
self.push_text(&format!("[^{name}]"), false);
self.push_generated_text(&format!("[^{name}]"), false);
},
Event::InlineMath(math) | Event::DisplayMath(math) => {
self.push_text(&math, true);
Event::InlineMath(math) => {
let source = self.source_ref(range, &math);
let styles = self.current_styles(false);
self.inline.push(FormattedTextFragment::math(
MathSource { source, mode: MathMode::Inline },
styles,
));
},
Event::DisplayMath(math) => {
let source = self.source_ref(range, &math);
self.flush_inline_as_paragraph();
self.push_block(FormattedTextLine::DisplayMath(MathSource {
source,
mode: MathMode::Display,
}));
},
}
}
@@ -147,11 +184,9 @@ impl Fold {
self.flush_pending_item();
self.flush_inline_as_paragraph();
let lang = match kind {
CodeBlockKind::Fenced(info) => info
.split_whitespace()
.next()
.unwrap_or_default()
.to_owned(),
CodeBlockKind::Fenced(info) => {
info.split_whitespace().next().unwrap_or_default().to_owned()
},
CodeBlockKind::Indented => String::new(),
};
let lang = if lang.is_empty() { "text".to_owned() } else { lang };
@@ -305,25 +340,56 @@ impl Fold {
/// Append text to the current inline run under the active styles,
/// merging into the previous fragment when the styles are identical.
fn push_text(&mut self, text: &str, inline_code: bool) {
if text.is_empty() || self.in_metadata {
fn push_source_text(&mut self, range: Range<usize>, text: &str, inline_code: bool) {
let text = self.source_ref(range, text);
self.push_text_ref(text, inline_code);
}
fn push_generated_text(&mut self, text: &str, inline_code: bool) {
self.push_text_ref(TextRef::generated(text.to_owned().into_boxed_str()), inline_code);
}
fn push_text_ref(&mut self, text: TextRef, inline_code: bool) {
if text.as_str().is_empty() || self.in_metadata {
return;
}
let styles = FormattedTextStyles {
let styles = self.current_styles(inline_code);
let text = if let Some(last) = self.inline.last_mut() {
if last.styles == styles {
match last.append_text(text) {
Ok(()) => return,
Err(text) => text,
}
} else {
text
}
} else {
text
};
self.inline.push(FormattedTextFragment::from_text(text, styles));
}
fn current_styles(&self, inline_code: bool) -> FormattedTextStyles {
FormattedTextStyles {
weight: (self.strong > 0).then_some(CustomWeight::Bold),
italic: self.emphasis > 0,
underline: false,
strikethrough: self.strike > 0,
inline_code,
hyperlink: self.links.last().map(|url| Hyperlink::Url(url.clone())),
};
if let Some(last) = self.inline.last_mut() {
if last.styles == styles {
last.text.push_str(text);
return;
}
}
self.inline.push(FormattedTextFragment { text: text.to_owned(), styles });
}
fn source_ref(&self, outer: Range<usize>, rendered: &str) -> TextRef {
let source_range = self.source.get(outer.clone()).and_then(|outer_text| {
outer_text.find(rendered).map(|offset| {
let start = outer.start + offset;
start..start + rendered.len()
})
});
source_range
.and_then(|range| TextRef::source(self.source.clone(), range))
.unwrap_or_else(|| TextRef::generated(rendered.to_owned().into_boxed_str()))
}
/// Emit the pending inline run as a plain paragraph block, if any.
@@ -346,8 +412,7 @@ impl Fold {
text,
}),
None => {
let indented_text =
FormattedIndentTextInline { indent_level: item.depth, text };
let indented_text = FormattedIndentTextInline { indent_level: item.depth, text };
match item.number {
Some(number) => {
FormattedTextLine::OrderedList(OrderedFormattedIndentTextInline {
@@ -374,13 +439,14 @@ impl Fold {
fn finish(mut self) -> FormattedText {
self.flush_inline_as_paragraph();
FormattedText { lines: self.out.into() }
FormattedText { source: self.source, lines: self.out.into() }
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::markdown::{FragmentContent, MathMode, TextRef};
fn parse(text: &str) -> Vec<FormattedTextLine> {
parse_markdown(text).lines.into_iter().collect()
@@ -390,6 +456,39 @@ mod tests {
FormattedTextFragment::plain_text(text)
}
#[test]
fn math_events_keep_mode_and_utf8_source_ranges() {
let document = parse_markdown("中文 $x^2$\n\n$$\\frac{1}{2}$$");
let FormattedTextLine::Line(inline) = &document.lines[0] else {
panic!("expected paragraph");
};
let FragmentContent::Math(inline_math) = &inline[1].content else {
panic!("expected inline math fragment: {:?}", inline[1]);
};
assert_eq!(inline_math.mode, MathMode::Inline);
assert_eq!(inline_math.source.as_str(), "x^2");
let FormattedTextLine::DisplayMath(display_math) = &document.lines[1] else {
panic!("expected display math block: {:?}", document.lines[1]);
};
assert_eq!(display_math.mode, MathMode::Display);
assert_eq!(display_math.source.as_str(), r"\frac{1}{2}");
let TextRef::Source { source, range } = &inline_math.source else {
panic!("math source should point into the document");
};
assert!(std::sync::Arc::ptr_eq(source, &document.source));
assert_eq!(&document.source[range.as_usize()], "x^2");
}
#[test]
fn escaped_dollars_and_code_do_not_become_math() {
let document = parse_markdown(r"escaped \$x$ and `$code$`");
let FormattedTextLine::Line(inline) = &document.lines[0] else { panic!() };
assert!(inline.iter().all(|fragment| !fragment.is_math()));
assert_eq!(document.raw_text().trim_end(), "escaped $x$ and $code$");
}
#[test]
fn heading_paragraph_and_inline_styles() {
let lines = parse("# Title\n\nplain **bold** *it* `code` ~~gone~~");
@@ -410,7 +509,7 @@ mod tests {
assert_eq!(inline[4], plain(" "));
assert_eq!(inline[5], FormattedTextFragment::inline_code("code"));
assert!(inline[7].styles.strikethrough);
assert_eq!(inline[7].text, "gone");
assert_eq!(inline[7].text(), "gone");
}
#[test]
@@ -507,7 +606,7 @@ mod tests {
// A hard break stays inside one paragraph as an embedded newline.
let lines = parse("one \ntwo");
let FormattedTextLine::Line(inline) = &lines[0] else { panic!() };
let text: String = inline.iter().map(|f| f.text.as_str()).collect();
let text: String = inline.iter().map(FormattedTextFragment::text).collect();
assert_eq!(text, "one\ntwo");
}
@@ -545,6 +644,6 @@ mod tests {
let lines = parse("a &amp; b");
let FormattedTextLine::Line(inline) = &lines[0] else { panic!() };
assert_eq!(inline.len(), 1);
assert_eq!(inline[0].text, "a & b");
assert_eq!(inline[0].text(), "a & b");
}
}