From 51ebaa5bf9fa07a857e1f90b2dabea24fda45685 Mon Sep 17 00:00:00 2001 From: kuddev Date: Sat, 18 Jul 2026 23:18:47 +0800 Subject: [PATCH] refactor(markdown): preserve source ranges for math --- nebula_app/src/display/markdown_view.rs | 18 ++- nebula_app/src/markdown/mod.rs | 185 +++++++++++++++++++++--- nebula_app/src/markdown/parser.rs | 177 ++++++++++++++++++----- 3 files changed, 321 insertions(+), 59 deletions(-) diff --git a/nebula_app/src/display/markdown_view.rs b/nebula_app/src/display/markdown_view.rs index 0fecd5a..d75ee61 100644 --- a/nebula_app/src/display/markdown_view.rs +++ b/nebula_app/src/display/markdown_view.rs @@ -262,7 +262,7 @@ fn wrap_inline(inline: &FormattedTextInline, max_cols: usize) -> Vec> .iter() .enumerate() .flat_map(|(frag, fragment)| { - fragment.text.chars().map(move |ch| WrapChar { + fragment.text().chars().map(move |ch| WrapChar { ch, frag, cols: ch.width().unwrap_or(0).max(1), @@ -439,6 +439,14 @@ impl LayoutCtx<'_> { } self.y += ch * 0.5; }, + // 原生数学布局接入前保留确定的源码回退，解析失败也不会吞掉文档内容。 + FormattedTextLine::DisplayMath(math) => { + self.y += ch * 0.35; + let mut span = Span::label(math.as_str(), false); + span.code = true; + self.push(ch * BODY_LINE, indent, 1.0, vec![span], decor); + self.y += ch * 0.5; + }, FormattedTextLine::UnorderedList(list) => { self.list_item(&list.text, list.indent_level, "• ", quote_depth, extra_indent); }, @@ -546,7 +554,11 @@ impl LayoutCtx<'_> { fn table(&mut self, table: &crate::markdown::FormattedTable, quote_depth: usize, indent: f32) { let ch = self.cell_h; let inline_cols = |inline: &FormattedTextInline| -> usize { - inline.iter().flat_map(|f| f.text.chars()).map(|c| c.width().unwrap_or(0).max(1)).sum() + inline + .iter() + .flat_map(|f| f.text().chars()) + .map(|c| c.width().unwrap_or(0).max(1)) + .sum() }; let col_count = table.headers.len().max(table.rows.iter().map(Vec::len).max().unwrap_or(0)).max(1); @@ -603,7 +615,7 @@ fn cell_spans( let mut used = 0usize; if let Some(inline) = cell { 'outer: for (frag, fragment) in inline.iter().enumerate() { - for ch in fragment.text.chars() { + for ch in fragment.text().chars() { let cols = ch.width().unwrap_or(0).max(1); if used + cols > width { // No room left: truncate with an ellipsis if one fits. diff --git a/nebula_app/src/markdown/mod.rs b/nebula_app/src/markdown/mod.rs index 0031e36..6f18a3a 100644 --- a/nebula_app/src/markdown/mod.rs +++ b/nebula_app/src/markdown/mod.rs @@ -12,6 +12,7 @@ use std::collections::VecDeque; use std::fmt; use std::ops::Range; +use std::sync::Arc; pub mod parser; @@ -42,12 +43,14 @@ impl CustomWeight { /// A parsed Markdown document: a flat sequence of block-level lines. #[derive(Debug, Clone, PartialEq, Eq, Default)] pub struct FormattedText { + /// Markdown 源码只保留这一份共享字节；fragment 用范围引用它，避免逐层复制。 + pub source: Arc, pub lines: VecDeque, } impl FormattedText { pub fn new(lines: impl Into>) -> Self { - Self { lines: lines.into() } + Self { source: Arc::from(""), lines: lines.into() } } /// The document's raw text, without any of the markdown markers. @@ -56,6 +59,108 @@ impl FormattedText { } } +/// UTF-8 源码中的半开字节范围。文档输入上限在解析入口保证可用 `u32` 表示。 +#[derive(Clone, Copy, Debug, Default, Eq, Hash, PartialEq)] +pub struct SourceRange { + pub start: u32, + pub end: u32, +} + +impl SourceRange { + pub fn new(range: Range) -> Option { + Some(Self { start: range.start.try_into().ok()?, end: range.end.try_into().ok()? }) + } + + pub fn as_usize(self) -> Range { + self.start as usize..self.end as usize + } +} + +/// 绝大多数内容引用 Markdown 源码；只有实体解码、软换行等合成文本单独分配。 +#[derive(Clone, Debug)] +pub enum TextRef { + Source { source: Arc, range: SourceRange }, + Generated(Box), +} + +impl Default for TextRef { + fn default() -> Self { + Self::Generated(Box::default()) + } +} + +impl PartialEq for TextRef { + fn eq(&self, other: &Self) -> bool { + self.as_str() == other.as_str() + } +} + +impl Eq for TextRef {} + +impl TextRef { + pub fn source(source: Arc, range: Range) -> Option { + let range = SourceRange::new(range)?; + source.get(range.as_usize())?; + Some(Self::Source { source, range }) + } + + pub fn generated(text: impl Into>) -> Self { + Self::Generated(text.into()) + } + + pub fn as_str(&self) -> &str { + match self { + Self::Source { source, range } => &source[range.as_usize()], + Self::Generated(text) => text, + } + } + + /// 只合并同一源码中的连续范围；跨 Markdown 标记的内容保持独立，避免错误扩大范围。 + fn append(&mut self, next: Self) -> Result<(), Self> { + match (self, next) { + ( + Self::Source { source, range }, + Self::Source { source: next_source, range: next_range }, + ) if Arc::ptr_eq(source, &next_source) && range.end == next_range.start => { + range.end = next_range.end; + Ok(()) + }, + (slot @ Self::Generated(_), Self::Generated(next)) => { + let mut combined = String::from(slot.as_str()); + combined.push_str(&next); + *slot = Self::Generated(combined.into_boxed_str()); + Ok(()) + }, + // 实体解码或反斜杠转义会在同样式文本中留下源码间隙；此时只合成这一小段， + // 既保持旧的 fragment 合并语义，也不扩大 range 到 Markdown 标记字节。 + (slot, next) => { + let mut combined = String::from(slot.as_str()); + combined.push_str(next.as_str()); + *slot = Self::Generated(combined.into_boxed_str()); + Ok(()) + }, + } + } +} + +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub enum MathMode { + Inline, + Display, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct MathSource { + pub source: TextRef, + pub mode: MathMode, +} + +impl MathSource { + pub fn as_str(&self) -> &str { + self.source.as_str() + } +} + /// One block-level element of the document. #[derive(Debug, Clone, Eq, PartialEq)] pub enum FormattedTextLine { @@ -69,16 +174,20 @@ pub enum FormattedTextLine { HorizontalRule, Image(FormattedImage), Table(FormattedTable), + DisplayMath(MathSource), /// A block nested inside `depth` levels of `>` quoting. Quoting wraps the /// inner block instead of forking every variant, so the flat list (and /// the virtual scroller's block indexing) survives. - Quote { depth: usize, line: Box }, + Quote { + depth: usize, + line: Box, + }, } impl FormattedTextLine { pub fn raw_text(&self) -> String { let join = |inline: &FormattedTextInline| -> String { - inline.iter().map(|fragment| fragment.text.as_str()).collect() + inline.iter().map(FormattedTextFragment::text).collect() }; let mut text = match self { Self::CodeBlock(text) => text.code.clone(), @@ -90,6 +199,7 @@ impl FormattedTextLine { Self::LineBreak | Self::HorizontalRule => "\n".to_string(), Self::Image(image) => format!("{}\n", image.alt_text), Self::Table(table) => table.to_plain_text(), + Self::DisplayMath(math) => math.as_str().to_owned(), Self::Quote { line, .. } => line.raw_text(), }; // Each `FormattedTextLine` unit represents a complete line. @@ -111,7 +221,8 @@ impl FormattedTextLine { | FormattedTextLine::LineBreak | FormattedTextLine::HorizontalRule | FormattedTextLine::Image(_) - | FormattedTextLine::Table(_) => None, + | FormattedTextLine::Table(_) + | FormattedTextLine::DisplayMath(_) => None, } } @@ -123,7 +234,7 @@ impl FormattedTextLine { let mut char_count = 0; for fragment in inline_fragments { let range_start = char_count; - char_count += fragment.text.chars().count(); + char_count += fragment.text().chars().count(); if let Some(Hyperlink::Url(url)) = &fragment.styles.hyperlink { hyperlinks.push((range_start..char_count, url.clone())); } @@ -196,7 +307,7 @@ impl FormattedTable { /// Serialize to GFM pipe-table markdown (used for raw-text extraction). pub fn to_plain_text(&self) -> String { fn inline_to_text(inline: &FormattedTextInline) -> String { - inline.iter().map(|f| f.text.as_str()).collect() + inline.iter().map(FormattedTextFragment::text).collect() } let mut lines = Vec::new(); @@ -222,10 +333,22 @@ impl FormattedTable { pub type FormattedTextInline = Vec; -/// A fragment of formatted text: the text itself plus formatting flags. +#[derive(Debug, Clone, Eq, PartialEq)] +pub enum FragmentContent { + Text(TextRef), + Math(MathSource), +} + +impl Default for FragmentContent { + fn default() -> Self { + Self::Text(TextRef::default()) + } +} + +/// A fragment of formatted text or an inline mathematical object plus formatting flags. #[derive(Debug, Clone, Eq, PartialEq, Default)] pub struct FormattedTextFragment { - pub text: String, + pub content: FragmentContent, pub styles: FormattedTextStyles, } @@ -249,29 +372,26 @@ pub struct FormattedTextStyles { impl FormattedTextFragment { pub fn plain_text(text: impl Into) -> Self { - Self { text: text.into(), styles: Default::default() } + Self::from_generated(text, Default::default()) } pub fn bold(text: impl Into) -> Self { Self { - text: text.into(), - styles: FormattedTextStyles { - weight: Some(CustomWeight::Bold), - ..Default::default() - }, + content: FragmentContent::Text(TextRef::generated(text.into().into_boxed_str())), + styles: FormattedTextStyles { weight: Some(CustomWeight::Bold), ..Default::default() }, } } pub fn italic(text: impl Into) -> Self { Self { - text: text.into(), + content: FragmentContent::Text(TextRef::generated(text.into().into_boxed_str())), styles: FormattedTextStyles { italic: true, ..Default::default() }, } } pub fn hyperlink(tag: impl Into, url: impl Into) -> Self { Self { - text: tag.into(), + content: FragmentContent::Text(TextRef::generated(tag.into().into_boxed_str())), styles: FormattedTextStyles { hyperlink: Some(Hyperlink::Url(url.into())), ..Default::default() @@ -281,10 +401,41 @@ impl FormattedTextFragment { pub fn inline_code(text: impl Into) -> Self { Self { - text: text.into(), + content: FragmentContent::Text(TextRef::generated(text.into().into_boxed_str())), styles: FormattedTextStyles { inline_code: true, ..Default::default() }, } } + + pub fn from_text(text: TextRef, styles: FormattedTextStyles) -> Self { + Self { content: FragmentContent::Text(text), styles } + } + + pub fn math(source: MathSource, styles: FormattedTextStyles) -> Self { + Self { content: FragmentContent::Math(source), styles } + } + + pub fn text(&self) -> &str { + match &self.content { + FragmentContent::Text(text) => text.as_str(), + FragmentContent::Math(math) => math.as_str(), + } + } + + pub fn is_math(&self) -> bool { + matches!(self.content, FragmentContent::Math(_)) + } + + pub(crate) fn append_text(&mut self, text: TextRef) -> Result<(), TextRef> { + match &mut self.content { + FragmentContent::Text(current) => current.append(text), + FragmentContent::Math(_) => Err(text), + } + } + + fn from_generated(text: impl Into, styles: FormattedTextStyles) -> Self { + let text = text.into().into_boxed_str(); + Self { content: FragmentContent::Text(TextRef::generated(text)), styles } + } } impl fmt::Debug for FormattedTextStyles { diff --git a/nebula_app/src/markdown/parser.rs b/nebula_app/src/markdown/parser.rs index ccd413b..8ddfd3e 100644 --- a/nebula_app/src/markdown/parser.rs +++ b/nebula_app/src/markdown/parser.rs @@ -6,15 +6,16 @@ //! This file owns ONLY the event folding. The AST lives in `super`, the //! on-screen layout (wrapping, virtual scrolling) in `display::markdown_view`. -use pulldown_cmark::{ - Alignment, CodeBlockKind, Event, Options, Parser, Tag, TagEnd, -}; +use std::ops::Range; +use std::sync::Arc; + +use pulldown_cmark::{Alignment, CodeBlockKind, Event, Options, Parser, Tag, TagEnd}; use super::{ CodeBlockText, CustomWeight, FormattedImage, FormattedIndentTextInline, FormattedTable, FormattedTaskList, FormattedText, FormattedTextFragment, FormattedTextHeader, - FormattedTextInline, FormattedTextLine, FormattedTextStyles, Hyperlink, - OrderedFormattedIndentTextInline, TableAlignment, + FormattedTextInline, FormattedTextLine, FormattedTextStyles, Hyperlink, MathMode, MathSource, + OrderedFormattedIndentTextInline, TableAlignment, TextRef, }; /// Parse a Markdown document. GFM tables, task lists and strikethrough are @@ -25,10 +26,11 @@ pub fn parse_markdown(text: &str) -> FormattedText { options.insert(Options::ENABLE_STRIKETHROUGH); options.insert(Options::ENABLE_TASKLISTS); options.insert(Options::ENABLE_YAML_STYLE_METADATA_BLOCKS); + options.insert(Options::ENABLE_MATH); - let mut fold = Fold::default(); - for event in Parser::new_ext(text, options) { - fold.event(event); + let mut fold = Fold::new(Arc::from(text)); + for (event, range) in Parser::new_ext(text, options).into_offset_iter() { + fold.event(event, range); } fold.finish() } @@ -55,8 +57,8 @@ struct TableFold { } /// Event-stream folding state. One instance per document. -#[derive(Default)] struct Fold { + source: Arc, out: Vec, /// Inline run being collected for the current paragraph / heading / list /// item / table cell. @@ -83,7 +85,27 @@ struct Fold { } impl Fold { - fn event(&mut self, event: Event<'_>) { + fn new(source: Arc) -> Self { + Self { + source, + out: Vec::new(), + inline: Vec::new(), + strong: 0, + emphasis: 0, + strike: 0, + links: Vec::new(), + list_stack: Vec::new(), + pending_item: None, + quote_depth: 0, + heading: None, + code: None, + table: None, + image: None, + in_metadata: false, + } + } + + fn event(&mut self, event: Event<'_>, range: Range) { match event { Event::Start(tag) => self.start(tag), Event::End(tag) => self.end(tag), @@ -96,22 +118,24 @@ impl Fold { } else if let Some(image) = &mut self.image { image.alt_text.push_str(&text); } else { - self.push_text(&text, false); + self.push_source_text(range, &text, false); } }, Event::Code(text) => { if let Some(image) = &mut self.image { image.alt_text.push_str(&text); } else { - self.push_text(&text, true); + self.push_source_text(range, &text, true); } }, // The viewer renders HTML as the literal text the author wrote. - Event::Html(html) | Event::InlineHtml(html) => self.push_text(&html, false), - Event::SoftBreak => self.push_text(" ", false), + Event::Html(html) | Event::InlineHtml(html) => { + self.push_source_text(range, &html, false) + }, + Event::SoftBreak => self.push_generated_text(" ", false), // A hard break stays inside the paragraph block: the layout pass // turns embedded '\n' into a forced visual line break. - Event::HardBreak => self.push_text("\n", false), + Event::HardBreak => self.push_generated_text("\n", false), Event::Rule => { self.flush_inline_as_paragraph(); self.push_block(FormattedTextLine::HorizontalRule); @@ -122,10 +146,23 @@ impl Fold { } }, Event::FootnoteReference(name) => { - self.push_text(&format!("[^{name}]"), false); + self.push_generated_text(&format!("[^{name}]"), false); }, - Event::InlineMath(math) | Event::DisplayMath(math) => { - self.push_text(&math, true); + Event::InlineMath(math) => { + let source = self.source_ref(range, &math); + let styles = self.current_styles(false); + self.inline.push(FormattedTextFragment::math( + MathSource { source, mode: MathMode::Inline }, + styles, + )); + }, + Event::DisplayMath(math) => { + let source = self.source_ref(range, &math); + self.flush_inline_as_paragraph(); + self.push_block(FormattedTextLine::DisplayMath(MathSource { + source, + mode: MathMode::Display, + })); }, } } @@ -147,11 +184,9 @@ impl Fold { self.flush_pending_item(); self.flush_inline_as_paragraph(); let lang = match kind { - CodeBlockKind::Fenced(info) => info - .split_whitespace() - .next() - .unwrap_or_default() - .to_owned(), + CodeBlockKind::Fenced(info) => { + info.split_whitespace().next().unwrap_or_default().to_owned() + }, CodeBlockKind::Indented => String::new(), }; let lang = if lang.is_empty() { "text".to_owned() } else { lang }; @@ -305,25 +340,56 @@ impl Fold { /// Append text to the current inline run under the active styles, /// merging into the previous fragment when the styles are identical. - fn push_text(&mut self, text: &str, inline_code: bool) { - if text.is_empty() || self.in_metadata { + fn push_source_text(&mut self, range: Range, text: &str, inline_code: bool) { + let text = self.source_ref(range, text); + self.push_text_ref(text, inline_code); + } + + fn push_generated_text(&mut self, text: &str, inline_code: bool) { + self.push_text_ref(TextRef::generated(text.to_owned().into_boxed_str()), inline_code); + } + + fn push_text_ref(&mut self, text: TextRef, inline_code: bool) { + if text.as_str().is_empty() || self.in_metadata { return; } - let styles = FormattedTextStyles { + let styles = self.current_styles(inline_code); + let text = if let Some(last) = self.inline.last_mut() { + if last.styles == styles { + match last.append_text(text) { + Ok(()) => return, + Err(text) => text, + } + } else { + text + } + } else { + text + }; + self.inline.push(FormattedTextFragment::from_text(text, styles)); + } + + fn current_styles(&self, inline_code: bool) -> FormattedTextStyles { + FormattedTextStyles { weight: (self.strong > 0).then_some(CustomWeight::Bold), italic: self.emphasis > 0, underline: false, strikethrough: self.strike > 0, inline_code, hyperlink: self.links.last().map(|url| Hyperlink::Url(url.clone())), - }; - if let Some(last) = self.inline.last_mut() { - if last.styles == styles { - last.text.push_str(text); - return; - } } - self.inline.push(FormattedTextFragment { text: text.to_owned(), styles }); + } + + fn source_ref(&self, outer: Range, rendered: &str) -> TextRef { + let source_range = self.source.get(outer.clone()).and_then(|outer_text| { + outer_text.find(rendered).map(|offset| { + let start = outer.start + offset; + start..start + rendered.len() + }) + }); + source_range + .and_then(|range| TextRef::source(self.source.clone(), range)) + .unwrap_or_else(|| TextRef::generated(rendered.to_owned().into_boxed_str())) } /// Emit the pending inline run as a plain paragraph block, if any. @@ -346,8 +412,7 @@ impl Fold { text, }), None => { - let indented_text = - FormattedIndentTextInline { indent_level: item.depth, text }; + let indented_text = FormattedIndentTextInline { indent_level: item.depth, text }; match item.number { Some(number) => { FormattedTextLine::OrderedList(OrderedFormattedIndentTextInline { @@ -374,13 +439,14 @@ impl Fold { fn finish(mut self) -> FormattedText { self.flush_inline_as_paragraph(); - FormattedText { lines: self.out.into() } + FormattedText { source: self.source, lines: self.out.into() } } } #[cfg(test)] mod tests { use super::*; + use crate::markdown::{FragmentContent, MathMode, TextRef}; fn parse(text: &str) -> Vec { parse_markdown(text).lines.into_iter().collect() @@ -390,6 +456,39 @@ mod tests { FormattedTextFragment::plain_text(text) } + #[test] + fn math_events_keep_mode_and_utf8_source_ranges() { + let document = parse_markdown("中文 $x^2$\n\n$$\\frac{1}{2}$$"); + let FormattedTextLine::Line(inline) = &document.lines[0] else { + panic!("expected paragraph"); + }; + let FragmentContent::Math(inline_math) = &inline[1].content else { + panic!("expected inline math fragment: {:?}", inline[1]); + }; + assert_eq!(inline_math.mode, MathMode::Inline); + assert_eq!(inline_math.source.as_str(), "x^2"); + + let FormattedTextLine::DisplayMath(display_math) = &document.lines[1] else { + panic!("expected display math block: {:?}", document.lines[1]); + }; + assert_eq!(display_math.mode, MathMode::Display); + assert_eq!(display_math.source.as_str(), r"\frac{1}{2}"); + + let TextRef::Source { source, range } = &inline_math.source else { + panic!("math source should point into the document"); + }; + assert!(std::sync::Arc::ptr_eq(source, &document.source)); + assert_eq!(&document.source[range.as_usize()], "x^2"); + } + + #[test] + fn escaped_dollars_and_code_do_not_become_math() { + let document = parse_markdown(r"escaped \$x$ and `$code$`"); + let FormattedTextLine::Line(inline) = &document.lines[0] else { panic!() }; + assert!(inline.iter().all(|fragment| !fragment.is_math())); + assert_eq!(document.raw_text().trim_end(), "escaped $x$ and $code$"); + } + #[test] fn heading_paragraph_and_inline_styles() { let lines = parse("# Title\n\nplain **bold** *it* `code` ~~gone~~"); @@ -410,7 +509,7 @@ mod tests { assert_eq!(inline[4], plain(" ")); assert_eq!(inline[5], FormattedTextFragment::inline_code("code")); assert!(inline[7].styles.strikethrough); - assert_eq!(inline[7].text, "gone"); + assert_eq!(inline[7].text(), "gone"); } #[test] @@ -507,7 +606,7 @@ mod tests { // A hard break stays inside one paragraph as an embedded newline. let lines = parse("one \ntwo"); let FormattedTextLine::Line(inline) = &lines[0] else { panic!() }; - let text: String = inline.iter().map(|f| f.text.as_str()).collect(); + let text: String = inline.iter().map(FormattedTextFragment::text).collect(); assert_eq!(text, "one\ntwo"); } @@ -545,6 +644,6 @@ mod tests { let lines = parse("a & b"); let FormattedTextLine::Line(inline) = &lines[0] else { panic!() }; assert_eq!(inline.len(), 1); - assert_eq!(inline[0].text, "a & b"); + assert_eq!(inline[0].text(), "a & b"); } }