meta_language/
mixed_regions.rs

1use std::collections::HashMap;
2
3use crate::configuration::{ParseConfiguration, RegionDetectionPolicy};
4use crate::language_catalog::canonical_language_name;
5use crate::line_index::LineIndex;
6use crate::link_network::{LinkId, LinkNetwork, LinkType};
7use crate::source::{ByteRange, SourceSpan};
8
9const TXT_LANGUAGE: &str = "txt";
10
11/// Embedded region discovered inside a mixed-language document.
12#[derive(Clone, Debug, PartialEq, Eq)]
13pub struct EmbeddedRegion {
14    language: String,
15    span: SourceSpan,
16}
17
18impl EmbeddedRegion {
19    pub(crate) const fn new(language: String, span: SourceSpan) -> Self {
20        Self { language, span }
21    }
22
23    /// Language detected for the embedded region.
24    #[must_use]
25    pub fn language(&self) -> &str {
26        &self.language
27    }
28
29    /// Source span covered by the embedded region.
30    #[must_use]
31    pub const fn span(&self) -> SourceSpan {
32        self.span
33    }
34}
35
36/// Embedded regions a host document delimits, in source order.
37///
38/// The whole buffer of a `txt` document is one region, and for HTML and
39/// Markdown hosts the regions are those their grammar CST (the host's
40/// `Syntax` links below `document`) delimits. Mirrors `detectEmbeddedRegionsInTree` in `js/src/regions.js`; `document`
41/// is the host's `Document` link in `network`.
42///
43/// - HTML: the `raw_text` of a `script_element` in the language its `type`
44///   attribute names (JavaScript when absent), the `raw_text` of a
45///   `style_element` as CSS, and the `attribute_value` of every `style`
46///   attribute as CSS.
47/// - Markdown: the `code_fence_content` of a `fenced_code_block` in the
48///   language `policy` selects from its info-string `language` and content,
49///   every `html_block` as HTML, and every inline `html_tag` as HTML, spanning
50///   from an opening tag to the matching closing tag among its siblings.
51#[must_use]
52pub fn detect_embedded_regions_in_tree(
53    network: &LinkNetwork,
54    document: LinkId,
55    text: &str,
56    language: &str,
57    policy: RegionDetectionPolicy,
58) -> Vec<EmbeddedRegion> {
59    let lines = LineIndex::new(text);
60    let host = canonical_language_name(language);
61    if host == Some(TXT_LANGUAGE) {
62        return vec![region_for(&lines, TXT_LANGUAGE.to_string(), 0, text.len())];
63    }
64    if !matches!(host, Some("HTML" | "Markdown")) {
65        return Vec::new();
66    }
67    let tree = HostTree::new(network, text);
68    let mut found = Vec::new();
69    let mut stack = vec![document];
70    while let Some(node) = stack.pop() {
71        if host == Some("HTML") {
72            tree.visit_html(node, &mut found);
73        } else {
74            tree.visit_markdown(node, policy, &mut found);
75        }
76        stack.extend(tree.children(node).iter().rev());
77    }
78    found.sort_by_key(|(_, start, _)| *start);
79    found
80        .into_iter()
81        .map(|(language, start, end)| region_for(&lines, language, start, end))
82        .collect()
83}
84
85/// Embedded regions of `text` in host `language`, in source order.
86///
87/// The host is parsed with its registered grammar and its CST read by
88/// [`detect_embedded_regions_in_tree`]. Mirrors `detectEmbeddedRegions` in
89/// `js/src/regions.js`: a `txt` document is one region, and a host other than
90/// HTML or Markdown has none.
91#[must_use]
92pub fn detect_embedded_regions(
93    text: &str,
94    language: &str,
95    policy: RegionDetectionPolicy,
96) -> Vec<EmbeddedRegion> {
97    let host = canonical_language_name(language);
98    if host == Some(TXT_LANGUAGE) {
99        let lines = LineIndex::new(text);
100        return vec![region_for(&lines, TXT_LANGUAGE.to_string(), 0, text.len())];
101    }
102    let Some(host @ ("HTML" | "Markdown")) = host else {
103        return Vec::new();
104    };
105    let configuration = ParseConfiguration::default().with_region_detection_policy(policy);
106    let network = LinkNetwork::parse(text, host, configuration);
107    let document = network.links().find(|link| {
108        link.metadata().link_type() == Some(LinkType::Document)
109            && link.metadata().language() == Some(host)
110    });
111    document.map_or_else(Vec::new, |document| {
112        detect_embedded_regions_in_tree(&network, document.id(), text, host, policy)
113    })
114}
115
116type FoundRegion = (String, usize, usize);
117
118/// The host grammar CST of a network: `Syntax` links in child order below
119/// their parent.
120struct HostTree<'a> {
121    network: &'a LinkNetwork,
122    text: &'a str,
123    children: HashMap<LinkId, Vec<LinkId>>,
124}
125
126impl<'a> HostTree<'a> {
127    fn new(network: &'a LinkNetwork, text: &'a str) -> Self {
128        let mut children: HashMap<LinkId, Vec<LinkId>> = HashMap::new();
129        for link in network.links() {
130            if link.metadata().link_type() == Some(LinkType::Syntax)
131                && let [parent] = link.references()
132            {
133                children.entry(*parent).or_default().push(link.id());
134            }
135        }
136        Self {
137            network,
138            text,
139            children,
140        }
141    }
142
143    fn children(&self, node: LinkId) -> &[LinkId] {
144        self.children.get(&node).map_or(&[], Vec::as_slice)
145    }
146
147    fn kind(&self, node: LinkId) -> Option<&str> {
148        self.network.link(node)?.metadata().term()
149    }
150
151    fn range(&self, node: LinkId) -> (usize, usize) {
152        self.network
153            .link(node)
154            .and_then(|link| link.metadata().span())
155            .map_or((0, 0), |span| {
156                (span.byte_range().start(), span.byte_range().end())
157            })
158    }
159
160    fn text(&self, node: LinkId) -> &'a str {
161        let (start, end) = self.range(node);
162        &self.text[start..end]
163    }
164
165    fn child_of_kind(&self, node: LinkId, kind: &str) -> Option<LinkId> {
166        self.children(node)
167            .iter()
168            .copied()
169            .find(|child| self.kind(*child) == Some(kind))
170    }
171
172    fn visit_html(&self, node: LinkId, found: &mut Vec<FoundRegion>) {
173        let content = self.child_of_kind(node, "raw_text");
174        match (self.kind(node), content) {
175            (Some("script_element"), Some(content)) => {
176                let type_value = self.attribute_value(node, "type");
177                if let Some(language) = script_language(type_value.unwrap_or_default()) {
178                    let (start, end) = self.range(content);
179                    found.push((language.to_string(), start, end));
180                }
181            }
182            (Some("style_element"), Some(content)) => {
183                let essence = mime_essence(self.attribute_value(node, "type").unwrap_or_default());
184                if essence.is_empty() || essence == "text/css" {
185                    let (start, end) = self.range(content);
186                    found.push(("CSS".to_string(), start, end));
187                }
188            }
189            (Some("attribute"), _) if self.attribute_name(node).as_deref() == Some("style") => {
190                if let Some(value) = self.attribute_value_node(node) {
191                    let (start, end) = self.range(value);
192                    found.push(("CSS".to_string(), start, end));
193                }
194            }
195            _ => {}
196        }
197    }
198
199    fn visit_markdown(
200        &self,
201        node: LinkId,
202        policy: RegionDetectionPolicy,
203        found: &mut Vec<FoundRegion>,
204    ) {
205        match self.kind(node) {
206            Some("fenced_code_block") => {
207                if let Some(content) = self.child_of_kind(node, "code_fence_content") {
208                    let tag = self
209                        .child_of_kind(node, "info_string")
210                        .and_then(|info| self.child_of_kind(info, "language"))
211                        .map_or("", |tag| self.text(tag));
212                    if let Some(language) = fence_language(tag, self.text(content), policy) {
213                        let (start, end) = self.range(content);
214                        found.push((language, start, end));
215                    }
216                }
217            }
218            Some("html_block") => {
219                let (start, end) = self.range(node);
220                found.push(("HTML".to_string(), start, end));
221            }
222            _ => {}
223        }
224        let tags: Vec<LinkId> = self
225            .children(node)
226            .iter()
227            .copied()
228            .filter(|child| self.kind(*child) == Some("html_tag"))
229            .collect();
230        let texts: Vec<&str> = tags.iter().map(|tag| self.text(*tag)).collect();
231        for (first, last) in pair_html_tags(&texts) {
232            found.push((
233                "HTML".to_string(),
234                self.range(tags[first]).0,
235                self.range(tags[last]).1,
236            ));
237        }
238    }
239
240    fn attribute_name(&self, attribute: LinkId) -> Option<String> {
241        self.child_of_kind(attribute, "attribute_name")
242            .map(|name| self.text(name).to_ascii_lowercase())
243    }
244
245    fn attribute_value_node(&self, attribute: LinkId) -> Option<LinkId> {
246        self.child_of_kind(attribute, "attribute_value")
247            .or_else(|| {
248                self.child_of_kind(attribute, "quoted_attribute_value")
249                    .and_then(|quoted| self.child_of_kind(quoted, "attribute_value"))
250            })
251    }
252
253    /// The value of an element's start-tag attribute, `""` when valueless.
254    fn attribute_value(&self, element: LinkId, name: &str) -> Option<&'a str> {
255        let start_tag = self.child_of_kind(element, "start_tag")?;
256        let attribute = self.children(start_tag).iter().copied().find(|child| {
257            self.kind(*child) == Some("attribute")
258                && self.attribute_name(*child).as_deref() == Some(name)
259        })?;
260        Some(
261            self.attribute_value_node(attribute)
262                .map_or("", |value| self.text(value)),
263        )
264    }
265}
266
267/// Void elements never have a closing tag.
268const HTML_VOID_ELEMENTS: &[&str] = &[
269    "area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "source", "track",
270    "wbr",
271];
272
273struct InlineTag {
274    name: Option<String>,
275    opening: bool,
276    closing: bool,
277}
278
279/// Groups sibling inline HTML tags into elements: an opening tag extends to
280/// its matching closing tag (same name, nesting counted); a void,
281/// self-closing, unmatched, comment, or declaration tag stands alone.
282fn pair_html_tags(texts: &[&str]) -> Vec<(usize, usize)> {
283    let tags: Vec<InlineTag> = texts
284        .iter()
285        .map(|text| {
286            let rest = &text[1.min(text.len())..];
287            let closing = rest.starts_with('/');
288            let rest = rest.strip_prefix('/').unwrap_or(rest);
289            let name: String = rest
290                .chars()
291                .take_while(|character| character.is_ascii_alphanumeric() || *character == '-')
292                .collect();
293            if !text.starts_with('<') || !name.starts_with(|c: char| c.is_ascii_alphabetic()) {
294                return InlineTag {
295                    name: None,
296                    opening: false,
297                    closing: false,
298                };
299            }
300            let name = name.to_ascii_lowercase();
301            let opening =
302                !closing && !text.ends_with("/>") && !HTML_VOID_ELEMENTS.contains(&name.as_str());
303            InlineTag {
304                name: Some(name),
305                opening,
306                closing,
307            }
308        })
309        .collect();
310    let mut groups = Vec::new();
311    let mut index = 0;
312    while index < tags.len() {
313        let mut last = index;
314        if tags[index].opening {
315            let mut depth = 0_usize;
316            for (next, tag) in tags.iter().enumerate().skip(index) {
317                if tag.name != tags[index].name {
318                    continue;
319                }
320                if tag.opening {
321                    depth += 1;
322                }
323                if tag.closing {
324                    depth -= 1;
325                }
326                if depth == 0 {
327                    last = next;
328                    break;
329                }
330            }
331        }
332        groups.push((index, last));
333        index = last + 1;
334    }
335    groups
336}
337
338/// <https://html.spec.whatwg.org/multipage/scripting.html#javascript-mime-type>
339const JAVASCRIPT_SCRIPT_TYPES: &[&str] = &[
340    "",
341    "module",
342    "application/ecmascript",
343    "application/javascript",
344    "application/x-ecmascript",
345    "application/x-javascript",
346    "text/ecmascript",
347    "text/javascript",
348    "text/javascript1.0",
349    "text/javascript1.1",
350    "text/javascript1.2",
351    "text/javascript1.3",
352    "text/javascript1.4",
353    "text/javascript1.5",
354    "text/jscript",
355    "text/livescript",
356    "text/x-ecmascript",
357    "text/x-javascript",
358];
359
360/// The language of a script element's content from its `type` attribute.
361///
362/// JavaScript for classic scripts and modules, JSON for import maps,
363/// speculation rules and JSON types, otherwise the registered language the
364/// MIME subtype names (such as `text/typescript`); `None` for other data
365/// blocks.
366#[must_use]
367pub fn script_language(type_attribute: &str) -> Option<&'static str> {
368    let essence = mime_essence(type_attribute);
369    if JAVASCRIPT_SCRIPT_TYPES.contains(&essence.as_str()) {
370        return Some("JavaScript");
371    }
372    if essence == "importmap"
373        || essence == "speculationrules"
374        || essence.ends_with("/json")
375        || essence.ends_with("+json")
376    {
377        return Some("JSON");
378    }
379    let (_, subtype) = essence.split_once('/')?;
380    canonical_language_name(subtype.strip_prefix("x-").unwrap_or(subtype))
381}
382
383fn mime_essence(type_attribute: &str) -> String {
384    type_attribute
385        .split(';')
386        .next()
387        .unwrap_or_default()
388        .trim()
389        .to_ascii_lowercase()
390}
391
392fn fence_language(tag: &str, content: &str, policy: RegionDetectionPolicy) -> Option<String> {
393    let named = (!tag.is_empty()).then(|| canonical_language_name(tag).unwrap_or(tag).to_string());
394    let sniffed = || {
395        let language = sniff_language(content).unwrap_or(TXT_LANGUAGE);
396        canonical_language_name(language)
397            .unwrap_or(language)
398            .to_string()
399    };
400    match policy {
401        RegionDetectionPolicy::NameDriven => named,
402        RegionDetectionPolicy::ContentDriven => Some(sniffed()),
403        RegionDetectionPolicy::Both => Some(named.unwrap_or_else(sniffed)),
404    }
405}
406
407/// The language a fenced block without an info string most likely holds,
408/// from the start of its content, as the JavaScript `sniffLanguage`.
409#[must_use]
410pub fn sniff_language(content: &str) -> Option<&'static str> {
411    let trimmed = content.trim_start();
412    let upper = trimmed.to_ascii_uppercase();
413
414    if trimmed.contains("fn main") {
415        Some("rust")
416    } else if trimmed.starts_with("def ") {
417        Some("Python")
418    } else if trimmed.starts_with('<') {
419        Some("HTML")
420    } else if trimmed.contains("function ")
421        || trimmed.contains("const ")
422        || trimmed.contains("let ")
423    {
424        Some("JavaScript")
425    } else if upper.starts_with("SELECT ") {
426        Some("sql-ansi")
427    } else {
428        None
429    }
430}
431
432fn region_for(lines: &LineIndex, language: String, start: usize, end: usize) -> EmbeddedRegion {
433    EmbeddedRegion::new(
434        language,
435        SourceSpan::new(
436            ByteRange::new(start, end),
437            lines.byte_point(start),
438            lines.byte_point(end),
439        ),
440    )
441}
meta_language/mixed_regions.rs

meta_language/
mixed_regions.rs
