meta_language/
tree_sitter_adapter.rs

1use std::borrow::Cow;
2
3use tree_sitter::{
4    InputEdit, Language, Node, Parser, Point as TreeSitterPoint, Range as TreeSitterRange, Tree,
5};
6
7#[allow(unsafe_code)]
8mod rocq_grammar {
9    use tree_sitter_language::LanguageFn;
10
11    unsafe extern "C" {
12        fn tree_sitter_rocq() -> *const ();
13    }
14
15    // SAFETY: build.rs compiles the generated parser from the pinned revision
16    // recorded in vendor/tree-sitter-rocq/NOTICE.md with this exact symbol.
17    pub const LANGUAGE: LanguageFn = unsafe { LanguageFn::from_raw(tree_sitter_rocq) };
18}
19
20#[allow(unsafe_code)]
21mod lean_grammar {
22    use tree_sitter_language::LanguageFn;
23
24    unsafe extern "C" {
25        fn tree_sitter_lean() -> *const ();
26    }
27
28    // SAFETY: build.rs compiles the generated parser and scanner from the
29    // pinned revision recorded in vendor/tree-sitter-lean/NOTICE.md with this
30    // exact symbol.
31    pub const LANGUAGE: LanguageFn = unsafe { LanguageFn::from_raw(tree_sitter_lean) };
32}
33
34#[allow(unsafe_code)]
35mod cmake_grammar {
36    use tree_sitter_language::LanguageFn;
37
38    unsafe extern "C" {
39        fn tree_sitter_cmake() -> *const ();
40    }
41
42    // SAFETY: build.rs compiles the generated parser and the patched scanner
43    // from the pinned revision recorded in vendor/tree-sitter-cmake/NOTICE.md
44    // with this exact symbol.
45    pub const LANGUAGE: LanguageFn = unsafe { LanguageFn::from_raw(tree_sitter_cmake) };
46}
47
48mod native;
49
50use crate::line_index::LineIndex;
51use crate::native_grammar_parser::{is_native_grammar, parse_native};
52use crate::{
53    ByteRange, LinkFlags, LinkId, LinkMetadata, LinkNetwork, LinkType, ParseConfiguration, Point,
54    SourceSpan,
55};
56use native::{convert_native_root, native_flags};
57
58pub fn parse(text: &str, language: &str, configuration: ParseConfiguration) -> Option<LinkNetwork> {
59    if let Some(id) = native_grammar_for_language(language) {
60        return Some(network_from_native(text, language, configuration, id));
61    }
62    let grammar = grammar_for_language(language)?;
63    let mut parser = Parser::new();
64    parser.set_language(&grammar).ok()?;
65    let parsed = parser.parse(text, None)?;
66
67    Some(network_from_tree(text, language, configuration, &parsed))
68}
69
70pub fn parse_incremental(
71    old_text: &str,
72    range: ByteRange,
73    replacement: &str,
74    language: &str,
75    configuration: ParseConfiguration,
76) -> Option<LinkNetwork> {
77    if let Some(id) = native_grammar_for_language(language) {
78        // The native executor parses the whole edited text.
79        let edited_text = apply_text_edit(old_text, range, replacement)?;
80        return Some(network_from_native(
81            &edited_text,
82            language,
83            configuration,
84            id,
85        ));
86    }
87    let grammar = grammar_for_language(language)?;
88    let edited_text = apply_text_edit(old_text, range, replacement)?;
89    let mut parser = Parser::new();
90    parser.set_language(&grammar).ok()?;
91    let mut old_tree = parser.parse(old_text, None)?;
92    old_tree.edit(&input_edit(old_text, range, replacement));
93    let mut parsed = parser.parse(&edited_text, Some(&old_tree))?;
94    // Error recovery that reuses subtrees of the old tree can settle on another
95    // tree than a fresh parse (for example an ERROR node that swallows an
96    // `else` clause), so a tree with errors is reparsed from scratch: the
97    // network of a text never depends on its edit history, and matches the
98    // JavaScript runtime, which always parses the whole text.
99    if parsed.root_node().has_error() {
100        parsed = parser.parse(&edited_text, None)?;
101    }
102
103    Some(network_from_tree(
104        &edited_text,
105        language,
106        configuration,
107        &parsed,
108    ))
109}
110
111/// Builds the network of `text` from the projected tree of the native
112/// grammar `id`, as `network_from_tree` builds it from a tree-sitter tree.
113fn network_from_native(
114    text: &str,
115    language: &str,
116    configuration: ParseConfiguration,
117    id: &str,
118) -> LinkNetwork {
119    let (mut network, document) = LinkNetwork::new_parse_document(text, language);
120    let root = parse_native(id, text);
121    let lines = LineIndex::new(text);
122    let context = ConvertContext::new(
123        text,
124        &lines,
125        language,
126        configuration,
127        SpanOffset::zero(),
128        text.len(),
129    );
130    let tree_parent = public_root(&mut network, document, context, native_flags(&root));
131    convert_native_root(&mut network, tree_parent, &root, context);
132    network.attach_embedded_regions(document, text, language, configuration);
133    network
134}
135
136/// Preserves the original public Lean root `file` above the grammar's
137/// `module` root, whichever grammar parsed the source, as `publicRoot` in
138/// `js/src/programming-language-parser.js` does. Consumers can query either
139/// layer; every other language's tree hangs from the document itself.
140fn public_root(
141    network: &mut LinkNetwork,
142    document: LinkId,
143    context: ConvertContext<'_>,
144    flags: LinkFlags,
145) -> LinkId {
146    let language = context.language;
147    if !language.eq_ignore_ascii_case("lean") && !language.eq_ignore_ascii_case("lean4") {
148        return document;
149    }
150    network.insert_link(
151        [document],
152        LinkMetadata::new()
153            .with_link_type(LinkType::Syntax)
154            .with_named(true)
155            .with_term("file")
156            .with_language(language)
157            .with_span(span_for_range(
158                context.lines,
159                0,
160                context.source_len,
161                SpanOffset::zero(),
162            ))
163            .with_flags(flags),
164    )
165}
166
167fn network_from_tree(
168    text: &str,
169    language: &str,
170    configuration: ParseConfiguration,
171    parsed: &Tree,
172) -> LinkNetwork {
173    let (mut network, document) = LinkNetwork::new_parse_document(text, language);
174    let root = parsed.root_node();
175    let lines = LineIndex::new(text);
176    let context = ConvertContext::new(
177        text,
178        &lines,
179        language,
180        configuration,
181        SpanOffset::zero(),
182        text.len(),
183    );
184    let tree_parent = public_root(&mut network, document, context, flags_for_node(root));
185    convert_root(&mut network, tree_parent, root, context);
186    network.attach_embedded_regions(document, text, language, configuration);
187    network
188}
189
190fn apply_text_edit(old_text: &str, range: ByteRange, replacement: &str) -> Option<String> {
191    if range.end() > old_text.len()
192        || !old_text.is_char_boundary(range.start())
193        || !old_text.is_char_boundary(range.end())
194    {
195        return None;
196    }
197
198    let mut edited =
199        String::with_capacity(old_text.len() - (range.end() - range.start()) + replacement.len());
200    edited.push_str(&old_text[..range.start()]);
201    edited.push_str(replacement);
202    edited.push_str(&old_text[range.end()..]);
203    Some(edited)
204}
205
206fn input_edit(old_text: &str, range: ByteRange, replacement: &str) -> InputEdit {
207    let lines = LineIndex::new(old_text);
208    let start_position = lines.byte_point(range.start());
209    let old_end_position = lines.byte_point(range.end());
210    let new_end_position = point_after_text(start_position, replacement);
211
212    InputEdit {
213        start_byte: range.start(),
214        old_end_byte: range.end(),
215        new_end_byte: range.start() + replacement.len(),
216        start_position: tree_sitter_point(start_position),
217        old_end_position: tree_sitter_point(old_end_position),
218        new_end_position: tree_sitter_point(new_end_position),
219    }
220}
221
222fn point_after_text(start: Point, text: &str) -> Point {
223    let mut row = start.row();
224    let mut column = start.column();
225    for byte in text.bytes() {
226        if byte == b'\n' {
227            row += 1;
228            column = 0;
229        } else {
230            column += 1;
231        }
232    }
233    Point::new(row, column)
234}
235
236const fn tree_sitter_point(point: Point) -> TreeSitterPoint {
237    TreeSitterPoint::new(point.row(), point.column())
238}
239
240pub fn parse_embedded_region_into(
241    network: &mut LinkNetwork,
242    region: LinkId,
243    text: &str,
244    language: &str,
245    span: SourceSpan,
246    configuration: ParseConfiguration,
247) -> Option<LinkId> {
248    let parse_text = embedded_parse_text(text, language);
249    if let Some(id) = native_grammar_for_language(language) {
250        let root = parse_native(id, parse_text.as_ref());
251        let lines = LineIndex::new(parse_text.as_ref());
252        let context = ConvertContext::new(
253            parse_text.as_ref(),
254            &lines,
255            language,
256            configuration,
257            SpanOffset::new(span.byte_range().start(), span.start_point()),
258            text.len(),
259        );
260        return Some(convert_native_root(network, region, &root, context));
261    }
262    let grammar = grammar_for_language(language)?;
263    let mut parser = Parser::new();
264    parser.set_language(&grammar).ok()?;
265    let parsed = parser.parse(parse_text.as_ref(), None)?;
266    let root = parsed.root_node();
267    let lines = LineIndex::new(parse_text.as_ref());
268    let context = ConvertContext::new(
269        parse_text.as_ref(),
270        &lines,
271        language,
272        configuration,
273        SpanOffset::new(span.byte_range().start(), span.start_point()),
274        text.len(),
275    );
276    Some(convert_root(network, region, root, context))
277}
278
279/// Converts a grammar root below `parent`. Tree-sitter starts the root after
280/// its leading padding, so the text outside the root is retained as gap
281/// tokens beside it, mirroring `parseGrammarCst` in
282/// `js/src/programming-language-parser.js`.
283fn convert_root(
284    network: &mut LinkNetwork,
285    parent: LinkId,
286    root: Node<'_>,
287    context: ConvertContext<'_>,
288) -> LinkId {
289    insert_gap_token(network, parent, 0, root.start_byte(), context);
290    let root_id = convert_node(network, parent, root, context);
291    insert_gap_token(
292        network,
293        parent,
294        root.end_byte(),
295        context.source_len,
296        context,
297    );
298    root_id
299}
300
301/// Returns the native grammar id the language catalog records as the primary
302/// default grammar of a language name or alias, when it is a native grammar.
303fn native_grammar_for_language(language: &str) -> Option<&'static str> {
304    let grammar = crate::language_catalog::language_entry(language)?
305        .grammars
306        .first()?;
307    is_native_grammar(&grammar.id).then_some(grammar.id.as_str())
308}
309
310/// Selects the primary default grammar the language catalog records for a
311/// language name or alias.
312fn grammar_for_language(language: &str) -> Option<Language> {
313    let grammar = crate::language_catalog::language_entry(language)?
314        .grammars
315        .first()?;
316    grammar_by_id(&grammar.id)
317}
318
319/// The node kind and field names a compiled default grammar declares.
320#[derive(Debug, Clone, PartialEq, Eq)]
321pub struct GrammarNames {
322    /// Node kind names by symbol id; bytes that are not UTF-8 read as U+FFFD.
323    pub node_kinds: Vec<String>,
324    /// Field names by field id, starting at field id 1.
325    pub fields: Vec<String>,
326}
327
328/// Returns the node kind and field names of a default grammar by its
329/// grammar-lock id, or `None` for an unknown id or an oracle grammar.
330///
331/// Both runtimes compile the same generated parser, so the names equal the
332/// WebAssembly grammar's. A C compiler that re-encodes the parser's non-ASCII
333/// names (MSVC without `/utf-8`) makes them differ instead of failing.
334#[must_use]
335#[allow(unsafe_code)]
336pub fn grammar_names(id: &str) -> Option<GrammarNames> {
337    use std::ffi::{CStr, c_char};
338    use tree_sitter::ffi;
339
340    let raw = grammar_by_id(id)?.into_raw();
341    let name = |pointer: *const c_char| {
342        if pointer.is_null() {
343            String::new()
344        } else {
345            // SAFETY: the grammar's names are NUL-terminated static strings.
346            unsafe { CStr::from_ptr(pointer) }
347                .to_string_lossy()
348                .into_owned()
349        }
350    };
351    // SAFETY: `raw` is the live language `into_raw` released, and the name
352    // functions return null for ids outside the grammar's tables.
353    let (node_kinds, fields) = unsafe {
354        let symbols = u16::try_from(ffi::ts_language_symbol_count(raw)).unwrap_or(u16::MAX);
355        let fields = u16::try_from(ffi::ts_language_field_count(raw)).unwrap_or(u16::MAX);
356        (
357            (0..symbols)
358                .map(|symbol| name(ffi::ts_language_symbol_name(raw, symbol)))
359                .collect(),
360            (1..=fields)
361                .map(|field| name(ffi::ts_language_field_name_for_id(raw, field)))
362                .collect(),
363        )
364    };
365    // SAFETY: returns the ownership `into_raw` released, so the language is freed.
366    drop(unsafe { Language::from_raw(raw) });
367    Some(GrammarNames { node_kinds, fields })
368}
369
370/// Returns the compiled grammar for a grammar-lock id. The oracle grammars of
371/// the languages a native grammar parses are development dependencies, so
372/// their ids return `None`.
373pub fn grammar_by_id(id: &str) -> Option<Language> {
374    Some(match id {
375        "agda" => tree_sitter_agda::LANGUAGE.into(),
376        "bash" => tree_sitter_bash::LANGUAGE.into(),
377        "cmake" => cmake_grammar::LANGUAGE.into(),
378        "cpp" => tree_sitter_cpp::LANGUAGE.into(),
379        "csharp" => tree_sitter_c_sharp::LANGUAGE.into(),
380        "css" => tree_sitter_css::LANGUAGE.into(),
381        "dart" => tree_sitter_dart::LANGUAGE.into(),
382        "dtd" => tree_sitter_xml::LANGUAGE_DTD.into(),
383        "elixir" => tree_sitter_elixir::LANGUAGE.into(),
384        "elm" => tree_sitter_elm::LANGUAGE.into(),
385        "erlang" => tree_sitter_erlang::LANGUAGE.into(),
386        "groovy" => tree_sitter_groovy::LANGUAGE.into(),
387        "haskell" => tree_sitter_haskell::LANGUAGE.into(),
388        "hcl" => tree_sitter_hcl::LANGUAGE.into(),
389        "html" => tree_sitter_html::LANGUAGE.into(),
390        "kotlin" => tree_sitter_kotlin_ng::LANGUAGE.into(),
391        "lean" => lean_grammar::LANGUAGE.into(),
392        "lua" => tree_sitter_lua::LANGUAGE.into(),
393        "markdown" => tree_sitter_md_025::LANGUAGE.into(),
394        "markdown_inline" => tree_sitter_md_025::INLINE_LANGUAGE.into(),
395        "matlab" => tree_sitter_matlab::LANGUAGE.into(),
396        "nix" => tree_sitter_nix::LANGUAGE.into(),
397        "ocaml" => tree_sitter_ocaml::LANGUAGE_OCAML.into(),
398        "ocaml_interface" => tree_sitter_ocaml::LANGUAGE_OCAML_INTERFACE.into(),
399        "odin" => tree_sitter_odin::LANGUAGE.into(),
400        "pascal" => tree_sitter_pascal::LANGUAGE.into(),
401        "perl" => ts_parser_perl::LANGUAGE.into(),
402        "php" => tree_sitter_php::LANGUAGE_PHP.into(),
403        "powershell" => tree_sitter_powershell::LANGUAGE.into(),
404        "python" => tree_sitter_python::LANGUAGE.into(),
405        "r" => tree_sitter_r::LANGUAGE.into(),
406        "rocq" => rocq_grammar::LANGUAGE.into(),
407        "ruby" => tree_sitter_ruby::LANGUAGE.into(),
408        "scala" => tree_sitter_scala::LANGUAGE.into(),
409        "sql" => tree_sitter_sequel::LANGUAGE.into(),
410        "swift" => tree_sitter_swift::LANGUAGE.into(),
411        "toml" => tree_sitter_toml_ng::LANGUAGE.into(),
412        "vb" => tree_sitter_vb_dotnet::LANGUAGE.into(),
413        "xml" => tree_sitter_xml::LANGUAGE_XML.into(),
414        "yaml" => tree_sitter_yaml::LANGUAGE.into(),
415        "zig" => tree_sitter_zig::LANGUAGE.into(),
416        _ => return None,
417    })
418}
419
420fn convert_node(
421    network: &mut LinkNetwork,
422    parent: LinkId,
423    node: Node<'_>,
424    context: ConvertContext<'_>,
425) -> LinkId {
426    convert_node_with(network, parent, node, &[], context)
427}
428
429/// Converts `node`, placing `injected` nodes from another tree (Markdown block
430/// continuations inside inline content) among its children. Mirrors
431/// `convertGrammarNode` in `js/src/programming-language-parser.js`.
432fn convert_node_with<'tree>(
433    network: &mut LinkNetwork,
434    parent: LinkId,
435    node: Node<'tree>,
436    injected: &[Node<'tree>],
437    context: ConvertContext<'_>,
438) -> LinkId {
439    let flags = flags_for_node(node);
440    let node_id = network.insert_link(
441        [parent],
442        LinkMetadata::new()
443            .with_link_type(LinkType::Syntax)
444            .with_named(node.is_named())
445            .with_term(node.kind())
446            .with_language(context.language)
447            .with_span(span_for_node(
448                node,
449                context.lines,
450                context.source_len,
451                context.offset,
452            ))
453            .with_flags(flags),
454    );
455
456    let inline_tree = is_markdown_inline_container(node, context.language)
457        .then(|| parse_markdown_inline(node, context.text));
458    let mut extra_children = injected.to_vec();
459    let own_children = inline_tree.as_ref().map_or_else(
460        || children_with_fields(node),
461        |tree| {
462            extra_children.extend(markdown_inline_excluded_children(node));
463            children_with_fields(tree.root_node())
464        },
465    );
466
467    if own_children.is_empty() && extra_children.is_empty() {
468        if is_rocq_identifier(node, context.language) {
469            let semantic_term =
470                rocq_identifier_term(&context.text[node.start_byte()..node.end_byte()]);
471            let semantic_id = network.insert_link(
472                [node_id],
473                LinkMetadata::new()
474                    .with_link_type(LinkType::Syntax)
475                    .with_named(node.is_named())
476                    .with_term(semantic_term)
477                    .with_language(context.language)
478                    .with_span(span_for_node(
479                        node,
480                        context.lines,
481                        context.source_len,
482                        context.offset,
483                    ))
484                    .with_flags(flags),
485            );
486            insert_leaf_token(network, semantic_id, node, context);
487        } else {
488            insert_leaf_token(network, node_id, node, context);
489        }
490        return node_id;
491    }
492
493    let mut covered_until = node.start_byte();
494    let mut child_has_error = false;
495    for child in distribute_injected_children(own_children, &extra_children) {
496        if context.has_synthetic_suffix() && child.node.start_byte() >= context.source_len {
497            break;
498        }
499        insert_gap_token(
500            network,
501            node_id,
502            covered_until,
503            child.node.start_byte(),
504            context,
505        );
506
507        let child_id = convert_node_with(network, node_id, child.node, &child.injected, context);
508        if let Some(label) = child.field {
509            network.insert_field(node_id, label, child_id);
510        }
511        child_has_error |= network
512            .link(child_id)
513            .is_some_and(|link| link.metadata().flags().has_error());
514        covered_until = covered_until.max(child.node.end_byte().min(context.source_len));
515    }
516
517    insert_gap_token(network, node_id, covered_until, node.end_byte(), context);
518    // A Markdown inline tree is parsed separately from its block container, so
519    // its errors reach the containing block nodes through their children.
520    if child_has_error && !flags.has_error() {
521        network.set_flags(node_id, flags.with_containing_error());
522    }
523    node_id
524}
525
526struct ChildNode<'tree> {
527    node: Node<'tree>,
528    field: Option<&'tree str>,
529    injected: Vec<Node<'tree>>,
530}
531
532fn children_with_fields(node: Node<'_>) -> Vec<ChildNode<'_>> {
533    (0..node.child_count())
534        .map(|index| ChildNode {
535            node: node
536                .child(index)
537                .expect("tree-sitter child index should be valid"),
538            field: node.field_name_for_child(index),
539            injected: Vec::new(),
540        })
541        .collect()
542}
543
544fn is_markdown_inline_container(node: Node<'_>, language: &str) -> bool {
545    (language.eq_ignore_ascii_case("markdown") || language.eq_ignore_ascii_case("md"))
546        && matches!(node.kind(), "inline" | "pipe_table_cell")
547}
548
549/// Named children of a Markdown inline container after the first, which the
550/// inline grammar skips, exactly as upstream's `MarkdownParser` does.
551fn markdown_inline_excluded_children(node: Node<'_>) -> Vec<Node<'_>> {
552    (1..node.child_count())
553        .filter_map(|index| node.child(index))
554        .filter(Node::is_named)
555        .collect()
556}
557
558/// tree-sitter-markdown parses block structure and inline content with two
559/// grammars. Like upstream's `MarkdownParser` (`bindings/rust/parser.rs` in
560/// tree-sitter-md), every `inline` and `pipe_table_cell` block node is parsed
561/// again with the inline grammar over the node's range minus its named
562/// children after the first (block continuations such as a quote's `> `).
563/// Mirrors `parseMarkdownInline` in `js/src/programming-language-parser.js`.
564fn parse_markdown_inline(node: Node<'_>, text: &str) -> Tree {
565    let mut range = node.range();
566    let mut ranges = Vec::new();
567    for child in markdown_inline_excluded_children(node) {
568        ranges.push(TreeSitterRange {
569            start_byte: range.start_byte,
570            start_point: range.start_point,
571            end_byte: child.start_byte(),
572            end_point: child.start_position(),
573        });
574        range.start_byte = child.end_byte();
575        range.start_point = child.end_position();
576    }
577    ranges.push(range);
578    let mut parser = Parser::new();
579    parser
580        .set_language(&tree_sitter_md_025::INLINE_LANGUAGE.into())
581        .expect("the Markdown inline grammar matches the tree-sitter ABI");
582    parser
583        .set_included_ranges(&ranges)
584        .expect("Markdown inline ranges follow the ordered block children");
585    parser
586        .parse(text, None)
587        .expect("tree-sitter parses without a timeout or cancellation")
588}
589
590/// Places nodes from another tree under the deepest child whose range contains
591/// them, and orders the rest among the children by start offset, block nodes
592/// first on ties.
593fn distribute_injected_children<'tree>(
594    mut children: Vec<ChildNode<'tree>>,
595    injected: &[Node<'tree>],
596) -> Vec<ChildNode<'tree>> {
597    let mut top = Vec::new();
598    for &node in injected {
599        if let Some(owner) = children
600            .iter_mut()
601            .find(|child| contains_node(child.node, node))
602        {
603            owner.injected.push(node);
604        } else {
605            top.push(ChildNode {
606                node,
607                field: None,
608                injected: Vec::new(),
609            });
610        }
611    }
612    if top.is_empty() {
613        return children;
614    }
615    top.extend(children);
616    top.sort_by_key(|child| child.node.start_byte());
617    top
618}
619
620fn contains_node(outer: Node<'_>, inner: Node<'_>) -> bool {
621    let (outer_start, outer_end) = (outer.start_byte(), outer.end_byte());
622    let (inner_start, inner_end) = (inner.start_byte(), inner.end_byte());
623    outer_start <= inner_start
624        && inner_end <= outer_end
625        && (inner_start < inner_end || (outer_start < inner_start && inner_start < outer_end))
626}
627
628const fn is_rocq_language(language: &str) -> bool {
629    language.eq_ignore_ascii_case("rocq") || language.eq_ignore_ascii_case("coq")
630}
631
632/// A Rocq `ident` with text, which also exposes it as a semantic leaf; a
633/// repaired, empty `ident` has none.
634fn is_rocq_identifier(node: Node<'_>, language: &str) -> bool {
635    is_rocq_language(language) && node.kind() == "ident" && node.start_byte() < node.end_byte()
636}
637
638fn rocq_identifier_term(text: &str) -> &'static str {
639    match text {
640        "bool" | "nat" | "Prop" | "Set" | "SProp" | "Type" | "Z" => "primitive_type",
641        _ => "identifier",
642    }
643}
644
645fn insert_leaf_token(
646    network: &mut LinkNetwork,
647    owner: LinkId,
648    node: Node<'_>,
649    context: ConvertContext<'_>,
650) {
651    let start = node.start_byte();
652    let end = node.end_byte().min(context.source_len);
653    if node.is_missing() || start >= end {
654        return;
655    }
656
657    let span = span_for_range(context.lines, start, end, context.offset);
658    let flags = flags_for_node(node);
659    let token = network.insert_link(
660        [owner],
661        LinkMetadata::new()
662            .with_link_type(LinkType::Token)
663            .with_named(node.is_named())
664            .with_term(&context.text[start..end])
665            .with_language(context.language)
666            .with_span(span)
667            .with_flags(flags),
668    );
669
670    if flags.is_extra() {
671        network.attach_trivia(
672            owner,
673            token,
674            span,
675            context.configuration.trivia_attachment_policy(),
676        );
677    }
678}
679
680/// Inserts the source text between visible tree-sitter children. That text is
681/// either lexer extras or text matched by hidden grammar rules (such as VB's
682/// `Module` keyword). Leading and trailing whitespace becomes extra trivia, and
683/// the text between them a non-extra token, mirroring `pushGapNodes` in
684/// `js/src/programming-language-parser.js`.
685fn insert_gap_token(
686    network: &mut LinkNetwork,
687    owner: LinkId,
688    start: usize,
689    end: usize,
690    context: ConvertContext<'_>,
691) {
692    let start = start.min(context.source_len);
693    let end = end.min(context.source_len);
694    if start >= end {
695        return;
696    }
697    let gap = &context.text[start..end];
698    let content_start = start + (gap.len() - gap.trim_start_matches(is_gap_whitespace).len());
699    let content_end = content_start + gap.trim_matches(is_gap_whitespace).len();
700    for (piece_start, piece_end, flags) in [
701        (start, content_start, LinkFlags::extra()),
702        (content_start, content_end, LinkFlags::clean()),
703        (content_end, end, LinkFlags::extra()),
704    ] {
705        if piece_start >= piece_end {
706            continue;
707        }
708        let span = span_for_range(context.lines, piece_start, piece_end, context.offset);
709        let token = network.insert_link(
710            [owner],
711            LinkMetadata::new()
712                .with_link_type(LinkType::Token)
713                .with_named(false)
714                .with_term(&context.text[piece_start..piece_end])
715                .with_language(context.language)
716                .with_span(span)
717                .with_flags(flags),
718        );
719        if flags.is_extra() {
720            network.attach_trivia(
721                owner,
722                token,
723                span,
724                context.configuration.trivia_attachment_policy(),
725            );
726        }
727    }
728}
729
730/// Whitespace around hidden-rule text: Unicode whitespace and the invisible
731/// format characters grammars lex as extras (tree-sitter-javascript's U+200B,
732/// U+2060 and U+FEFF).
733const fn is_gap_whitespace(character: char) -> bool {
734    character.is_whitespace() || matches!(character, '\u{200B}' | '\u{2060}' | '\u{FEFF}')
735}
736
737fn flags_for_node(node: Node<'_>) -> LinkFlags {
738    let mut flags = LinkFlags::clean();
739    if node.is_error() {
740        flags = flags.with_error();
741    }
742    // Mirrors tree-sitter's `ts_node_has_error`, which is true for error and
743    // missing nodes themselves as well as for their ancestors.
744    if node.has_error() || node.is_error() || node.is_missing() {
745        flags = flags.with_containing_error();
746    }
747    if node.is_missing() {
748        flags = flags.with_missing();
749    }
750    if node.is_extra() {
751        flags = flags.with_extra();
752    }
753    flags
754}
755
756fn span_for_node(
757    node: Node<'_>,
758    lines: &LineIndex,
759    source_len: usize,
760    offset: SpanOffset,
761) -> SourceSpan {
762    let start = node.start_byte().min(source_len);
763    let end = node.end_byte().min(source_len);
764    span_for_range(lines, start, end, offset)
765}
766
767fn span_for_range(lines: &LineIndex, start: usize, end: usize, offset: SpanOffset) -> SourceSpan {
768    SourceSpan::new(
769        ByteRange::new(offset.byte + start, offset.byte + end),
770        offset.point(lines.byte_point(start)),
771        offset.point(lines.byte_point(end)),
772    )
773}
774
775fn embedded_parse_text<'a>(text: &'a str, language: &str) -> Cow<'a, str> {
776    if language.eq_ignore_ascii_case("css") && css_declaration_list_needs_semicolon(text) {
777        Cow::Owned(format!("{text};"))
778    } else {
779        Cow::Borrowed(text)
780    }
781}
782
783fn css_declaration_list_needs_semicolon(text: &str) -> bool {
784    let trimmed = text.trim_end();
785    !trimmed.is_empty()
786        && !trimmed.ends_with(';')
787        && !trimmed.ends_with('}')
788        && !trimmed.contains('{')
789}
790
791/// Position of a parsed text inside its host document, used to translate
792/// region-relative spans into document spans.
793#[derive(Clone, Copy, Debug, PartialEq, Eq)]
794pub struct SpanOffset {
795    byte: usize,
796    point: Point,
797}
798
799impl SpanOffset {
800    pub const fn new(byte: usize, point: Point) -> Self {
801        Self { byte, point }
802    }
803
804    pub const fn zero() -> Self {
805        Self::new(0, Point::new(0, 0))
806    }
807
808    pub const fn byte(self, byte: usize) -> usize {
809        self.byte + byte
810    }
811
812    pub const fn point(self, point: Point) -> Point {
813        let row = self.point.row() + point.row();
814        let column = if point.row() == 0 {
815            self.point.column() + point.column()
816        } else {
817            point.column()
818        };
819        Point::new(row, column)
820    }
821}
822
823#[derive(Clone, Copy, Debug, PartialEq, Eq)]
824struct ConvertContext<'a> {
825    text: &'a str,
826    lines: &'a LineIndex,
827    language: &'a str,
828    configuration: ParseConfiguration,
829    offset: SpanOffset,
830    source_len: usize,
831}
832
833impl<'a> ConvertContext<'a> {
834    const fn new(
835        text: &'a str,
836        lines: &'a LineIndex,
837        language: &'a str,
838        configuration: ParseConfiguration,
839        offset: SpanOffset,
840        source_len: usize,
841    ) -> Self {
842        Self {
843            text,
844            lines,
845            language,
846            configuration,
847            offset,
848            source_len,
849        }
850    }
851
852    const fn has_synthetic_suffix(self) -> bool {
853        self.source_len < self.text.len()
854    }
855}
856
857/// The top-level nodes of `text` in `language`'s default grammar as
858/// `(kind, start, end)` byte ranges, or `None` when no grammar parses it.
859pub fn top_level_nodes(text: &str, language: &str) -> Option<Vec<(String, usize, usize)>> {
860    if let Some(id) = native_grammar_for_language(language) {
861        let root = parse_native(id, text);
862        return Some(
863            root.children
864                .iter()
865                .map(|(child, _)| (child.term.clone(), child.start, child.end))
866                .collect(),
867        );
868    }
869    let grammar = grammar_for_language(language)?;
870    let mut parser = Parser::new();
871    parser.set_language(&grammar).ok()?;
872    let parsed = parser.parse(text, None)?;
873    let root = parsed.root_node();
874    let mut cursor = root.walk();
875    Some(
876        root.children(&mut cursor)
877            .map(|child| {
878                (
879                    child.kind().to_owned(),
880                    child.start_byte(),
881                    child.end_byte(),
882                )
883            })
884            .collect(),
885    )
886}
meta_language/tree_sitter_adapter.rs

meta_language/
tree_sitter_adapter.rs
