meta_language/grammar/import/
tree_sitter_json.rs

1use serde_json::{Map, Value};
2
3use super::{parse_error, unsupported_error, GrammarImportError};
4use crate::grammar::{CharClassItem, Grammar, GrammarExpr, GrammarFormat, GrammarRule, RuleKind};
5
6const FORMAT: GrammarFormat = GrammarFormat::TreeSitter;
7
8/// Parses tree-sitter `grammar.json` text into the grammar IR.
9///
10/// # Errors
11///
12/// Returns [`GrammarImportError`] when the JSON cannot be parsed or validated,
13/// or when a node type cannot be represented in the grammar IR.
14pub fn import_tree_sitter_json(text: &str) -> Result<Grammar, GrammarImportError> {
15    let value = serde_json::from_str::<Value>(text)
16        .map_err(|error| parse_error(FORMAT, error.to_string()))?;
17    let root = value
18        .as_object()
19        .ok_or_else(|| parse_error(FORMAT, "grammar.json root must be an object"))?;
20    let rules = required_object(root, "rules")?;
21    let rule_names = source_ordered_rule_names(text, rules)?;
22    if rule_names.is_empty() {
23        return Err(parse_error(FORMAT, "rules object must not be empty"));
24    }
25
26    let mut grammar = Grammar::new().with_source_format(FORMAT);
27    for name in rule_names {
28        let node = rules
29            .get(&name)
30            .ok_or_else(|| parse_error(FORMAT, format!("rule {name:?} is missing")))?;
31        grammar.add_rule(lower_rule(&name, node)?);
32    }
33
34    if let Some(start) = grammar.rules().first().map(|rule| rule.name.clone()) {
35        grammar.set_start(start);
36    }
37    if let Some(extras) = root.get("extras") {
38        if let Some(expr) = lower_extras(extras)? {
39            grammar.add_rule(GrammarRule::new("_extras", expr).with_kind(RuleKind::Silent));
40        }
41    }
42
43    Ok(grammar)
44}
45
46fn lower_rule(name: &str, node: &Value) -> Result<GrammarRule, GrammarImportError> {
47    match node_type(node)? {
48        "TOKEN" => Ok(GrammarRule::new(name, lower_content(node)?).with_kind(RuleKind::Token)),
49        "IMMEDIATE_TOKEN" => Ok(GrammarRule::new(
50            name,
51            GrammarExpr::capture("immediate_token", lower_content(node)?),
52        )
53        .with_kind(RuleKind::Token)),
54        _ => lower_node(node).map(|expr| GrammarRule::new(name, expr)),
55    }
56}
57
58fn lower_node(node: &Value) -> Result<GrammarExpr, GrammarImportError> {
59    match node_type(node)? {
60        "SYMBOL" => string_field(node, "name").map(GrammarExpr::NonTerminal),
61        "STRING" => string_field(node, "value").map(GrammarExpr::Terminal),
62        "PATTERN" => string_field(node, "value").map(|value| lower_pattern(&value)),
63        "BLANK" => Ok(GrammarExpr::Empty),
64        "SEQ" => lower_sequence(array_field(node, "members")?),
65        "CHOICE" => lower_choice(array_field(node, "members")?),
66        "REPEAT" => lower_content(node).map(GrammarExpr::zero_or_more),
67        "REPEAT1" => lower_content(node).map(GrammarExpr::one_or_more),
68        "PREC" => lower_precedence(node, "prec"),
69        "PREC_LEFT" => lower_precedence(node, "prec_left"),
70        "PREC_RIGHT" => lower_precedence(node, "prec_right"),
71        "PREC_DYNAMIC" => lower_precedence(node, "prec_dynamic"),
72        "TOKEN" => lower_content(node).map(|expr| GrammarExpr::capture("token", expr)),
73        "IMMEDIATE_TOKEN" => {
74            lower_content(node).map(|expr| GrammarExpr::capture("immediate_token", expr))
75        }
76        "FIELD" => {
77            let label = string_field(node, "name")?;
78            lower_content(node).map(|expr| GrammarExpr::capture(label, expr))
79        }
80        "ALIAS" => {
81            let value = string_field(node, "value")?;
82            lower_content(node).map(|expr| GrammarExpr::capture(format!("alias:{value}"), expr))
83        }
84        "RESERVED" => lower_reserved(node),
85        construct => Err(unsupported_error(FORMAT, construct)),
86    }
87}
88
89fn lower_sequence(members: &[Value]) -> Result<GrammarExpr, GrammarImportError> {
90    let mut lowered = Vec::new();
91    for member in members {
92        push_sequence_item(&mut lowered, lower_node(member)?);
93    }
94
95    Ok(match lowered.len() {
96        0 => GrammarExpr::Empty,
97        1 => lowered.remove(0),
98        _ => GrammarExpr::Sequence(lowered),
99    })
100}
101
102fn push_sequence_item(items: &mut Vec<GrammarExpr>, item: GrammarExpr) {
103    match item {
104        GrammarExpr::Empty => {}
105        GrammarExpr::Sequence(nested) => {
106            for item in nested {
107                push_sequence_item(items, item);
108            }
109        }
110        item => items.push(item),
111    }
112}
113
114fn lower_choice(members: &[Value]) -> Result<GrammarExpr, GrammarImportError> {
115    let mut alternatives = Vec::new();
116    for member in members {
117        alternatives.push(lower_node(member)?);
118    }
119
120    if alternatives.len() == 2 {
121        if alternatives[0] == GrammarExpr::Empty {
122            return Ok(GrammarExpr::optional(alternatives.remove(1)));
123        }
124        if alternatives[1] == GrammarExpr::Empty {
125            return Ok(GrammarExpr::optional(alternatives.remove(0)));
126        }
127    }
128
129    Ok(match alternatives.len() {
130        0 => GrammarExpr::Empty,
131        1 => alternatives.remove(0),
132        _ => GrammarExpr::Choice {
133            ordered: false,
134            alternatives,
135        },
136    })
137}
138
139fn lower_precedence(node: &Value, label: &str) -> Result<GrammarExpr, GrammarImportError> {
140    let value = precedence_value(node)?;
141    lower_content(node).map(|expr| GrammarExpr::capture(format!("{label}={value}"), expr))
142}
143
144fn lower_reserved(node: &Value) -> Result<GrammarExpr, GrammarImportError> {
145    let label = node
146        .get("context_name")
147        .and_then(Value::as_str)
148        .map_or_else(
149            || "reserved".to_string(),
150            |context| format!("reserved:{context}"),
151        );
152    lower_content(node).map(|expr| GrammarExpr::capture(label, expr))
153}
154
155fn lower_content(node: &Value) -> Result<GrammarExpr, GrammarImportError> {
156    lower_node(
157        node.as_object()
158            .and_then(|object| object.get("content"))
159            .ok_or_else(|| parse_error(FORMAT, "node is missing content"))?,
160    )
161}
162
163fn lower_extras(node: &Value) -> Result<Option<GrammarExpr>, GrammarImportError> {
164    let members = node
165        .as_array()
166        .ok_or_else(|| parse_error(FORMAT, "extras must be an array"))?;
167    if members.is_empty() {
168        return Ok(None);
169    }
170    lower_choice(members).map(Some)
171}
172
173fn lower_pattern(value: &str) -> GrammarExpr {
174    parse_char_class_pattern(value)
175        .unwrap_or_else(|| GrammarExpr::capture("regex", GrammarExpr::Terminal(value.to_string())))
176}
177
178fn parse_char_class_pattern(pattern: &str) -> Option<GrammarExpr> {
179    if !pattern.starts_with('[') || !pattern.ends_with(']') {
180        return None;
181    }
182    let content = &pattern[1..pattern.len() - 1];
183    let (negated, content) = content
184        .strip_prefix('^')
185        .map_or((false, content), |content| (true, content));
186    let chars = content.chars().collect::<Vec<_>>();
187    let mut items = Vec::new();
188    let mut index = 0;
189    while index < chars.len() {
190        let start = read_class_char(&chars, &mut index)?;
191        if index + 1 < chars.len() && chars[index] == '-' {
192            index += 1;
193            let end = read_class_char(&chars, &mut index)?;
194            if start > end {
195                return None;
196            }
197            items.push(CharClassItem::Range(start, end));
198        } else {
199            items.push(CharClassItem::Char(start));
200        }
201    }
202    if items.is_empty() {
203        return None;
204    }
205    Some(GrammarExpr::CharClass { negated, items })
206}
207
208fn read_class_char(chars: &[char], index: &mut usize) -> Option<char> {
209    let character = *chars.get(*index)?;
210    *index += 1;
211    if character != '\\' {
212        return Some(character);
213    }
214    let escaped = *chars.get(*index)?;
215    *index += 1;
216    match escaped {
217        'n' => Some('\n'),
218        'r' => Some('\r'),
219        't' => Some('\t'),
220        '\\' | '"' | '\'' | '[' | ']' | '-' | '^' => Some(escaped),
221        _ => None,
222    }
223}
224
225fn node_type(node: &Value) -> Result<&str, GrammarImportError> {
226    node.as_object()
227        .ok_or_else(|| parse_error(FORMAT, "grammar node must be an object"))?
228        .get("type")
229        .and_then(Value::as_str)
230        .ok_or_else(|| parse_error(FORMAT, "grammar node is missing string type"))
231}
232
233fn string_field(node: &Value, field: &str) -> Result<String, GrammarImportError> {
234    node.as_object()
235        .and_then(|object| object.get(field))
236        .and_then(Value::as_str)
237        .map(ToString::to_string)
238        .ok_or_else(|| parse_error(FORMAT, format!("node is missing string {field}")))
239}
240
241fn array_field<'node>(
242    node: &'node Value,
243    field: &str,
244) -> Result<&'node [Value], GrammarImportError> {
245    node.as_object()
246        .and_then(|object| object.get(field))
247        .and_then(Value::as_array)
248        .map(Vec::as_slice)
249        .ok_or_else(|| parse_error(FORMAT, format!("node is missing array {field}")))
250}
251
252fn precedence_value(node: &Value) -> Result<String, GrammarImportError> {
253    let value = node
254        .as_object()
255        .and_then(|object| object.get("value"))
256        .ok_or_else(|| parse_error(FORMAT, "precedence node is missing value"))?;
257    match value {
258        Value::Number(number) => Ok(number.to_string()),
259        Value::String(value) => Ok(value.clone()),
260        _ => Err(parse_error(
261            FORMAT,
262            "precedence value must be a number or string",
263        )),
264    }
265}
266
267fn required_object<'value>(
268    root: &'value Map<String, Value>,
269    field: &str,
270) -> Result<&'value Map<String, Value>, GrammarImportError> {
271    root.get(field)
272        .and_then(Value::as_object)
273        .ok_or_else(|| parse_error(FORMAT, format!("grammar.json must contain object {field}")))
274}
275
276fn source_ordered_rule_names(
277    text: &str,
278    rules: &Map<String, Value>,
279) -> Result<Vec<String>, GrammarImportError> {
280    let names = JsonScanner::new(text).top_level_object_keys("rules")?;
281    if names.len() != rules.len() || names.iter().any(|name| !rules.contains_key(name)) {
282        return Err(parse_error(
283            FORMAT,
284            "rules object in source text does not match parsed JSON object",
285        ));
286    }
287    Ok(names)
288}
289
290#[derive(Clone, Debug)]
291struct JsonScanner<'text> {
292    text: &'text str,
293    cursor: usize,
294}
295
296impl<'text> JsonScanner<'text> {
297    const fn new(text: &'text str) -> Self {
298        Self { text, cursor: 0 }
299    }
300
301    fn top_level_object_keys(mut self, property: &str) -> Result<Vec<String>, GrammarImportError> {
302        self.skip_whitespace();
303        self.consume_byte(b'{')?;
304        self.skip_whitespace();
305        if self.try_consume_byte(b'}') {
306            return Err(parse_error(FORMAT, format!("missing {property} object")));
307        }
308
309        loop {
310            let key = self.parse_string()?;
311            self.skip_whitespace();
312            self.consume_byte(b':')?;
313            self.skip_whitespace();
314            if key == property {
315                return self.object_keys();
316            }
317            self.skip_value()?;
318            self.skip_whitespace();
319            if self.try_consume_byte(b'}') {
320                return Err(parse_error(FORMAT, format!("missing {property} object")));
321            }
322            self.consume_byte(b',')?;
323            self.skip_whitespace();
324        }
325    }
326
327    fn object_keys(&mut self) -> Result<Vec<String>, GrammarImportError> {
328        self.consume_byte(b'{')?;
329        self.skip_whitespace();
330        let mut keys = Vec::new();
331        if self.try_consume_byte(b'}') {
332            return Ok(keys);
333        }
334
335        loop {
336            keys.push(self.parse_string()?);
337            self.skip_whitespace();
338            self.consume_byte(b':')?;
339            self.skip_whitespace();
340            self.skip_value()?;
341            self.skip_whitespace();
342            if self.try_consume_byte(b'}') {
343                return Ok(keys);
344            }
345            self.consume_byte(b',')?;
346            self.skip_whitespace();
347        }
348    }
349
350    fn skip_value(&mut self) -> Result<(), GrammarImportError> {
351        self.skip_whitespace();
352        match self.peek_byte() {
353            Some(b'"') => self.parse_string().map(|_| ()),
354            Some(b'{') => self.skip_object(),
355            Some(b'[') => self.skip_array(),
356            Some(_) => {
357                while let Some(byte) = self.peek_byte() {
358                    if byte.is_ascii_whitespace() || matches!(byte, b',' | b'}' | b']') {
359                        break;
360                    }
361                    self.cursor += 1;
362                }
363                Ok(())
364            }
365            None => Err(parse_error(FORMAT, "unexpected end of JSON value")),
366        }
367    }
368
369    fn skip_object(&mut self) -> Result<(), GrammarImportError> {
370        self.consume_byte(b'{')?;
371        self.skip_whitespace();
372        if self.try_consume_byte(b'}') {
373            return Ok(());
374        }
375        loop {
376            self.parse_string()?;
377            self.skip_whitespace();
378            self.consume_byte(b':')?;
379            self.skip_value()?;
380            self.skip_whitespace();
381            if self.try_consume_byte(b'}') {
382                return Ok(());
383            }
384            self.consume_byte(b',')?;
385            self.skip_whitespace();
386        }
387    }
388
389    fn skip_array(&mut self) -> Result<(), GrammarImportError> {
390        self.consume_byte(b'[')?;
391        self.skip_whitespace();
392        if self.try_consume_byte(b']') {
393            return Ok(());
394        }
395        loop {
396            self.skip_value()?;
397            self.skip_whitespace();
398            if self.try_consume_byte(b']') {
399                return Ok(());
400            }
401            self.consume_byte(b',')?;
402            self.skip_whitespace();
403        }
404    }
405
406    fn parse_string(&mut self) -> Result<String, GrammarImportError> {
407        self.skip_whitespace();
408        let start = self.cursor;
409        self.consume_byte(b'"')?;
410        let bytes = self.text.as_bytes();
411        while let Some(byte) = bytes.get(self.cursor).copied() {
412            self.cursor += 1;
413            match byte {
414                b'\\' => {
415                    if self.cursor >= bytes.len() {
416                        return Err(parse_error(FORMAT, "unterminated JSON string escape"));
417                    }
418                    self.cursor += 1;
419                }
420                b'"' => {
421                    let text = &self.text[start..self.cursor];
422                    return serde_json::from_str::<String>(text)
423                        .map_err(|error| parse_error(FORMAT, error.to_string()));
424                }
425                _ => {}
426            }
427        }
428        Err(parse_error(FORMAT, "unterminated JSON string"))
429    }
430
431    fn skip_whitespace(&mut self) {
432        while self
433            .peek_byte()
434            .is_some_and(|byte| byte.is_ascii_whitespace())
435        {
436            self.cursor += 1;
437        }
438    }
439
440    fn consume_byte(&mut self, expected: u8) -> Result<(), GrammarImportError> {
441        self.skip_whitespace();
442        if self.try_consume_byte(expected) {
443            Ok(())
444        } else {
445            Err(parse_error(
446                FORMAT,
447                format!("expected JSON byte {:?}", char::from(expected)),
448            ))
449        }
450    }
451
452    fn try_consume_byte(&mut self, expected: u8) -> bool {
453        if self.peek_byte() == Some(expected) {
454            self.cursor += 1;
455            true
456        } else {
457            false
458        }
459    }
460
461    fn peek_byte(&self) -> Option<u8> {
462        self.text.as_bytes().get(self.cursor).copied()
463    }
464}
meta_language/grammar/import/tree_sitter_json.rs

meta_language/grammar/import/
tree_sitter_json.rs
