meta_language/grammar/emit/
tree_sitter_json.rs

1use crate::grammar::{CharClassItem, Grammar, GrammarExpr, GrammarFormat, GrammarRule, RuleKind};
2
3use super::{EmitReport, GrammarEmitError, ordered_rules, unsupported_error};
4
5const FORMAT: GrammarFormat = GrammarFormat::TreeSitter;
6const EXTRAS_RULE: &str = "_extras";
7const PRECEDENCE_TYPES: [(&str, &str); 4] = [
8    ("prec=", "PREC"),
9    ("prec_left=", "PREC_LEFT"),
10    ("prec_right=", "PREC_RIGHT"),
11    ("prec_dynamic=", "PREC_DYNAMIC"),
12];
13const MAX_SAFE_INTEGER: i64 = 9_007_199_254_740_991;
14
15/// Emits the declarative tree-sitter `grammar.json` form, the inverse of
16/// [`crate::grammar::import_tree_sitter_json`].
17///
18/// Captures produced by the importer map back to their `FIELD`, `ALIAS`,
19/// `PREC*`, `TOKEN`, `IMMEDIATE_TOKEN`, `RESERVED`, and `PATTERN` nodes. The
20/// text matches JavaScript `JSON.stringify(document, null, 2)` so both runtimes
21/// emit byte-identical documents.
22pub fn emit_tree_sitter_json(grammar: &Grammar) -> Result<(String, EmitReport), GrammarEmitError> {
23    let mut report = EmitReport::default();
24    let mut rules = Vec::new();
25    let mut inline = Vec::new();
26    let mut extras = None;
27    for rule in ordered_rules(grammar) {
28        if rule.name() == EXTRAS_RULE && rule.kind() == RuleKind::Silent {
29            extras = Some(extras_members(rule.expr(), &mut report)?);
30            continue;
31        }
32        if rule.kind() == RuleKind::Silent {
33            report.add_lossy(format!(
34                "tree-sitter emits RuleKind::Silent rule {:?} in the inline list",
35                rule.name()
36            ));
37            inline.push(Json::string(rule.name()));
38        }
39        rules.push((rule.name().to_string(), emit_rule(rule, &mut report)?));
40    }
41
42    let mut document = vec![
43        ("name".to_string(), Json::string(&grammar_name(grammar))),
44        ("rules".to_string(), Json::Object(rules)),
45    ];
46    if let Some(extras) = extras {
47        document.push(("extras".to_string(), Json::Array(extras)));
48    }
49    if !inline.is_empty() {
50        document.push(("inline".to_string(), Json::Array(inline)));
51    }
52    let mut output = String::new();
53    Json::Object(document).write(&mut output, 0);
54    output.push('\n');
55    Ok((output, report))
56}
57
58fn emit_rule(rule: &GrammarRule, report: &mut EmitReport) -> Result<Json, GrammarEmitError> {
59    if rule.kind() == RuleKind::Token
60        && let GrammarExpr::Capture {
61            label: Some(label),
62            expr,
63        } = rule.expr()
64        && label == "immediate_token"
65    {
66        return Ok(node(
67            "IMMEDIATE_TOKEN",
68            vec![("content", emit_node(expr, report)?)],
69        ));
70    }
71    if rule.kind() == RuleKind::Atomic {
72        report.add_lossy(format!(
73            "tree-sitter emits RuleKind::Atomic rule {:?} as a token",
74            rule.name()
75        ));
76    }
77    let body = emit_node(rule.expr(), report)?;
78    Ok(match rule.kind() {
79        RuleKind::Token | RuleKind::Atomic => node("TOKEN", vec![("content", body)]),
80        RuleKind::Normal | RuleKind::Silent => body,
81    })
82}
83
84fn extras_members(
85    expr: &GrammarExpr,
86    report: &mut EmitReport,
87) -> Result<Vec<Json>, GrammarEmitError> {
88    match expr {
89        GrammarExpr::Choice {
90            ordered: false,
91            alternatives,
92        } => emit_nodes(alternatives, report),
93        expr => Ok(vec![emit_node(expr, report)?]),
94    }
95}
96
97fn emit_nodes(
98    items: &[GrammarExpr],
99    report: &mut EmitReport,
100) -> Result<Vec<Json>, GrammarEmitError> {
101    items.iter().map(|item| emit_node(item, report)).collect()
102}
103
104fn emit_node(expr: &GrammarExpr, report: &mut EmitReport) -> Result<Json, GrammarEmitError> {
105    Ok(match expr {
106        GrammarExpr::Empty => blank(),
107        GrammarExpr::Terminal(value) => node("STRING", vec![("value", Json::string(value))]),
108        GrammarExpr::TerminalInsensitive(value) => {
109            report.add_lossy(format!(
110                "tree-sitter expands case-insensitive terminal {value:?} to a pattern"
111            ));
112            pattern(case_insensitive_pattern(value))
113        }
114        GrammarExpr::CharRange(start, end) => pattern(format!(
115            "[{}-{}]",
116            escape_class_char(*start),
117            escape_class_char(*end)
118        )),
119        GrammarExpr::CharClass { negated, items } => pattern(char_class_pattern(*negated, items)?),
120        GrammarExpr::AnyChar => pattern(".".to_string()),
121        GrammarExpr::NonTerminal(name) => node("SYMBOL", vec![("name", Json::string(name))]),
122        GrammarExpr::Choice {
123            ordered,
124            alternatives,
125        } => {
126            if alternatives.is_empty() {
127                return Err(unsupported_error(FORMAT, "empty Choice"));
128            }
129            if *ordered {
130                report.add_lossy("tree-sitter treats ordered choice as unordered choice");
131            }
132            node(
133                "CHOICE",
134                vec![("members", Json::Array(emit_nodes(alternatives, report)?))],
135            )
136        }
137        GrammarExpr::Sequence(items) => node(
138            "SEQ",
139            vec![("members", Json::Array(emit_nodes(items, report)?))],
140        ),
141        GrammarExpr::Optional(inner) => optional(emit_node(inner, report)?),
142        GrammarExpr::ZeroOrMore(inner) => {
143            node("REPEAT", vec![("content", emit_node(inner, report)?)])
144        }
145        GrammarExpr::OneOrMore(inner) => {
146            node("REPEAT1", vec![("content", emit_node(inner, report)?)])
147        }
148        GrammarExpr::Repeat { expr, min, max } => emit_repeat(expr, *min, *max, report)?,
149        GrammarExpr::And(_) | GrammarExpr::Not(_) => {
150            return Err(unsupported_error(FORMAT, "predicate"));
151        }
152        GrammarExpr::Capture { label, expr } => emit_capture(label.as_deref(), expr, report)?,
153        GrammarExpr::Feature(feature) => {
154            return Err(unsupported_error(GrammarFormat::TreeSitter, feature.head()));
155        }
156    })
157}
158
159fn emit_repeat(
160    expr: &GrammarExpr,
161    min: usize,
162    max: Option<usize>,
163    report: &mut EmitReport,
164) -> Result<Json, GrammarEmitError> {
165    if let Some(max) = max
166        && max < min
167    {
168        return Err(unsupported_error(
169            FORMAT,
170            format!("Repeat with min {min} greater than max Some({max})"),
171        ));
172    }
173    report.add_lossy(format!(
174        "tree-sitter desugared Repeat with min {min} and max {max:?}"
175    ));
176    let content = emit_node(expr, report)?;
177    let mut members = vec![content.clone(); min];
178    match max {
179        None => members.push(node("REPEAT", vec![("content", content)])),
180        Some(max) => {
181            for _ in min..max {
182                members.push(optional(content.clone()));
183            }
184        }
185    }
186    Ok(match members.len() {
187        0 => blank(),
188        1 => members.remove(0),
189        _ => node("SEQ", vec![("members", Json::Array(members))]),
190    })
191}
192
193fn emit_capture(
194    label: Option<&str>,
195    expr: &GrammarExpr,
196    report: &mut EmitReport,
197) -> Result<Json, GrammarEmitError> {
198    let Some(label) = label else {
199        report.add_lossy("tree-sitter dropped anonymous capture");
200        return emit_node(expr, report);
201    };
202    if label == "regex"
203        && let GrammarExpr::Terminal(value) = expr
204    {
205        return Ok(pattern(value.clone()));
206    }
207    for (prefix, kind) in PRECEDENCE_TYPES {
208        if let Some(value) = label.strip_prefix(prefix) {
209            return Ok(node(
210                kind,
211                vec![
212                    ("value", precedence_value(value)),
213                    ("content", emit_node(expr, report)?),
214                ],
215            ));
216        }
217    }
218    if let Some(value) = label.strip_prefix("alias:") {
219        return Ok(node(
220            "ALIAS",
221            vec![
222                ("content", emit_node(expr, report)?),
223                ("named", Json::Bool(is_identifier(value))),
224                ("value", Json::string(value)),
225            ],
226        ));
227    }
228    let content = emit_node(expr, report)?;
229    Ok(match label {
230        "token" => node("TOKEN", vec![("content", content)]),
231        "immediate_token" => node("IMMEDIATE_TOKEN", vec![("content", content)]),
232        "reserved" => node("RESERVED", vec![("content", content)]),
233        label => match label.strip_prefix("reserved:") {
234            Some(context_name) => node(
235                "RESERVED",
236                vec![
237                    ("context_name", Json::string(context_name)),
238                    ("content", content),
239                ],
240            ),
241            None => node(
242                "FIELD",
243                vec![("name", Json::string(label)), ("content", content)],
244            ),
245        },
246    })
247}
248
249fn char_class_pattern(negated: bool, items: &[CharClassItem]) -> Result<String, GrammarEmitError> {
250    if items.is_empty() {
251        return Err(unsupported_error(FORMAT, "empty CharClass"));
252    }
253    let mut output = String::from("[");
254    if negated {
255        output.push('^');
256    }
257    for item in items {
258        match item {
259            CharClassItem::Char(value) => output.push_str(&escape_class_char(*value)),
260            CharClassItem::Range(start, end) => {
261                output.push_str(&escape_class_char(*start));
262                output.push('-');
263                output.push_str(&escape_class_char(*end));
264            }
265        }
266    }
267    output.push(']');
268    Ok(output)
269}
270
271fn case_insensitive_pattern(value: &str) -> String {
272    value
273        .chars()
274        .map(|character| {
275            let lower = character.to_lowercase().collect::<Vec<_>>();
276            let upper = character.to_uppercase().collect::<Vec<_>>();
277            if lower == upper || lower.len() != 1 || upper.len() != 1 {
278                escape_pattern_char(character)
279            } else {
280                format!(
281                    "[{}{}]",
282                    escape_class_char(lower[0]),
283                    escape_class_char(upper[0])
284                )
285            }
286        })
287        .collect()
288}
289
290fn escape_class_char(character: char) -> String {
291    match character {
292        '\n' => "\\n".to_string(),
293        '\r' => "\\r".to_string(),
294        '\t' => "\\t".to_string(),
295        '\\' | ']' | '[' | '-' | '^' => format!("\\{character}"),
296        character => character.to_string(),
297    }
298}
299
300fn escape_pattern_char(character: char) -> String {
301    match character {
302        '\n' => "\\n".to_string(),
303        '\r' => "\\r".to_string(),
304        '\t' => "\\t".to_string(),
305        '\\' | '^' | '$' | '.' | '|' | '?' | '*' | '+' | '(' | ')' | '[' | ']' | '{' | '}'
306        | '/' => format!("\\{character}"),
307        character => character.to_string(),
308    }
309}
310
311fn precedence_value(value: &str) -> Json {
312    let digits = value.strip_prefix('-').unwrap_or(value);
313    let canonical = digits == "0"
314        || (digits.starts_with(|c: char| ('1'..='9').contains(&c))
315            && digits.chars().all(|c| c.is_ascii_digit()));
316    match value.parse::<i64>() {
317        Ok(number) if canonical && number.abs() <= MAX_SAFE_INTEGER => Json::Number(number),
318        _ => Json::string(value),
319    }
320}
321
322fn grammar_name(grammar: &Grammar) -> String {
323    let source = grammar
324        .start_rule()
325        .or_else(|| grammar.rules().first())
326        .map_or("grammar", GrammarRule::name);
327    let name = source
328        .chars()
329        .map(|c| {
330            if c.is_ascii_alphanumeric() || c == '_' {
331                c
332            } else {
333                '_'
334            }
335        })
336        .collect::<String>();
337    if name.starts_with(|c: char| c.is_ascii_alphabetic() || c == '_') {
338        name
339    } else {
340        format!("_{name}")
341    }
342}
343
344fn is_identifier(value: &str) -> bool {
345    value.starts_with(|c: char| c.is_ascii_alphabetic() || c == '_')
346        && value.chars().all(|c| c.is_ascii_alphanumeric() || c == '_')
347}
348
349fn node(kind: &str, fields: Vec<(&str, Json)>) -> Json {
350    let mut entries = vec![("type".to_string(), Json::string(kind))];
351    entries.extend(
352        fields
353            .into_iter()
354            .map(|(key, value)| (key.to_string(), value)),
355    );
356    Json::Object(entries)
357}
358
359fn blank() -> Json {
360    node("BLANK", Vec::new())
361}
362
363fn pattern(value: String) -> Json {
364    node("PATTERN", vec![("value", Json::String(value))])
365}
366
367fn optional(content: Json) -> Json {
368    node(
369        "CHOICE",
370        vec![("members", Json::Array(vec![content, blank()]))],
371    )
372}
373
374/// Insertion-ordered JSON value rendered like `JSON.stringify(value, null, 2)`.
375#[derive(Clone, Debug)]
376enum Json {
377    String(String),
378    Number(i64),
379    Bool(bool),
380    Array(Vec<Self>),
381    Object(Vec<(String, Self)>),
382}
383
384impl Json {
385    fn string(value: &str) -> Self {
386        Self::String(value.to_string())
387    }
388
389    fn write(&self, output: &mut String, indent: usize) {
390        match self {
391            Self::String(value) => write_json_string(output, value),
392            Self::Number(value) => output.push_str(&value.to_string()),
393            Self::Bool(value) => output.push_str(if *value { "true" } else { "false" }),
394            Self::Array(items) if items.is_empty() => output.push_str("[]"),
395            Self::Object(entries) if entries.is_empty() => output.push_str("{}"),
396            Self::Array(items) => {
397                output.push_str("[\n");
398                for (index, item) in items.iter().enumerate() {
399                    push_separator(output, index, indent + 2);
400                    item.write(output, indent + 2);
401                }
402                push_close(output, indent, ']');
403            }
404            Self::Object(entries) => {
405                output.push_str("{\n");
406                for (index, (key, value)) in entries.iter().enumerate() {
407                    push_separator(output, index, indent + 2);
408                    write_json_string(output, key);
409                    output.push_str(": ");
410                    value.write(output, indent + 2);
411                }
412                push_close(output, indent, '}');
413            }
414        }
415    }
416}
417
418fn push_separator(output: &mut String, index: usize, indent: usize) {
419    if index > 0 {
420        output.push_str(",\n");
421    }
422    output.push_str(&" ".repeat(indent));
423}
424
425fn push_close(output: &mut String, indent: usize, close: char) {
426    output.push('\n');
427    output.push_str(&" ".repeat(indent));
428    output.push(close);
429}
430
431fn write_json_string(output: &mut String, value: &str) {
432    // serde_json escapes strings exactly like ECMAScript JSON.stringify for
433    // well-formed Unicode: short escapes plus lowercase `\u00xx` controls.
434    output.push_str(&serde_json::to_string(value).unwrap_or_else(|_| format!("{value:?}")));
435}
meta_language/grammar/emit/tree_sitter_json.rs

meta_language/grammar/emit/
tree_sitter_json.rs
