meta_language/grammar/interchange/
mod.rs

1//! Grammar interchange commands shared by the Rust and JavaScript tools.
2//!
3//! The commands import, validate, convert, merge, rename, export and round
4//! trip grammars. [`run_grammar_command`] is the whole command-line behavior
5//! with file reading injected, so the library and the `meta-language` binary
6//! report the same standard output, standard error and exit status. It mirrors
7//! `js/src/grammar-interchange.js`, and the native grammar listing read by
8//! [`parse_native_grammar`] and written by [`render_native_grammar`] mirrors
9//! the same file.
10
11use std::collections::{BTreeMap, BTreeSet};
12
13use super::Grammar;
14use super::emit::{
15    EmitReport, GrammarEmitError, emit_abnf, emit_antlr, emit_bnf, emit_ebnf, emit_gbnf, emit_lark,
16    emit_pest, emit_tree_sitter_json,
17};
18use super::import::{
19    GrammarImportError, import_abnf, import_antlr, import_bnf, import_ebnf, import_gbnf,
20    import_lark, import_pest, import_tree_sitter_json,
21};
22use super::merge::{GrammarMergeOptions, GrammarMergeSource, merge_grammars, rename_grammar_rule};
23use super::round_trip::{GrammarRoundTrip, GrammarRoundTripError, check_grammar_round_trip};
24use super::validate::{Severity, validate};
25
26mod feature_forms;
27mod feature_links;
28mod json;
29mod links;
30mod lossless;
31mod lowering;
32mod native;
33
34pub use feature_links::{render_declaration_links, render_rule_fields};
35pub use json::{deserialize_grammar, serialize_grammar};
36pub use links::{
37    parse_grammar_links, parse_links_expression, percent_decode_links_text,
38    percent_encode_links_text, render_grammar_links, render_links_expression, render_rule_link,
39};
40pub use lossless::{
41    GRAMMAR_LOSSLESS_FORMATS, GrammarLayout, GrammarLayoutDefinition, GrammarLayoutImplicit,
42    GrammarLosslessError, GrammarSourceDefinition, GrammarSourceSplit, capture_grammar_layout,
43    emit_grammar_lossless, import_grammar_lossless, parse_grammar_layout_links,
44    render_grammar_layout_links, split_grammar_source,
45};
46pub use lowering::{
47    GRAMMAR_LOWERING_FORMATS, GrammarLowering, GrammarLoweringEncoding, GrammarLoweringError,
48    GrammarLoweringFailure, GrammarLoweringFailureKind, GrammarLoweringMetadata,
49    GrammarLoweringOptions, GrammarLoweringReport, GrammarLoweringStatus, GrammarLoweringStep,
50    MAX_LOWERED_CHARACTERS, check_grammar_lowering, dropped_grammar_features, lower_grammar,
51    parse_lowering_metadata, reconstruct_grammar, render_lowering_metadata,
52};
53pub use native::{
54    parse_native_expression, parse_native_grammar, render_native_expression, render_native_feature,
55    render_native_grammar,
56};
57
58/// Imports a grammar from one notation.
59pub type GrammarImporter = fn(&str) -> Result<Grammar, GrammarImportError>;
60
61/// Emits a grammar in one notation, with its fidelity report.
62pub type GrammarEmitter = fn(&Grammar) -> Result<(String, EmitReport), GrammarEmitError>;
63
64/// The formats [`grammar_importer`] and the `--from` options read.
65pub const GRAMMAR_IMPORT_FORMATS: &[&str] = &[
66    "abnf",
67    "antlr",
68    "bnf",
69    "ebnf",
70    "gbnf",
71    "lark",
72    "native",
73    "pest",
74    "tree-sitter-json",
75];
76
77/// The formats [`grammar_emitter`] and the `--to` options write.
78pub const GRAMMAR_EXPORT_FORMATS: &[&str] = &[
79    "abnf",
80    "antlr",
81    "bnf",
82    "ebnf",
83    "gbnf",
84    "lark",
85    "native",
86    "pest",
87    "tree-sitter-json",
88];
89
90/// The importer of one of [`GRAMMAR_IMPORT_FORMATS`].
91#[must_use]
92pub fn grammar_importer(format: &str) -> Option<GrammarImporter> {
93    let importer: GrammarImporter = match format {
94        "abnf" => import_abnf,
95        "antlr" => import_antlr,
96        "bnf" => import_bnf,
97        "ebnf" => import_ebnf,
98        "gbnf" => import_gbnf,
99        "lark" => import_lark,
100        "native" => parse_native_grammar,
101        "pest" => import_pest,
102        "tree-sitter-json" => import_tree_sitter_json,
103        _ => return None,
104    };
105    Some(importer)
106}
107
108/// The emitter of one of [`GRAMMAR_EXPORT_FORMATS`].
109#[must_use]
110pub fn grammar_emitter(format: &str) -> Option<GrammarEmitter> {
111    let emitter: GrammarEmitter = match format {
112        "abnf" => emit_abnf,
113        "antlr" => emit_antlr,
114        "bnf" => emit_bnf,
115        "ebnf" => emit_ebnf,
116        "gbnf" => emit_gbnf,
117        "lark" => emit_lark,
118        "native" => |grammar| Ok((render_native_grammar(grammar), EmitReport::default())),
119        "pest" => emit_pest,
120        "tree-sitter-json" => emit_tree_sitter_json,
121        _ => return None,
122    };
123    Some(emitter)
124}
125
126/// The help text of `meta-language grammar`, the same in both runtimes.
127pub const GRAMMAR_COMMAND_USAGE: &str = "usage: meta-language grammar <command> [options]
128
129commands:
130  formats
131      list the import and export formats
132  import --from FORMAT FILE
133      print the grammar as a native listing
134  validate --from FORMAT FILE
135      report grammar diagnostics; exit 1 when one is an error
136  convert --from FORMAT --to FORMAT FILE
137      translate a grammar to another format; lossy steps go to standard error
138  export --to FORMAT FILE
139      export a native listing to another format
140  merge --source FORMAT:FILE... [--language NAME] [--require A=B]... [--to FORMAT]
141      merge grammars of one language; earlier sources take precedence
142  rename --from FORMAT --rule OLD --name NEW [--namespace NS] [--to FORMAT] FILE
143      rename a rule and every reference to it
144  round-trip --from FORMAT [--accept TEXT]... [--reject TEXT]... FILE
145      export and re-import a mutated grammar; exit 1 when it is not preserved
146  help
147      print this help
148
149exit status: 0 success, 1 a problem was found, 2 a usage or input error
150";
151
152/// What one `meta-language grammar` run printed and its exit status.
153#[derive(Clone, Debug, PartialEq, Eq)]
154pub struct GrammarCommandOutput {
155    /// 0 success, 1 a problem was found, 2 a usage or input error.
156    pub exit_code: i32,
157    /// Standard output.
158    pub stdout: String,
159    /// Standard error.
160    pub stderr: String,
161}
162
163/// Reads a file's text for [`run_grammar_command`].
164pub type GrammarFileReader<'a> = &'a dyn Fn(&str) -> std::io::Result<String>;
165
166#[derive(Clone, Copy, PartialEq, Eq)]
167enum Arity {
168    One,
169    Many,
170}
171
172struct CommandSpec {
173    options: &'static [(&'static str, Arity)],
174    required: &'static [&'static str],
175    files: usize,
176    run: fn(&mut Context<'_>) -> Result<i32, CommandError>,
177}
178
179struct CommandError {
180    message: String,
181    detail: Option<String>,
182}
183
184impl CommandError {
185    fn new(message: impl Into<String>) -> Self {
186        Self {
187            message: message.into(),
188            detail: None,
189        }
190    }
191
192    fn with_detail(message: impl Into<String>, detail: impl Into<String>) -> Self {
193        Self {
194            message: message.into(),
195            detail: Some(detail.into()),
196        }
197    }
198}
199
200struct Context<'a> {
201    options: BTreeMap<&'static str, Vec<String>>,
202    files: Vec<String>,
203    read_file: GrammarFileReader<'a>,
204    stdout: Vec<String>,
205    stderr: Vec<String>,
206}
207
208impl Context<'_> {
209    fn one(&self, option: &str) -> Option<&str> {
210        self.options
211            .get(option)
212            .and_then(|values| values.first())
213            .map(String::as_str)
214    }
215
216    fn many(&self, option: &str) -> &[String] {
217        self.options.get(option).map_or(&[], Vec::as_slice)
218    }
219
220    fn required(&self, option: &str) -> &str {
221        self.one(option).unwrap_or_default()
222    }
223}
224
225fn command_spec(name: &str) -> Option<CommandSpec> {
226    use Arity::{Many, One};
227    Some(match name {
228        "formats" => CommandSpec {
229            options: &[],
230            required: &[],
231            files: 0,
232            run: formats_command,
233        },
234        "import" => CommandSpec {
235            options: &[("from", One)],
236            required: &["from"],
237            files: 1,
238            run: import_command,
239        },
240        "validate" => CommandSpec {
241            options: &[("from", One)],
242            required: &["from"],
243            files: 1,
244            run: validate_command,
245        },
246        "convert" => CommandSpec {
247            options: &[("from", One), ("to", One)],
248            required: &["from", "to"],
249            files: 1,
250            run: convert_command,
251        },
252        "export" => CommandSpec {
253            options: &[("to", One)],
254            required: &["to"],
255            files: 1,
256            run: export_command,
257        },
258        "merge" => CommandSpec {
259            options: &[
260                ("source", Many),
261                ("language", One),
262                ("require", Many),
263                ("to", One),
264            ],
265            required: &["source"],
266            files: 0,
267            run: merge_command,
268        },
269        "rename" => CommandSpec {
270            options: &[
271                ("from", One),
272                ("rule", One),
273                ("name", One),
274                ("namespace", One),
275                ("to", One),
276            ],
277            required: &["from", "rule", "name"],
278            files: 1,
279            run: rename_command,
280        },
281        "round-trip" => CommandSpec {
282            options: &[("from", One), ("accept", Many), ("reject", Many)],
283            required: &["from"],
284            files: 1,
285            run: round_trip_command,
286        },
287        _ => return None,
288    })
289}
290
291/// Runs `meta-language grammar <command> ...` over `args`.
292///
293/// `read_file` returns a file's text. The output, messages and exit status
294/// are the same as those of `runGrammarCommand` in the JavaScript package.
295#[must_use]
296pub fn run_grammar_command(
297    args: &[String],
298    read_file: GrammarFileReader<'_>,
299) -> GrammarCommandOutput {
300    let Some((name, rest)) = args.split_first() else {
301        return GrammarCommandOutput {
302            exit_code: 2,
303            stdout: String::new(),
304            stderr: GRAMMAR_COMMAND_USAGE.to_owned(),
305        };
306    };
307    if matches!(name.as_str(), "help" | "--help" | "-h") {
308        return GrammarCommandOutput {
309            exit_code: 0,
310            stdout: GRAMMAR_COMMAND_USAGE.to_owned(),
311            stderr: String::new(),
312        };
313    }
314    let mut context = Context {
315        options: BTreeMap::new(),
316        files: Vec::new(),
317        read_file,
318        stdout: Vec::new(),
319        stderr: Vec::new(),
320    };
321    let result = command_spec(name)
322        .ok_or_else(|| {
323            CommandError::new(format!(
324                "unknown command {name}; run meta-language grammar help"
325            ))
326        })
327        .and_then(|spec| {
328            parse_arguments(name, &spec, rest, &mut context)?;
329            (spec.run)(&mut context)
330        });
331    let exit_code = result.unwrap_or_else(|error| {
332        context.stderr.push(format!("error: {}", error.message));
333        context.stderr.extend(error.detail);
334        2
335    });
336    GrammarCommandOutput {
337        exit_code,
338        stdout: join_lines(&context.stdout),
339        stderr: join_lines(&context.stderr),
340    }
341}
342
343fn join_lines(lines: &[String]) -> String {
344    lines
345        .iter()
346        .map(|line| {
347            if line.ends_with('\n') {
348                line.clone()
349            } else {
350                format!("{line}\n")
351            }
352        })
353        .collect()
354}
355
356fn parse_arguments(
357    name: &str,
358    spec: &CommandSpec,
359    rest: &[String],
360    context: &mut Context<'_>,
361) -> Result<(), CommandError> {
362    let mut arguments = rest.iter();
363    while let Some(argument) = arguments.next() {
364        let Some(option) = argument.strip_prefix("--") else {
365            context.files.push(argument.clone());
366            continue;
367        };
368        let Some(&(key, arity)) = spec.options.iter().find(|(key, _)| *key == option) else {
369            return Err(CommandError::new(format!(
370                "unknown option {argument} for grammar {name}"
371            )));
372        };
373        let Some(value) = arguments.next() else {
374            return Err(CommandError::new(format!(
375                "option {argument} needs a value"
376            )));
377        };
378        let values = context.options.entry(key).or_default();
379        if arity == Arity::One && !values.is_empty() {
380            return Err(CommandError::new(format!(
381                "option {argument} is given twice"
382            )));
383        }
384        values.push(value.clone());
385    }
386    for option in spec.required {
387        if !context.options.contains_key(option) {
388            return Err(CommandError::new(format!(
389                "grammar {name} needs --{option}"
390            )));
391        }
392    }
393    if context.files.len() > spec.files {
394        return Err(CommandError::new(format!(
395            "unexpected argument {}",
396            context.files[spec.files]
397        )));
398    }
399    if context.files.len() < spec.files {
400        return Err(CommandError::new(format!(
401            "grammar {name} needs a grammar file"
402        )));
403    }
404    Ok(())
405}
406
407fn import_format(format: &str) -> Result<&str, CommandError> {
408    if GRAMMAR_IMPORT_FORMATS.contains(&format) {
409        Ok(format)
410    } else {
411        Err(CommandError::new(format!(
412            "unknown import format {format} (expected one of: {})",
413            GRAMMAR_IMPORT_FORMATS.join(", ")
414        )))
415    }
416}
417
418fn export_format(format: &str) -> Result<&str, CommandError> {
419    if GRAMMAR_EXPORT_FORMATS.contains(&format) {
420        Ok(format)
421    } else {
422        Err(CommandError::new(format!(
423            "unknown export format {format} (expected one of: {})",
424            GRAMMAR_EXPORT_FORMATS.join(", ")
425        )))
426    }
427}
428
429fn read(context: &Context<'_>, file: &str) -> Result<String, CommandError> {
430    (context.read_file)(file).map_err(|_| CommandError::new(format!("cannot read {file}")))
431}
432
433fn import_failure(file: &str, format: &str, error: &GrammarImportError) -> CommandError {
434    let category = match error {
435        GrammarImportError::Unsupported { .. } => "unsupported construct",
436        GrammarImportError::Parse { .. } => "parse error",
437    };
438    CommandError::with_detail(
439        format!("cannot import {file} as {format} ({category})"),
440        error.to_string(),
441    )
442}
443
444fn emit_failure(format: &str, error: &GrammarEmitError) -> CommandError {
445    CommandError::with_detail(format!("cannot export as {format}"), error.to_string())
446}
447
448fn load(context: &Context<'_>, file: &str, format: &str) -> Result<Grammar, CommandError> {
449    let source = read(context, file)?;
450    let import = grammar_importer(format).expect("checked import format");
451    import(&source).map_err(|error| import_failure(file, format, &error))
452}
453
454fn emit(context: &mut Context<'_>, format: &str, grammar: &Grammar) -> Result<(), CommandError> {
455    let emit = grammar_emitter(format).expect("checked export format");
456    let (source, report) = emit(grammar).map_err(|error| emit_failure(format, &error))?;
457    context.stdout.push(source);
458    context
459        .stderr
460        .extend(report.lossy.iter().map(|note| format!("lossy: {note}")));
461    Ok(())
462}
463
464#[allow(
465    clippy::unnecessary_wraps,
466    reason = "every command shares the runner signature"
467)]
468fn formats_command(context: &mut Context<'_>) -> Result<i32, CommandError> {
469    context
470        .stdout
471        .push(format!("import: {}", GRAMMAR_IMPORT_FORMATS.join(" ")));
472    context
473        .stdout
474        .push(format!("export: {}", GRAMMAR_EXPORT_FORMATS.join(" ")));
475    Ok(0)
476}
477
478fn import_command(context: &mut Context<'_>) -> Result<i32, CommandError> {
479    let format = import_format(context.required("from"))?;
480    let grammar = load(context, &context.files[0], format)?;
481    context.stdout.push(render_native_grammar(&grammar));
482    Ok(0)
483}
484
485fn validate_command(context: &mut Context<'_>) -> Result<i32, CommandError> {
486    let format = import_format(context.required("from"))?;
487    let diagnostics = validate(&load(context, &context.files[0], format)?);
488    for diagnostic in &diagnostics {
489        context.stdout.push(format!(
490            "{} {} {}: {}",
491            diagnostic.severity.as_str(),
492            diagnostic.kind.as_str(),
493            diagnostic.location.rule,
494            diagnostic.message
495        ));
496    }
497    let errors = diagnostics
498        .iter()
499        .filter(|diagnostic| diagnostic.severity == Severity::Error)
500        .count();
501    context.stdout.push(format!(
502        "{errors} error(s), {} warning(s)",
503        diagnostics.len() - errors
504    ));
505    Ok(i32::from(errors > 0))
506}
507
508fn convert_command(context: &mut Context<'_>) -> Result<i32, CommandError> {
509    let to = export_format(context.required("to"))?.to_owned();
510    let format = import_format(context.required("from"))?;
511    let grammar = load(context, &context.files[0], format)?;
512    emit(context, &to, &grammar)?;
513    Ok(0)
514}
515
516fn export_command(context: &mut Context<'_>) -> Result<i32, CommandError> {
517    let to = export_format(context.required("to"))?.to_owned();
518    let grammar = load(context, &context.files[0], "native")?;
519    emit(context, &to, &grammar)?;
520    Ok(0)
521}
522
523fn merge_command(context: &mut Context<'_>) -> Result<i32, CommandError> {
524    let to = export_format(context.one("to").unwrap_or("native"))?.to_owned();
525    let language = context.one("language").unwrap_or("grammar").to_owned();
526    let mut ids = BTreeSet::new();
527    let mut sources = Vec::new();
528    for (precedence, argument) in context.many("source").iter().enumerate() {
529        let Some((format, file)) = argument.split_once(':') else {
530            return Err(CommandError::new(format!(
531                "a merge source is FORMAT:FILE, not {argument}"
532            )));
533        };
534        let format = import_format(format)?;
535        let id = base_name(file);
536        if !ids.insert(id.to_owned()) {
537            return Err(CommandError::new(format!(
538                "merge sources need distinct file names, {id} is given twice"
539            )));
540        }
541        let precedence = u32::try_from(precedence).unwrap_or(u32::MAX);
542        sources.push(
543            GrammarMergeSource::new(id, language.as_str(), load(context, file, format)?)
544                .with_precedence(precedence),
545        );
546    }
547    let mut required_equivalences = Vec::new();
548    for pair in context.many("require") {
549        let Some((first, second)) = pair.split_once('=') else {
550            return Err(CommandError::new(format!(
551                "a required equivalence is SOURCE:RULE=SOURCE:RULE, not {pair}"
552            )));
553        };
554        required_equivalences.push((first.to_owned(), second.to_owned()));
555    }
556    let options = GrammarMergeOptions {
557        required_equivalences,
558        ..GrammarMergeOptions::default()
559    };
560    let result =
561        merge_grammars(&sources, &options).map_err(|error| CommandError::new(error.to_string()))?;
562    let group = &result.groups[0];
563    emit(context, &to, &group.grammar)?;
564    for decision in &group.decisions {
565        context.stderr.push(format!(
566            "{} {}: {} ({})",
567            decision.kind.as_str(),
568            decision.name,
569            decision.members.join(" "),
570            decision.basis
571        ));
572    }
573    for alternative in group.alternatives.iter().chain(&result.alternatives) {
574        context.stderr.push(format!(
575            "alternative {} {}: {}",
576            alternative.reason.as_str(),
577            alternative.name,
578            alternative.options.join(" ")
579        ));
580    }
581    for failure in &result.failures {
582        context.stderr.push(format!(
583            "unresolved {}: {}",
584            failure.members.join(" = "),
585            failure.reason.as_str()
586        ));
587    }
588    Ok(i32::from(result.status() != "complete"))
589}
590
591fn base_name(file: &str) -> &str {
592    file.rsplit(['/', '\\']).next().unwrap_or(file)
593}
594
595fn rename_command(context: &mut Context<'_>) -> Result<i32, CommandError> {
596    let to = export_format(context.one("to").unwrap_or("native"))?.to_owned();
597    let format = import_format(context.required("from"))?;
598    let grammar = load(context, &context.files[0], format)?;
599    let renamed = rename_grammar_rule(
600        &grammar,
601        context.required("rule"),
602        context.required("name"),
603        context.one("namespace"),
604        &[],
605    )
606    .map_err(|error| CommandError::new(error.to_string()))?;
607    emit(context, &to, &renamed.grammar)?;
608    for alias in &renamed.aliases {
609        context
610            .stderr
611            .push(format!("alias {} = {}", alias.canonical, alias.original));
612    }
613    Ok(0)
614}
615
616fn round_trip_command(context: &mut Context<'_>) -> Result<i32, CommandError> {
617    let format = import_format(context.required("from"))?.to_owned();
618    if !GRAMMAR_EXPORT_FORMATS.contains(&format.as_str()) {
619        return Err(CommandError::new(format!(
620            "round-trip needs a format that is both imported and exported (one of: {})",
621            GRAMMAR_EXPORT_FORMATS.join(", ")
622        )));
623    }
624    let file = context.files[0].clone();
625    let source = read(context, &file)?;
626    let import = grammar_importer(&format).expect("checked import format");
627    let emit = grammar_emitter(&format).expect("checked export format");
628    let (accepts, rejects) = (context.many("accept"), context.many("reject"));
629    let report = check_grammar_round_trip(
630        &source,
631        &GrammarRoundTrip::new(&import, &emit).with_samples(accepts, rejects),
632    )
633    .map_err(|error| match error {
634        GrammarRoundTripError::Import(error) => import_failure(&file, &format, &error),
635        GrammarRoundTripError::Emit(error) => emit_failure(&format, &error),
636        error @ (GrammarRoundTripError::Normalize(_) | GrammarRoundTripError::MissingStartRule) => {
637            CommandError::with_detail(format!("cannot round-trip {file}"), error.to_string())
638        }
639    })?;
640    context.stdout.push(report.status.as_str().to_owned());
641    for failure in &report.failures {
642        context.stdout.push(format!(
643            "{} {}: {}",
644            failure.kind.as_str(),
645            failure.stage.as_str(),
646            failure.detail
647        ));
648    }
649    Ok(i32::from(report.status.as_str() != "preserved"))
650}
meta_language/grammar/interchange/mod.rs

meta_language/grammar/interchange/
mod.rs
