1use crate::line_index::LineIndex;
2use crate::link_flags::LinkFlags;
3use crate::link_network::{LinkId, LinkMetadata, LinkNetwork, LinkType};
4use crate::source::{ByteRange, SourceSpan};
5
6#[derive(Clone, Copy, Debug, PartialEq, Eq)]
8pub struct NaturalLanguageGrammarFixture {
9 language: &'static str,
10 grammatical_source: &'static str,
11 ungrammatical_source: &'static str,
12 provenance: &'static str,
13}
14
15impl NaturalLanguageGrammarFixture {
16 #[must_use]
18 pub const fn language(&self) -> &'static str {
19 self.language
20 }
21
22 #[must_use]
24 pub const fn grammatical_source(&self) -> &'static str {
25 self.grammatical_source
26 }
27
28 #[must_use]
30 pub const fn ungrammatical_source(&self) -> &'static str {
31 self.ungrammatical_source
32 }
33
34 #[must_use]
36 pub const fn provenance(&self) -> &'static str {
37 self.provenance
38 }
39}
40
41const STARTER_GRAMMAR_PROVENANCE: &str = "repo-authored starter pass/fail sentence; license: Unlicense; \
42 morphosyntax tag names use Universal Dependencies v2 UPOS/UFeats/deprel vocabulary; \
43 no UD treebank sentence data imported";
44
45pub const NATURAL_LANGUAGE_GRAMMAR_FIXTURES: &[NaturalLanguageGrammarFixture] = &[
47 NaturalLanguageGrammarFixture {
48 language: "English",
49 grammatical_source: "Hawaii is a state.\n",
50 ungrammatical_source: "Hawaii are a state.\n",
51 provenance: STARTER_GRAMMAR_PROVENANCE,
52 },
53 NaturalLanguageGrammarFixture {
54 language: "Mandarin Chinese",
55 grammatical_source: "你好。\n",
56 ungrammatical_source: "你好 的。\n",
57 provenance: STARTER_GRAMMAR_PROVENANCE,
58 },
59 NaturalLanguageGrammarFixture {
60 language: "Hindi",
61 grammatical_source: "नमस्ते।\n",
62 ungrammatical_source: "नमस्ते है।\n",
63 provenance: STARTER_GRAMMAR_PROVENANCE,
64 },
65 NaturalLanguageGrammarFixture {
66 language: "Spanish",
67 grammatical_source: "Hawaii es un estado.\n",
68 ungrammatical_source: "Hawaii son un estado.\n",
69 provenance: STARTER_GRAMMAR_PROVENANCE,
70 },
71 NaturalLanguageGrammarFixture {
72 language: "Modern Standard Arabic",
73 grammatical_source: "مرحبا.\n",
74 ungrammatical_source: "مرحبا هو.\n",
75 provenance: STARTER_GRAMMAR_PROVENANCE,
76 },
77 NaturalLanguageGrammarFixture {
78 language: "French",
79 grammatical_source: "Hawaii est un etat.\n",
80 ungrammatical_source: "Hawaii sont un etat.\n",
81 provenance: STARTER_GRAMMAR_PROVENANCE,
82 },
83 NaturalLanguageGrammarFixture {
84 language: "Bengali",
85 grammatical_source: "নমস্কার।\n",
86 ungrammatical_source: "নমস্কার আছে।\n",
87 provenance: STARTER_GRAMMAR_PROVENANCE,
88 },
89 NaturalLanguageGrammarFixture {
90 language: "Portuguese",
91 grammatical_source: "Hawaii e um estado.\n",
92 ungrammatical_source: "Hawaii sao um estado.\n",
93 provenance: STARTER_GRAMMAR_PROVENANCE,
94 },
95 NaturalLanguageGrammarFixture {
96 language: "Russian",
97 grammatical_source: "Гавайи это штат.\n",
98 ungrammatical_source: "Гавайи это штаты.\n",
99 provenance: STARTER_GRAMMAR_PROVENANCE,
100 },
101 NaturalLanguageGrammarFixture {
102 language: "Urdu",
103 grammatical_source: "سلام۔\n",
104 ungrammatical_source: "سلام ہے۔\n",
105 provenance: STARTER_GRAMMAR_PROVENANCE,
106 },
107];
108
109#[derive(Clone, Debug, PartialEq, Eq)]
110struct GrammarToken {
111 text: String,
112 range: ByteRange,
113}
114
115#[derive(Clone, Copy, Debug, PartialEq, Eq)]
116struct MorphAnalysis {
117 upos: &'static str,
118 features: &'static [&'static str],
119 deprel: &'static str,
120}
121
122#[derive(Clone, Copy, Debug, PartialEq, Eq)]
123struct LexiconEntry {
124 language: &'static str,
125 surface: &'static str,
126 analysis: MorphAnalysis,
127}
128
129#[derive(Clone, Copy, Debug, PartialEq, Eq)]
130struct SentenceGrammar {
131 language: &'static str,
132 accepted: &'static [&'static str],
133}
134
135const NO_FEATURES: &[&str] = &[];
136const NUMBER_SING: &[&str] = &["Number=Sing"];
137const NUMBER_PLUR: &[&str] = &["Number=Plur"];
138const INDEFINITE_ARTICLE: &[&str] = &["Definite=Ind", "PronType=Art"];
139const AUX_SINGULAR: &[&str] = &[
140 "Mood=Ind",
141 "Number=Sing",
142 "Person=3",
143 "Tense=Pres",
144 "VerbForm=Fin",
145];
146const AUX_PLURAL: &[&str] = &[
147 "Mood=Ind",
148 "Number=Plur",
149 "Person=3",
150 "Tense=Pres",
151 "VerbForm=Fin",
152];
153const PRON_SINGULAR: &[&str] = &["Number=Sing", "Person=3", "PronType=Prs"];
154
155const PUNCT_ANALYSIS: MorphAnalysis = MorphAnalysis {
156 upos: "PUNCT",
157 features: NO_FEATURES,
158 deprel: "punct",
159};
160
161const SENTENCE_GRAMMARS: &[SentenceGrammar] = &[
162 SentenceGrammar {
163 language: "English",
164 accepted: &["Hawaii", "is", "a", "state", "."],
165 },
166 SentenceGrammar {
167 language: "Mandarin Chinese",
168 accepted: &["你好", "\u{3002}"],
169 },
170 SentenceGrammar {
171 language: "Hindi",
172 accepted: &["नमस्ते", "\u{0964}"],
173 },
174 SentenceGrammar {
175 language: "Spanish",
176 accepted: &["Hawaii", "es", "un", "estado", "."],
177 },
178 SentenceGrammar {
179 language: "Modern Standard Arabic",
180 accepted: &["مرحبا", "."],
181 },
182 SentenceGrammar {
183 language: "French",
184 accepted: &["Hawaii", "est", "un", "etat", "."],
185 },
186 SentenceGrammar {
187 language: "Bengali",
188 accepted: &["নমস্কার", "\u{0964}"],
189 },
190 SentenceGrammar {
191 language: "Portuguese",
192 accepted: &["Hawaii", "e", "um", "estado", "."],
193 },
194 SentenceGrammar {
195 language: "Russian",
196 accepted: &["Гавайи", "это", "штат", "."],
197 },
198 SentenceGrammar {
199 language: "Urdu",
200 accepted: &["سلام", "\u{06d4}"],
201 },
202];
203
204const LEXICON: &[LexiconEntry] = &[
205 LexiconEntry {
206 language: "English",
207 surface: "Hawaii",
208 analysis: MorphAnalysis {
209 upos: "PROPN",
210 features: NUMBER_SING,
211 deprel: "nsubj",
212 },
213 },
214 LexiconEntry {
215 language: "English",
216 surface: "is",
217 analysis: MorphAnalysis {
218 upos: "AUX",
219 features: AUX_SINGULAR,
220 deprel: "cop",
221 },
222 },
223 LexiconEntry {
224 language: "English",
225 surface: "are",
226 analysis: MorphAnalysis {
227 upos: "AUX",
228 features: AUX_PLURAL,
229 deprel: "cop",
230 },
231 },
232 LexiconEntry {
233 language: "English",
234 surface: "a",
235 analysis: MorphAnalysis {
236 upos: "DET",
237 features: INDEFINITE_ARTICLE,
238 deprel: "det",
239 },
240 },
241 LexiconEntry {
242 language: "English",
243 surface: "state",
244 analysis: MorphAnalysis {
245 upos: "NOUN",
246 features: NUMBER_SING,
247 deprel: "root",
248 },
249 },
250 LexiconEntry {
251 language: "Mandarin Chinese",
252 surface: "你好",
253 analysis: MorphAnalysis {
254 upos: "INTJ",
255 features: NO_FEATURES,
256 deprel: "root",
257 },
258 },
259 LexiconEntry {
260 language: "Mandarin Chinese",
261 surface: "的",
262 analysis: MorphAnalysis {
263 upos: "PART",
264 features: NO_FEATURES,
265 deprel: "mark",
266 },
267 },
268 LexiconEntry {
269 language: "Hindi",
270 surface: "नमस्ते",
271 analysis: MorphAnalysis {
272 upos: "INTJ",
273 features: NO_FEATURES,
274 deprel: "root",
275 },
276 },
277 LexiconEntry {
278 language: "Hindi",
279 surface: "है",
280 analysis: MorphAnalysis {
281 upos: "AUX",
282 features: AUX_SINGULAR,
283 deprel: "cop",
284 },
285 },
286 LexiconEntry {
287 language: "Spanish",
288 surface: "Hawaii",
289 analysis: MorphAnalysis {
290 upos: "PROPN",
291 features: NUMBER_SING,
292 deprel: "nsubj",
293 },
294 },
295 LexiconEntry {
296 language: "Spanish",
297 surface: "es",
298 analysis: MorphAnalysis {
299 upos: "AUX",
300 features: AUX_SINGULAR,
301 deprel: "cop",
302 },
303 },
304 LexiconEntry {
305 language: "Spanish",
306 surface: "son",
307 analysis: MorphAnalysis {
308 upos: "AUX",
309 features: AUX_PLURAL,
310 deprel: "cop",
311 },
312 },
313 LexiconEntry {
314 language: "Spanish",
315 surface: "un",
316 analysis: MorphAnalysis {
317 upos: "DET",
318 features: INDEFINITE_ARTICLE,
319 deprel: "det",
320 },
321 },
322 LexiconEntry {
323 language: "Spanish",
324 surface: "estado",
325 analysis: MorphAnalysis {
326 upos: "NOUN",
327 features: NUMBER_SING,
328 deprel: "root",
329 },
330 },
331 LexiconEntry {
332 language: "Modern Standard Arabic",
333 surface: "مرحبا",
334 analysis: MorphAnalysis {
335 upos: "INTJ",
336 features: NO_FEATURES,
337 deprel: "root",
338 },
339 },
340 LexiconEntry {
341 language: "Modern Standard Arabic",
342 surface: "هو",
343 analysis: MorphAnalysis {
344 upos: "PRON",
345 features: PRON_SINGULAR,
346 deprel: "dep",
347 },
348 },
349 LexiconEntry {
350 language: "French",
351 surface: "Hawaii",
352 analysis: MorphAnalysis {
353 upos: "PROPN",
354 features: NUMBER_SING,
355 deprel: "nsubj",
356 },
357 },
358 LexiconEntry {
359 language: "French",
360 surface: "est",
361 analysis: MorphAnalysis {
362 upos: "AUX",
363 features: AUX_SINGULAR,
364 deprel: "cop",
365 },
366 },
367 LexiconEntry {
368 language: "French",
369 surface: "sont",
370 analysis: MorphAnalysis {
371 upos: "AUX",
372 features: AUX_PLURAL,
373 deprel: "cop",
374 },
375 },
376 LexiconEntry {
377 language: "French",
378 surface: "un",
379 analysis: MorphAnalysis {
380 upos: "DET",
381 features: INDEFINITE_ARTICLE,
382 deprel: "det",
383 },
384 },
385 LexiconEntry {
386 language: "French",
387 surface: "etat",
388 analysis: MorphAnalysis {
389 upos: "NOUN",
390 features: NUMBER_SING,
391 deprel: "root",
392 },
393 },
394 LexiconEntry {
395 language: "Bengali",
396 surface: "নমস্কার",
397 analysis: MorphAnalysis {
398 upos: "INTJ",
399 features: NO_FEATURES,
400 deprel: "root",
401 },
402 },
403 LexiconEntry {
404 language: "Bengali",
405 surface: "আছে",
406 analysis: MorphAnalysis {
407 upos: "AUX",
408 features: AUX_SINGULAR,
409 deprel: "cop",
410 },
411 },
412 LexiconEntry {
413 language: "Portuguese",
414 surface: "Hawaii",
415 analysis: MorphAnalysis {
416 upos: "PROPN",
417 features: NUMBER_SING,
418 deprel: "nsubj",
419 },
420 },
421 LexiconEntry {
422 language: "Portuguese",
423 surface: "e",
424 analysis: MorphAnalysis {
425 upos: "AUX",
426 features: AUX_SINGULAR,
427 deprel: "cop",
428 },
429 },
430 LexiconEntry {
431 language: "Portuguese",
432 surface: "sao",
433 analysis: MorphAnalysis {
434 upos: "AUX",
435 features: AUX_PLURAL,
436 deprel: "cop",
437 },
438 },
439 LexiconEntry {
440 language: "Portuguese",
441 surface: "um",
442 analysis: MorphAnalysis {
443 upos: "DET",
444 features: INDEFINITE_ARTICLE,
445 deprel: "det",
446 },
447 },
448 LexiconEntry {
449 language: "Portuguese",
450 surface: "estado",
451 analysis: MorphAnalysis {
452 upos: "NOUN",
453 features: NUMBER_SING,
454 deprel: "root",
455 },
456 },
457 LexiconEntry {
458 language: "Russian",
459 surface: "Гавайи",
460 analysis: MorphAnalysis {
461 upos: "PROPN",
462 features: NUMBER_SING,
463 deprel: "nsubj",
464 },
465 },
466 LexiconEntry {
467 language: "Russian",
468 surface: "это",
469 analysis: MorphAnalysis {
470 upos: "PRON",
471 features: PRON_SINGULAR,
472 deprel: "cop",
473 },
474 },
475 LexiconEntry {
476 language: "Russian",
477 surface: "штат",
478 analysis: MorphAnalysis {
479 upos: "NOUN",
480 features: NUMBER_SING,
481 deprel: "root",
482 },
483 },
484 LexiconEntry {
485 language: "Russian",
486 surface: "штаты",
487 analysis: MorphAnalysis {
488 upos: "NOUN",
489 features: NUMBER_PLUR,
490 deprel: "root",
491 },
492 },
493 LexiconEntry {
494 language: "Urdu",
495 surface: "سلام",
496 analysis: MorphAnalysis {
497 upos: "INTJ",
498 features: NO_FEATURES,
499 deprel: "root",
500 },
501 },
502 LexiconEntry {
503 language: "Urdu",
504 surface: "ہے",
505 analysis: MorphAnalysis {
506 upos: "AUX",
507 features: AUX_SINGULAR,
508 deprel: "cop",
509 },
510 },
511];
512
513pub fn annotate_morphosyntax(
514 network: &mut LinkNetwork,
515 region: LinkId,
516 text: &str,
517 lines: &LineIndex,
518 language: &str,
519 span: SourceSpan,
520) {
521 let grammar_tokens = grammar_tokens(text);
522 if grammar_tokens.is_empty() {
523 return;
524 }
525
526 let sentence_is_accepted = sentence_grammar(language)
527 .is_some_and(|grammar| token_surfaces_match(&grammar_tokens, grammar.accepted));
528 let should_report_errors =
529 is_registered_grammar_fixture(language, text) && !sentence_is_accepted;
530 let sentence_flags = if sentence_is_accepted {
531 LinkFlags::clean()
532 } else if should_report_errors {
533 LinkFlags::containing_error()
534 } else {
535 LinkFlags::clean()
536 };
537 let sentence = network.insert_link(
538 [region],
539 LinkMetadata::new()
540 .with_link_type(LinkType::Syntax)
541 .with_named(true)
542 .with_term("natural-language:sentence")
543 .with_language(language)
544 .with_span(span)
545 .with_flags(sentence_flags),
546 );
547
548 for token in &grammar_tokens {
549 let token_span = span_for_range(lines, token.range.start(), token.range.end());
550 let form = network.insert_link(
551 [sentence],
552 LinkMetadata::new()
553 .with_link_type(LinkType::Syntax)
554 .with_named(true)
555 .with_term(format!("form:{}", token.text))
556 .with_language(language)
557 .with_span(token_span),
558 );
559
560 if let Some(analysis) = morphology_for(language, &token.text) {
561 insert_upos_link(network, form, language, token_span, analysis.upos);
562 for feature in analysis.features {
563 insert_ufeat_link(network, form, language, token_span, feature);
564 }
565 insert_deprel_link(
566 network,
567 sentence,
568 form,
569 language,
570 token_span,
571 analysis.deprel,
572 );
573 } else if should_report_errors {
574 insert_error_link(
575 network,
576 [sentence, form],
577 "natural-language:error:unknown-token",
578 language,
579 token_span,
580 );
581 }
582 }
583
584 if should_report_errors {
585 insert_error_link(
586 network,
587 [sentence],
588 "natural-language:error:grammar",
589 language,
590 span,
591 );
592 }
593}
594
595fn insert_upos_link(
596 network: &mut LinkNetwork,
597 form: LinkId,
598 language: &str,
599 span: SourceSpan,
600 upos: &str,
601) -> LinkId {
602 network.insert_link(
603 [form],
604 LinkMetadata::new()
605 .with_link_type(LinkType::Syntax)
606 .with_named(true)
607 .with_term(format!("upos:{upos}"))
608 .with_language(language)
609 .with_span(span),
610 )
611}
612
613fn insert_ufeat_link(
614 network: &mut LinkNetwork,
615 form: LinkId,
616 language: &str,
617 span: SourceSpan,
618 feature: &str,
619) -> LinkId {
620 network.insert_link(
621 [form],
622 LinkMetadata::new()
623 .with_link_type(LinkType::Syntax)
624 .with_named(true)
625 .with_term(format!("ufeat:{feature}"))
626 .with_language(language)
627 .with_span(span),
628 )
629}
630
631fn insert_deprel_link(
632 network: &mut LinkNetwork,
633 sentence: LinkId,
634 form: LinkId,
635 language: &str,
636 span: SourceSpan,
637 deprel: &str,
638) -> LinkId {
639 network.insert_link(
640 [sentence, form],
641 LinkMetadata::new()
642 .with_link_type(LinkType::Syntax)
643 .with_named(true)
644 .with_term(format!("deprel:{deprel}"))
645 .with_language(language)
646 .with_span(span),
647 )
648}
649
650fn insert_error_link<const N: usize>(
651 network: &mut LinkNetwork,
652 references: [LinkId; N],
653 term: &'static str,
654 language: &str,
655 span: SourceSpan,
656) -> LinkId {
657 network.insert_link(
658 references,
659 LinkMetadata::new()
660 .with_link_type(LinkType::Syntax)
661 .with_named(true)
662 .with_term(term)
663 .with_language(language)
664 .with_span(span)
665 .with_flags(LinkFlags::error()),
666 )
667}
668
669fn sentence_grammar(language: &str) -> Option<SentenceGrammar> {
670 SENTENCE_GRAMMARS
671 .iter()
672 .find(|grammar| grammar.language == language)
673 .copied()
674}
675
676fn token_surfaces_match(tokens: &[GrammarToken], expected: &[&str]) -> bool {
677 tokens.len() == expected.len()
678 && tokens
679 .iter()
680 .zip(expected.iter().copied())
681 .all(|(token, expected)| token.text == expected)
682}
683
684fn morphology_for(language: &str, surface: &str) -> Option<MorphAnalysis> {
685 if is_sentence_punctuation_token(surface) {
686 return Some(PUNCT_ANALYSIS);
687 }
688
689 LEXICON
690 .iter()
691 .find(|entry| entry.language == language && entry.surface == surface)
692 .map(|entry| entry.analysis)
693}
694
695fn is_registered_grammar_fixture(language: &str, text: &str) -> bool {
696 NATURAL_LANGUAGE_GRAMMAR_FIXTURES
697 .iter()
698 .filter(|fixture| fixture.language == language)
699 .any(|fixture| text == fixture.grammatical_source || text == fixture.ungrammatical_source)
700}
701
702fn grammar_tokens(text: &str) -> Vec<GrammarToken> {
703 let mut tokens = Vec::new();
704 let mut word_start = None;
705
706 for (index, character) in text.char_indices() {
707 if character.is_whitespace() {
708 push_pending_word_token(&mut tokens, text, &mut word_start, index);
709 } else if is_sentence_punctuation(character) {
710 push_pending_word_token(&mut tokens, text, &mut word_start, index);
711 tokens.push(GrammarToken {
712 text: character.to_string(),
713 range: ByteRange::new(index, index + character.len_utf8()),
714 });
715 } else if word_start.is_none() {
716 word_start = Some(index);
717 }
718 }
719
720 push_pending_word_token(&mut tokens, text, &mut word_start, text.len());
721 tokens
722}
723
724fn push_pending_word_token(
725 tokens: &mut Vec<GrammarToken>,
726 text: &str,
727 word_start: &mut Option<usize>,
728 end: usize,
729) {
730 let Some(start) = word_start.take() else {
731 return;
732 };
733 if start == end {
734 return;
735 }
736 tokens.push(GrammarToken {
737 text: text[start..end].to_string(),
738 range: ByteRange::new(start, end),
739 });
740}
741
742fn is_sentence_punctuation_token(surface: &str) -> bool {
743 let mut characters = surface.chars();
744 let Some(character) = characters.next() else {
745 return false;
746 };
747 characters.next().is_none() && is_sentence_punctuation(character)
748}
749
750const fn is_sentence_punctuation(character: char) -> bool {
751 matches!(
752 character,
753 '.' | '!' | '?' | '\u{0964}' | '\u{3002}' | '\u{06d4}'
754 )
755}
756
757fn span_for_range(lines: &LineIndex, start: usize, end: usize) -> SourceSpan {
758 SourceSpan::new(
759 ByteRange::new(start, end),
760 lines.byte_point(start),
761 lines.byte_point(end),
762 )
763}