1use std::collections::{BTreeMap, BTreeSet};
2
3use crate::concept_records::{ConceptRecord, concept_record, concept_records};
4use crate::grammar::GRAMMAR_CONCEPTS;
5use crate::link_network::{Link, LinkId, LinkMetadata, LinkNetwork, LinkType};
6use crate::lino_serialization::LinoSerializationError;
7
8mod semantic_lexicon;
9
10use semantic_lexicon::{
11 SemanticLexiconConcept, is_wikidata_qid, is_wordnet_cili_id, semantic_lexicon,
12};
13
14const EXTERNAL_IDENTIFIER_VOCABULARY_PREFIX: &str = "external-identifier:";
15const FORMER_EXTERNAL_IDENTIFIER_VOCABULARY_PREFIX: &str = "external-id:";
17
18pub const FORMER_CONCEPT_ID_VOCABULARY: &str = "meta-language";
20
21pub const FORMER_CONCEPT_IDS: &[(&str, &str)] = &[
28 ("grammar.repetition", "grammar.counted-repetition"),
29 ("grammar.zero-or-more", "grammar.zero-or-more-repetition"),
30 ("grammar.one-or-more", "grammar.one-or-more-repetition"),
31 ("grammar.optional", "grammar.optional-expression"),
32 ("grammar.non-terminal", "grammar.nonterminal"),
33 ("grammar.char-class", "grammar.character-class"),
34 ("grammar.char-range", "grammar.character-range"),
35 ("grammar.any-char", "grammar.any-character"),
36 ("grammar.empty", "grammar.empty-expression"),
37 ("grammar.boolean", "grammar.boolean-value"),
38 ("sequence", "sequential-composition"),
39 ("strong", "strong-emphasis"),
40 ("blockquote", "block-quote"),
41];
42
43#[must_use]
45pub fn current_concept_id(id: &str) -> &str {
46 FORMER_CONCEPT_IDS
47 .iter()
48 .find(|(former, _)| *former == id)
49 .map_or(id, |(_, current)| current)
50}
51
52#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
54pub struct ConceptOntologyImportReport {
55 concepts: usize,
56 assigned: usize,
57 renamed: usize,
58 alias_links: usize,
59 syntax_mappings: usize,
60}
61
62impl ConceptOntologyImportReport {
63 #[must_use]
65 pub const fn concepts(self) -> usize {
66 self.concepts
67 }
68
69 #[must_use]
71 pub const fn assigned(self) -> usize {
72 self.assigned
73 }
74
75 #[must_use]
77 pub const fn renamed(self) -> usize {
78 self.renamed
79 }
80
81 #[must_use]
83 pub const fn alias_links(self) -> usize {
84 self.alias_links
85 }
86
87 #[must_use]
89 pub const fn syntax_mappings(self) -> usize {
90 self.syntax_mappings
91 }
92}
93
94#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
96pub struct ConceptRecordSeedReport {
97 concepts: usize,
98 links: usize,
99}
100
101impl ConceptRecordSeedReport {
102 #[must_use]
104 pub const fn concepts(self) -> usize {
105 self.concepts
106 }
107
108 #[must_use]
110 pub const fn links(self) -> usize {
111 self.links
112 }
113}
114
115#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
117pub struct ConceptOntologySeedReport {
118 lexicon_concepts: usize,
119 structural_concepts: usize,
120 grammar_concepts: usize,
121 formatting_concepts: usize,
122 alias_links: usize,
123 syntax_mappings: usize,
124}
125
126impl ConceptOntologySeedReport {
127 const fn new(
128 lexicon_concepts: usize,
129 structural_concepts: usize,
130 grammar_concepts: usize,
131 formatting_concepts: usize,
132 alias_links: usize,
133 syntax_mappings: usize,
134 ) -> Self {
135 Self {
136 lexicon_concepts,
137 structural_concepts,
138 grammar_concepts,
139 formatting_concepts,
140 alias_links,
141 syntax_mappings,
142 }
143 }
144
145 #[must_use]
147 pub const fn lexicon_concepts(self) -> usize {
148 self.lexicon_concepts
149 }
150
151 #[must_use]
153 pub const fn structural_concepts(self) -> usize {
154 self.structural_concepts
155 }
156
157 #[must_use]
159 pub const fn grammar_concepts(self) -> usize {
160 self.grammar_concepts
161 }
162
163 #[must_use]
165 pub const fn formatting_concepts(self) -> usize {
166 self.formatting_concepts
167 }
168
169 #[must_use]
171 pub const fn alias_links(self) -> usize {
172 self.alias_links
173 }
174
175 #[must_use]
177 pub const fn syntax_mappings(self) -> usize {
178 self.syntax_mappings
179 }
180}
181
182struct StructuralConcept {
183 id: &'static str,
184 definition: &'static str,
185 syntax: &'static [(&'static str, &'static str)],
186}
187
188#[derive(Clone, Copy, Debug, PartialEq, Eq)]
189pub struct StatehoodConceptIds {
190 pub proposition: LinkId,
191 pub subject: LinkId,
192 pub object: LinkId,
193}
194
195const STATEHOOD_PROPOSITION_SYNTAX: &[(&str, &str)] = &[
196 ("English", "Hawaii is a state."),
197 ("en", "Hawaii is a state."),
198 ("Russian", "Гавайи это штат."),
199 ("ru", "Гавайи это штат."),
200];
201
202const HAWAII_ENTITY_SYNTAX: &[(&str, &str)] = &[
203 ("English", "Hawaii"),
204 ("en", "Hawaii"),
205 ("Russian", "Гавайи"),
206 ("ru", "Гавайи"),
207];
208
209const UNITED_STATES_STATE_SYNTAX: &[(&str, &str)] = &[
210 ("English", "state"),
211 ("en", "state"),
212 ("Russian", "штат"),
213 ("ru", "штат"),
214];
215
216const STRUCTURAL_CONCEPTS: &[StructuralConcept] = &[
217 StructuralConcept {
218 id: "function",
219 definition: "Reusable computation with parameters and a result boundary.",
220 syntax: &[
221 ("Rust", "fn"),
222 ("Python", "def"),
223 ("JavaScript", "function"),
224 ("C", "function"),
225 ("C++", "function"),
226 ("C#", "method"),
227 ("Java", "method"),
228 ("Visual Basic", "Function"),
229 ("R", "function"),
230 ("sql-ansi", "CREATE FUNCTION"),
231 ("Delphi/Object Pascal", "function"),
232 ],
233 },
234 StructuralConcept {
235 id: "binding",
236 definition: "Association between a name and a value or computation.",
237 syntax: &[
238 ("Rust", "let"),
239 ("Python", "="),
240 ("JavaScript", "let"),
241 ("C", "="),
242 ("C++", "="),
243 ("C#", "="),
244 ("Java", "="),
245 ("Visual Basic", "Dim"),
246 ("R", "<-"),
247 ("sql-ansi", "AS"),
248 ("Delphi/Object Pascal", ":="),
249 ],
250 },
251 StructuralConcept {
252 id: "application",
253 definition: "Application of a callable expression to arguments.",
254 syntax: &[
255 ("Rust", "call(...)"),
256 ("Python", "call(...)"),
257 ("JavaScript", "call(...)"),
258 ("C", "call(...)"),
259 ("C++", "call(...)"),
260 ("C#", "call(...)"),
261 ("Java", "call(...)"),
262 ("Visual Basic", "Call"),
263 ("R", "call(...)"),
264 ("sql-ansi", "CALL"),
265 ("Delphi/Object Pascal", "call(...)"),
266 ],
267 },
268 StructuralConcept {
269 id: "sequential-composition",
270 definition: "Ordered execution or evaluation of multiple operations.",
271 syntax: &[
272 ("Rust", ";"),
273 ("Python", "newline"),
274 ("JavaScript", ";"),
275 ("C", ";"),
276 ("C++", ";"),
277 ("C#", ";"),
278 ("Java", ";"),
279 ("Visual Basic", "newline"),
280 ("R", ";"),
281 ("sql-ansi", ";"),
282 ("Delphi/Object Pascal", "begin ... end"),
283 ],
284 },
285 StructuralConcept {
286 id: "branch",
287 definition: "Conditional selection among alternative operations.",
288 syntax: &[
289 ("Rust", "if"),
290 ("Python", "if"),
291 ("JavaScript", "if"),
292 ("C", "if"),
293 ("C++", "if"),
294 ("C#", "if"),
295 ("Java", "if"),
296 ("Visual Basic", "If"),
297 ("R", "if"),
298 ("sql-ansi", "CASE"),
299 ("Delphi/Object Pascal", "if"),
300 ],
301 },
302 StructuralConcept {
303 id: "loop",
304 definition: "Repeated execution or evaluation over a condition or iterable.",
305 syntax: &[
306 ("Rust", "loop"),
307 ("Python", "for"),
308 ("JavaScript", "for"),
309 ("C", "for"),
310 ("C++", "for"),
311 ("C#", "for"),
312 ("Java", "for"),
313 ("Visual Basic", "For"),
314 ("R", "for"),
315 ("sql-ansi", "WHILE"),
316 ("Delphi/Object Pascal", "for"),
317 ],
318 },
319 StructuralConcept {
320 id: "parameter",
321 definition: "Named input accepted by a function abstraction.",
322 syntax: &[
323 ("Rust", "parameter"),
324 ("Python", "parameter"),
325 ("JavaScript", "parameter"),
326 ("C", "parameter"),
327 ("C++", "parameter"),
328 ("C#", "parameter"),
329 ("Java", "parameter"),
330 ("Visual Basic", "parameter"),
331 ("R", "parameter"),
332 ("sql-ansi", "parameter"),
333 ("Delphi/Object Pascal", "parameter"),
334 ],
335 },
336 StructuralConcept {
337 id: "argument",
338 definition: "Concrete input supplied to a function application.",
339 syntax: &[
340 ("Rust", "argument"),
341 ("Python", "argument"),
342 ("JavaScript", "argument"),
343 ("C", "argument"),
344 ("C++", "argument"),
345 ("C#", "argument"),
346 ("Java", "argument"),
347 ("Visual Basic", "argument"),
348 ("R", "argument"),
349 ("sql-ansi", "argument"),
350 ("Delphi/Object Pascal", "argument"),
351 ],
352 },
353 StructuralConcept {
354 id: "return",
355 definition: "Transfer of a function result to its caller.",
356 syntax: &[
357 ("Rust", "return"),
358 ("Python", "return"),
359 ("JavaScript", "return"),
360 ("C", "return"),
361 ("C++", "return"),
362 ("C#", "return"),
363 ("Java", "return"),
364 ("Visual Basic", "Return"),
365 ("R", "return"),
366 ("sql-ansi", "RETURN"),
367 ("Delphi/Object Pascal", "Result"),
368 ],
369 },
370 StructuralConcept {
371 id: "assignment",
372 definition: "Update that stores a value into a named location.",
373 syntax: &[
374 ("Rust", "="),
375 ("Python", "="),
376 ("JavaScript", "="),
377 ("C", "="),
378 ("C++", "="),
379 ("C#", "="),
380 ("Java", "="),
381 ("Visual Basic", "="),
382 ("R", "<-"),
383 ("sql-ansi", "="),
384 ("Delphi/Object Pascal", ":="),
385 ],
386 },
387];
388
389impl LinkNetwork {
390 pub(crate) fn seed_statehood_worked_example(&mut self) -> StatehoodConceptIds {
391 let proposition = self.insert_typed_point(
392 "statehood",
393 LinkType::Concept,
394 Some("Statehood proposition connecting Hawaii (Q782) to U.S. state (Q35657)."),
395 );
396 let subject = self.insert_typed_point(
397 "Q782",
398 LinkType::Concept,
399 Some("Wikidata Q782; Hawaii; state of the United States."),
400 );
401 let object = self.insert_typed_point(
402 "Q35657",
403 LinkType::Concept,
404 Some("Wikidata Q35657; state of the United States."),
405 );
406
407 for (language, syntax) in STATEHOOD_PROPOSITION_SYNTAX {
408 self.insert_concept_syntax_mapping(proposition, "statehood", language, syntax, true);
409 }
410 for (language, syntax) in HAWAII_ENTITY_SYNTAX {
411 self.insert_concept_syntax_mapping(subject, "Q782", language, syntax, true);
412 }
413 for (language, syntax) in UNITED_STATES_STATE_SYNTAX {
414 self.insert_concept_syntax_mapping(object, "Q35657", language, syntax, true);
415 }
416
417 StatehoodConceptIds {
418 proposition,
419 subject,
420 object,
421 }
422 }
423
424 #[must_use]
430 pub fn seed_common_concept_ontology(&mut self) -> ConceptOntologySeedReport {
431 let lexicon = semantic_lexicon();
432 let mut alias_links = 0;
433 let mut syntax_mappings = 0;
434
435 for concept in &lexicon.concepts {
436 let definition = concept.definition();
437 let concept_link = self.intern_concept(concept.id(), Some(&definition));
438 alias_links += self.insert_external_aliases(concept_link, concept);
439
440 for entry in concept.syntax_entries() {
441 self.insert_concept_syntax_mapping(
442 concept_link,
443 concept.id(),
444 entry.language,
445 entry.syntax,
446 entry.canonical,
447 );
448 syntax_mappings += 1;
449 }
450 }
451
452 let mut structural_concepts = BTreeSet::new();
453 for concept in STRUCTURAL_CONCEPTS {
454 structural_concepts.insert(concept.id);
455 let concept_link = self.intern_concept(concept.id, Some(concept.definition));
456 alias_links += self.insert_former_concept_ids(concept_link, concept.id);
457
458 for (language, syntax) in concept.syntax {
459 self.insert_concept_syntax_mapping(
460 concept_link,
461 concept.id,
462 language,
463 syntax,
464 true,
465 );
466 syntax_mappings += 1;
467 }
468 }
469
470 let grammar_concepts = self.seed_grammar_concept_ontology();
471 syntax_mappings += GRAMMAR_CONCEPTS
472 .iter()
473 .map(|concept| concept.syntax.len())
474 .sum::<usize>();
475
476 let formatting = self.seed_document_formatting_concepts();
477 syntax_mappings += formatting.syntax_mappings();
478
479 let statehood = self.seed_statehood_worked_example();
480 for (concept_link, external_id) in
481 [(statehood.subject, "Q782"), (statehood.object, "Q35657")]
482 {
483 let (_alias, inserted) =
484 self.insert_concept_alias_link(concept_link, "Wikidata", external_id);
485 if inserted {
486 alias_links += 1;
487 }
488 }
489 syntax_mappings += STATEHOOD_PROPOSITION_SYNTAX.len()
490 + HAWAII_ENTITY_SYNTAX.len()
491 + UNITED_STATES_STATE_SYNTAX.len();
492
493 ConceptOntologySeedReport::new(
494 lexicon.concept_count,
495 structural_concepts.len(),
496 grammar_concepts,
497 formatting.concepts(),
498 alias_links,
499 syntax_mappings,
500 )
501 }
502
503 pub fn intern_concept(&mut self, exact_id: &str, definition: Option<&str>) -> LinkId {
509 self.insert_typed_point(exact_id, LinkType::Concept, definition)
510 }
511
512 pub fn insert_concept_expression(
517 &mut self,
518 concept: &str,
519 language: &str,
520 expression: &str,
521 ) -> LinkId {
522 let concept_link = self.find_term(concept).unwrap_or_else(|| {
523 self.intern_concept(
524 concept,
525 Some("A language-free concept shared by exact interlingual id."),
526 )
527 });
528 self.insert_concept_syntax_mapping(concept_link, concept, language, expression, true)
529 }
530
531 pub fn insert_concept_mapping(
533 &mut self,
534 concept: &str,
535 language: &str,
536 syntax: &str,
537 ) -> LinkId {
538 self.insert_concept_expression(concept, language, syntax)
539 }
540
541 pub fn insert_concept_alias(
543 &mut self,
544 concept_link: LinkId,
545 vocabulary: &str,
546 external_id: &str,
547 ) -> LinkId {
548 self.insert_concept_alias_link(concept_link, vocabulary, external_id)
549 .0
550 }
551
552 pub fn import_concept_ontology_lino(
558 &mut self,
559 text: &str,
560 ) -> Result<ConceptOntologyImportReport, LinoSerializationError> {
561 let source = Self::from_lino(text)?;
562 Ok(self.import_concept_ontology(&source))
563 }
564
565 pub fn import_concept_ontology(&mut self, source: &Self) -> ConceptOntologyImportReport {
573 let mut concept_links: BTreeMap<LinkId, (LinkId, String)> = BTreeMap::new();
574 let mut report = ConceptOntologyImportReport::default();
575
576 for link in source.links() {
577 if link.metadata().link_type() != Some(LinkType::Concept) {
578 continue;
579 }
580 let Some(term) = link.metadata().term() else {
581 continue;
582 };
583 let id = current_concept_id(term);
584 let concept_link = if let Some(record) = concept_record(id) {
585 report.assigned += 1;
586 self.insert_concept_record(record)
587 } else {
588 self.intern_concept(id, link.metadata().definition())
589 };
590 if id != term {
591 self.insert_concept_alias_link(concept_link, FORMER_CONCEPT_ID_VOCABULARY, term);
592 report.renamed += 1;
593 }
594 concept_links.insert(link.id(), (concept_link, id.to_string()));
595 report.concepts += 1;
596 }
597
598 for link in source.links() {
599 if link.metadata().link_type() != Some(LinkType::Semantic) {
600 continue;
601 }
602 let [source_concept, source_context] = link.references() else {
603 continue;
604 };
605 let Some((target_concept, concept_id)) = concept_links.get(source_concept) else {
606 continue;
607 };
608 let Some(context) = source.link(*source_context) else {
609 continue;
610 };
611 let Some(term) = link.metadata().term() else {
612 continue;
613 };
614
615 match context.metadata().link_type() {
616 Some(LinkType::Language) => {
617 if let Some(language) = link
618 .metadata()
619 .language()
620 .or_else(|| context.metadata().term())
621 {
622 self.insert_concept_syntax_mapping(
623 *target_concept,
624 concept_id,
625 language,
626 term,
627 false,
628 );
629 report.syntax_mappings += 1;
630 }
631 }
632 Some(LinkType::Type) => {
633 let vocabulary = link.metadata().language().or_else(|| {
634 context
635 .metadata()
636 .term()
637 .and_then(external_vocabulary_from_term)
638 });
639 if let Some(vocabulary) = vocabulary {
640 self.insert_concept_alias(*target_concept, vocabulary, term);
641 report.alias_links += 1;
642 }
643 }
644 _ => {}
645 }
646 }
647
648 report
649 }
650
651 pub fn insert_concept_record(&mut self, record: &ConceptRecord) -> LinkId {
657 let concept_link = self.intern_concept(&record.id, Some(&record.definition));
658 self.insert_concept_syntax_mapping(concept_link, &record.id, "en", &record.phrase, false);
659 for alias in &record.source_aliases {
660 self.insert_concept_alias_link(concept_link, &alias.source, &alias.name);
661 }
662 for former in &record.former_names {
663 self.insert_concept_alias_link(concept_link, FORMER_CONCEPT_ID_VOCABULARY, former);
664 }
665 concept_link
666 }
667
668 pub fn seed_concept_records(&mut self) -> ConceptRecordSeedReport {
670 let before = self.len();
671 for record in concept_records() {
672 self.insert_concept_record(record);
673 }
674 ConceptRecordSeedReport {
675 concepts: concept_records().len(),
676 links: self.len() - before,
677 }
678 }
679
680 fn insert_external_aliases(
681 &mut self,
682 concept_link: LinkId,
683 concept: &SemanticLexiconConcept,
684 ) -> usize {
685 let mut aliases = BTreeSet::new();
686 if let Some(vocabulary) = external_vocabulary_for_id(concept.id()) {
687 aliases.insert((vocabulary, concept.id()));
688 }
689 if let Some(entity_id) = concept.entity_id.as_deref()
690 && let Some(vocabulary) = external_vocabulary_for_id(entity_id)
691 {
692 aliases.insert((vocabulary, entity_id));
693 }
694
695 aliases
696 .into_iter()
697 .filter(|(vocabulary, external_id)| {
698 let (_alias, inserted) =
699 self.insert_concept_alias_link(concept_link, vocabulary, external_id);
700 inserted
701 })
702 .count()
703 }
704
705 pub(crate) fn insert_former_concept_ids(
707 &mut self,
708 concept_link: LinkId,
709 concept_id: &str,
710 ) -> usize {
711 FORMER_CONCEPT_IDS
712 .iter()
713 .filter(|(former, current)| {
714 *current == concept_id
715 && self
716 .insert_concept_alias_link(
717 concept_link,
718 FORMER_CONCEPT_ID_VOCABULARY,
719 former,
720 )
721 .1
722 })
723 .count()
724 }
725
726 fn insert_concept_alias_link(
727 &mut self,
728 concept_link: LinkId,
729 vocabulary: &str,
730 external_id: &str,
731 ) -> (LinkId, bool) {
732 let vocabulary_term = external_vocabulary_term(vocabulary);
733 let vocabulary_link = self.insert_typed_point(
734 &vocabulary_term,
735 LinkType::Type,
736 Some("External concept identifier vocabulary."),
737 );
738
739 if let Some(existing) =
740 self.find_concept_alias(concept_link, vocabulary_link, vocabulary, external_id)
741 {
742 return (existing, false);
743 }
744
745 (
746 self.insert_link(
747 [concept_link, vocabulary_link],
748 LinkMetadata::new()
749 .with_link_type(LinkType::Semantic)
750 .with_named(true)
751 .with_term(external_id)
752 .with_language(vocabulary),
753 ),
754 true,
755 )
756 }
757
758 pub(crate) fn insert_concept_syntax_mapping(
759 &mut self,
760 concept_link: LinkId,
761 concept: &str,
762 language: &str,
763 syntax: &str,
764 update_reconstruction: bool,
765 ) -> LinkId {
766 let language_link = self.insert_typed_point(language, LinkType::Language, None);
767 self.cache_concept_syntax(concept, language, syntax, update_reconstruction);
768
769 if let Some(existing) =
770 self.find_concept_syntax_mapping(concept_link, language_link, syntax, language)
771 {
772 return existing;
773 }
774
775 self.insert_link(
776 [concept_link, language_link],
777 LinkMetadata::new()
778 .with_link_type(LinkType::Semantic)
779 .with_named(true)
780 .with_term(syntax)
781 .with_language(language),
782 )
783 }
784
785 fn find_concept_syntax_mapping(
786 &self,
787 concept_link: LinkId,
788 language_link: LinkId,
789 syntax: &str,
790 language: &str,
791 ) -> Option<LinkId> {
792 self.links()
793 .find(|link| {
794 let references = link.references();
795 link.metadata().link_type() == Some(LinkType::Semantic)
796 && references.len() == 2
797 && references[0] == concept_link
798 && references[1] == language_link
799 && link.metadata().term() == Some(syntax)
800 && link.metadata().language() == Some(language)
801 })
802 .map(Link::id)
803 }
804
805 fn find_concept_alias(
806 &self,
807 concept_link: LinkId,
808 vocabulary_link: LinkId,
809 vocabulary: &str,
810 external_id: &str,
811 ) -> Option<LinkId> {
812 self.links()
813 .find(|link| {
814 let references = link.references();
815 link.metadata().link_type() == Some(LinkType::Semantic)
816 && references.len() == 2
817 && references[0] == concept_link
818 && references[1] == vocabulary_link
819 && link.metadata().term() == Some(external_id)
820 && link.metadata().language() == Some(vocabulary)
821 })
822 .map(Link::id)
823 }
824}
825
826fn external_vocabulary_for_id(value: &str) -> Option<&'static str> {
827 if is_wikidata_qid(value) {
828 Some("Wikidata")
829 } else if is_wordnet_cili_id(value) {
830 Some("WordNet CILI")
831 } else {
832 None
833 }
834}
835
836fn external_vocabulary_term(vocabulary: &str) -> String {
837 format!("{EXTERNAL_IDENTIFIER_VOCABULARY_PREFIX}{vocabulary}")
838}
839
840fn external_vocabulary_from_term(term: &str) -> Option<&str> {
841 term.strip_prefix(EXTERNAL_IDENTIFIER_VOCABULARY_PREFIX)
842 .or_else(|| term.strip_prefix(FORMER_EXTERNAL_IDENTIFIER_VOCABULARY_PREFIX))
843}