meta_language/
binary_format.rs

1use std::fmt::Write as _;
2
3use crate::{ByteRange, LinkMetadata, LinkNetwork, LinkType, Point, SourceSpan};
4
5const BYTE_CHUNK_SIZE: usize = 4096;
6
7impl LinkNetwork {
8    /// Stores an arbitrary file as a lossless byte-token network.
9    ///
10    /// Unlike [`Self::parse`], this boundary does not require UTF-8 and does not
11    /// interpret the file. `format` is an opaque media type, extension, or other
12    /// caller-defined format identifier. Format-specific parsers can enrich the
13    /// returned network without changing its lossless byte layer.
14    #[must_use]
15    pub fn parse_bytes(bytes: &[u8], format: &str) -> Self {
16        let mut network = Self::self_describing();
17        let format_link = network.insert_typed_point(format, LinkType::Language, None);
18        let document_span = SourceSpan::new(
19            ByteRange::new(0, bytes.len()),
20            Point::new(0, 0),
21            Point::new(0, bytes.len()),
22        );
23        let document = network.insert_link(
24            [format_link],
25            LinkMetadata::new()
26                .with_link_type(LinkType::Document)
27                .with_named(true)
28                .with_term(format!("{format} document"))
29                .with_language(format)
30                .with_span(document_span),
31        );
32
33        for (chunk_index, chunk) in bytes.chunks(BYTE_CHUNK_SIZE).enumerate() {
34            let offset = chunk_index * BYTE_CHUNK_SIZE;
35            let encoded = encode_bytes(chunk);
36            network.insert_link(
37                [document],
38                LinkMetadata::new()
39                    .with_link_type(LinkType::Token)
40                    .with_named(true)
41                    .with_term(format!("bytes:{encoded}"))
42                    .with_language(format)
43                    .with_span(SourceSpan::new(
44                        ByteRange::new(offset, offset + chunk.len()),
45                        Point::new(0, offset),
46                        Point::new(0, offset + chunk.len()),
47                    )),
48            );
49        }
50        network
51    }
52
53    /// Reconstructs bytes stored by [`Self::parse_bytes`] in source order.
54    #[must_use]
55    pub fn reconstruct_bytes(&self) -> Vec<u8> {
56        let mut tokens = self
57            .links()
58            .filter(|link| link.metadata().link_type() == Some(LinkType::Token))
59            .filter_map(|link| {
60                Some((
61                    link.metadata().span()?.byte_range().start(),
62                    link.id().as_u64(),
63                    decode_bytes(link.metadata().term()?.strip_prefix("bytes:")?)?,
64                ))
65            })
66            .collect::<Vec<_>>();
67        tokens.sort_by_key(|(offset, id, _bytes)| (*offset, *id));
68        tokens
69            .into_iter()
70            .flat_map(|(_offset, _id, bytes)| bytes)
71            .collect()
72    }
73}
74
75fn encode_bytes(bytes: &[u8]) -> String {
76    bytes.iter().fold(
77        String::with_capacity(bytes.len() * 2),
78        |mut encoded, byte| {
79            write!(encoded, "{byte:02x}").expect("writing to a String cannot fail");
80            encoded
81        },
82    )
83}
84
85fn decode_bytes(encoded: &str) -> Option<Vec<u8>> {
86    if encoded.len() % 2 != 0 {
87        return None;
88    }
89    (0..encoded.len())
90        .step_by(2)
91        .map(|offset| u8::from_str_radix(&encoded[offset..offset + 2], 16))
92        .collect::<Result<_, _>>()
93        .ok()
94}
meta_language/binary_format.rs

meta_language/
binary_format.rs
