expede.wtf
This repository has no description
WIP
author
Brooklyn Zelenka
date
(Dec 22, 2023, 4:03 PM +0200)
commit
825920f0
825920f073956ad424a9396b6eaddc34308fb6fc
parent
6024f7cf
6024f7cfff7d5eadefbbfec1fd14de7d73ef2fee
+209
-32
default_13059096625533786817_0_3671.profraw
default_13059096625533786817_0_3671.profraw
This is a binary file and will not be displayed.
default_13059096625533786817_0_3677.profraw
default_13059096625533786817_0_3677.profraw
This is a binary file and will not be displayed.
default_4043864716203302617_0_3668.profraw
default_4043864716203302617_0_3668.profraw
This is a binary file and will not be displayed.
default_4313509673237537478_0_3668.profraw
default_4313509673237537478_0_3668.profraw
This is a binary file and will not be displayed.
default_650759500492820796_0_3668.profraw
default_650759500492820796_0_3668.profraw
This is a binary file and will not be displayed.
default_7045038967156446105_0_3668.profraw
default_7045038967156446105_0_3668.profraw
This is a binary file and will not be displayed.
default_8341410385769770431_0_3668.profraw
default_8341410385769770431_0_3668.profraw
This is a binary file and will not be displayed.
+2
-2
flake.nix
+2
-2
flake.nix
···
1
1
{
2
-
description = "ipld-inline";
2
+
description = "ipld_inline";
3
3
4
4
inputs = {
5
5
nixpkgs.url = "nixpkgs/nixos-23.05";
···
76
76
77
77
in rec {
78
78
devShells.default = pkgs.devshell.mkShell {
79
-
name = "ipld-inline";
79
+
name = "ipld_inline";
80
80
packages = [
81
81
# For nightly rustfmt to be used instead of the rustfmt provided by `rust-toolchain`, it must appear first in the list
82
82
# nightly-rustfmt
+29
-3
src/cid.rs
+29
-3
src/cid.rs
···
1
+
//! Helpers for working with [`Cid`]s
1
2
use libipld::{
2
3
cid,
3
4
cid::Cid,
···
8
9
use multihash::MultihashDigest;
9
10
use thiserror::Error;
10
11
12
+
/// Create a [`Cid`] for some [`Ipld`]
13
+
///
14
+
/// # Arguments
15
+
///
16
+
/// * `ipld` - [`Ipld`] to create the [`Cid`] for
17
+
/// * `codec` - The [`Codec`] the [`Ipld`] is encoded with
18
+
/// * `digester` - The [`MultihashDigest`] used in the [`Cid`]
19
+
/// * `version` - The [`Cid`] version
20
+
///
21
+
/// # Examples
22
+
///
23
+
/// ```
24
+
/// # use ipld_inline::cid;
25
+
/// # use libipld::{ipld, cid::Version};
26
+
/// # use libipld_cbor::DagCborCodec;
27
+
/// # use multihash::Code::Sha2_256;
28
+
/// # use std::{collections::BTreeMap, str::FromStr};
29
+
/// #
30
+
/// let observed = cid::new(&ipld!([1, 2, 3]), DagCborCodec, &Sha2_256, Version::V1).unwrap();
31
+
/// let expected = FromStr::from_str("bafyreickxqyrg7hhhdm2z24kduovd4k4vvbmfmenzn7nc6pxg6qzjm2v44").unwrap();
32
+
/// assert_eq!(observed, expected);
33
+
/// ```
11
34
pub fn new<C: Codec, D: MultihashDigest<64>>(
12
35
ipld: &Ipld,
13
36
codec: C,
···
17
40
where
18
41
Ipld: Encode<C>,
19
42
{
20
-
let encoded = codec.encode(ipld).map_err(Error::EncodingError)?;
43
+
let encoded = codec.encode(ipld).map_err(Error::IpldEncodingError)?;
21
44
let multihash = digester.digest(encoded.as_slice());
22
45
CidGeneric::new(version, codec.into(), multihash).map_err(Error::ConstructionError)
23
46
}
24
47
48
+
/// [`Cid`] construction errors
25
49
#[derive(Debug, Error)]
26
50
pub enum Error {
51
+
/// Error encoding the [`Ipld`] to make a [`Cid`] from
27
52
#[error(transparent)]
28
-
EncodingError(#[from] libipld::error::Error),
53
+
IpldEncodingError(#[from] libipld::error::Error),
29
54
30
-
#[error("unable to convert to `Cid`")]
55
+
/// Unable to construct a [`Cid`]
56
+
#[error("unable to generate to `Cid`")]
31
57
ConstructionError(#[from] cid::Error),
32
58
}
+15
-2
src/extractor.rs
+15
-2
src/extractor.rs
···
1
+
//! Strategies for decomposing inlined [`Ipld`] to a DAG
1
2
use crate::cid;
2
3
use crate::iterator::post_order::{is_delimiter_next, PostOrderIpldIter};
3
4
use core::iter::Peekable;
···
8
9
};
9
10
use multihash::MultihashDigest;
10
11
12
+
// FIXME consider a newtype to mark inlined ipld as such. Can then wrap that with an `extract` etc
13
+
14
+
/// The general [`Ipld`] extraction strategy
15
+
///
16
+
/// Converts Inline IPLD into "regular" [`Ipld`]. This does a series of graph nodes.
11
17
#[derive(Clone, Debug)]
12
18
pub struct Extractor<'a, C, D>
13
19
where
···
25
31
where
26
32
Ipld: Encode<C>,
27
33
{
34
+
/// Initialize an [`Extractor`]
35
+
///
36
+
/// # Arguments
37
+
///
38
+
/// * `ipld` - The inline [`Ipld`] to extract
39
+
/// * `codec` - The [`Codec`] to fall back to if the inline IPLD doesn't contain a [`Cid`]
40
+
/// * `digester` - The hash digest function to use if the inline IPLD doesn't contain a [`Cid`]
41
+
/// * `cid_version` - The [`Cid`] version to use if the inline IPLD doesn't contain a [`Cid`]
28
42
pub fn new(ipld: Ipld, codec: C, digester: &'a D, cid_version: Version) -> Self {
29
43
Extractor {
30
44
iterator: <Ipld as Into<PostOrderIpldIter>>::into(ipld).peekable(),
31
45
stack: vec![],
32
-
33
46
codec,
34
47
digester,
35
48
cid_version,
···
116
129
use libipld::{cid::CidGeneric, ipld};
117
130
use libipld_cbor::DagCborCodec;
118
131
use multihash::Code::Sha2_256;
119
-
use pretty_assertions::{assert_eq, assert_ne};
132
+
use pretty_assertions::assert_eq;
120
133
use proptest::prelude::*;
121
134
use std::collections::BTreeMap;
122
135
+1
src/inliner.rs
+1
src/inliner.rs
+11
-2
src/inliner/at_most_once.rs
+11
-2
src/inliner/at_most_once.rs
···
1
+
//! Inlining each subgraph at most once (deduplicated)
1
2
use super::{exactly_once::ExactlyOnce, quiet::Quiet};
2
3
use crate::store::traits::Store;
3
4
use libipld::{cid::Cid, ipld::Ipld};
4
5
use std::collections::HashSet;
5
6
7
+
/// [`Ipld`] inliner that only inlines a [`Cid`] once, if avalaible
8
+
///
9
+
/// If a subgraph isn't available by the required [`Cid`], it's merely skipped
6
10
#[derive(Debug)]
7
11
pub struct AtMostOnce<'a, S: Store + ?Sized> {
8
12
exactly_once: ExactlyOnce<'a, S>,
···
10
14
}
11
15
12
16
impl<'a, S: Store + ?Sized> AtMostOnce<'a, S> {
17
+
/// Initialize a new [`AtMostOnce`] inliner
18
+
///
19
+
/// # Arguments
20
+
///
21
+
/// - `ipld` - The [`Ipld`] to inline
22
+
/// - `store` - The content addressed [`Store`] to draw graphs from
13
23
pub fn new(ipld: Ipld, store: &'a mut S) -> Self {
14
-
let exactly_once = ExactlyOnce::new(ipld, store);
15
24
AtMostOnce {
16
-
exactly_once,
25
+
exactly_once: ExactlyOnce::new(ipld, store),
17
26
seen: HashSet::new(),
18
27
}
19
28
}
+95
-19
src/inliner/exactly_once.rs
+95
-19
src/inliner/exactly_once.rs
···
8
8
// Structs //
9
9
/////////////
10
10
11
+
/// [`Ipld`] inliner that only inlines a [`Cid`] once
12
+
///
13
+
/// Unlike [`AtMostOnce`][crate::inliner::at_most_once::AtMostOnce], this inliner will stops if the [`Cid`] is not availale from the [`Store`].
11
14
#[derive(Debug)]
12
15
pub struct ExactlyOnce<'a, S: Store + ?Sized> {
13
16
quiet: Quiet<'a, S>,
14
17
stuck_at: Option<Cid>,
15
18
}
16
19
20
+
/// Error state if a [`Cid`] is not available from the [`ExactlyOnce`]'s [`Store`]
21
+
///
22
+
/// This struct can be [resolved][Stuck::resolve] to continue inlining.
17
23
#[derive(Debug)]
18
24
pub struct Stuck<'a, S: Store + ?Sized> {
19
25
needs: Cid,
···
77
83
////////////////////////////
78
84
79
85
impl<'a, S: Store + ?Sized> Stuck<'a, S> {
86
+
pub fn ignore(&'a mut self) -> &'a mut ExactlyOnce<'a, S> {
87
+
self.iterator.quiet.push(Ipld::Link(self.needs.clone()));
88
+
self.iterator
89
+
}
90
+
80
91
pub fn stub(&'a mut self, ipld: Ipld) -> &'a mut ExactlyOnce<'a, S> {
81
-
self.iterator.quiet.push(ipld.clone()); // FIXME Needs tests
92
+
self.iterator.quiet.push(ipld.clone());
82
93
self.iterator
83
94
}
84
95
···
93
104
}
94
105
95
106
impl<'a, S: Store + ?Sized> ExactlyOnce<'a, S> {
107
+
/// Initialize a new [`ExactlyOnce`]
96
108
pub fn new(ipld: Ipld, store: &'a mut S) -> Self {
97
109
ExactlyOnce {
98
110
quiet: Quiet::new(ipld, store),
···
100
112
}
101
113
}
102
114
115
+
// /// Returns the [`Cid`] that the iterator got stuck at
116
+
// ///
117
+
// /// # Examples
118
+
// ///
119
+
// /// ```
120
+
// /// # use ipld_inline::inliner::exactly_once::ExactlyOnce;
121
+
// /// # use ipld_inline::store::memory::MemoryStore;
122
+
// /// # use libipld::{ipld, cid::CidGeneric};
123
+
// /// #
124
+
// /// let mut store = MemoryStore::new();
125
+
// /// let missing_cid = CidGeneric::try_from(
126
+
// /// "bafyreihnubkcms63243zlfgnwiugmk6ijitz63me7bqf455ia2fpbn4ceq".to_string(),
127
+
// /// )
128
+
// /// .unwrap();
129
+
// /// let mut exactly_once = ExactlyOnce::new(ipld!({"a": 1, "b": missing_cid}), &mut store);
130
+
// /// {
131
+
// /// let mut foo = exactly_once;
132
+
// /// foo.last();
133
+
// /// }
134
+
// /// // let stuck_at = exactly_once.last().unwrap();
135
+
// /// exactly_once.ignore();
136
+
// /// assert_eq!(exactly_once.wants(), Some(missing_cid));
137
+
// /// ```
103
138
pub fn wants(&self) -> Option<Cid> {
104
139
self.stuck_at
105
140
}
106
141
142
+
// /// FIXME
143
+
// ///
144
+
// /// # Examples
145
+
// ///
146
+
// /// ```
147
+
// /// # use ipld_inline::inliner::exactly_once::ExactlyOnce;
148
+
// /// # use ipld_inline::store::memory::MemoryStore;
149
+
// /// # use libipld::{ipld, cid::CidGeneric};
150
+
// /// # use std::str::FromStr;
151
+
// /// #
152
+
// /// let mut store = MemoryStore::new();
153
+
// /// let missing_cid = FromStr::from_str("bafyreihnubkcms63243zlfgnwiugmk6ijitz63me7bqf455ia2fpbn4ceq").unwrap();
154
+
// ///
155
+
// /// let mut exactly_once = ExactlyOnce::new(ipld!({"a": 1, "b": missing_cid}), &mut store);
156
+
// /// exactly_once.last();
157
+
// /// exactly_once.ignore();
158
+
// ///
159
+
// /// assert_eq!(exactly_once.next(), Some(Err(missing_cid)));
160
+
// /// ```
107
161
pub fn ignore(&mut self) -> Option<()> {
108
162
self.stuck_at.map(|cid| {
109
163
self.quiet.push(Ipld::Link(cid));
···
131
185
}
132
186
}
133
187
134
-
// fn happy_next(&'a mut self) -> Option<Result<Ipld, Stuck<'a, S>>> {
135
-
// if self.stuck_at.is_some() {
136
-
// return None;
137
-
// }
138
-
//
139
-
// match self.quiet.next() {
140
-
// None => None,
141
-
// Some(Err(cid)) => {
142
-
// self.stuck_at = Some(cid);
143
-
// Some(Err(Stuck {
144
-
// needs: cid,
145
-
// it: self,
146
-
// }))
147
-
// }
148
-
// Some(Ok(ipld)) => Some(Ok(ipld.clone())),
149
-
// }
150
-
// }
188
+
/// Returns the [`Cid`] that the iterator got stuck at
189
+
///
190
+
/// # Examples
191
+
///
192
+
/// ```
193
+
/// # use ipld_inline::inliner::exactly_once::ExactlyOnce;
194
+
/// # use ipld_inline::store::memory::MemoryStore;
195
+
/// # use libipld::{ipld, cid::CidGeneric};
196
+
/// #
197
+
/// let mut store = MemoryStore::new();
198
+
/// let missing_cid = CidGeneric::try_from(
199
+
/// "bafyreihnubkcms63243zlfgnwiugmk6ijitz63me7bqf455ia2fpbn4ceq".to_string(),
200
+
/// )
201
+
/// .unwrap();
202
+
/// let mut exactly_once = ExactlyOnce::new(ipld!({"a": 1, "b": missing_cid}), &mut store);
203
+
/// let mut stuck = match exactly_once.happy_next().unwrap() {
204
+
/// Ok(_) => todo!(),
205
+
/// Err(s) => s
206
+
/// };
207
+
/// let eio = stuck.ignore();
208
+
/// assert_eq!(eio.wants(), Some(missing_cid));
209
+
/// ```
210
+
pub fn happy_next(&'a mut self) -> Option<Result<Ipld, Stuck<'a, S>>> {
211
+
if self.stuck_at.is_some() {
212
+
return None;
213
+
}
214
+
215
+
match self.quiet.next() {
216
+
None => None,
217
+
Some(Err(cid)) => {
218
+
self.stuck_at = Some(cid);
219
+
Some(Err(Stuck {
220
+
needs: cid,
221
+
iterator: self,
222
+
}))
223
+
}
224
+
Some(Ok(ipld)) => Some(Ok(ipld.clone())),
225
+
}
226
+
}
151
227
}
152
228
153
229
#[cfg(test)]
···
155
231
use super::*;
156
232
use crate::store::memory::MemoryStore;
157
233
use libipld::ipld;
158
-
use pretty_assertions::{assert_eq, assert_ne};
234
+
use pretty_assertions::assert_eq;
159
235
160
236
// FIXME
161
237
#[test]
+7
src/inliner/quiet.rs
+7
src/inliner/quiet.rs
···
1
+
//! Naive inlining
2
+
//!
3
+
//! This inlining strategy tries its best, but:
4
+
//! - Doesn't attempt to deduplicate DAGs
5
+
//! - Doesn't stop if a [`Cid`] is not available in the attached [`Store`]
6
+
7
+
// FIXME rename this module Naive
1
8
use crate::iterator::post_order::PostOrderIpldIter;
2
9
use crate::store::traits::Store;
3
10
use libipld::{cid::Cid, ipld::Ipld};
+3
src/iterator.rs
+3
src/iterator.rs
+3
-1
src/iterator/post_order.rs
+3
-1
src/iterator/post_order.rs
···
1
+
//! Post-order [`Ipld`] iteration
1
2
use core::iter::Peekable;
2
3
use libipld::ipld::Ipld;
3
4
5
+
/// A post-order [`Ipld`] iterator
4
6
#[derive(Debug, Default, PartialEq, Clone)]
5
7
pub struct PostOrderIpldIter {
6
8
inbound: Vec<Ipld>,
···
68
70
mod tests {
69
71
use super::*;
70
72
use libipld::{cid::CidGeneric, ipld};
71
-
use pretty_assertions::{assert_eq, assert_ne};
73
+
use pretty_assertions::assert_eq;
72
74
73
75
#[test]
74
76
fn poii_test() {
+4
src/lib.rs
+4
src/lib.rs
+3
-2
src/store/memory.rs
+3
-2
src/store/memory.rs
···
1
+
//! An in-memory [`Ipld`] store
1
2
use super::traits::Store;
2
3
use libipld::error::BlockNotFound;
3
4
use libipld::{Cid, Ipld};
4
5
use std::collections::BTreeMap;
5
6
6
-
// More convenient to use and clearer to read than BTree
7
-
7
+
/// A basic in-memory [`Ipld`] store
8
8
#[derive(Clone, Debug, Default)]
9
9
pub struct MemoryStore {
10
10
store: BTreeMap<Cid, Ipld>,
11
11
}
12
12
13
13
impl MemoryStore {
14
+
/// Initialize an empty [`MemoryStore`]
14
15
pub const fn new() -> Self {
15
16
MemoryStore {
16
17
store: BTreeMap::new(),
+35
-1
src/store/traits.rs
+35
-1
src/store/traits.rs
···
1
+
//! Content-addressed store trait
1
2
use crate::cid;
2
3
use crate::extractor::Extractor;
3
4
use crate::inliner::at_most_once::AtMostOnce;
···
14
15
use std::collections::BTreeMap;
15
16
use thiserror::Error;
16
17
17
-
/// A trait for interacting with content addressed stores
18
+
/// A trait describing inline/extract-capable content addressed stores
18
19
pub trait Store {
19
20
/// Retrieve a block by CID
20
21
///
···
164
165
Ok(buffer)
165
166
}
166
167
168
+
/// Try to inline all CIDs based on the contents of the [`Store`]
169
+
///
170
+
/// # Arguments
171
+
///
172
+
/// * `self` - The block store
173
+
/// * `ipld` - The [`Ipld`] to to attempt to inline
174
+
///
175
+
/// # Examples
176
+
///
177
+
/// ```
178
+
/// # use ipld_inline::store::traits::Store;
179
+
/// #
180
+
/// # use libipld::{ipld, cid::Version};
181
+
/// # use libipld_cbor::DagCborCodec;
182
+
/// # use multihash::Code::Sha2_256;
183
+
/// # use std::{collections::BTreeMap, str::FromStr};
184
+
/// #
185
+
/// let mut store = BTreeMap::new();
186
+
/// let inner_cid = store.put(ipld!([4, 5, 6]), DagCborCodec, &Sha2_256, Version::V1).unwrap();
187
+
///
188
+
/// let observed = store.try_inline(ipld!({"a": 123, "b": inner_cid})).unwrap();
189
+
/// let expected = ipld!({
190
+
/// "a": 123,
191
+
/// "b": {
192
+
/// "/": {
193
+
/// "data": ipld!([4, 5, 6]),
194
+
/// "link": inner_cid
195
+
/// }
196
+
/// }
197
+
/// });
198
+
///
199
+
/// assert_eq!(observed, expected);
200
+
/// ```
167
201
fn try_inline(&mut self, ipld: Ipld) -> Result<Ipld, Cid> {
168
202
//FIXME attack of the clones, clippy recommends switching to borriwng the Ipld
169
203
match ExactlyOnce::new(ipld.clone(), self).last() {