gazagnaire.org
Content-defined chunking (FastCDC) in pure OCaml
cdc: new package for content-defined chunking
FastCDC (Xia et al., USENIX ATC 2016) in pure OCaml: a gear rolling
hash with normalized chunking splits byte streams into variable-size
chunks whose boundaries depend on content, not position, so an edit
early in a stream leaves every later chunk boundary (and hash)
intact. Cut over strings, plus a streaming fold over bytesrw readers
with memory bounded by the maximum chunk size.
The gear table is a fixed splitmix64-derived constant: boundaries are
stable across runs and versions (pinned by a regression vector, since
changing them silently degrades deduplication), though no
cross-implementation boundary standard exists. Tests cover exact
tiling, size bounds, boundary suffix stability, streaming/whole-string
agreement at several slice sizes and degenerate all-zero input; the
same invariants run under alcobar on arbitrary bytes.
Separate package rather than part of ocaml-vcdiff: chunking is the
dedup/sync primitive and shares nothing with the delta codec; both
were the missing pieces for delta-based storage and transfer.
author
Thomas Gazagnaire
date
(Jun 11, 2026, 4:32 PM -0700)
commit
cf6698bb
cf6698bb69fcef512fd51d4901dbde594e7f7114
+666
+15
LICENSE.md
+15
LICENSE.md
···
1
+
ISC License
2
+
3
+
Copyright (c) 2026 Thomas Gazagnaire <thomas@gazagnaire.org>
4
+
5
+
Permission to use, copy, modify, and distribute this software for any
6
+
purpose with or without fee is hereby granted, provided that the above
7
+
copyright notice and this permission notice appear in all copies.
8
+
9
+
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
10
+
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
11
+
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
12
+
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
13
+
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
14
+
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
15
+
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+69
README.md
+69
README.md
···
1
+
# cdc
2
+
3
+
Content-defined chunking (FastCDC) in pure OCaml.
4
+
5
+
## Overview
6
+
7
+
Content-defined chunking splits a byte stream into variable-size chunks
8
+
whose boundaries are decided by the bytes around them, not by absolute
9
+
positions. Insert or delete a few bytes early in a stream and only the
10
+
chunks around the edit change; every later chunk keeps its boundary and
11
+
therefore its hash. That property is the building block of deduplicating
12
+
storage and delta synchronization.
13
+
14
+
The algorithm is [FastCDC][fastcdc] (Xia et al., USENIX ATC 2016): a gear
15
+
rolling hash with normalized chunking -- a harder boundary test before the
16
+
expected chunk size and an easier one after it, narrowing the chunk-size
17
+
distribution around the expected size.
18
+
19
+
- `cut` -- chunk boundaries of a string
20
+
- `fold` -- streaming chunker over a `Bytesrw.Bytes.Reader.t`, memory
21
+
bounded by the maximum chunk size
22
+
23
+
The gear table is a fixed constant of this implementation: boundaries are
24
+
stable across runs and versions, but there is no cross-implementation
25
+
standard for them.
26
+
27
+
[fastcdc]: https://www.usenix.org/conference/atc16/technical-sessions/presentation/xia
28
+
29
+
## Installation
30
+
31
+
<!-- $MDX skip -->
32
+
```sh
33
+
$ opam install cdc
34
+
```
35
+
36
+
## Usage
37
+
38
+
```ocaml
39
+
# let p = Cdc.params ~min_size:64 ~avg_size:256 ~max_size:1024 ();;
40
+
val p : Cdc.params = <abstr>
41
+
# let gen () = String.init 20000 (fun i -> Char.chr ((i * 31 + (i / 7)) land 0xff));;
42
+
val gen : unit -> string = <fun>
43
+
# let data = gen () in
44
+
let chunks = Cdc.cut p data in
45
+
List.for_all
46
+
(fun (c : Cdc.chunk) -> c.len >= 1 && c.len <= 1024)
47
+
chunks
48
+
- : bool = true
49
+
# let data = gen () in
50
+
let chunks = Cdc.cut p data in
51
+
String.concat ""
52
+
(List.map (fun (c : Cdc.chunk) -> String.sub data c.off c.len) chunks)
53
+
= data
54
+
- : bool = true
55
+
```
56
+
57
+
Chunk a stream without materializing it:
58
+
59
+
```ocaml
60
+
# let data = gen () in
61
+
let r = Bytesrw.Bytes.Reader.of_string ~slice_length:4096 data in
62
+
Cdc.fold p (fun n ~off:_ chunk -> n + String.length chunk) 0 r
63
+
= String.length data
64
+
- : bool = true
65
+
```
66
+
67
+
## License
68
+
69
+
ISC -- see [LICENSE.md](LICENSE.md).
+45
cdc.opam
+45
cdc.opam
···
1
+
# This file is generated by dune, edit dune-project instead
2
+
opam-version: "2.0"
3
+
synopsis: "Content-defined chunking (FastCDC) in pure OCaml"
4
+
description: """
5
+
Split byte streams into variable-size chunks whose boundaries are
6
+
decided by content, not position, so an insertion early in a stream
7
+
only changes the chunks around the edit. Implements the FastCDC
8
+
gear-hash algorithm with normalized chunking, over strings or
9
+
bytesrw readers. The building block for deduplicating storage and
10
+
delta synchronization."""
11
+
maintainer: ["Thomas Gazagnaire"]
12
+
authors: ["Thomas Gazagnaire"]
13
+
license: "ISC"
14
+
tags: ["org:blacksun" "codec.binary"]
15
+
homepage: "https://tangled.org/gazagnaire.org/ocaml-cdc"
16
+
bug-reports: "https://tangled.org/gazagnaire.org/ocaml-cdc/issues"
17
+
depends: [
18
+
"dune" {>= "3.21"}
19
+
"ocaml" {>= "5.1"}
20
+
"bytesrw" {>= "0.2"}
21
+
"odoc" {with-doc}
22
+
"alcotest" {with-test}
23
+
"alcobar" {with-test}
24
+
"fmt" {with-test}
25
+
"mdx" {with-test}
26
+
]
27
+
build: [
28
+
["dune" "subst"] {dev}
29
+
[
30
+
"dune"
31
+
"build"
32
+
"-p"
33
+
name
34
+
"-j"
35
+
jobs
36
+
"@install"
37
+
"@runtest" {with-test}
38
+
"@doc" {with-doc}
39
+
]
40
+
]
41
+
dev-repo: "git+https://tangled.org/gazagnaire.org/ocaml-cdc"
42
+
x-maintenance-intent: ["(latest)"]
43
+
x-quality-build: "2026-06-11"
44
+
x-quality-fuzz: "2026-06-11"
45
+
x-quality-test: "2026-06-11"
+3
cdc.opam.template
+3
cdc.opam.template
+7
dune
+7
dune
+32
dune-project
+32
dune-project
···
1
+
(lang dune 3.21)
2
+
(using mdx 0.4)
3
+
(name cdc)
4
+
5
+
(generate_opam_files true)
6
+
(implicit_transitive_deps false)
7
+
8
+
(source (tangled gazagnaire.org/ocaml-cdc))
9
+
10
+
(maintainers "Thomas Gazagnaire")
11
+
(authors "Thomas Gazagnaire")
12
+
(license ISC)
13
+
14
+
(package
15
+
(name cdc)
16
+
(synopsis "Content-defined chunking (FastCDC) in pure OCaml")
17
+
(tags (org:blacksun codec.binary))
18
+
(description
19
+
"Split byte streams into variable-size chunks whose boundaries are
20
+
decided by content, not position, so an insertion early in a stream
21
+
only changes the chunks around the edit. Implements the FastCDC
22
+
gear-hash algorithm with normalized chunking, over strings or
23
+
bytesrw readers. The building block for deduplicating storage and
24
+
delta synchronization.")
25
+
(depends
26
+
(ocaml (>= 5.1))
27
+
(bytesrw (>= 0.2))
28
+
(odoc :with-doc)
29
+
(alcotest :with-test)
30
+
(alcobar :with-test)
31
+
(fmt :with-test)
32
+
(mdx :with-test)))
+25
fuzz/dune
+25
fuzz/dune
···
1
+
(executable
2
+
(name fuzz)
3
+
(libraries cdc bytesrw alcobar fmt))
4
+
5
+
; Quick check with Crowbar (no AFL instrumentation)
6
+
7
+
(rule
8
+
(alias runtest)
9
+
(enabled_if
10
+
(<> %{profile} afl))
11
+
(deps fuzz.exe)
12
+
(action
13
+
(run %{exe:fuzz.exe})))
14
+
15
+
; AFL-instrumented build target (use with --profile=afl)
16
+
17
+
(rule
18
+
(alias fuzz)
19
+
(enabled_if
20
+
(= %{profile} afl))
21
+
(deps fuzz.exe)
22
+
(action
23
+
(progn
24
+
(run %{exe:fuzz.exe} --gen-corpus corpus)
25
+
(run afl-fuzz -V 60 -i corpus -o _fuzz -- %{exe:fuzz.exe} @@))))
+1
fuzz/fuzz.ml
+1
fuzz/fuzz.ml
···
1
+
let () = Alcobar.run "cdc" [ Fuzz_cdc.suite ]
+55
fuzz/fuzz_cdc.ml
+55
fuzz/fuzz_cdc.ml
···
1
+
open Alcobar
2
+
3
+
(* Small bounds so short fuzz inputs still produce several chunks. *)
4
+
let p = Cdc.params ~min_size:8 ~avg_size:64 ~max_size:256 ()
5
+
6
+
(* Chunks tile the input exactly and respect the size bounds. *)
7
+
let test_tiling s =
8
+
let chunks = Cdc.cut p s in
9
+
let off = ref 0 in
10
+
List.iteri
11
+
(fun i (c : Cdc.chunk) ->
12
+
if c.off <> !off then fail "chunks not contiguous";
13
+
if c.len <= 0 || c.len > Cdc.max_size p then
14
+
fail "chunk size out of bounds";
15
+
if i < List.length chunks - 1 && c.len < Cdc.min_size p then
16
+
fail "non-final chunk below min size";
17
+
off := !off + c.len)
18
+
chunks;
19
+
if !off <> String.length s then fail "chunks do not cover the input"
20
+
21
+
(* Cutting the suffix that starts at the first boundary yields the remaining
22
+
chunks: boundaries only depend on bytes since the previous boundary. *)
23
+
let test_suffix_stability s =
24
+
match Cdc.cut p s with
25
+
| first :: rest ->
26
+
let suffix = String.sub s first.len (String.length s - first.len) in
27
+
let again = Cdc.cut p suffix in
28
+
if
29
+
not
30
+
(List.for_all2
31
+
(fun (a : Cdc.chunk) (b : Cdc.chunk) ->
32
+
a.off - first.len = b.off && a.len = b.len)
33
+
rest again)
34
+
then fail "suffix chunks differ"
35
+
| [] -> if s <> "" then fail "no chunks for non-empty input"
36
+
37
+
(* The streaming fold and the whole-string cut agree, whatever the slice
38
+
size. *)
39
+
let test_fold_matches s =
40
+
let r = Bytesrw.Bytes.Reader.of_string ~slice_length:13 s in
41
+
let folded = List.rev (Cdc.fold p (fun acc ~off c -> (off, c) :: acc) [] r) in
42
+
let expected =
43
+
List.map
44
+
(fun (c : Cdc.chunk) -> (c.off, String.sub s c.off c.len))
45
+
(Cdc.cut p s)
46
+
in
47
+
if folded <> expected then fail "fold and cut disagree"
48
+
49
+
let suite =
50
+
( "cdc",
51
+
[
52
+
test_case "tiling" [ bytes ] test_tiling;
53
+
test_case "suffix stability" [ bytes ] test_suffix_stability;
54
+
test_case "fold matches cut" [ bytes ] test_fold_matches;
55
+
] )
+5
fuzz/fuzz_cdc.mli
+5
fuzz/fuzz_cdc.mli
+183
lib/cdc.ml
+183
lib/cdc.ml
···
1
+
(*---------------------------------------------------------------------------
2
+
Copyright (c) 2026 Thomas Gazagnaire <thomas@gazagnaire.org>
3
+
SPDX-License-Identifier: ISC
4
+
---------------------------------------------------------------------------*)
5
+
6
+
(* FastCDC (Xia et al., USENIX ATC 2016): gear rolling hash + normalized
7
+
chunking. The fingerprint lives in the 62 low bits of a native int (the
8
+
shift-left accumulation discards overflow), and the boundary masks select
9
+
high fingerprint bits, where the gear hash concentrates entropy. *)
10
+
11
+
(* Gear table: 256 pseudo-random 62-bit constants from splitmix64 with a
12
+
fixed seed, so boundaries are deterministic across runs and versions. *)
13
+
let gear =
14
+
let state = ref 0x9E3779B97F4A7C15L in
15
+
let next () =
16
+
state := Int64.add !state 0x9E3779B97F4A7C15L;
17
+
let z = !state in
18
+
let z =
19
+
Int64.mul
20
+
(Int64.logxor z (Int64.shift_right_logical z 30))
21
+
0xBF58476D1CE4E5B9L
22
+
in
23
+
let z =
24
+
Int64.mul
25
+
(Int64.logxor z (Int64.shift_right_logical z 27))
26
+
0x94D049BB133111EBL
27
+
in
28
+
Int64.logxor z (Int64.shift_right_logical z 31)
29
+
in
30
+
Array.init 256 (fun _ ->
31
+
Int64.to_int (Int64.logand (next ()) 0x3FFF_FFFF_FFFF_FFFFL))
32
+
33
+
type params = {
34
+
min_size : int;
35
+
avg_size : int;
36
+
max_size : int;
37
+
mask_s : int; (* avg bits + 2: harder test before the expected size *)
38
+
mask_l : int; (* avg bits - 2: easier test after it *)
39
+
}
40
+
41
+
let min_size p = p.min_size
42
+
let avg_size p = p.avg_size
43
+
let max_size p = p.max_size
44
+
let is_pow2 n = n > 0 && n land (n - 1) = 0
45
+
46
+
let log2 n =
47
+
let rec go k n = if n <= 1 then k else go (k + 1) (n lsr 1) in
48
+
go 0 n
49
+
50
+
(* A mask of [bits] high fingerprint bits within the 62-bit range. *)
51
+
let mask_of_bits bits = ((1 lsl bits) - 1) lsl (62 - bits)
52
+
53
+
let params ?min_size ?(avg_size = 8192) ?max_size () =
54
+
if (not (is_pow2 avg_size)) || avg_size < 64 then
55
+
invalid_arg "Cdc.params: avg_size must be a power of two >= 64";
56
+
let min_size = Option.value min_size ~default:(avg_size / 4) in
57
+
let max_size = Option.value max_size ~default:(avg_size * 8) in
58
+
if min_size <= 0 || min_size > avg_size || avg_size > max_size then
59
+
invalid_arg "Cdc.params: need 0 < min_size <= avg_size <= max_size";
60
+
let bits = log2 avg_size in
61
+
{
62
+
min_size;
63
+
avg_size;
64
+
max_size;
65
+
mask_s = mask_of_bits (bits + 2);
66
+
mask_l = mask_of_bits (bits - 2);
67
+
}
68
+
69
+
type chunk = { off : int; len : int }
70
+
71
+
(* Roll the gear fingerprint over s.[off+i) for i in [from, upto) and return
72
+
the boundary i+1 of the first position where the masked fingerprint is
73
+
zero, or 0 when none is. The fingerprint threads through [fp] so the
74
+
normal-size and large-size scans compose. *)
75
+
let scan s ~off ~mask fp ~from ~upto =
76
+
let result = ref 0 in
77
+
let i = ref from in
78
+
while !result = 0 && !i < upto do
79
+
fp :=
80
+
(!fp lsl 1)
81
+
+ Array.unsafe_get gear (Char.code (String.unsafe_get s (off + !i)));
82
+
if !fp land mask = 0 then result := !i + 1;
83
+
incr i
84
+
done;
85
+
!result
86
+
87
+
(* Length of the first chunk of [s.[off .. off+len)] (FastCDC's cut-point
88
+
function). Always in [1 .. max_size] and >= min_size unless the input is
89
+
shorter than that. *)
90
+
let cut_one p s ~off ~len =
91
+
if len <= p.min_size then len
92
+
else begin
93
+
let n = if len > p.max_size then p.max_size else len in
94
+
let normal = if n < p.avg_size then n else p.avg_size in
95
+
let fp = ref 0 in
96
+
match scan s ~off ~mask:p.mask_s fp ~from:p.min_size ~upto:normal with
97
+
| 0 -> (
98
+
match scan s ~off ~mask:p.mask_l fp ~from:normal ~upto:n with
99
+
| 0 -> n
100
+
| boundary -> boundary)
101
+
| boundary -> boundary
102
+
end
103
+
104
+
let cut p s =
105
+
let len = String.length s in
106
+
let rec go off acc =
107
+
if off >= len then List.rev acc
108
+
else begin
109
+
let n = cut_one p s ~off ~len:(len - off) in
110
+
go (off + n) ({ off; len = n } :: acc)
111
+
end
112
+
in
113
+
go 0 []
114
+
115
+
(* Pending bytes of the streaming chunker: a growing buffer with the
116
+
not-yet-chunked region at [start, filled) and [emitted] the absolute
117
+
offset of the next chunk. *)
118
+
type pending = {
119
+
mutable buf : bytes;
120
+
mutable start : int;
121
+
mutable filled : int;
122
+
mutable emitted : int;
123
+
}
124
+
125
+
(* Compact or grow so [extra] more bytes fit behind [filled]. *)
126
+
let reserve w extra =
127
+
if w.filled + extra > Bytes.length w.buf then begin
128
+
let pending = w.filled - w.start in
129
+
if pending + extra > Bytes.length w.buf then begin
130
+
let cap = ref (2 * Bytes.length w.buf) in
131
+
while pending + extra > !cap do
132
+
cap := 2 * !cap
133
+
done;
134
+
let nb = Bytes.create !cap in
135
+
Bytes.blit w.buf w.start nb 0 pending;
136
+
w.buf <- nb
137
+
end
138
+
else Bytes.blit w.buf w.start w.buf 0 pending;
139
+
w.start <- 0;
140
+
w.filled <- pending
141
+
end
142
+
143
+
let push w slice =
144
+
let n = Bytesrw.Bytes.Slice.length slice in
145
+
reserve w n;
146
+
Bytes.blit
147
+
(Bytesrw.Bytes.Slice.bytes slice)
148
+
(Bytesrw.Bytes.Slice.first slice)
149
+
w.buf w.filled n;
150
+
w.filled <- w.filled + n
151
+
152
+
(* Streaming chunker: accumulate reader slices and emit chunks as soon as
153
+
[max_size] bytes are pending (a cut point never depends on bytes beyond
154
+
that). *)
155
+
let fold p f acc r =
156
+
let w =
157
+
{ buf = Bytes.create (2 * p.max_size); start = 0; filled = 0; emitted = 0 }
158
+
in
159
+
let acc = ref acc in
160
+
let emit_one () =
161
+
let s = Bytes.unsafe_to_string w.buf in
162
+
let n = cut_one p s ~off:w.start ~len:(w.filled - w.start) in
163
+
let chunk = Bytes.sub_string w.buf w.start n in
164
+
acc := f !acc ~off:w.emitted chunk;
165
+
w.emitted <- w.emitted + n;
166
+
w.start <- w.start + n
167
+
in
168
+
let rec drain () =
169
+
let slice = Bytesrw.Bytes.Reader.read r in
170
+
if Bytesrw.Bytes.Slice.is_eod slice then
171
+
while w.filled - w.start > 0 do
172
+
emit_one ()
173
+
done
174
+
else begin
175
+
push w slice;
176
+
while w.filled - w.start >= p.max_size do
177
+
emit_one ()
178
+
done;
179
+
drain ()
180
+
end
181
+
in
182
+
drain ();
183
+
!acc
+70
lib/cdc.mli
+70
lib/cdc.mli
···
1
+
(*---------------------------------------------------------------------------
2
+
Copyright (c) 2026 Thomas Gazagnaire <thomas@gazagnaire.org>
3
+
SPDX-License-Identifier: ISC
4
+
---------------------------------------------------------------------------*)
5
+
6
+
(** Content-defined chunking.
7
+
8
+
Splits byte streams into variable-size chunks whose boundaries depend on the
9
+
{e content} around them, not on absolute positions: inserting or deleting
10
+
bytes early in a stream only changes the chunks around the edit, while every
11
+
later chunk keeps its boundaries (and therefore its hash). This is the
12
+
building block of deduplicating storage and delta synchronization.
13
+
14
+
The algorithm is FastCDC (Xia et al., USENIX ATC 2016): a gear rolling hash
15
+
with normalized chunking — a harder boundary test before the expected chunk
16
+
size and an easier one after it, which narrows the chunk size distribution
17
+
around the expected size.
18
+
19
+
A boundary decision depends only on the bytes since the previous boundary,
20
+
so for any cut point [p] of [chunks ps s], the chunks of the suffix
21
+
[String.sub s p (String.length s - p)] are exactly the remaining chunks of
22
+
[s].
23
+
24
+
The gear table is a fixed constant of this implementation: chunk boundaries
25
+
are stable across runs and versions, but there is no cross-implementation
26
+
standard for them. *)
27
+
28
+
(** {1:params Parameters} *)
29
+
30
+
type params
31
+
(** Chunk size bounds: minimum, expected and maximum size in bytes. *)
32
+
33
+
val params : ?min_size:int -> ?avg_size:int -> ?max_size:int -> unit -> params
34
+
(** [params ()] are chunking parameters. The expected chunk size {!val-avg_size}
35
+
must be a power of two, at least 64; it defaults to [8192]. {!val-min_size}
36
+
defaults to a quarter of it, {!val-max_size} to eight times it, and the
37
+
minimum must be positive and no larger than the expected size, which must be
38
+
no larger than the maximum. Raises [Invalid_argument] otherwise. *)
39
+
40
+
val min_size : params -> int
41
+
(** [min_size ps] is the minimum chunk size in bytes. *)
42
+
43
+
val avg_size : params -> int
44
+
(** [avg_size ps] is the expected chunk size in bytes. *)
45
+
46
+
val max_size : params -> int
47
+
(** [max_size ps] is the maximum chunk size in bytes. *)
48
+
49
+
(** {1:chunks Chunking} *)
50
+
51
+
type chunk = { off : int; len : int }
52
+
(** A chunk: {!field-len} bytes starting at the absolute input offset
53
+
{!field-off}. *)
54
+
55
+
val cut : params -> string -> chunk list
56
+
(** [cut ps s] are the chunks of [s], in order. Their concatenation is exactly
57
+
[s]; every chunk except the last is between {!val-min_size} and
58
+
{!val-max_size}; the last chunk is non-empty (unless [s] is empty) and no
59
+
larger than the maximum. *)
60
+
61
+
val fold :
62
+
params ->
63
+
('acc -> off:int -> string -> 'acc) ->
64
+
'acc ->
65
+
Bytesrw.Bytes.Reader.t ->
66
+
'acc
67
+
(** [fold ps f acc r] folds [f] over the chunks of the stream [r], passing each
68
+
chunk's bytes and absolute offset. Boundaries are identical to {!cut} on the
69
+
whole stream; memory is bounded by {!val-max_size} plus the reader's slice
70
+
size. *)
+1
test/test.ml
+1
test/test.ml
···
1
+
let () = Alcotest.run (fst Test_cdc.suite) [ Test_cdc.suite ]
+142
test/test_cdc.ml
+142
test/test_cdc.ml
···
1
+
(* Deterministic pseudo-random input (xorshift), so every assertion below is
2
+
exact. *)
3
+
let random_string n =
4
+
let state = ref 0x12345678 in
5
+
String.init n (fun _ ->
6
+
state := !state lxor (!state lsl 13);
7
+
state := !state lxor (!state lsr 7);
8
+
state := !state lxor (!state lsl 17);
9
+
Char.chr (!state land 0xff))
10
+
11
+
let small = Cdc.params ~min_size:64 ~avg_size:256 ~max_size:1024 ()
12
+
let input = random_string 20_000
13
+
14
+
let check_coverage name ps s =
15
+
let chunks = Cdc.cut ps s in
16
+
let total = List.fold_left (fun n (c : Cdc.chunk) -> n + c.len) 0 chunks in
17
+
Alcotest.(check int) (name ^ ": total length") (String.length s) total;
18
+
let _ =
19
+
List.fold_left
20
+
(fun expect (c : Cdc.chunk) ->
21
+
Alcotest.(check int) (name ^ ": contiguous") expect c.off;
22
+
c.off + c.len)
23
+
0 chunks
24
+
in
25
+
let rec check_sizes = function
26
+
| [] -> ()
27
+
| [ (last : Cdc.chunk) ] ->
28
+
if String.length s > 0 then
29
+
Alcotest.(check bool)
30
+
(name ^ ": last chunk in (0, max]")
31
+
true
32
+
(last.len > 0 && last.len <= Cdc.max_size ps)
33
+
| (c : Cdc.chunk) :: rest ->
34
+
Alcotest.(check bool)
35
+
(name ^ ": chunk in [min, max]")
36
+
true
37
+
(c.len >= Cdc.min_size ps && c.len <= Cdc.max_size ps);
38
+
check_sizes rest
39
+
in
40
+
check_sizes chunks
41
+
42
+
let test_coverage () =
43
+
check_coverage "random" small input;
44
+
check_coverage "empty" small "";
45
+
check_coverage "tiny" small "abc";
46
+
check_coverage "all zeros" small (String.make 10_000 '\000');
47
+
check_coverage "min-sized" small (random_string (Cdc.min_size small));
48
+
check_coverage "max-sized" small (random_string (Cdc.max_size small))
49
+
50
+
(* A boundary decision depends only on the bytes since the previous boundary:
51
+
chunking the suffix that starts at any cut point yields exactly the
52
+
remaining chunks. *)
53
+
let test_suffix_stability () =
54
+
let chunks = Cdc.cut small input in
55
+
match chunks with
56
+
| first :: rest when rest <> [] ->
57
+
let suffix =
58
+
String.sub input first.len (String.length input - first.len)
59
+
in
60
+
let suffix_chunks = Cdc.cut small suffix in
61
+
Alcotest.(check int)
62
+
"same number of chunks" (List.length rest)
63
+
(List.length suffix_chunks);
64
+
List.iter2
65
+
(fun (a : Cdc.chunk) (b : Cdc.chunk) ->
66
+
Alcotest.(check int) "same offset" (a.off - first.len) b.off;
67
+
Alcotest.(check int) "same length" a.len b.len)
68
+
rest suffix_chunks
69
+
| _ -> Alcotest.fail "expected several chunks"
70
+
71
+
(* All-zero input never matches a boundary mask (gear[0] accumulates into a
72
+
constant-ish fingerprint that the masks reject), so every chunk maxes
73
+
out. This pins the degenerate-input behavior: max-size chunks, not a
74
+
pathological flood of tiny ones. *)
75
+
let test_zeros_max_out () =
76
+
let zeros = String.make 10_000 '\000' in
77
+
let chunks = Cdc.cut small zeros in
78
+
List.iteri
79
+
(fun i (c : Cdc.chunk) ->
80
+
if i < List.length chunks - 1 then
81
+
Alcotest.(check int) "max-size chunk" (Cdc.max_size small) c.len)
82
+
chunks
83
+
84
+
let test_fold_matches_cut () =
85
+
List.iter
86
+
(fun slice_length ->
87
+
let r = Bytesrw.Bytes.Reader.of_string ~slice_length input in
88
+
let folded =
89
+
List.rev (Cdc.fold small (fun acc ~off s -> (off, s) :: acc) [] r)
90
+
in
91
+
let expected =
92
+
List.map
93
+
(fun (c : Cdc.chunk) -> (c.off, String.sub input c.off c.len))
94
+
(Cdc.cut small input)
95
+
in
96
+
Alcotest.(check (list (pair int string)))
97
+
(Printf.sprintf "slice length %d" slice_length)
98
+
expected folded)
99
+
[ 1; 7; 4096; 65536 ]
100
+
101
+
let test_params_validation () =
102
+
let invalid f = Alcotest.check_raises "rejects" (Invalid_argument "") f in
103
+
let invalid f =
104
+
ignore invalid;
105
+
match f () with
106
+
| exception Invalid_argument _ -> ()
107
+
| _ -> Alcotest.fail "expected Invalid_argument"
108
+
in
109
+
invalid (fun () -> Cdc.params ~avg_size:1000 () (* not a power of two *));
110
+
invalid (fun () -> Cdc.params ~avg_size:32 () (* too small *));
111
+
invalid (fun () -> Cdc.params ~min_size:0 ());
112
+
invalid (fun () -> Cdc.params ~min_size:9000 ~avg_size:8192 ());
113
+
invalid (fun () -> Cdc.params ~max_size:4096 ~avg_size:8192 ());
114
+
let p = Cdc.params () in
115
+
Alcotest.(check int) "default avg" 8192 (Cdc.avg_size p);
116
+
Alcotest.(check int) "default min" 2048 (Cdc.min_size p);
117
+
Alcotest.(check int) "default max" 65536 (Cdc.max_size p)
118
+
119
+
(* Regression vector: the first chunk lengths of the fixed input above. The
120
+
gear table and masks are constants of the implementation; if these values
121
+
change, previously stored chunk hashes no longer match and deduplication
122
+
silently degrades, so a change here is a breaking change. *)
123
+
let test_pinned_boundaries () =
124
+
let lens =
125
+
List.filteri (fun i _ -> i < 8) (Cdc.cut small input)
126
+
|> List.map (fun (c : Cdc.chunk) -> c.len)
127
+
in
128
+
Alcotest.(check (list int))
129
+
"first chunk lengths"
130
+
[ 283; 142; 257; 295; 345; 269; 79; 290 ]
131
+
lens
132
+
133
+
let suite =
134
+
( "cdc",
135
+
[
136
+
Alcotest.test_case "coverage and bounds" `Quick test_coverage;
137
+
Alcotest.test_case "suffix stability" `Quick test_suffix_stability;
138
+
Alcotest.test_case "all zeros max out" `Quick test_zeros_max_out;
139
+
Alcotest.test_case "fold matches cut" `Quick test_fold_matches_cut;
140
+
Alcotest.test_case "params validation" `Quick test_params_validation;
141
+
Alcotest.test_case "pinned boundaries" `Quick test_pinned_boundaries;
142
+
] )
+6
test/test_cdc.mli
+6
test/test_cdc.mli
···
1
+
(** Tests for {!Cdc}: chunk coverage and size bounds, boundary suffix stability,
2
+
streaming/whole-string agreement, parameter validation, and a pinned
3
+
boundary regression vector. *)
4
+
5
+
val suite : string * unit Alcotest.test_case list
6
+
(** [suite] is the Alcotest suite for {!Cdc}. *)