nandi.uk
Fork of daniellemaywood.uk/gleam — Wasm codegen work
Allow endianness specifiers for multi-byte codepoint segments
author
GearsDatapacks
committer
Louis Pilfold
date
(Jul 6, 2025, 1:29 PM +0100)
commit
17f0f7b8
17f0f7b84c987b1b542aca6e6b7280879396105c
parent
7e15dd2e
7e15dd2e6e1651dae5f2fd1a0b0ccb95a1bfd086
+264
-2
+4
CHANGELOG.md
+4
CHANGELOG.md
···
196
196
197
197
([Louis Pilfold](https://github.com/lpil))
198
198
199
+
- The compiler now allows using bit array options to specify endianness when
200
+
construction or pattern matching on bit arrays.
201
+
([Surya Rose](https://github.com/GearsDatapacks))
202
+
199
203
### Build tool
200
204
201
205
- `gleam update`, `gleam deps update`, and `gleam deps download` will now print
+12
-2
compiler-core/src/bit_array.rs
+12
-2
compiler-core/src/bit_array.rs
···
219
219
}
220
220
}
221
221
222
-
// Endianness is only valid for int, utf16, utf32 and float
222
+
// Endianness is only valid for int, utf16, utf16_codepoint, utf32,
223
+
// utf32_codepoint and float
223
224
match categories {
224
225
SegmentOptionCategories {
225
-
type_: None | Some(Int { .. } | Utf16 { .. } | Utf32 { .. } | Float { .. }),
226
+
type_:
227
+
None
228
+
| Some(
229
+
Int { .. }
230
+
| Utf16 { .. }
231
+
| Utf32 { .. }
232
+
| Utf16Codepoint { .. }
233
+
| Utf32Codepoint { .. }
234
+
| Float { .. },
235
+
),
226
236
..
227
237
} => {}
228
238
+46
compiler-core/src/erlang/tests/bit_arrays.rs
+46
compiler-core/src/erlang/tests/bit_arrays.rs
···
209
209
"
210
210
);
211
211
}
212
+
213
+
#[test]
214
+
fn utf16_codepoint_little_endian() {
215
+
assert_erl!(
216
+
"
217
+
pub fn go(codepoint) {
218
+
<<codepoint:utf16_codepoint-little>>
219
+
}
220
+
"
221
+
);
222
+
}
223
+
224
+
#[test]
225
+
fn utf32_codepoint_little_endian() {
226
+
assert_erl!(
227
+
"
228
+
pub fn go(codepoint) {
229
+
<<codepoint:utf32_codepoint-little>>
230
+
}
231
+
"
232
+
);
233
+
}
234
+
235
+
#[test]
236
+
fn pattern_match_utf16_codepoint_little_endian() {
237
+
assert_erl!(
238
+
"
239
+
pub fn go(x) {
240
+
let assert <<codepoint:utf16_codepoint-little>> = x
241
+
codepoint
242
+
}
243
+
"
244
+
);
245
+
}
246
+
247
+
#[test]
248
+
fn pattern_match_utf32_codepoint_little_endian() {
249
+
assert_erl!(
250
+
"
251
+
pub fn go(x) {
252
+
let assert <<codepoint:utf32_codepoint-little>> = x
253
+
codepoint
254
+
}
255
+
"
256
+
);
257
+
}
+37
compiler-core/src/erlang/tests/snapshots/gleam_core__erlang__tests__bit_arrays__pattern_match_utf16_codepoint_little_endian.snap
+37
compiler-core/src/erlang/tests/snapshots/gleam_core__erlang__tests__bit_arrays__pattern_match_utf16_codepoint_little_endian.snap
···
1
+
---
2
+
source: compiler-core/src/erlang/tests/bit_arrays.rs
3
+
expression: "\npub fn go(x) {\n let assert <<codepoint:utf16_codepoint-little>> = x\n codepoint\n}\n"
4
+
---
5
+
----- SOURCE CODE
6
+
7
+
pub fn go(x) {
8
+
let assert <<codepoint:utf16_codepoint-little>> = x
9
+
codepoint
10
+
}
11
+
12
+
13
+
----- COMPILED ERLANG
14
+
-module(my@mod).
15
+
-compile([no_auto_import, nowarn_unused_vars, nowarn_unused_function, nowarn_nomatch]).
16
+
-define(FILEPATH, "project/test/my/mod.gleam").
17
+
-export([go/1]).
18
+
19
+
-file("project/test/my/mod.gleam", 2).
20
+
-spec go(bitstring()) -> integer().
21
+
go(X) ->
22
+
Codepoint@1 = case X of
23
+
<<Codepoint/utf16-little>> -> Codepoint;
24
+
_assert_fail ->
25
+
erlang:error(#{gleam_error => let_assert,
26
+
message => <<"Pattern match failed, no pattern matched the value."/utf8>>,
27
+
file => <<?FILEPATH/utf8>>,
28
+
module => <<"my/mod"/utf8>>,
29
+
function => <<"go"/utf8>>,
30
+
line => 3,
31
+
value => _assert_fail,
32
+
start => 18,
33
+
'end' => 69,
34
+
pattern_start => 29,
35
+
pattern_end => 65})
36
+
end,
37
+
Codepoint@1.
+37
compiler-core/src/erlang/tests/snapshots/gleam_core__erlang__tests__bit_arrays__pattern_match_utf32_codepoint_little_endian.snap
+37
compiler-core/src/erlang/tests/snapshots/gleam_core__erlang__tests__bit_arrays__pattern_match_utf32_codepoint_little_endian.snap
···
1
+
---
2
+
source: compiler-core/src/erlang/tests/bit_arrays.rs
3
+
expression: "\npub fn go(x) {\n let assert <<codepoint:utf32_codepoint-little>> = x\n codepoint\n}\n"
4
+
---
5
+
----- SOURCE CODE
6
+
7
+
pub fn go(x) {
8
+
let assert <<codepoint:utf32_codepoint-little>> = x
9
+
codepoint
10
+
}
11
+
12
+
13
+
----- COMPILED ERLANG
14
+
-module(my@mod).
15
+
-compile([no_auto_import, nowarn_unused_vars, nowarn_unused_function, nowarn_nomatch]).
16
+
-define(FILEPATH, "project/test/my/mod.gleam").
17
+
-export([go/1]).
18
+
19
+
-file("project/test/my/mod.gleam", 2).
20
+
-spec go(bitstring()) -> integer().
21
+
go(X) ->
22
+
Codepoint@1 = case X of
23
+
<<Codepoint/utf32-little>> -> Codepoint;
24
+
_assert_fail ->
25
+
erlang:error(#{gleam_error => let_assert,
26
+
message => <<"Pattern match failed, no pattern matched the value."/utf8>>,
27
+
file => <<?FILEPATH/utf8>>,
28
+
module => <<"my/mod"/utf8>>,
29
+
function => <<"go"/utf8>>,
30
+
line => 3,
31
+
value => _assert_fail,
32
+
start => 18,
33
+
'end' => 69,
34
+
pattern_start => 29,
35
+
pattern_end => 65})
36
+
end,
37
+
Codepoint@1.
+21
compiler-core/src/erlang/tests/snapshots/gleam_core__erlang__tests__bit_arrays__utf16_codepoint_little_endian.snap
+21
compiler-core/src/erlang/tests/snapshots/gleam_core__erlang__tests__bit_arrays__utf16_codepoint_little_endian.snap
···
1
+
---
2
+
source: compiler-core/src/erlang/tests/bit_arrays.rs
3
+
expression: "\npub fn go(codepoint) {\n <<codepoint:utf16_codepoint-little>>\n}\n"
4
+
---
5
+
----- SOURCE CODE
6
+
7
+
pub fn go(codepoint) {
8
+
<<codepoint:utf16_codepoint-little>>
9
+
}
10
+
11
+
12
+
----- COMPILED ERLANG
13
+
-module(my@mod).
14
+
-compile([no_auto_import, nowarn_unused_vars, nowarn_unused_function, nowarn_nomatch]).
15
+
-define(FILEPATH, "project/test/my/mod.gleam").
16
+
-export([go/1]).
17
+
18
+
-file("project/test/my/mod.gleam", 2).
19
+
-spec go(integer()) -> bitstring().
20
+
go(Codepoint) ->
21
+
<<Codepoint/utf16-little>>.
+21
compiler-core/src/erlang/tests/snapshots/gleam_core__erlang__tests__bit_arrays__utf32_codepoint_little_endian.snap
+21
compiler-core/src/erlang/tests/snapshots/gleam_core__erlang__tests__bit_arrays__utf32_codepoint_little_endian.snap
···
1
+
---
2
+
source: compiler-core/src/erlang/tests/bit_arrays.rs
3
+
expression: "\npub fn go(codepoint) {\n <<codepoint:utf32_codepoint-little>>\n}\n"
4
+
---
5
+
----- SOURCE CODE
6
+
7
+
pub fn go(codepoint) {
8
+
<<codepoint:utf32_codepoint-little>>
9
+
}
10
+
11
+
12
+
----- COMPILED ERLANG
13
+
-module(my@mod).
14
+
-compile([no_auto_import, nowarn_unused_vars, nowarn_unused_function, nowarn_nomatch]).
15
+
-define(FILEPATH, "project/test/my/mod.gleam").
16
+
-export([go/1]).
17
+
18
+
-file("project/test/my/mod.gleam", 2).
19
+
-spec go(integer()) -> bitstring().
20
+
go(Codepoint) ->
21
+
<<Codepoint/utf32-little>>.
+22
compiler-core/src/javascript/tests/bit_arrays.rs
+22
compiler-core/src/javascript/tests/bit_arrays.rs
···
2539
2539
"
2540
2540
);
2541
2541
}
2542
+
2543
+
#[test]
2544
+
fn utf16_codepoint_little_endian() {
2545
+
assert_js!(
2546
+
"
2547
+
pub fn go(codepoint) {
2548
+
<<codepoint:utf16_codepoint-little>>
2549
+
}
2550
+
"
2551
+
);
2552
+
}
2553
+
2554
+
#[test]
2555
+
fn utf32_codepoint_little_endian() {
2556
+
assert_js!(
2557
+
"
2558
+
pub fn go(codepoint) {
2559
+
<<codepoint:utf32_codepoint-little>>
2560
+
}
2561
+
"
2562
+
);
2563
+
}
+17
compiler-core/src/javascript/tests/snapshots/gleam_core__javascript__tests__bit_arrays__utf16_codepoint_little_endian.snap
+17
compiler-core/src/javascript/tests/snapshots/gleam_core__javascript__tests__bit_arrays__utf16_codepoint_little_endian.snap
···
1
+
---
2
+
source: compiler-core/src/javascript/tests/bit_arrays.rs
3
+
expression: "\npub fn go(codepoint) {\n <<codepoint:utf16_codepoint-little>>\n}\n"
4
+
---
5
+
----- SOURCE CODE
6
+
7
+
pub fn go(codepoint) {
8
+
<<codepoint:utf16_codepoint-little>>
9
+
}
10
+
11
+
12
+
----- COMPILED JAVASCRIPT
13
+
import { toBitArray, codepointToUtf16 } from "../gleam.mjs";
14
+
15
+
export function go(codepoint) {
16
+
return toBitArray([codepointToUtf16(codepoint, false)]);
17
+
}
+17
compiler-core/src/javascript/tests/snapshots/gleam_core__javascript__tests__bit_arrays__utf32_codepoint_little_endian.snap
+17
compiler-core/src/javascript/tests/snapshots/gleam_core__javascript__tests__bit_arrays__utf32_codepoint_little_endian.snap
···
1
+
---
2
+
source: compiler-core/src/javascript/tests/bit_arrays.rs
3
+
expression: "\npub fn go(codepoint) {\n <<codepoint:utf32_codepoint-little>>\n}\n"
4
+
---
5
+
----- SOURCE CODE
6
+
7
+
pub fn go(codepoint) {
8
+
<<codepoint:utf32_codepoint-little>>
9
+
}
10
+
11
+
12
+
----- COMPILED JAVASCRIPT
13
+
import { toBitArray, codepointToUtf32 } from "../gleam.mjs";
14
+
15
+
export function go(codepoint) {
16
+
return toBitArray([codepointToUtf32(codepoint, false)]);
17
+
}
+30
test/language/test/language_test.gleam
+30
test/language/test/language_test.gleam
···
1094
1094
h
1095
1095
})
1096
1096
}),
1097
+
"pattern-match UTF-16 codepoint little-endian"
1098
+
|> example(fn() {
1099
+
assert_equal(ffi.utf_codepoint(127_757), {
1100
+
let assert <<codepoint:utf16_codepoint-little>> = <<"🌍":utf16-little>>
1101
+
codepoint
1102
+
})
1103
+
}),
1104
+
"pattern-match UTF-32 codepoint little-endian"
1105
+
|> example(fn() {
1106
+
assert_equal(ffi.utf_codepoint(127_757), {
1107
+
let assert <<codepoint:utf32_codepoint-little>> = <<"🌍":utf32-little>>
1108
+
codepoint
1109
+
})
1110
+
}),
1097
1111
]
1098
1112
}
1099
1113
···
1707
1721
<<codepoint:utf16_codepoint>>
1708
1722
})
1709
1723
}),
1724
+
"UTF-16 codepoint little-endian"
1725
+
|> example(fn() {
1726
+
assert_equal(<<60, 216, 13, 223>>, {
1727
+
// 🌍
1728
+
let codepoint = ffi.utf_codepoint(127_757)
1729
+
<<codepoint:utf16_codepoint-little>>
1730
+
})
1731
+
}),
1710
1732
"UTF-32 codepoint"
1711
1733
|> example(fn() {
1712
1734
assert_equal(<<0, 1, 243, 13>>, {
1713
1735
// 🌍
1714
1736
let codepoint = ffi.utf_codepoint(127_757)
1715
1737
<<codepoint:utf32_codepoint>>
1738
+
})
1739
+
}),
1740
+
"UTF-32 codepoint little-endian"
1741
+
|> example(fn() {
1742
+
assert_equal(<<13, 243, 1, 0>>, {
1743
+
// 🌍
1744
+
let codepoint = ffi.utf_codepoint(127_757)
1745
+
<<codepoint:utf32_codepoint-little>>
1716
1746
})
1717
1747
}),
1718
1748
]