mirror of
https://github.com/zjs81/meshcore-open.git
synced 2026-08-11 18:26:27 +10:00
Add ability to send images over lora
This commit is contained in:
Zach
@@ -0,0 +1,137 @@
|
||||
import 'dart:convert';
|
||||
import 'dart:io';
|
||||
import 'dart:typed_data';
|
||||
|
||||
import 'package:flutter_test/flutter_test.dart';
|
||||
import 'package:meshcore_open/services/entropy_tables.dart';
|
||||
|
||||
/// Structural checks on the shipped CDF table file. The invariants here are the
|
||||
/// ones the rANS coder relies on; if any of them breaks, encoding silently
|
||||
/// produces garbage rather than failing loudly.
|
||||
void main() {
|
||||
final Directory goldenDir = _resolveGoldenDir();
|
||||
final Uint8List raw = File(
|
||||
'${goldenDir.path}/aeic_cdf_ft32.bin',
|
||||
).readAsBytesSync();
|
||||
final Map<String, dynamic> manifest =
|
||||
jsonDecode(File('${goldenDir.path}/manifest.json').readAsStringSync())
|
||||
as Map<String, dynamic>;
|
||||
final EntropyTables tables = EntropyTables.parse(raw);
|
||||
|
||||
test('header matches the manifest', () {
|
||||
expect(raw.length, manifest['table_bytes']);
|
||||
expect(tables.version, 1);
|
||||
expect(tables.precision, manifest['precision']);
|
||||
expect(tables.bypassPrecision, manifest['bypass_precision']);
|
||||
expect(tables.streamParts, manifest['stream_parts']);
|
||||
expect(tables.groups.length, 2);
|
||||
});
|
||||
|
||||
test('group shapes match the manifest', () {
|
||||
final List<Map<String, dynamic>> meta =
|
||||
(manifest['table_groups'] as List<dynamic>)
|
||||
.cast<Map<String, dynamic>>();
|
||||
for (var g = 0; g < 2; g++) {
|
||||
final CdfGroup group = tables.groups[g];
|
||||
expect(group.numCdfs, meta[g]['rows'], reason: 'group $g rows');
|
||||
expect(group.cdfWidth, meta[g]['width'], reason: 'group $g width');
|
||||
expect(group.quantizedCdf.length, group.numCdfs * group.cdfWidth);
|
||||
|
||||
var lenMin = 1 << 30, lenMax = -(1 << 30);
|
||||
var offMin = 1 << 30, offMax = -(1 << 30);
|
||||
for (var r = 0; r < group.numCdfs; r++) {
|
||||
lenMin = group.cdfLength[r] < lenMin ? group.cdfLength[r] : lenMin;
|
||||
lenMax = group.cdfLength[r] > lenMax ? group.cdfLength[r] : lenMax;
|
||||
offMin = group.offset[r] < offMin ? group.offset[r] : offMin;
|
||||
offMax = group.offset[r] > offMax ? group.offset[r] : offMax;
|
||||
}
|
||||
expect(lenMin, meta[g]['cdf_length_min']);
|
||||
expect(lenMax, meta[g]['cdf_length_max']);
|
||||
expect(offMin, meta[g]['offset_min']);
|
||||
expect(offMax, meta[g]['offset_max']);
|
||||
}
|
||||
expect(tables.zGroup.numCdfs, 128);
|
||||
expect(tables.zGroup.cdfWidth, 19);
|
||||
expect(tables.yGroup.numCdfs, 64);
|
||||
expect(tables.yGroup.cdfWidth, 3133);
|
||||
});
|
||||
|
||||
test('every CDF row is a valid, gap-free distribution', () {
|
||||
for (var g = 0; g < tables.groups.length; g++) {
|
||||
final CdfGroup group = tables.groups[g];
|
||||
for (var r = 0; r < group.numCdfs; r++) {
|
||||
final int n = group.cdfLength[r];
|
||||
expect(n, greaterThanOrEqualTo(2), reason: 'group $g row $r length');
|
||||
expect(n, lessThanOrEqualTo(group.cdfWidth));
|
||||
expect(group.cdfAt(r, 0), 0, reason: 'group $g row $r first');
|
||||
expect(
|
||||
group.cdfAt(r, n - 1),
|
||||
1 << 16,
|
||||
reason: 'group $g row $r terminal',
|
||||
);
|
||||
for (var c = 0; c + 1 < n; c++) {
|
||||
final int gap = group.cdfAt(r, c + 1) - group.cdfAt(r, c);
|
||||
expect(
|
||||
gap,
|
||||
greaterThanOrEqualTo(1),
|
||||
reason: 'group $g row $r has a zero-frequency symbol at $c',
|
||||
);
|
||||
}
|
||||
for (var c = n; c < group.cdfWidth; c++) {
|
||||
expect(group.cdfAt(r, c), 0, reason: 'group $g row $r padding at $c');
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('index-quantizer block parses', () {
|
||||
final IndexQuantizerParams p = tables.indexQuantizer;
|
||||
expect(p.scalesLevels, 64);
|
||||
expect(p.scaleTable.length, 64);
|
||||
expect(p.logScaleMin, closeTo(-2.2072749131897207, 1e-15));
|
||||
expect(p.logScaleStep, closeTo(0.12305479932808384, 1e-15));
|
||||
expect(p.scaleThreshold, closeTo(0.08, 1e-7));
|
||||
expect(p.scaleFloor, closeTo(1e-5, 1e-11));
|
||||
expect(p.scaleTable.first, closeTo(0.11, 1e-5));
|
||||
expect(p.scaleTable.last, closeTo(256.0, 1e-3));
|
||||
for (var i = 1; i < p.scaleTable.length; i++) {
|
||||
expect(p.scaleTable[i], greaterThan(p.scaleTable[i - 1]));
|
||||
}
|
||||
});
|
||||
|
||||
test('rejects a corrupt magic', () {
|
||||
final Uint8List bad = Uint8List.fromList(raw.sublist(0, 4096));
|
||||
bad[3] ^= 0xFF;
|
||||
expect(
|
||||
() => EntropyTables.parse(bad),
|
||||
throwsA(isA<EntropyTableFormatException>()),
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects a truncated file', () {
|
||||
expect(
|
||||
() => EntropyTables.parse(Uint8List.fromList(raw.sublist(0, 1024))),
|
||||
throwsA(isA<EntropyTableFormatException>()),
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects trailing garbage', () {
|
||||
final Uint8List extra = Uint8List(raw.length + 1)..setRange(0, raw.length, raw);
|
||||
expect(
|
||||
() => EntropyTables.parse(extra),
|
||||
throwsA(isA<EntropyTableFormatException>()),
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
Directory _resolveGoldenDir() {
|
||||
for (final String candidate in <String>[
|
||||
'test/services/golden',
|
||||
'../test/services/golden',
|
||||
'golden',
|
||||
]) {
|
||||
final Directory d = Directory(candidate);
|
||||
if (d.existsSync()) return d;
|
||||
}
|
||||
return Directory('test/services/golden');
|
||||
}
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,33 @@
|
||||
{
|
||||
"format": "aeic-entropy-e2e-recording",
|
||||
"version": 1,
|
||||
"checkpoint": "AEIC_SE_ft32.pkl",
|
||||
"size": 512,
|
||||
"files": [
|
||||
{
|
||||
"file": "kodim01.aeicrec",
|
||||
"bytes": 7378887,
|
||||
"sha256": "dd4278523e2a51d031e11b3d71e90f9ffe863d1244bf3de3896525ff0da9c049"
|
||||
},
|
||||
{
|
||||
"file": "kodim02.aeicrec",
|
||||
"bytes": 7378887,
|
||||
"sha256": "8cc3043664c7c6d993baf6fd37db04f0e22954af114c1463b2000a5e230c2e20"
|
||||
},
|
||||
{
|
||||
"file": "kodim05.aeicrec",
|
||||
"bytes": 7378927,
|
||||
"sha256": "535eef3213b0fdb0238bebad7e3baa041fe62aae1623fd9ec2deebd06c11ce1c"
|
||||
},
|
||||
{
|
||||
"file": "image2.aeicrec",
|
||||
"bytes": 7378902,
|
||||
"sha256": "2779478763c22955bc0e61c22fb555dd0f016f3a5d3629a2272a66d1000d485e"
|
||||
},
|
||||
{
|
||||
"file": "images.aeicrec",
|
||||
"bytes": 7378878,
|
||||
"sha256": "d7153bfd8bb9a1f7bd764eb7fcb43b1932205ac1b43be5b2c6ef6c8a5c57edb1"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,486 @@
|
||||
{
|
||||
"checkpoint": "AEIC_SE_ft32.pkl",
|
||||
"size": 512,
|
||||
"precision": 16,
|
||||
"bypass_precision": 2,
|
||||
"stream_parts": 2,
|
||||
"table_file": "aeic_cdf_ft32.bin",
|
||||
"table_bytes": 813648,
|
||||
"table_sha256": "4089fde2af16c340642a5c857be42f6d0f21caf71dd5b4f32d62efcd41c77bd5",
|
||||
"table_groups": [
|
||||
{
|
||||
"group": "z",
|
||||
"rows": 128,
|
||||
"width": 19,
|
||||
"cdf_length_min": 19,
|
||||
"cdf_length_max": 19,
|
||||
"offset_min": -8,
|
||||
"offset_max": -8,
|
||||
"cdf_max": 65536
|
||||
},
|
||||
{
|
||||
"group": "y",
|
||||
"rows": 64,
|
||||
"width": 3133,
|
||||
"cdf_length_min": 5,
|
||||
"cdf_length_max": 3133,
|
||||
"offset_min": -1565,
|
||||
"offset_max": -1,
|
||||
"cdf_max": 65536
|
||||
}
|
||||
],
|
||||
"z_cdf_group_index": 0,
|
||||
"y_cdf_group_index": 1,
|
||||
"images": [
|
||||
{
|
||||
"image": "kodim01.png",
|
||||
"stem": "kodim01",
|
||||
"bitstream_file": "kodim01.bin",
|
||||
"bitstream_bytes_stat": 136,
|
||||
"bitstream_sha256": "771be01642e69c277e6dd6a1b5fe926d45990883e0a2d56a2fca67f5e29b385d",
|
||||
"container_flag": 17,
|
||||
"container_header_bytes": 2,
|
||||
"substream_sizes": [
|
||||
81,
|
||||
52
|
||||
],
|
||||
"substream_sha256": [
|
||||
"7803b0726155984d86876d260c72c7376fd1e24b49fa31e279e6c88b0b0b3e73",
|
||||
"8b7e01993f42cd08cb08f3f75d2f94e186ea03e772d47a09a58796bf7db33b0b"
|
||||
],
|
||||
"vector_file": "kodim01.gv",
|
||||
"vector_bytes_stat": 270592,
|
||||
"n_z_symbols": 2048,
|
||||
"n_y_symbols_each": 16384,
|
||||
"z_q_min": -1,
|
||||
"z_q_max": 1,
|
||||
"y_q_min": -1,
|
||||
"y_q_max": 1,
|
||||
"y_index_min": 0,
|
||||
"y_index_max": 13,
|
||||
"n_skipped_y_indexes": 0,
|
||||
"roundtrip_bitexact": true
|
||||
},
|
||||
{
|
||||
"image": "kodim02.png",
|
||||
"stem": "kodim02",
|
||||
"bitstream_file": "kodim02.bin",
|
||||
"bitstream_bytes_stat": 135,
|
||||
"bitstream_sha256": "6d158062f4e940a09098fdae516eba541cf1c6c3c4aee39d95031add3bf33d7a",
|
||||
"container_flag": 17,
|
||||
"container_header_bytes": 2,
|
||||
"substream_sizes": [
|
||||
90,
|
||||
42
|
||||
],
|
||||
"substream_sha256": [
|
||||
"a9ab7f345e0733c799da943ac065a21bd407a00e946fab48ec1e1dd8b7bee468",
|
||||
"7f765eebbaf5dfde47e1d1f70eb8a1407d55f59e0edf5839b378b9c3d7a39dd3"
|
||||
],
|
||||
"vector_file": "kodim02.gv",
|
||||
"vector_bytes_stat": 270592,
|
||||
"n_z_symbols": 2048,
|
||||
"n_y_symbols_each": 16384,
|
||||
"z_q_min": -1,
|
||||
"z_q_max": 1,
|
||||
"y_q_min": -1,
|
||||
"y_q_max": 1,
|
||||
"y_index_min": 0,
|
||||
"y_index_max": 13,
|
||||
"n_skipped_y_indexes": 0,
|
||||
"roundtrip_bitexact": true
|
||||
},
|
||||
{
|
||||
"image": "kodim05.png",
|
||||
"stem": "kodim05",
|
||||
"bitstream_file": "kodim05.bin",
|
||||
"bitstream_bytes_stat": 173,
|
||||
"bitstream_sha256": "65ba06f964e1f726f7a72f2c4e634282d60ed16232747bb4a84c1d73ed8411bc",
|
||||
"container_flag": 17,
|
||||
"container_header_bytes": 2,
|
||||
"substream_sizes": [
|
||||
119,
|
||||
51
|
||||
],
|
||||
"substream_sha256": [
|
||||
"12a97d8cd41b03930fa735d867f31756ba499e2d7220ea10f9b5ea96d1f15de3",
|
||||
"c30c0be1d858848fcd6833a66bc6c3fae8b3dedcac463a598621794cef4ff894"
|
||||
],
|
||||
"vector_file": "kodim05.gv",
|
||||
"vector_bytes_stat": 270592,
|
||||
"n_z_symbols": 2048,
|
||||
"n_y_symbols_each": 16384,
|
||||
"z_q_min": -1,
|
||||
"z_q_max": 1,
|
||||
"y_q_min": -1,
|
||||
"y_q_max": 1,
|
||||
"y_index_min": 0,
|
||||
"y_index_max": 14,
|
||||
"n_skipped_y_indexes": 0,
|
||||
"roundtrip_bitexact": true
|
||||
},
|
||||
{
|
||||
"image": "kodim08.png",
|
||||
"stem": "kodim08",
|
||||
"bitstream_file": "kodim08.bin",
|
||||
"bitstream_bytes_stat": 209,
|
||||
"bitstream_sha256": "9e1747db3ec84a993e28b52923140948afbf93df483b4f20be93a0f79206ebc4",
|
||||
"container_flag": 17,
|
||||
"container_header_bytes": 2,
|
||||
"substream_sizes": [
|
||||
136,
|
||||
70
|
||||
],
|
||||
"substream_sha256": [
|
||||
"965f5fd268c6f367b998c827f95f090ba2d945d7e4ebe4003fb604c48711cd03",
|
||||
"19fd4a704675ad1cd3f97aaf2670e7818d465d3fc800a03621dd11e167a77a9f"
|
||||
],
|
||||
"vector_file": "kodim08.gv",
|
||||
"vector_bytes_stat": 270592,
|
||||
"n_z_symbols": 2048,
|
||||
"n_y_symbols_each": 16384,
|
||||
"z_q_min": -1,
|
||||
"z_q_max": 1,
|
||||
"y_q_min": -1,
|
||||
"y_q_max": 1,
|
||||
"y_index_min": 0,
|
||||
"y_index_max": 14,
|
||||
"n_skipped_y_indexes": 0,
|
||||
"roundtrip_bitexact": true
|
||||
},
|
||||
{
|
||||
"image": "kodim13.png",
|
||||
"stem": "kodim13",
|
||||
"bitstream_file": "kodim13.bin",
|
||||
"bitstream_bytes_stat": 118,
|
||||
"bitstream_sha256": "75e13bfca9c13400c62aaab08648fa408a4e4cdb13d02ce3e64767f08728cc64",
|
||||
"container_flag": 17,
|
||||
"container_header_bytes": 2,
|
||||
"substream_sizes": [
|
||||
64,
|
||||
51
|
||||
],
|
||||
"substream_sha256": [
|
||||
"5b1b045e0c850df548bfdc3cf5777bcc1050d226365a5a3ab3dc1cdfd76cd077",
|
||||
"81b80624237a8796034d94281be0aa3088eebf8124218d7f4a02257473d2fbcb"
|
||||
],
|
||||
"vector_file": "kodim13.gv",
|
||||
"vector_bytes_stat": 270592,
|
||||
"n_z_symbols": 2048,
|
||||
"n_y_symbols_each": 16384,
|
||||
"z_q_min": -1,
|
||||
"z_q_max": 1,
|
||||
"y_q_min": -1,
|
||||
"y_q_max": 1,
|
||||
"y_index_min": 0,
|
||||
"y_index_max": 14,
|
||||
"n_skipped_y_indexes": 0,
|
||||
"roundtrip_bitexact": true
|
||||
},
|
||||
{
|
||||
"image": "kodim19.png",
|
||||
"stem": "kodim19",
|
||||
"bitstream_file": "kodim19.bin",
|
||||
"bitstream_bytes_stat": 170,
|
||||
"bitstream_sha256": "0ac1e2f5981c7afb8c3efd5cc2c14acf4b53d949e4dd2f2701cf377068d62e0e",
|
||||
"container_flag": 17,
|
||||
"container_header_bytes": 2,
|
||||
"substream_sizes": [
|
||||
104,
|
||||
63
|
||||
],
|
||||
"substream_sha256": [
|
||||
"d9f566fbde8f3da32f738004a7e88764cf52d8df04c7905f53c0018057899e6f",
|
||||
"790c717eb572cb07ac17c4324424293559a127de8299ffd4eaeb02f2d2e4f200"
|
||||
],
|
||||
"vector_file": "kodim19.gv",
|
||||
"vector_bytes_stat": 270592,
|
||||
"n_z_symbols": 2048,
|
||||
"n_y_symbols_each": 16384,
|
||||
"z_q_min": -1,
|
||||
"z_q_max": 1,
|
||||
"y_q_min": -1,
|
||||
"y_q_max": 1,
|
||||
"y_index_min": 0,
|
||||
"y_index_max": 13,
|
||||
"n_skipped_y_indexes": 0,
|
||||
"roundtrip_bitexact": true
|
||||
},
|
||||
{
|
||||
"image": "kodim23.png",
|
||||
"stem": "kodim23",
|
||||
"bitstream_file": "kodim23.bin",
|
||||
"bitstream_bytes_stat": 206,
|
||||
"bitstream_sha256": "1728c415e5a35f9fd501a35369166db8435e3fcfa2fabe99dcd38f21782f6ed7",
|
||||
"container_flag": 17,
|
||||
"container_header_bytes": 2,
|
||||
"substream_sizes": [
|
||||
135,
|
||||
68
|
||||
],
|
||||
"substream_sha256": [
|
||||
"b169be22ee0554ce83b47118534a25066c4dbb21b3c11e16e69c36eb905d485c",
|
||||
"c3dd848ad555641febdbad1d5cfe7516965362e735eb36172fed096c1f8fa69e"
|
||||
],
|
||||
"vector_file": "kodim23.gv",
|
||||
"vector_bytes_stat": 270592,
|
||||
"n_z_symbols": 2048,
|
||||
"n_y_symbols_each": 16384,
|
||||
"z_q_min": -2,
|
||||
"z_q_max": 1,
|
||||
"y_q_min": -2,
|
||||
"y_q_max": 2,
|
||||
"y_index_min": -1,
|
||||
"y_index_max": 14,
|
||||
"n_skipped_y_indexes": 1,
|
||||
"roundtrip_bitexact": true
|
||||
},
|
||||
{
|
||||
"image": "kodim24.png",
|
||||
"stem": "kodim24",
|
||||
"bitstream_file": "kodim24.bin",
|
||||
"bitstream_bytes_stat": 154,
|
||||
"bitstream_sha256": "9c397ca0212cc6aeaea687c2fc8dc3b88a95a6c153108a056e4fc7254a21b371",
|
||||
"container_flag": 17,
|
||||
"container_header_bytes": 2,
|
||||
"substream_sizes": [
|
||||
91,
|
||||
60
|
||||
],
|
||||
"substream_sha256": [
|
||||
"4d83ea0c3d9f58795068962f02a9eea9d7ad91afaebe0e330c20e95f695e22c1",
|
||||
"65c7cdc5f581934b21ea8a16aff1a67dd36118c58933997fc776b99c7f317525"
|
||||
],
|
||||
"vector_file": "kodim24.gv",
|
||||
"vector_bytes_stat": 270592,
|
||||
"n_z_symbols": 2048,
|
||||
"n_y_symbols_each": 16384,
|
||||
"z_q_min": -1,
|
||||
"z_q_max": 1,
|
||||
"y_q_min": -1,
|
||||
"y_q_max": 2,
|
||||
"y_index_min": 0,
|
||||
"y_index_max": 14,
|
||||
"n_skipped_y_indexes": 0,
|
||||
"roundtrip_bitexact": true
|
||||
},
|
||||
{
|
||||
"image": "image2.webp",
|
||||
"stem": "image2",
|
||||
"bitstream_file": "image2.bin",
|
||||
"bitstream_bytes_stat": 147,
|
||||
"bitstream_sha256": "258fe68ff7bb9ad41e1b3c7f885c56d051b3d55d313e5e5cc04b4faaf3b181e0",
|
||||
"container_flag": 17,
|
||||
"container_header_bytes": 2,
|
||||
"substream_sizes": [
|
||||
88,
|
||||
56
|
||||
],
|
||||
"substream_sha256": [
|
||||
"60c3ec394f474c060bfe51f4a086ab40d6f409ab4694ff8b02b68f15029116be",
|
||||
"d874295cab097586ca1da2026a89a694051baf832fe3e4cd52f03fafbdf896c8"
|
||||
],
|
||||
"vector_file": "image2.gv",
|
||||
"vector_bytes_stat": 270592,
|
||||
"n_z_symbols": 2048,
|
||||
"n_y_symbols_each": 16384,
|
||||
"z_q_min": -3,
|
||||
"z_q_max": 2,
|
||||
"y_q_min": -1,
|
||||
"y_q_max": 1,
|
||||
"y_index_min": -1,
|
||||
"y_index_max": 16,
|
||||
"n_skipped_y_indexes": 1,
|
||||
"roundtrip_bitexact": true
|
||||
},
|
||||
{
|
||||
"image": "images.jpeg",
|
||||
"stem": "images",
|
||||
"bitstream_file": "images.bin",
|
||||
"bitstream_bytes_stat": 128,
|
||||
"bitstream_sha256": "2ad2224c85c25c394daf0c6e35d0c8fba194b689863f42ebe85a0d67dd632869",
|
||||
"container_flag": 17,
|
||||
"container_header_bytes": 2,
|
||||
"substream_sizes": [
|
||||
75,
|
||||
50
|
||||
],
|
||||
"substream_sha256": [
|
||||
"706a74709539f2d63f0b9b58d78920c9dfe995a804a4af19ded31b0c51f9adde",
|
||||
"d35ab3320032adaad85343918c5e118fcb3aea20168024e433b608acc6d918ab"
|
||||
],
|
||||
"vector_file": "images.gv",
|
||||
"vector_bytes_stat": 270592,
|
||||
"n_z_symbols": 2048,
|
||||
"n_y_symbols_each": 16384,
|
||||
"z_q_min": -1,
|
||||
"z_q_max": 1,
|
||||
"y_q_min": -1,
|
||||
"y_q_max": 1,
|
||||
"y_index_min": 0,
|
||||
"y_index_max": 13,
|
||||
"n_skipped_y_indexes": 0,
|
||||
"roundtrip_bitexact": true
|
||||
}
|
||||
],
|
||||
"synthetic": [
|
||||
{
|
||||
"name": "z_escape_exact",
|
||||
"cdf_group": 0,
|
||||
"n": 640,
|
||||
"n_filler_each_end": 256,
|
||||
"bitstream_file": "synth_z_escape_exact.bin",
|
||||
"bitstream_bytes_stat": 307,
|
||||
"bitstream_sha256": "43918b9fc2cd685ce619632b83cb00d99b763e7476e2ea08367e1a88d1040c44",
|
||||
"container_flag": 17,
|
||||
"container_header_bytes": 2,
|
||||
"substream_sizes": [
|
||||
152,
|
||||
152
|
||||
],
|
||||
"vector_file": "synth_z_escape_exact.gv",
|
||||
"vector_bytes_stat": 2624,
|
||||
"expected_decode_sha256": "230e5130f0690bc31122a475f2d04495244ea1c2dcc663c74e9d4182425d9255",
|
||||
"sym_min": 0,
|
||||
"sym_max": 9,
|
||||
"n_skipped": 0,
|
||||
"roundtrip_exact": true
|
||||
},
|
||||
{
|
||||
"name": "z_escape_mixed",
|
||||
"cdf_group": 0,
|
||||
"n": 256,
|
||||
"n_filler_each_end": 64,
|
||||
"bitstream_file": "synth_z_escape_mixed.bin",
|
||||
"bitstream_bytes_stat": 443,
|
||||
"bitstream_sha256": "fc3088f9608c0c8e56876d0034d1c8ab45ec34c83dd6491c26981836b4f8075f",
|
||||
"container_flag": 17,
|
||||
"container_header_bytes": 2,
|
||||
"substream_sizes": [
|
||||
211,
|
||||
229
|
||||
],
|
||||
"vector_file": "synth_z_escape_mixed.gv",
|
||||
"vector_bytes_stat": 1088,
|
||||
"expected_decode_sha256": "a428f456dc7f31ecae1c1fc5fed8517bcd585783f03f3e3423fbf3649745501a",
|
||||
"sym_min": -72,
|
||||
"sym_max": 138,
|
||||
"n_skipped": 0,
|
||||
"roundtrip_exact": true
|
||||
},
|
||||
{
|
||||
"name": "z_dense_normal",
|
||||
"cdf_group": 0,
|
||||
"n": 10240,
|
||||
"n_filler_each_end": 4096,
|
||||
"bitstream_file": "synth_z_dense_normal.bin",
|
||||
"bitstream_bytes_stat": 3841,
|
||||
"bitstream_sha256": "0d2c604369c89081df11ac39a2c3218f3e4bcfe208b32fb9a716ea2ea292be8a",
|
||||
"container_flag": 17,
|
||||
"container_header_bytes": 2,
|
||||
"substream_sizes": [
|
||||
1891,
|
||||
1947
|
||||
],
|
||||
"vector_file": "synth_z_dense_normal.gv",
|
||||
"vector_bytes_stat": 41024,
|
||||
"expected_decode_sha256": "a293328cba494459297083a0e7c465af5f8ee339c701e5d84bef749ace2d9786",
|
||||
"sym_min": -8,
|
||||
"sym_max": 7,
|
||||
"n_skipped": 0,
|
||||
"roundtrip_exact": true
|
||||
},
|
||||
{
|
||||
"name": "y_bypass_long",
|
||||
"cdf_group": 1,
|
||||
"n": 640,
|
||||
"n_filler_each_end": 64,
|
||||
"bitstream_file": "synth_y_bypass_long.bin",
|
||||
"bitstream_bytes_stat": 2251,
|
||||
"bitstream_sha256": "faa8b40f9ce682dc10c1cbb6471c684182ed4fa32023ce915513598147f24c99",
|
||||
"container_flag": 17,
|
||||
"container_header_bytes": 2,
|
||||
"substream_sizes": [
|
||||
1163,
|
||||
1085
|
||||
],
|
||||
"vector_file": "synth_y_bypass_long.gv",
|
||||
"vector_bytes_stat": 2624,
|
||||
"expected_decode_sha256": "f842f136f187644884733ffd83db78647ead2faeec29d2be3d53322159e2589c",
|
||||
"sym_min": -32000,
|
||||
"sym_max": 32000,
|
||||
"n_skipped": 0,
|
||||
"roundtrip_exact": true
|
||||
},
|
||||
{
|
||||
"name": "y_skip_indexes",
|
||||
"cdf_group": 1,
|
||||
"n": 384,
|
||||
"n_filler_each_end": 128,
|
||||
"bitstream_file": "synth_y_skip_indexes.bin",
|
||||
"bitstream_bytes_stat": 160,
|
||||
"bitstream_sha256": "4144dd2271f54d7ceab9001dbb43601839a08176d576cad15a3fe02502361be7",
|
||||
"container_flag": 17,
|
||||
"container_header_bytes": 2,
|
||||
"substream_sizes": [
|
||||
78,
|
||||
79
|
||||
],
|
||||
"vector_file": "synth_y_skip_indexes.gv",
|
||||
"vector_bytes_stat": 1600,
|
||||
"expected_decode_sha256": "8e17e97e31d0e687efec4ae1f6bd3d7ddd80d73f6a3bbf1bcf72072062069ae3",
|
||||
"sym_min": -1564,
|
||||
"sym_max": 12345,
|
||||
"n_skipped": 43,
|
||||
"roundtrip_exact": true
|
||||
},
|
||||
{
|
||||
"name": "y_edges_normal",
|
||||
"cdf_group": 1,
|
||||
"n": 1280,
|
||||
"n_filler_each_end": 512,
|
||||
"bitstream_file": "synth_y_edges_normal.bin",
|
||||
"bitstream_bytes_stat": 470,
|
||||
"bitstream_sha256": "ae336f2000c2f6548e897078cf429e85fa1efd6df7973a6257b6495072e5f8ec",
|
||||
"container_flag": 17,
|
||||
"container_header_bytes": 2,
|
||||
"substream_sizes": [
|
||||
207,
|
||||
260
|
||||
],
|
||||
"vector_file": "synth_y_edges_normal.gv",
|
||||
"vector_bytes_stat": 5184,
|
||||
"expected_decode_sha256": "ff25d3624640925c557155e83d02fd2fec0b25de8f4b3320fcc6c1db2590c6ac",
|
||||
"sym_min": -1565,
|
||||
"sym_max": 1564,
|
||||
"n_skipped": 0,
|
||||
"roundtrip_exact": true
|
||||
},
|
||||
{
|
||||
"name": "y_tiny",
|
||||
"cdf_group": 1,
|
||||
"n": 130,
|
||||
"n_filler_each_end": 64,
|
||||
"bitstream_file": "synth_y_tiny.bin",
|
||||
"bitstream_bytes_stat": 13,
|
||||
"bitstream_sha256": "b5df63c5644dc0b7c599a2009de025721c3ff439eed06b427deee1439dc5a84c",
|
||||
"container_flag": 17,
|
||||
"container_header_bytes": 2,
|
||||
"substream_sizes": [
|
||||
6,
|
||||
4
|
||||
],
|
||||
"vector_file": "synth_y_tiny.gv",
|
||||
"vector_bytes_stat": 584,
|
||||
"expected_decode_sha256": "e851be60ef0e9dc7488caaf7ba6e35ecccb933ce6775829d0fbd9dc3010d1d49",
|
||||
"sym_min": -1,
|
||||
"sym_max": 0,
|
||||
"n_skipped": 0,
|
||||
"roundtrip_exact": true
|
||||
}
|
||||
],
|
||||
"reference_port_selfcheck": {
|
||||
"ok": 17,
|
||||
"total": 17
|
||||
}
|
||||
}
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,3 @@
|
||||
‹Ζ;Ιa�/ώ��Ϋ����ςpώ�ο=��� �χ��υ�� φ��Υa��/ώ��Ϋ����ςpώ�ο=��� �χ��υ�� φ��Υa��/ώ��Ϋ����ςpώ�ο=��� �χ��υ�� φ��Υa��/ώ��Ϋ����ςpώ�ο=��� �χ��υ�� φ��Υa��―~ώ��»����κpώ�ο;����ώχ��ν��� φ��Νa��―~ώ��»����κpώ�ο;����ώχ��ν��� φ��Νa��―~ώ��»����κpώ�ο;����ώχ��ν��� φ��Νa��―~ώ��»����κpώ�ο;����ώχ��ν��� φ��Νa��―~ώ��»����κpώ�ο;����ώχ��ν��� φ��Νa��/~ώ��›����βpώ�ο9��� ώχ��ε�� φ��Εa��/~ώ��›����βpώ�ο9��� ώχ��ε�� φ��Εa��‹����ζη��/η��›Γω�ίψί��—�ύ� φ��Εa��/~ώ��›����βpώ�ο9��� ώχ��ε�� φ��Εa��―}ώ��{����Ϊpώ�ο7����ύχ��έ���φ��½a��―}ώ��{����Ϊpώ�ο7����ύχ��έ���φ��½a��/}ώ��[�����pώ�ο5��� ύχ��Υ�� φ��µa��/}ώ?�[�����pώ�ο5��� ύχ��Υ�� φΟ�µaώ�+�ί�ο|Ύ�Κpφώο η?�Ιϋ�7�ω��¶�ίφ��Βη��ορω��/ η��Γω��Α��_όχ��΅a��_φ��Βη��ορω��/ η�Γω��Α��_όχ��΅a��_¶�οξω��ϋ�οΏο.�ύ�Ύpξ��ϋχ�ίοί�ίf/��aώ�²ηηώo{>Ώο,�ώ�¶pΦ�Ηώε�_ϋ‡� ζ�_φ��Άη��οιω��/
|
||||
η��›Βω��΅��_ϊχ���a��_φ���η��οηω��― η��{Βω��™��ίωχ��ya��ίφ��’η��οεω��/ η��[Βω��‘��_ωχ��qa��_φ��‚η��οαω��/η��Βω�����_ψχ��aa�ώ_φ�/w>Ώοέωύ�rpξ�vp®�Ηύυ�Χύ�G…�Ua��/vώ��›����bpώ�ο��� φχ��e�� φ?�Eaη�JηΏϋ;��+Α9�οg��τ·ώίτW�§„ύ�·„ύ�οΞω��οsώ��λΐω��>pώ�ίΞί��χόύ�ίFΨ��w„ύ�οΘω��orώ��‹ΐω��&pώ�ίΘί��—όύ�ί@Ψ?�„}�ηΏ�η�� ΐωχΐω�ό½�ό}�α`ΰ 9��o��βζ��οΉω��/ώζ��›Ώω��α~��_ξχ��Α`��_ φ��Βζ��ο±ω��/όζ��Ώω��Α~��_μχ��΅`��_
|
||||
φ��’ζ��ο¥ω��/ωζ��[Ύω��‘~��_ιχ��q`��_φ��jζ��ο›ω��―φζ��»½ω��i~��ίζχ��I`��ίφ��2ζ��ο�ω��/σζ�ΫΌω��1~��_γχ��`�ύ_φϋλ—�σϋ—�ϋλ»ωχϋ»ω�ηχyίίό�ύυ�wύο²ε�οmω�﬛ίο›ϋίlί�Χφύ�‘_�ίεΧ��«•���nε��ο�›��οζζ��§υύ�ί[ί��'}ύ�ίΣΧ��K”���ε��ο„›��oαζ��Gτύ�ίEί��Η{ύϊ�½γΛ’�Ϋ’Οο²mR�¶mjί,ίϋ_Λ�‘^�_ιuί ‘οFδ_οBm οQ›‡ Δw�ρA_�Χ�—xΥώ―<ώοσψω�ΚlκώοΜζ��ς^�_σή�_j—��]��Bγ�����/Δζ�ο›��A{��E{��!]��%]��/*ώ��›����Άkώ�οι��� �χ��¥z�� Θυ��…\��― ώ�»‡���κjώ�ο»�����χ��νy���Όυ �Ν[ω�"ασ�›„��/Ά¦��&jή�‡δυ�—δε� °U�_°υ��Bΰ��οψ��/”ζ��¥ω��Ax��_„χ��!Z��_Άυ�―σύ�οΟχ��:hώ�ϋ ω��sχ�ίsχ��‘υ�ί‘υ�―αύ�ο‡χ��gώ�{�ω��aχ�ίaχ��υ�ίυ��Κά��ο3χ��―\ζ�;—ω��Ιt��ίLχ��©V�ώίjΥ�―µ=�οΧφϋο™ς�^d��gΝω�wΝΥ�ηTΩ�χTύ�Λf��o›½�ο¬���¶bώ�ΗΖύ�ΧΖύ�GNύ�WNρ�K_Ώώo}ύΏο4�ψ�Φ`ϊ� ύv�_ύ–� u�_ϊ/[½ώo[=?ο¬—τ�¶^Ζ�Η¶ύ�µmυ�‘Oρ�•OχοΤτι�VΣί�R\Ζ�[q™�G½�Wε�1Mρ�5Mύο&τχ��Πί�kfyόο™ε�_&Z� 'Z�_��ώ��’�οbσ�οcσϊ��Vξ�;Z©�'–ρ�7–΅�§ ¥�mGν�ΚΣ�[(_�KLΙύo1ε�_„�_!6ώ ?„�_?τώ―bΌο‹ρϋ�«<Ή�οςδ���Ώ�-^χ� @ω�
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,84 @@
|
||||
import 'package:flutter_test/flutter_test.dart';
|
||||
import 'package:meshcore_open/models/image_codec_support.dart';
|
||||
import 'package:meshcore_open/services/image_codec_session_io.dart';
|
||||
|
||||
/// Connection tests for the codec worker's boot payload.
|
||||
///
|
||||
/// Two bugs in this feature had the same shape and both survived a green suite:
|
||||
/// `imageCodecRansCoderBuilder` was declared, documented and consumed but never
|
||||
/// assigned; and `entropyDecodeGraphPath` was resolved on the main isolate but
|
||||
/// never put in the list handed to the worker, so every decode threw
|
||||
/// [ImageCodecBundleIncomplete] while `canDecode` cheerfully reported true.
|
||||
///
|
||||
/// Neither was a broken component. Both were missing *connections*, and unit
|
||||
/// tests that exercise each side in isolation cannot see them. These tests
|
||||
/// assert the payload itself: what spawn() sends is exactly what the worker
|
||||
/// needs to rebuild a bundle that can decode.
|
||||
void main() {
|
||||
ImageCodecBundle fiveAssetBundle() => const ImageCodecBundle(
|
||||
decoderGraphPath: '/models/aeic_decoder_qdq_conv_pct.onnx',
|
||||
entropyGraphPath: '/models/aeic_entropy_side_fp32_op17.onnx',
|
||||
entropyDecodeGraphPath: '/models/aeic_entropy_decode_fp32_op17.onnx',
|
||||
tablesPath: '/models/aeic_cdf_ft32.bin',
|
||||
ratePoint: AeicRatePoint.ft32,
|
||||
);
|
||||
|
||||
group('codec worker boot payload', () {
|
||||
test('a five-asset bundle survives the round trip and can still decode', () {
|
||||
final sent = fiveAssetBundle();
|
||||
expect(sent.supportsDecode, isTrue, reason: 'precondition');
|
||||
|
||||
final rebuilt = debugBundleFromBootPayload(
|
||||
debugBootPayloadFor(sent),
|
||||
);
|
||||
|
||||
expect(rebuilt.decoderGraphPath, sent.decoderGraphPath);
|
||||
expect(rebuilt.entropyGraphPath, sent.entropyGraphPath);
|
||||
expect(rebuilt.tablesPath, sent.tablesPath);
|
||||
expect(rebuilt.ratePoint, sent.ratePoint);
|
||||
// The one that was missing. Without it the worker builds a bundle whose
|
||||
// supportsDecode is false and every decode throws, on a correct install.
|
||||
expect(rebuilt.entropyDecodeGraphPath, sent.entropyDecodeGraphPath);
|
||||
expect(rebuilt.supportsDecode, isTrue);
|
||||
});
|
||||
|
||||
test('no field is silently reindexed by the positional layout', () {
|
||||
// The payload is a positional List. Inserting a slot anywhere but the end
|
||||
// shifts every field after it, and the casts are permissive enough that
|
||||
// tablesPath could arrive as the rate point without throwing. Pin each
|
||||
// slot to its meaning so a future insert fails here rather than in the
|
||||
// field, where it would read as a mysterious wrong-model error.
|
||||
final payload = debugBootPayloadFor(fiveAssetBundle());
|
||||
expect(payload[1], '/models/aeic_decoder_qdq_conv_pct.onnx');
|
||||
expect(payload[2], '/models/aeic_entropy_side_fp32_op17.onnx');
|
||||
expect(payload[3], '/models/aeic_cdf_ft32.bin');
|
||||
expect(payload[4], AeicRatePoint.ft32.wireValue);
|
||||
expect(payload[6], '/models/aeic_entropy_decode_fp32_op17.onnx');
|
||||
});
|
||||
|
||||
test('a send-only bundle rebuilds as send-only rather than half-decoding',
|
||||
() {
|
||||
const sendOnly = ImageCodecBundle(
|
||||
decoderGraphPath: '/models/decoder.onnx',
|
||||
entropyGraphPath: '/models/entropy.onnx',
|
||||
tablesPath: '/models/tables.bin',
|
||||
ratePoint: AeicRatePoint.ft32,
|
||||
);
|
||||
final rebuilt = debugBundleFromBootPayload(
|
||||
debugBootPayloadFor(sendOnly),
|
||||
);
|
||||
expect(rebuilt.entropyDecodeGraphPath, isNull);
|
||||
expect(rebuilt.supportsDecode, isFalse);
|
||||
});
|
||||
|
||||
test('a short payload does not crash the worker', () {
|
||||
// An older sender, or a truncated message, must degrade to "cannot
|
||||
// decode" rather than throwing a RangeError inside the isolate where the
|
||||
// failure would surface as an opaque spawn error.
|
||||
final short = debugBootPayloadFor(fiveAssetBundle()).sublist(0, 6);
|
||||
final rebuilt = debugBundleFromBootPayload(short);
|
||||
expect(rebuilt.entropyDecodeGraphPath, isNull);
|
||||
expect(rebuilt.supportsDecode, isFalse);
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,723 @@
|
||||
import 'dart:convert';
|
||||
import 'dart:io';
|
||||
import 'dart:typed_data';
|
||||
|
||||
import 'package:crypto/crypto.dart' as crypto;
|
||||
import 'package:flutter_test/flutter_test.dart';
|
||||
import 'package:http/http.dart' as http;
|
||||
import 'package:http/testing.dart';
|
||||
import 'package:meshcore_open/models/image_codec_support.dart';
|
||||
import 'package:meshcore_open/services/app_settings_service.dart';
|
||||
import 'package:meshcore_open/services/image_codec_file_store.dart';
|
||||
import 'package:meshcore_open/services/image_codec_service.dart';
|
||||
import 'package:meshcore_open/services/image_codec_settings_store.dart';
|
||||
|
||||
/// Real file store, redirected at a temp directory.
|
||||
///
|
||||
/// Subclassed rather than faked on purpose: the resume logic's whole premise is
|
||||
/// that a partial file's *length on disk* is its progress marker, so a test that
|
||||
/// mocked the filesystem would be testing the mock. Only the one method that
|
||||
/// needs `path_provider` (unavailable in a unit test) is overridden.
|
||||
class _TempFileStore extends ImageCodecFileStore {
|
||||
final String root;
|
||||
|
||||
_TempFileStore(this.root);
|
||||
|
||||
@override
|
||||
Future<String> modelDirectoryPath() async => root;
|
||||
}
|
||||
|
||||
/// Deterministic pseudo-random bytes, so a sliced Range response can be checked
|
||||
/// byte-for-byte against the source.
|
||||
Uint8List _body(int length, [int seed = 7]) {
|
||||
final bytes = Uint8List(length);
|
||||
var state = seed | 1;
|
||||
for (var i = 0; i < length; i++) {
|
||||
state = (state * 1103515245 + 12345) & 0x7FFFFFFF;
|
||||
bytes[i] = (state >> 16) & 0xFF;
|
||||
}
|
||||
return bytes;
|
||||
}
|
||||
|
||||
String _sha256(List<int> bytes) => crypto.sha256.convert(bytes).toString();
|
||||
|
||||
/// A record of every range the client asked for, as `'start-end'`.
|
||||
class _Log {
|
||||
final List<String> ranges = [];
|
||||
int headCount = 0;
|
||||
}
|
||||
|
||||
/// Serves [assets] (by last path segment) over Range requests.
|
||||
///
|
||||
/// [failAtOffset] makes exactly one range request die half-way through its body,
|
||||
/// which is what an interrupted 872 MB transfer looks like from Dart's side.
|
||||
http.Client Function() _server(
|
||||
Map<String, Uint8List> assets,
|
||||
_Log log, {
|
||||
int? failAtOffset,
|
||||
bool acceptRanges = true,
|
||||
}) {
|
||||
return () => MockClient.streaming((request, _) async {
|
||||
final name = request.url.pathSegments.last;
|
||||
final data = assets[name];
|
||||
if (data == null) {
|
||||
return http.StreamedResponse(const Stream<List<int>>.empty(), 404);
|
||||
}
|
||||
if (request.method == 'HEAD') {
|
||||
log.headCount++;
|
||||
return http.StreamedResponse(
|
||||
const Stream<List<int>>.empty(),
|
||||
200,
|
||||
contentLength: data.length,
|
||||
headers: acceptRanges ? {'accept-ranges': 'bytes'} : const {},
|
||||
);
|
||||
}
|
||||
final range = request.headers['Range'];
|
||||
if (range == null) {
|
||||
return http.StreamedResponse(
|
||||
Stream<List<int>>.value(data),
|
||||
200,
|
||||
contentLength: data.length,
|
||||
);
|
||||
}
|
||||
final match = RegExp(r'bytes=(\d+)-(\d+)').firstMatch(range)!;
|
||||
final start = int.parse(match.group(1)!);
|
||||
final end = int.parse(match.group(2)!);
|
||||
log.ranges.add('$start-$end');
|
||||
final slice = data.sublist(start, end + 1);
|
||||
|
||||
Stream<List<int>> stream() async* {
|
||||
if (failAtOffset == start) {
|
||||
yield slice.sublist(0, slice.length ~/ 2);
|
||||
throw const SocketException('connection reset by peer');
|
||||
}
|
||||
// Several chunks, so a mid-stream failure is a realistic partial write.
|
||||
const pieces = 4;
|
||||
final step = (slice.length / pieces).ceil();
|
||||
for (var i = 0; i < slice.length; i += step) {
|
||||
yield slice.sublist(i, (i + step).clamp(0, slice.length));
|
||||
}
|
||||
}
|
||||
|
||||
return http.StreamedResponse(
|
||||
stream(),
|
||||
206,
|
||||
contentLength: slice.length,
|
||||
headers: {'content-range': 'bytes $start-$end/${data.length}'},
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
void main() {
|
||||
late Directory tempDir;
|
||||
late _TempFileStore store;
|
||||
|
||||
setUp(() async {
|
||||
tempDir = await Directory.systemTemp.createTemp('image_codec_dl');
|
||||
store = _TempFileStore(tempDir.path);
|
||||
});
|
||||
|
||||
tearDown(() async {
|
||||
if (tempDir.existsSync()) {
|
||||
await tempDir.delete(recursive: true);
|
||||
}
|
||||
});
|
||||
|
||||
ImageCodecService serviceWith(http.Client Function() clientFactory) {
|
||||
return ImageCodecService(
|
||||
AppSettingsService(),
|
||||
fileStore: store,
|
||||
settingsStore: InMemoryImageCodecSettingsStore(),
|
||||
clientFactory: clientFactory,
|
||||
);
|
||||
}
|
||||
|
||||
// 12 MiB clears the 10 MB threshold, so this asset takes the 8-way ranged
|
||||
// path; the three small ones take the plain GET path. Both are exercised.
|
||||
// The shape mirrors the real bundle: a small decoder graph, a huge weights
|
||||
// sibling, a mid-size entropy graph and a tiny table file.
|
||||
final large = _body(12 * 1024 * 1024, 3);
|
||||
final small = _body(4096, 11);
|
||||
final entropy = _body(65536, 23);
|
||||
final entropyDecode = _body(32768, 31);
|
||||
final tables = _body(2048, 29);
|
||||
|
||||
/// The five-role spec. Digests default to empty (verification skipped), which
|
||||
/// is the shipping state until the weights are published.
|
||||
ImageCodecModelSpec spec({
|
||||
String? largeDigest,
|
||||
String? smallDigest,
|
||||
String? entropyDigest,
|
||||
String? entropyDecodeDigest,
|
||||
String? tablesDigest,
|
||||
}) {
|
||||
return ImageCodecModelSpec(
|
||||
id: 'test-model',
|
||||
label: 'Test model',
|
||||
ratePoint: AeicRatePoint.ft32,
|
||||
assets: [
|
||||
ImageCodecModelAsset(
|
||||
role: ImageCodecAssetRole.decoderGraph,
|
||||
fileName: 'model.onnx',
|
||||
sourceUrl: 'https://example.invalid/repo/model.onnx',
|
||||
sizeBytes: small.length,
|
||||
sha256: smallDigest ?? '',
|
||||
),
|
||||
ImageCodecModelAsset(
|
||||
role: ImageCodecAssetRole.decoderWeights,
|
||||
fileName: 'model.onnx.data',
|
||||
sourceUrl: 'https://example.invalid/repo/model.onnx.data',
|
||||
sizeBytes: large.length,
|
||||
sha256: largeDigest ?? '',
|
||||
),
|
||||
ImageCodecModelAsset(
|
||||
role: ImageCodecAssetRole.entropyGraph,
|
||||
fileName: 'entropy.onnx',
|
||||
sourceUrl: 'https://example.invalid/repo/entropy.onnx',
|
||||
sizeBytes: entropy.length,
|
||||
sha256: entropyDigest ?? '',
|
||||
),
|
||||
ImageCodecModelAsset(
|
||||
role: ImageCodecAssetRole.entropyDecodeGraph,
|
||||
fileName: 'entropy_decode.onnx',
|
||||
sourceUrl: 'https://example.invalid/repo/entropy_decode.onnx',
|
||||
sizeBytes: entropyDecode.length,
|
||||
sha256: entropyDecodeDigest ?? '',
|
||||
),
|
||||
ImageCodecModelAsset(
|
||||
role: ImageCodecAssetRole.cdfTables,
|
||||
fileName: 'cdf.bin',
|
||||
sourceUrl: 'https://example.invalid/repo/cdf.bin',
|
||||
sizeBytes: tables.length,
|
||||
sha256: tablesDigest ?? '',
|
||||
),
|
||||
],
|
||||
);
|
||||
}
|
||||
|
||||
ImageCodecModelSpec verifiedSpec() => spec(
|
||||
smallDigest: _sha256(small),
|
||||
largeDigest: _sha256(large),
|
||||
entropyDigest: _sha256(entropy),
|
||||
entropyDecodeDigest: _sha256(entropyDecode),
|
||||
tablesDigest: _sha256(tables),
|
||||
);
|
||||
|
||||
final serverAssets = <String, Uint8List>{
|
||||
'model.onnx': small,
|
||||
'model.onnx.data': large,
|
||||
'entropy.onnx': entropy,
|
||||
'entropy_decode.onnx': entropyDecode,
|
||||
'cdf.bin': tables,
|
||||
};
|
||||
|
||||
group('downloadPresetModel', () {
|
||||
test('fetches all five assets of the bundle and verifies each', () async {
|
||||
final log = _Log();
|
||||
final service = serviceWith(_server(serverAssets, log));
|
||||
addTearDown(service.dispose);
|
||||
|
||||
final record = await service.downloadPresetModel(verifiedSpec());
|
||||
|
||||
// The record points at the DECODER GRAPH, not the weights and not
|
||||
// whichever asset happened to be first: that is the path ONNX Runtime is
|
||||
// handed.
|
||||
expect(record.name, 'model.onnx');
|
||||
expect(record.localPath, '${tempDir.path}/model.onnx');
|
||||
// ...but the recorded size is the whole bundle, because that is what the
|
||||
// user gave up on their device.
|
||||
expect(
|
||||
record.fileSizeBytes,
|
||||
small.length +
|
||||
large.length +
|
||||
entropy.length +
|
||||
entropyDecode.length +
|
||||
tables.length,
|
||||
);
|
||||
expect(record.assetFileNames, [
|
||||
'model.onnx',
|
||||
'model.onnx.data',
|
||||
'entropy.onnx',
|
||||
'entropy_decode.onnx',
|
||||
'cdf.bin',
|
||||
]);
|
||||
expect(record.bundleVersion, kImageCodecBundleVersion);
|
||||
|
||||
final graph = File('${tempDir.path}/model.onnx');
|
||||
final weights = File('${tempDir.path}/model.onnx.data');
|
||||
expect(graph.existsSync(), isTrue);
|
||||
expect(weights.existsSync(), isTrue);
|
||||
expect(await graph.readAsBytes(), small);
|
||||
expect(await weights.length(), large.length);
|
||||
expect(_sha256(await weights.readAsBytes()), _sha256(large));
|
||||
expect(
|
||||
await File('${tempDir.path}/entropy.onnx').readAsBytes(),
|
||||
entropy,
|
||||
);
|
||||
// Two different entropy exports land side by side; neither may overwrite
|
||||
// or be mistaken for the other.
|
||||
expect(
|
||||
await File('${tempDir.path}/entropy_decode.onnx').readAsBytes(),
|
||||
entropyDecode,
|
||||
);
|
||||
expect(entropyDecode, isNot(entropy));
|
||||
expect(await File('${tempDir.path}/cdf.bin').readAsBytes(), tables);
|
||||
|
||||
// The external-weights sibling MUST keep its exact name or the graph's
|
||||
// relative reference will not resolve.
|
||||
expect(weights.uri.pathSegments.last, 'model.onnx.data');
|
||||
|
||||
// Resume state is swept once an asset verifies.
|
||||
final leftovers = tempDir
|
||||
.listSync()
|
||||
.whereType<File>()
|
||||
.map((f) => f.uri.pathSegments.last)
|
||||
.where((n) => n.startsWith('.'))
|
||||
.toList();
|
||||
expect(leftovers, isEmpty);
|
||||
|
||||
expect(service.selectedModel?.id, 'test-model');
|
||||
});
|
||||
|
||||
test('progress is one bar across the set, not four', () async {
|
||||
final log = _Log();
|
||||
final service = serviceWith(_server(serverAssets, log));
|
||||
addTearDown(service.dispose);
|
||||
|
||||
final progress = <double>[];
|
||||
final names = <String>{};
|
||||
service.addListener(() {
|
||||
final value = service.downloadProgress;
|
||||
if (value != null) progress.add(value);
|
||||
final name = service.downloadFileName;
|
||||
if (name != null) names.add(name);
|
||||
});
|
||||
|
||||
await service.downloadPresetModel(verifiedSpec());
|
||||
|
||||
expect(progress, isNotEmpty);
|
||||
// Monotonic: a per-file bar would snap back to 0 three times.
|
||||
for (var i = 1; i < progress.length; i++) {
|
||||
expect(
|
||||
progress[i],
|
||||
greaterThanOrEqualTo(progress[i - 1]),
|
||||
reason: 'progress went backwards at $i',
|
||||
);
|
||||
}
|
||||
expect(progress.last, closeTo(1.0, 0.001));
|
||||
// Every asset was named while it was in flight, so the UI can say which
|
||||
// of the four files a 900 MB transfer is on.
|
||||
expect(
|
||||
names,
|
||||
containsAll(<String>['model.onnx', 'model.onnx.data', 'cdf.bin']),
|
||||
);
|
||||
});
|
||||
|
||||
test('refuses a spec that is missing a bundle role', () async {
|
||||
final service = serviceWith(_server(serverAssets, _Log()));
|
||||
addTearDown(service.dispose);
|
||||
|
||||
// Decoder-only: it would install a codec that can render a latent and
|
||||
// nothing else, which is exactly the state this work removes.
|
||||
final full = spec();
|
||||
await expectLater(
|
||||
service.downloadPresetModel(
|
||||
ImageCodecModelSpec(
|
||||
id: 'decoder-only',
|
||||
label: 'Decoder only',
|
||||
assets: full.assets.take(2).toList(),
|
||||
),
|
||||
),
|
||||
throwsA(isA<StateError>()),
|
||||
);
|
||||
expect(tempDir.listSync(), isEmpty);
|
||||
});
|
||||
|
||||
test('discardPartialDownload sweeps every asset of the bundle', () async {
|
||||
final service = serviceWith(_server(serverAssets, _Log()));
|
||||
addTearDown(service.dispose);
|
||||
final target = spec();
|
||||
for (final asset in target.assets) {
|
||||
await File(
|
||||
await store.chunkFilePath('${asset.fileName}.99', 0),
|
||||
).writeAsBytes(const [1, 2, 3]);
|
||||
}
|
||||
final other = File(await store.chunkFilePath('unrelated.onnx.99', 0));
|
||||
await other.writeAsBytes(const [1]);
|
||||
|
||||
await service.discardPartialDownload(target);
|
||||
|
||||
final leftovers = tempDir
|
||||
.listSync()
|
||||
.whereType<File>()
|
||||
.map((f) => f.uri.pathSegments.last)
|
||||
.toList();
|
||||
expect(leftovers, ['.unrelated.onnx.99_chunk_0']);
|
||||
});
|
||||
|
||||
test('refuses a spec whose URLs are placeholders', () async {
|
||||
final service = serviceWith(_server(serverAssets, _Log()));
|
||||
addTearDown(service.dispose);
|
||||
|
||||
await expectLater(
|
||||
service.downloadPresetModel(
|
||||
ImageCodecModelSpec(
|
||||
id: 'placeholder',
|
||||
label: 'Placeholder',
|
||||
urlsArePlaceholders: true,
|
||||
assets: spec().assets,
|
||||
),
|
||||
),
|
||||
throwsA(isA<StateError>()),
|
||||
);
|
||||
expect(tempDir.listSync(), isEmpty);
|
||||
});
|
||||
|
||||
test('the shipped preset points at published, verifiable assets', () {
|
||||
// The weights are published, so the old "still a placeholder" guard is
|
||||
// inverted: what matters now is that nothing ships half-wired. A real URL
|
||||
// with an empty digest is worse than a placeholder, because the download
|
||||
// succeeds and the integrity check silently passes.
|
||||
expect(imageCodecPresetModels, hasLength(1));
|
||||
for (final preset in imageCodecPresetModels) {
|
||||
expect(preset.urlsArePlaceholders, isFalse);
|
||||
expect(preset.assets, isNotEmpty);
|
||||
for (final asset in preset.assets) {
|
||||
expect(
|
||||
asset.sourceUrl,
|
||||
startsWith('https://huggingface.co/'),
|
||||
reason: asset.fileName,
|
||||
);
|
||||
expect(
|
||||
asset.sourceUrl,
|
||||
contains(asset.fileName),
|
||||
reason: '${asset.fileName} url must name its own file',
|
||||
);
|
||||
expect(asset.sizeBytes, greaterThan(0), reason: asset.fileName);
|
||||
expect(
|
||||
asset.sha256,
|
||||
matches(RegExp(r'^[0-9a-f]{64}$')),
|
||||
reason: '${asset.fileName} needs a real digest',
|
||||
);
|
||||
}
|
||||
// Five assets, one bundle: decoder graph + weights, both entropy
|
||||
// graphs, and the CDF tables.
|
||||
expect(preset.assets, hasLength(5));
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
group('resume', () {
|
||||
test('a broken transfer resumes from the bytes already on disk', () async {
|
||||
final chunkSize = (large.length / 8).ceil();
|
||||
final victimStart = chunkSize * 3;
|
||||
|
||||
final firstLog = _Log();
|
||||
final first = serviceWith(
|
||||
_server(serverAssets, firstLog, failAtOffset: victimStart),
|
||||
);
|
||||
addTearDown(first.dispose);
|
||||
|
||||
await expectLater(
|
||||
first.downloadPresetModel(spec()),
|
||||
throwsA(isA<SocketException>()),
|
||||
);
|
||||
|
||||
// Seven chunks landed whole, one is half-written, and none were deleted.
|
||||
final partials = tempDir
|
||||
.listSync()
|
||||
.whereType<File>()
|
||||
.where((f) => f.uri.pathSegments.last.startsWith('.'))
|
||||
.toList();
|
||||
expect(partials, hasLength(8));
|
||||
final victimPath = partials.firstWhere(
|
||||
(f) => f.uri.pathSegments.last.endsWith('_chunk_3'),
|
||||
);
|
||||
final resumeFrom = await victimPath.length();
|
||||
expect(resumeFrom, greaterThan(0));
|
||||
expect(resumeFrom, lessThan(chunkSize));
|
||||
expect(File('${tempDir.path}/model.onnx.data').existsSync(), isFalse);
|
||||
|
||||
final secondLog = _Log();
|
||||
final second = serviceWith(_server(serverAssets, secondLog));
|
||||
addTearDown(second.dispose);
|
||||
await second.downloadPresetModel(verifiedSpec());
|
||||
|
||||
// Exactly one range was re-requested, and it started where the partial
|
||||
// file ended rather than at the chunk boundary.
|
||||
expect(secondLog.ranges, hasLength(1));
|
||||
expect(
|
||||
secondLog.ranges.single,
|
||||
startsWith('${victimStart + resumeFrom}-'),
|
||||
);
|
||||
|
||||
final weights = File('${tempDir.path}/model.onnx.data');
|
||||
expect(await weights.length(), large.length);
|
||||
expect(_sha256(await weights.readAsBytes()), _sha256(large));
|
||||
});
|
||||
|
||||
test('an already-complete verified asset is not re-fetched', () async {
|
||||
final log = _Log();
|
||||
final service = serviceWith(_server(serverAssets, log));
|
||||
addTearDown(service.dispose);
|
||||
final digested = verifiedSpec();
|
||||
|
||||
await service.downloadPresetModel(digested);
|
||||
final rangesAfterFirst = log.ranges.length;
|
||||
expect(rangesAfterFirst, greaterThan(0));
|
||||
|
||||
await service.downloadPresetModel(digested);
|
||||
expect(log.ranges.length, rangesAfterFirst, reason: 'no re-download');
|
||||
});
|
||||
|
||||
test('resume state does not survive a change in upstream length', () async {
|
||||
// Chunk keys embed the total size, so offsets computed for one length can
|
||||
// never be spliced onto a file of another length.
|
||||
final a = await store.chunkFilePath('model.onnx.data.1000', 3);
|
||||
final b = await store.chunkFilePath('model.onnx.data.2000', 3);
|
||||
expect(a, isNot(b));
|
||||
});
|
||||
});
|
||||
|
||||
group('integrity', () {
|
||||
test('a wrong digest fails loudly and removes the file', () async {
|
||||
final service = serviceWith(_server(serverAssets, _Log()));
|
||||
addTearDown(service.dispose);
|
||||
|
||||
await expectLater(
|
||||
service.downloadPresetModel(
|
||||
spec(smallDigest: _sha256(utf8.encode('not the model'))),
|
||||
),
|
||||
throwsA(isA<ImageCodecIntegrityFailure>()),
|
||||
);
|
||||
expect(File('${tempDir.path}/model.onnx').existsSync(), isFalse);
|
||||
expect(service.selectedModel, isNull);
|
||||
});
|
||||
|
||||
test('sha256OfFile streams the same digest as an in-memory hash', () async {
|
||||
final path = '${tempDir.path}/blob.bin';
|
||||
await File(path).writeAsBytes(large);
|
||||
expect(await store.sha256OfFile(path), _sha256(large));
|
||||
});
|
||||
|
||||
test('a missing digest is skipped rather than treated as a match', () {
|
||||
const withDigest = ImageCodecModelAsset(
|
||||
role: ImageCodecAssetRole.decoderGraph,
|
||||
fileName: 'a',
|
||||
sourceUrl: 'https://example.invalid/a',
|
||||
sizeBytes: 1,
|
||||
sha256:
|
||||
'0000000000000000000000000000000000000000000000000000000000000000',
|
||||
);
|
||||
const without = ImageCodecModelAsset(
|
||||
role: ImageCodecAssetRole.cdfTables,
|
||||
fileName: 'b',
|
||||
sourceUrl: 'https://example.invalid/b',
|
||||
sizeBytes: 1,
|
||||
);
|
||||
expect(withDigest.hasChecksum, isTrue);
|
||||
expect(without.hasChecksum, isFalse);
|
||||
});
|
||||
});
|
||||
|
||||
group('scanDownloadedModels', () {
|
||||
test('preserves chunk files but reaps other hidden junk', () async {
|
||||
final chunk = File(await store.chunkFilePath('model.onnx.data.99', 2));
|
||||
await chunk.writeAsBytes(const [1, 2, 3]);
|
||||
final junk = File('${tempDir.path}/.DS_Store');
|
||||
await junk.writeAsBytes(const [0]);
|
||||
await File('${tempDir.path}/model.onnx').writeAsBytes(small);
|
||||
|
||||
final found = await store.scanDownloadedModels();
|
||||
|
||||
expect(found.map((m) => m.name), ['model.onnx']);
|
||||
expect(chunk.existsSync(), isTrue, reason: 'resume state must survive');
|
||||
expect(junk.existsSync(), isFalse);
|
||||
});
|
||||
|
||||
test('deletePartialDownloads sweeps only the named model', () async {
|
||||
final mine = File(await store.chunkFilePath('model.onnx.data.99', 0));
|
||||
final other = File(await store.chunkFilePath('other.onnx.99', 0));
|
||||
await mine.writeAsBytes(const [1]);
|
||||
await other.writeAsBytes(const [1]);
|
||||
|
||||
await store.deletePartialDownloads('model.onnx.data');
|
||||
|
||||
expect(mine.existsSync(), isFalse);
|
||||
expect(other.existsSync(), isTrue);
|
||||
});
|
||||
});
|
||||
|
||||
group('non-ranged servers', () {
|
||||
test('fall back to a single GET', () async {
|
||||
final log = _Log();
|
||||
final service = serviceWith(
|
||||
_server(serverAssets, log, acceptRanges: false),
|
||||
);
|
||||
addTearDown(service.dispose);
|
||||
|
||||
await service.downloadPresetModel(verifiedSpec());
|
||||
|
||||
expect(log.ranges, isEmpty);
|
||||
expect(
|
||||
await File('${tempDir.path}/model.onnx.data').length(),
|
||||
large.length,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
group('installedBundle', () {
|
||||
test(
|
||||
'a fresh install resolves the decoder, entropy and table paths',
|
||||
() async {
|
||||
final service = serviceWith(_server(serverAssets, _Log()));
|
||||
addTearDown(service.dispose);
|
||||
|
||||
await service.downloadPresetModel(verifiedSpec());
|
||||
|
||||
final bundle = service.installedBundle;
|
||||
expect(bundle, isNotNull);
|
||||
expect(bundle!.decoderGraphPath, '${tempDir.path}/model.onnx');
|
||||
expect(bundle.entropyGraphPath, '${tempDir.path}/entropy.onnx');
|
||||
// Resolved by ROLE. Both entropy files end in `.onnx`, so a
|
||||
// position- or extension-based guess would be a coin flip, and handing
|
||||
// the decode-side graph to the encoder fails at the first run.
|
||||
expect(
|
||||
bundle.entropyDecodeGraphPath,
|
||||
'${tempDir.path}/entropy_decode.onnx',
|
||||
);
|
||||
expect(bundle.tablesPath, '${tempDir.path}/cdf.bin');
|
||||
expect(bundle.isComplete, isTrue);
|
||||
expect(bundle.supportsDecode, isTrue);
|
||||
expect(service.needsModelUpgrade, isFalse);
|
||||
expect(service.needsModelDownload, isFalse);
|
||||
},
|
||||
);
|
||||
|
||||
test('a bundle-version-1 install can send but not receive', () async {
|
||||
// The record the previous release wrote: four assets, no decode-side
|
||||
// graph. Encoding still works, decoding does not, and the remedy is a
|
||||
// re-download rather than "your device cannot do this".
|
||||
final v1 = ImageCodecModelRecord(
|
||||
id: 'test-model',
|
||||
name: 'model.onnx',
|
||||
sourceUrl: 'https://example.invalid/repo/model.onnx',
|
||||
localPath: '${tempDir.path}/model.onnx',
|
||||
downloadedAt: DateTime.fromMillisecondsSinceEpoch(1730000000000),
|
||||
fileSizeBytes: 1,
|
||||
assetFileNames: const [
|
||||
'model.onnx',
|
||||
'model.onnx.data',
|
||||
'entropy.onnx',
|
||||
'cdf.bin',
|
||||
],
|
||||
bundleVersion: 1,
|
||||
);
|
||||
final service = ImageCodecService(
|
||||
AppSettingsService(),
|
||||
fileStore: store,
|
||||
settingsStore: InMemoryImageCodecSettingsStore(
|
||||
ImageCodecPreferences(
|
||||
enabled: true,
|
||||
selectedModelId: v1.id,
|
||||
downloadedModels: [v1],
|
||||
),
|
||||
),
|
||||
clientFactory: _server(serverAssets, _Log()),
|
||||
);
|
||||
addTearDown(service.dispose);
|
||||
|
||||
final bundle = service.installedBundle;
|
||||
expect(bundle, isNotNull);
|
||||
expect(bundle!.entropyGraphPath, '${tempDir.path}/entropy.onnx');
|
||||
// No spec asset name is present in the record, and the heuristic must NOT
|
||||
// invent one: a filename that was never downloaded resolves to an opaque
|
||||
// ORT failure instead of a download prompt.
|
||||
expect(bundle.entropyDecodeGraphPath, isNull);
|
||||
expect(bundle.isComplete, isTrue);
|
||||
expect(bundle.supportsDecode, isFalse);
|
||||
expect(service.needsModelUpgrade, isTrue);
|
||||
expect(service.needsModelDownload, isFalse);
|
||||
expect(service.canDecode, isFalse);
|
||||
expect(service.statusReason, isNotNull);
|
||||
});
|
||||
|
||||
test('a pre-bundle install is an upgrade, not a broken build', () {
|
||||
// The decoder-only record the shipped build wrote: no asset list, no
|
||||
// bundle version. It must stay loadable, report an incomplete bundle,
|
||||
// and ask for a download rather than declaring the device incapable.
|
||||
final legacy = ImageCodecModelRecord(
|
||||
id: 'aeic-se-decoder-qdq-conv-pct-novae',
|
||||
name: 'aeic_decoder_qdq_conv_pct_novae.onnx',
|
||||
sourceUrl: 'https://example.invalid/x.onnx',
|
||||
localPath: '${tempDir.path}/aeic_decoder_qdq_conv_pct_novae.onnx',
|
||||
downloadedAt: DateTime.fromMillisecondsSinceEpoch(1730000000000),
|
||||
fileSizeBytes: 2909610,
|
||||
);
|
||||
final service = ImageCodecService(
|
||||
AppSettingsService(),
|
||||
fileStore: store,
|
||||
settingsStore: InMemoryImageCodecSettingsStore(
|
||||
ImageCodecPreferences(
|
||||
enabled: true,
|
||||
selectedModelId: legacy.id,
|
||||
downloadedModels: [legacy],
|
||||
),
|
||||
),
|
||||
clientFactory: _server(serverAssets, _Log()),
|
||||
);
|
||||
addTearDown(service.dispose);
|
||||
|
||||
expect(service.needsModelDownload, isFalse, reason: 'a model IS present');
|
||||
expect(service.needsModelUpgrade, isTrue);
|
||||
final bundle = service.installedBundle;
|
||||
expect(bundle, isNotNull);
|
||||
expect(bundle!.decoderGraphPath, legacy.localPath);
|
||||
expect(bundle.entropyGraphPath, isNull);
|
||||
expect(bundle.entropyDecodeGraphPath, isNull);
|
||||
expect(bundle.tablesPath, isNull);
|
||||
expect(bundle.isComplete, isFalse);
|
||||
expect(bundle.supportsDecode, isFalse);
|
||||
expect(service.canEncode, isFalse);
|
||||
expect(service.canDecode, isFalse);
|
||||
// An incomplete install is a download away, so the user always gets a
|
||||
// sentence explaining what to do.
|
||||
expect(service.statusReason, isNotNull);
|
||||
});
|
||||
|
||||
test('nothing installed means no bundle and a download prompt', () {
|
||||
final service = ImageCodecService(
|
||||
AppSettingsService(),
|
||||
fileStore: store,
|
||||
settingsStore: InMemoryImageCodecSettingsStore(
|
||||
const ImageCodecPreferences(enabled: true),
|
||||
),
|
||||
clientFactory: _server(serverAssets, _Log()),
|
||||
);
|
||||
addTearDown(service.dispose);
|
||||
|
||||
expect(service.installedBundle, isNull);
|
||||
expect(service.needsModelDownload, isTrue);
|
||||
expect(service.needsModelUpgrade, isFalse);
|
||||
expect(service.canEncode, isFalse);
|
||||
});
|
||||
|
||||
test('statusReason is a superset of unavailableReason', () {
|
||||
final service = serviceWith(_server(serverAssets, _Log()));
|
||||
addTearDown(service.dispose);
|
||||
|
||||
// While kImageCodecBitstreamPathAvailable is false this is the build
|
||||
// sentence; when the gate flips, the remaining branches (switched off,
|
||||
// not downloaded, needs upgrade) take over. Either way it is non-empty
|
||||
// whenever the codec is not ready, which is the contract the compose
|
||||
// sheet's banner depends on.
|
||||
final status = service.statusReason;
|
||||
expect(status, isNotNull);
|
||||
expect(status!.trim(), isNotEmpty);
|
||||
final permanent = service.unavailableReason;
|
||||
if (permanent != null) {
|
||||
expect(status, permanent);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,407 @@
|
||||
import 'dart:convert';
|
||||
import 'dart:io';
|
||||
import 'dart:typed_data';
|
||||
|
||||
import 'package:crypto/crypto.dart' show sha256;
|
||||
import 'package:flutter_test/flutter_test.dart';
|
||||
import 'package:meshcore_open/services/entropy_tables.dart';
|
||||
import 'package:meshcore_open/services/image_codec_backend.dart';
|
||||
import 'package:meshcore_open/services/image_codec_entropy.dart';
|
||||
|
||||
/// End-to-end cross-language conformance for the AEIC entropy path.
|
||||
///
|
||||
/// This is deliberately **not** a Dart-encodes-then-Dart-decodes round trip.
|
||||
/// That shape passes happily with a completely wrong wire format — a swapped
|
||||
/// mask permutation, a reversed squeeze, an off-by-one in `my_build_indexes` —
|
||||
/// because both halves make the same mistake. Every assertion here compares
|
||||
/// Dart against bytes and tensors that Python/ORT/C++ produced:
|
||||
///
|
||||
/// ENCODE: recorded encode-graph outputs -> real Dart four-stage loop ->
|
||||
/// real Dart rANS ==> byte-identical to the recorded C++ bitstream.
|
||||
/// DECODE: recorded C++ bitstream -> real Dart rANS -> real Dart four-stage
|
||||
/// loop (replaying the recorded decode-side network calls)
|
||||
/// ==> y_hat exactly equal, element for element, to the recorded one.
|
||||
///
|
||||
/// The only thing faked is [AeicEntropyNetwork]: the neural half is replayed
|
||||
/// from `.aeicrec` recordings made by `aic/exp/record_entropy_io.py`. The fake
|
||||
/// asserts on its *inputs* as well as returning outputs — in particular the
|
||||
/// `base` tensor handed to each decode stage must match the recorded one bit
|
||||
/// for bit, which is what localises a wrong mask / mergeContext / squeeze to
|
||||
/// the stage that broke instead of to a garbled final image.
|
||||
void main() {
|
||||
final Directory goldenDir = _resolveGoldenDir();
|
||||
final Directory e2eDir = Directory('${goldenDir.path}/e2e');
|
||||
final EntropyTables tables = EntropyTables.parse(
|
||||
File('${goldenDir.path}/aeic_cdf_ft32.bin').readAsBytesSync(),
|
||||
);
|
||||
final AeicRansCoderFactory coders = AeicRansCoders(tables);
|
||||
final Map<String, dynamic> manifest =
|
||||
jsonDecode(File('${e2eDir.path}/manifest.json').readAsStringSync())
|
||||
as Map<String, dynamic>;
|
||||
final List<Map<String, dynamic>> files = (manifest['files'] as List<dynamic>)
|
||||
.cast<Map<String, dynamic>>();
|
||||
|
||||
test('recording corpus is present and unmodified', () {
|
||||
expect(manifest['format'], 'aeic-entropy-e2e-recording');
|
||||
expect(manifest['version'], 1);
|
||||
expect(manifest['checkpoint'], 'AEIC_SE_ft32.pkl');
|
||||
expect(manifest['size'], 512);
|
||||
expect(files.length, 5);
|
||||
for (final Map<String, dynamic> rec in files) {
|
||||
final File f = File('${e2eDir.path}/${rec['file']}');
|
||||
expect(f.existsSync(), isTrue, reason: '${rec['file']} missing');
|
||||
final Uint8List raw = f.readAsBytesSync();
|
||||
expect(raw.length, rec['bytes'], reason: '${rec['file']} size');
|
||||
expect(
|
||||
sha256.convert(raw).toString(),
|
||||
rec['sha256'],
|
||||
reason: '${rec['file']} sha256',
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
for (final Map<String, dynamic> rec in files) {
|
||||
final String name = rec['file'] as String;
|
||||
group(name, () {
|
||||
late _Recording r;
|
||||
late AeicEntropyGeometry geometry;
|
||||
late AeicMaskSet masks;
|
||||
|
||||
setUpAll(() {
|
||||
r = _Recording.load('${e2eDir.path}/$name');
|
||||
geometry = AeicEntropyGeometry.forResolution(
|
||||
r.meta['size'] as int,
|
||||
yChannels: (r.meta['y_shape'] as List<dynamic>)[1] as int,
|
||||
);
|
||||
masks = AeicMaskSet(geometry);
|
||||
});
|
||||
|
||||
test('recording shape matches the geometry the codec derives', () {
|
||||
expect(r.meta['checkpoint'], 'AEIC_SE_ft32.pkl');
|
||||
expect(r.meta['z_cdf_group'], kAeicZCdfGroup);
|
||||
expect(r.meta['y_cdf_group'], kAeicYCdfGroup);
|
||||
expect(r.meta['byte_order'], 'little');
|
||||
expect(r.f32('enc/z_q').length, geometry.zElements);
|
||||
expect(r.f32('enc/y_hat').length, geometry.yElements);
|
||||
expect(r.f32('dec/y_hat').length, geometry.yElements);
|
||||
expect(r.u8('enc/bitstream').length, r.meta['bitstream_bytes']);
|
||||
expect(r.calls.length, 5);
|
||||
expect(r.calls[0]['kind'], 'hyper_synthesis');
|
||||
for (var i = 0; i < 4; i++) {
|
||||
expect(r.calls[i + 1]['kind'], 'stage');
|
||||
expect(r.calls[i + 1]['stage'], i);
|
||||
}
|
||||
});
|
||||
|
||||
// The pieces the Dart entropy layer computes on its own between the
|
||||
// graph and the coder. Checked against Python directly so a divergence
|
||||
// here is attributed to squeeze / my_build_indexes rather than to rANS.
|
||||
test('symbols and indexes match the recorded integer arrays', () {
|
||||
_expectSameInts(
|
||||
aeicToSymbols(r.f32('enc/z_q')),
|
||||
r.i16('enc/z_symbols'),
|
||||
'z symbols',
|
||||
);
|
||||
_expectSameInts(
|
||||
aeicZIndexes(geometry),
|
||||
r.i16('enc/z_indexes'),
|
||||
'z indexes',
|
||||
);
|
||||
for (var s = 0; s < 4; s++) {
|
||||
_expectSameInts(
|
||||
aeicToSymbols(masks.squeeze(r.f32('enc/yq$s'))),
|
||||
r.i16('enc/symbols$s'),
|
||||
'stage $s symbols',
|
||||
);
|
||||
_expectSameInts(
|
||||
aeicBuildIndexes(masks.squeeze(r.f32('enc/sc$s'))),
|
||||
r.i16('enc/indexes$s'),
|
||||
'stage $s indexes',
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test('ENCODE: Dart bitstream is byte-identical to the C++ bitstream',
|
||||
() async {
|
||||
final _ReplayNetwork network = _ReplayNetwork(r);
|
||||
final AeicEntropyCodec codec = AeicEntropyCodec(
|
||||
geometry: geometry,
|
||||
network: network,
|
||||
coders: coders,
|
||||
);
|
||||
final Uint8List got = await codec.encode(
|
||||
Uint8List(geometry.resolution * geometry.resolution * 3),
|
||||
);
|
||||
expect(network.encodeCalls, 1);
|
||||
_expectSameBytes(got, r.u8('enc/bitstream'), name);
|
||||
expect(
|
||||
sha256.convert(got).toString(),
|
||||
r.meta['bitstream_sha256'],
|
||||
reason: '$name: bitstream sha256',
|
||||
);
|
||||
});
|
||||
|
||||
test('DECODE: y_hat from the C++ bitstream is exactly the recorded y_hat',
|
||||
() async {
|
||||
final _ReplayNetwork network = _ReplayNetwork(r);
|
||||
final AeicEntropyCodec codec = AeicEntropyCodec(
|
||||
geometry: geometry,
|
||||
network: network,
|
||||
coders: coders,
|
||||
);
|
||||
final Float32List got = await codec.decodeToLatent(
|
||||
r.u8('enc/bitstream'),
|
||||
);
|
||||
expect(network.hyperCalls, 1);
|
||||
expect(network.stageCalls, <int>[0, 1, 2, 3]);
|
||||
_expectSameFloats(got, r.f32('dec/y_hat'), '$name: y_hat');
|
||||
// The recording asserts the decoder's latent equals the encoder's; if
|
||||
// that holds in Python it must hold here too.
|
||||
expect(r.meta['decoded_y_hat_equals_encoder_y_hat'], isTrue);
|
||||
_expectSameFloats(got, r.f32('enc/y_hat'), '$name: y_hat vs encoder');
|
||||
});
|
||||
|
||||
test('a single flipped bitstream byte does not still pass', () async {
|
||||
// Byte 3 is the first payload byte of sub-stream 0 (1-byte flag +
|
||||
// 2-byte size header), which the decoder loads straight into the rANS
|
||||
// state, so flipping it must change the output. The *last* byte is a
|
||||
// poor choice: renormalisation does not always consume the tail, and
|
||||
// on two of these five recordings flipping it is genuinely a no-op.
|
||||
final Uint8List mutated = Uint8List.fromList(r.u8('enc/bitstream'));
|
||||
expect(mutated.length, greaterThan(4));
|
||||
mutated[3] ^= 0x01;
|
||||
final AeicEntropyCodec codec = AeicEntropyCodec(
|
||||
geometry: geometry,
|
||||
network: _networkForMutation(r),
|
||||
coders: coders,
|
||||
);
|
||||
Float32List? got;
|
||||
try {
|
||||
got = await codec.decodeToLatent(mutated);
|
||||
} catch (_) {
|
||||
// Desync raising is an acceptable outcome; silently matching is not.
|
||||
return;
|
||||
}
|
||||
expect(
|
||||
_sameFloats(got, r.f32('dec/y_hat')),
|
||||
isFalse,
|
||||
reason: '$name: corrupting the stream changed nothing — the '
|
||||
'comparison is vacuous',
|
||||
);
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/// A replay network for the mutated-stream test: it must NOT assert on its
|
||||
/// inputs, because a desynchronised decode legitimately feeds it a different
|
||||
/// `base`. It returns the recorded outputs regardless.
|
||||
AeicEntropyNetwork _networkForMutation(_Recording r) =>
|
||||
_ReplayNetwork(r, strict: false);
|
||||
|
||||
/// [AeicEntropyNetwork] that replays one `.aeicrec` recording.
|
||||
///
|
||||
/// Returns the ORT tensors Python captured, and — when [strict] — asserts that
|
||||
/// the tensors the Dart loop hands it are bit-for-bit the ones Python's own
|
||||
/// decode loop handed the real graph.
|
||||
class _ReplayNetwork implements AeicEntropyNetwork {
|
||||
_ReplayNetwork(this.r, {this.strict = true});
|
||||
|
||||
final _Recording r;
|
||||
final bool strict;
|
||||
|
||||
int encodeCalls = 0;
|
||||
int hyperCalls = 0;
|
||||
final List<int> stageCalls = <int>[];
|
||||
|
||||
@override
|
||||
bool get supportsDecodeSide => true;
|
||||
|
||||
@override
|
||||
Future<AeicEncodeSideTensors> runEncodeSide(Float32List imageChw) async {
|
||||
encodeCalls++;
|
||||
return AeicEncodeSideTensors(
|
||||
zQ: r.f32('enc/z_q'),
|
||||
yQ: <Float32List>[for (var i = 0; i < 4; i++) r.f32('enc/yq$i')],
|
||||
scales: <Float32List>[for (var i = 0; i < 4; i++) r.f32('enc/sc$i')],
|
||||
);
|
||||
}
|
||||
|
||||
@override
|
||||
Future<Float32List> runHyperSynthesis(Float32List zQ) async {
|
||||
hyperCalls++;
|
||||
final Map<String, dynamic> call = r.calls[0];
|
||||
if (strict) {
|
||||
_expectSameFloats(
|
||||
zQ,
|
||||
r.f32((call['inputs'] as Map<String, dynamic>)['z_q'] as String),
|
||||
'hyper_synthesis input z_q',
|
||||
);
|
||||
}
|
||||
return r.f32((call['outputs'] as Map<String, dynamic>)['base0'] as String);
|
||||
}
|
||||
|
||||
@override
|
||||
Future<AeicStageParams> runStage(int stage, Float32List base) async {
|
||||
stageCalls.add(stage);
|
||||
final Map<String, dynamic> call = r.calls[stage + 1];
|
||||
expect(call['stage'], stage, reason: 'call table is positional');
|
||||
if (strict) {
|
||||
_expectSameFloats(
|
||||
base,
|
||||
r.f32((call['inputs'] as Map<String, dynamic>)['base'] as String),
|
||||
'stage $stage input base',
|
||||
);
|
||||
}
|
||||
final Map<String, dynamic> outputs = call['outputs'] as Map<String, dynamic>;
|
||||
return AeicStageParams(
|
||||
meansSupp: r.f32(outputs['means'] as String),
|
||||
scalesSupp: r.f32(outputs['scales'] as String),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Reader for the `.aeicrec` container (magic "AEICREC1", little-endian):
|
||||
/// 32-byte header, an 8-byte-aligned tensor blob, then a UTF-8 JSON index.
|
||||
class _Recording {
|
||||
_Recording(this.index, this.bytes)
|
||||
: _entries = <String, Map<String, dynamic>>{
|
||||
for (final Map<String, dynamic> e
|
||||
in (index['entries'] as List<dynamic>)
|
||||
.cast<Map<String, dynamic>>())
|
||||
e['name'] as String: e,
|
||||
};
|
||||
|
||||
final Map<String, dynamic> index;
|
||||
final Uint8List bytes;
|
||||
final Map<String, Map<String, dynamic>> _entries;
|
||||
|
||||
static _Recording load(String path) {
|
||||
final Uint8List bytes = File(path).readAsBytesSync();
|
||||
final ByteData bd = ByteData.sublistView(bytes);
|
||||
final String magic = ascii.decode(bytes.sublist(0, 8));
|
||||
if (magic != 'AEICREC1') {
|
||||
throw FormatException('bad .aeicrec magic "$magic" in $path');
|
||||
}
|
||||
final int version = bd.getUint32(8, Endian.little);
|
||||
if (version != 1) {
|
||||
throw FormatException('.aeicrec version $version in $path');
|
||||
}
|
||||
final int indexOffset = bd.getUint64(16, Endian.little);
|
||||
final int indexLength = bd.getUint32(24, Endian.little);
|
||||
final Map<String, dynamic> index =
|
||||
jsonDecode(
|
||||
utf8.decode(bytes.sublist(indexOffset, indexOffset + indexLength)),
|
||||
)
|
||||
as Map<String, dynamic>;
|
||||
return _Recording(index, bytes);
|
||||
}
|
||||
|
||||
Map<String, dynamic> get meta => index['meta'] as Map<String, dynamic>;
|
||||
|
||||
List<Map<String, dynamic>> get calls =>
|
||||
(index['calls'] as List<dynamic>).cast<Map<String, dynamic>>();
|
||||
|
||||
Map<String, dynamic> _entry(String name, String dtype) {
|
||||
final Map<String, dynamic>? e = _entries[name];
|
||||
if (e == null) {
|
||||
throw StateError('no entry "$name" in recording');
|
||||
}
|
||||
if (e['dtype'] != dtype) {
|
||||
throw StateError('entry "$name" is ${e['dtype']}, wanted $dtype');
|
||||
}
|
||||
return e;
|
||||
}
|
||||
|
||||
Float32List f32(String name) {
|
||||
final Map<String, dynamic> e = _entry(name, 'f32');
|
||||
return Float32List.sublistView(
|
||||
bytes,
|
||||
e['offset'] as int,
|
||||
(e['offset'] as int) + (e['length'] as int),
|
||||
);
|
||||
}
|
||||
|
||||
Int16List i16(String name) {
|
||||
final Map<String, dynamic> e = _entry(name, 'i16');
|
||||
return Int16List.sublistView(
|
||||
bytes,
|
||||
e['offset'] as int,
|
||||
(e['offset'] as int) + (e['length'] as int),
|
||||
);
|
||||
}
|
||||
|
||||
Uint8List u8(String name) {
|
||||
final Map<String, dynamic> e = _entry(name, 'u8');
|
||||
return Uint8List.sublistView(
|
||||
bytes,
|
||||
e['offset'] as int,
|
||||
(e['offset'] as int) + (e['length'] as int),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
void _expectSameBytes(Uint8List got, Uint8List want, String label) {
|
||||
final int n = got.length < want.length ? got.length : want.length;
|
||||
for (var i = 0; i < n; i++) {
|
||||
if (got[i] != want[i]) {
|
||||
fail(
|
||||
'$label: first byte divergence at offset $i of ${want.length} '
|
||||
'(got 0x${got[i].toRadixString(16)}, '
|
||||
'want 0x${want[i].toRadixString(16)})',
|
||||
);
|
||||
}
|
||||
}
|
||||
expect(
|
||||
got.length,
|
||||
want.length,
|
||||
reason: '$label: length differs (prefix matched)',
|
||||
);
|
||||
}
|
||||
|
||||
void _expectSameInts(List<int> got, List<int> want, String label) {
|
||||
expect(got.length, want.length, reason: '$label: length');
|
||||
for (var i = 0; i < got.length; i++) {
|
||||
if (got[i] != want[i]) {
|
||||
fail(
|
||||
'$label: first divergence at index $i of ${want.length} '
|
||||
'(got ${got[i]}, want ${want[i]})',
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Exact equality, element for element — no tolerance. These are integers
|
||||
/// carried in float32 (symbols + means), so "close" is not the bar.
|
||||
void _expectSameFloats(Float32List got, Float32List want, String label) {
|
||||
expect(got.length, want.length, reason: '$label: length');
|
||||
for (var i = 0; i < got.length; i++) {
|
||||
if (got[i] != want[i]) {
|
||||
fail(
|
||||
'$label: first divergence at index $i of ${want.length} '
|
||||
'(got ${got[i]}, want ${want[i]}, '
|
||||
'diff ${(got[i] - want[i]).abs()})',
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bool _sameFloats(Float32List a, Float32List b) {
|
||||
if (a.length != b.length) return false;
|
||||
for (var i = 0; i < a.length; i++) {
|
||||
if (a[i] != b[i]) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
Directory _resolveGoldenDir() {
|
||||
for (final String candidate in <String>[
|
||||
'test/services/golden',
|
||||
'../test/services/golden',
|
||||
'golden',
|
||||
]) {
|
||||
final Directory d = Directory(candidate);
|
||||
if (d.existsSync()) return d;
|
||||
}
|
||||
return Directory('test/services/golden');
|
||||
}
|
||||
@@ -0,0 +1,754 @@
|
||||
import 'dart:io';
|
||||
import 'dart:math' as math;
|
||||
import 'dart:typed_data';
|
||||
|
||||
import 'package:flutter_test/flutter_test.dart';
|
||||
import 'package:meshcore_open/services/entropy_tables.dart';
|
||||
import 'package:meshcore_open/services/image_codec_backend.dart'
|
||||
show AeicRansCoders;
|
||||
import 'package:meshcore_open/services/image_codec_entropy.dart';
|
||||
|
||||
/// Golden vectors for the entropy layer — the arithmetic between the ONNX
|
||||
/// tensors and the rANS coder.
|
||||
///
|
||||
/// GENERATED BY: `aic/exp/export_entropy_layer_golden.py`, which runs the real
|
||||
/// `torch` ops from `aic/aeic/src/codec/codec_practical.py`
|
||||
/// (`get_mask_four_parts`, `sequeeze`, `torch.round`, `my_build_indexes`) on
|
||||
/// inputs built from a closed-form integer recipe. Every constant in that
|
||||
/// recipe is a power-of-two fraction or a plain float64 division, so Dart and
|
||||
/// numpy reproduce the float32 inputs bit-for-bit and the comparison is real
|
||||
/// rather than a re-implementation checking itself.
|
||||
///
|
||||
/// Regenerate with:
|
||||
/// cd /Users/Zach/Documents/mycode/aic
|
||||
/// AEIC_DEVICE=cpu .venv/bin/python exp/export_entropy_layer_golden.py
|
||||
///
|
||||
/// WHY THIS MATTERS: none of these failures are loud. A wrong mask permutation,
|
||||
/// a `sequeeze` that folds the wrong channels, or a rounding tie resolved away
|
||||
/// from zero instead of to even does not throw — it desynchronises rANS and
|
||||
/// produces a sharp, plausible, wrong image.
|
||||
class _Golden {
|
||||
// --- small case: C = 8, H = 4, W = 4, squeezed length 32 ---
|
||||
|
||||
/// `mask_i` flattened over `[1, 8, 4, 4]`, '1' where live. Straight from
|
||||
/// `get_mask_four_parts(1, 8, 4, 4)`.
|
||||
static const List<String> masks = <String>[
|
||||
'10100000101000001010000010100000010100000101000001010000010100000000101000001010000010100000101000000101000001010000010100000101',
|
||||
'00000101000001010000010100000101000010100000101000001010000010100101000001010000010100000101000010100000101000001010000010100000',
|
||||
'00001010000010100000101000001010000001010000010100000101000001011010000010100000101000001010000001010000010100000101000001010000',
|
||||
'01010000010100000101000001010000101000001010000010100000101000000000010100000101000001010000010100001010000010100000101000001010',
|
||||
];
|
||||
|
||||
static const List<List<int>> symbols = <List<int>>[
|
||||
<int>[-1, -1, -2, 4, 0, 0, 6, -3, 2, 1, 1, -1, -5, 1, 0, 0, -1, -1, -2, 4,
|
||||
0, 0, 5, -3, 2, 1, 1, -1, -5, 1, 0, 0],
|
||||
<int>[-1, -1, 4, 4, 0, 0, 0, 5, 2, 1, -1, -1, 3, -5, 1, 0, -1, -2, 4, 4,
|
||||
0, 0, 0, 5, 1, 1, -1, -1, 3, -5, 1, 0],
|
||||
<int>[-1, -2, -2, 4, 1, 0, 0, 5, 2, 0, -1, -1, 3, -5, 1, 0, -1, -2, -2, 4,
|
||||
0, 0, 0, 5, 2, -1, -1, -1, 3, -5, 1, 0],
|
||||
<int>[-1, -1, -2, -2, 0, 0, 5, -3, 2, 1, -1, -1, -5, -5, 0, 0, -1, -1, -2,
|
||||
-2, 0, 0, 5, -3, 2, 1, -1, -1, -5, 1, 0, 0],
|
||||
];
|
||||
|
||||
static const List<List<int>> indexes = <List<int>>[
|
||||
<int>[-1, 6, -1, 8, 12, 15, 13, 16, 0, 9, 3, 10, 13, 16, 14, 17, 0, 9, 3,
|
||||
10, 13, 16, 14, 17, 5, 11, 7, 12, 15, 17, 15, 18],
|
||||
<int>[14, 11, 15, 12, 7, 0, 8, 2, 15, 13, 16, 14, 9, 4, 11, 6, 15, 13, 16,
|
||||
14, 10, 4, 11, 6, 16, 14, 17, 15, 11, 7, 12, 9],
|
||||
<int>[11, 14, 12, 15, -1, 8, 0, 9, 12, 16, 13, 16, 2, 10, 5, 11, 13, 16,
|
||||
13, 16, 3, 10, 5, 11, 14, 17, 15, 17, 7, 12, 8, 13],
|
||||
<int>[5, -1, 7, 0, 15, 12, 15, 13, 8, 1, 10, 4, 16, 14, 16, 14, 8, 2, 10,
|
||||
4, 16, 14, 16, 14, 11, 6, 12, 8, 17, 15, 17, 16],
|
||||
];
|
||||
|
||||
// --- full case: C = 256, H = W = 16 (the shipping geometry) ---
|
||||
//
|
||||
// 16,384 symbols per stage is too much to embed, so the golden is a
|
||||
// fingerprint: total, extremes, and the first and last twelve values. Any
|
||||
// permutation error moves the head or the tail; any arithmetic error moves
|
||||
// the sum.
|
||||
static const List<int> fullSymbolSum = <int>[-17, 40, -10, 29];
|
||||
static const List<int> fullIndexSum = <int>[412240, 412285, 412231, 412219];
|
||||
static const List<List<int>> fullSymbolHead = <List<int>>[
|
||||
<int>[-1, -4, -2, 1, 4, -1, -5, -2, 0, 3, -1, -5],
|
||||
<int>[-3, 0, 3, 5, 0, -4, -1, 2, 4, 1, -4, -2],
|
||||
<int>[0, 3, -1, 2, -3, 0, 2, -1, 1, -4, -1, 2],
|
||||
<int>[-3, -1, 2, -2, 1, 3, -2, 1, 4, 0, 3, 5],
|
||||
];
|
||||
static const List<List<int>> fullSymbolTail = <List<int>>[
|
||||
<int>[-4, -2, 1, 4, -2, 1, -3, 0, 3, -2, -6, -3],
|
||||
<int>[-2, 0, -3, -1, 2, 5, 0, -4, -1, 1, 4, 6],
|
||||
<int>[-5, -2, 0, 3, -1, -6, -3, 0, 2, 5, 1, -4],
|
||||
<int>[-1, 1, 4, 0, -4, -2, 1, 3, 6, 2, -2, 0],
|
||||
];
|
||||
static const List<List<int>> fullIndexHead = <List<int>>[
|
||||
<int>[-1, -1, -1, 1, 0, 4, 4, 6, 6, 8, 8, 9],
|
||||
<int>[0, 0, 4, 3, 6, 6, 8, 7, 9, 9, 10, 10],
|
||||
<int>[0, 2, 2, 5, 4, 7, 7, 8, 8, 10, 10, 11],
|
||||
<int>[-1, -1, 0, 0, 3, 2, 5, 5, 7, 7, 9, 8],
|
||||
];
|
||||
static const List<List<int>> fullIndexTail = <List<int>>[
|
||||
<int>[16, 17, 17, 18, 17, 18, 18, 18, 18, 19, 19, 19],
|
||||
<int>[16, 16, 17, 16, 17, 17, 18, 18, 18, 18, 19, 18],
|
||||
<int>[16, 16, 16, 17, 17, 17, 17, 18, 18, 18, 18, 19],
|
||||
<int>[17, 17, 17, 17, 18, 18, 18, 18, 19, 19, 19, 19],
|
||||
];
|
||||
|
||||
/// `my_build_indexes` on a spread of scales, isolated from everything else.
|
||||
static const List<double> probeScales = <double>[
|
||||
0.0,
|
||||
9.999999974752427e-07,
|
||||
0.05000000074505806,
|
||||
0.07989999651908875,
|
||||
0.07999999821186066,
|
||||
0.10999999940395355,
|
||||
0.11000010371208191,
|
||||
0.5,
|
||||
1.0,
|
||||
2.0,
|
||||
7.5,
|
||||
63.900001525878906,
|
||||
255.89999389648438,
|
||||
256.0,
|
||||
10000.0,
|
||||
];
|
||||
static const List<int> probeIndexes = <int>[
|
||||
-1, -1, -1, -1, 0, 0, 0, 12, 17, 23, 34, 51, 62, 63, 63,
|
||||
];
|
||||
}
|
||||
|
||||
/// The same closed-form recipe the generator uses, in Dart.
|
||||
///
|
||||
/// `y`, `means_supp` and `scales_supp` for a `[1, C, H, W]` tensor. Kept
|
||||
/// byte-identical to `recipe()` in `export_entropy_layer_golden.py`.
|
||||
({Float32List y, Float32List means, Float32List scales}) _recipe(
|
||||
int channels,
|
||||
int height,
|
||||
int width,
|
||||
) {
|
||||
final n = channels * height * width;
|
||||
final y = Float32List(n);
|
||||
final means = Float32List(n);
|
||||
final scales = Float32List(n);
|
||||
var i = 0;
|
||||
for (var c = 0; c < channels; c++) {
|
||||
for (var h = 0; h < height; h++) {
|
||||
for (var w = 0; w < width; w++, i++) {
|
||||
y[i] = ((c * 3 + h * 17 + w * 11) % 61 - 30) * 0.125;
|
||||
means[i] = ((c * 7 + h * 13 + w * 29) % 97 - 48) * 0.0625;
|
||||
scales[i] = ((c * 11 + h * 5 + w * 3) % 700) * 0.01 + 0.001;
|
||||
}
|
||||
}
|
||||
}
|
||||
return (y: y, means: means, scales: scales);
|
||||
}
|
||||
|
||||
/// One stage of `compress()`: mask, quantize, fold, index.
|
||||
({Int16List symbols, Int16List indexes}) _runStage(
|
||||
AeicMaskSet masks,
|
||||
Float32List y,
|
||||
Float32List meansSupp,
|
||||
Float32List scalesSupp,
|
||||
int stage,
|
||||
) {
|
||||
final means = masks.applyMask(meansSupp, stage);
|
||||
final scales = masks.applyMask(scalesSupp, stage);
|
||||
final maskedY = masks.applyMask(y, stage);
|
||||
final yq = Float32List(y.length);
|
||||
for (var i = 0; i < y.length; i++) {
|
||||
yq[i] = roundHalfToEven(f32(maskedY[i] - means[i]));
|
||||
}
|
||||
return (
|
||||
symbols: aeicToSymbols(masks.squeeze(yq)),
|
||||
indexes: aeicBuildIndexes(masks.squeeze(scales)),
|
||||
);
|
||||
}
|
||||
|
||||
void main() {
|
||||
group('AeicEntropyGeometry', () {
|
||||
test('512x512 ft32 matches the shapes the bitstream format assumes', () {
|
||||
final g = AeicEntropyGeometry.forResolution(512);
|
||||
expect(g.yShape, <int>[1, 256, 16, 16]);
|
||||
expect(g.zShape, <int>[1, 128, 4, 4]);
|
||||
expect(g.squeezedChannels, 64);
|
||||
// From aic/results/rans_port_spec.md §1.
|
||||
expect(g.zElements, 2048);
|
||||
expect(g.symbolsPerStage, 16384);
|
||||
expect(g.totalEntries, 67584);
|
||||
});
|
||||
|
||||
test('z is ceil(y/4), not floor', () {
|
||||
// 320/32 = 10 -> z must be 3, matching compress()'s reflect padding.
|
||||
final g = AeicEntropyGeometry.forResolution(320);
|
||||
expect(g.yHeight, 10);
|
||||
expect(g.zHeight, 3);
|
||||
});
|
||||
|
||||
test('rejects a resolution g_a cannot downsample by 32', () {
|
||||
expect(() => AeicEntropyGeometry.forResolution(500), throwsArgumentError);
|
||||
expect(() => AeicEntropyGeometry.forResolution(0), throwsArgumentError);
|
||||
});
|
||||
});
|
||||
|
||||
group('AeicMaskSet', () {
|
||||
test('the four masks are get_mask_four_parts, element for element', () {
|
||||
final masks = AeicMaskSet(
|
||||
AeicEntropyGeometry.forResolution(128, yChannels: 8),
|
||||
);
|
||||
// 128/32 = 4, so this is exactly the C=8 H=W=4 case in the golden.
|
||||
for (var stage = 0; stage < 4; stage++) {
|
||||
final tensor = masks.maskTensor(stage);
|
||||
final actual = tensor.map((v) => v == 1.0 ? '1' : '0').join();
|
||||
expect(actual, _Golden.masks[stage], reason: 'mask_$stage');
|
||||
}
|
||||
});
|
||||
|
||||
test('every position is claimed by exactly one stage, per channel', () {
|
||||
final geometry = AeicEntropyGeometry.forResolution(512);
|
||||
final masks = AeicMaskSet(geometry);
|
||||
final counts = Uint8List(geometry.yElements);
|
||||
for (var stage = 0; stage < 4; stage++) {
|
||||
final tensor = masks.maskTensor(stage);
|
||||
for (var i = 0; i < tensor.length; i++) {
|
||||
counts[i] += tensor[i].toInt();
|
||||
}
|
||||
}
|
||||
expect(counts.every((c) => c == 1), isTrue);
|
||||
});
|
||||
|
||||
test('each mask carries exactly a quarter of the tensor', () {
|
||||
final geometry = AeicEntropyGeometry.forResolution(512);
|
||||
final masks = AeicMaskSet(geometry);
|
||||
for (var stage = 0; stage < 4; stage++) {
|
||||
final live = masks
|
||||
.maskTensor(stage)
|
||||
.where((v) => v == 1.0)
|
||||
.length;
|
||||
expect(live, geometry.symbolsPerStage);
|
||||
}
|
||||
});
|
||||
|
||||
test('squeeze then unsqueeze is the identity on a masked tensor', () {
|
||||
final geometry = AeicEntropyGeometry.forResolution(512);
|
||||
final masks = AeicMaskSet(geometry);
|
||||
final recipe = _recipe(256, 16, 16);
|
||||
for (var stage = 0; stage < 4; stage++) {
|
||||
final masked = masks.applyMask(recipe.y, stage);
|
||||
final restored = masks.unsqueeze(masks.squeeze(masked), stage);
|
||||
expect(restored, masked, reason: 'stage $stage');
|
||||
}
|
||||
});
|
||||
|
||||
test('mergeContext replaces exactly the stage mask', () {
|
||||
final geometry = AeicEntropyGeometry.forResolution(512);
|
||||
final masks = AeicMaskSet(geometry);
|
||||
final base = _recipe(256, 16, 16).y;
|
||||
final stageLatent = masks.applyMask(_recipe(256, 16, 16).means, 2);
|
||||
final merged = masks.mergeContext(base, stageLatent, 2);
|
||||
final mask = masks.maskTensor(2);
|
||||
for (var i = 0; i < merged.length; i++) {
|
||||
expect(merged[i], mask[i] == 1.0 ? stageLatent[i] : base[i]);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
group('roundHalfToEven', () {
|
||||
test('ties go to even, unlike Dart round()', () {
|
||||
expect(roundHalfToEven(0.5), 0.0);
|
||||
expect(roundHalfToEven(1.5), 2.0);
|
||||
expect(roundHalfToEven(2.5), 2.0);
|
||||
expect(roundHalfToEven(-0.5), 0.0);
|
||||
expect(roundHalfToEven(-1.5), -2.0);
|
||||
expect(roundHalfToEven(-2.5), -2.0);
|
||||
// Dart disagrees on every one of those ties.
|
||||
expect((-0.5).roundToDouble(), -1.0);
|
||||
});
|
||||
|
||||
test('non-ties are ordinary rounding', () {
|
||||
expect(roundHalfToEven(0.49), 0.0);
|
||||
expect(roundHalfToEven(0.51), 1.0);
|
||||
expect(roundHalfToEven(-1.51), -2.0);
|
||||
expect(roundHalfToEven(-1.49), -1.0);
|
||||
expect(roundHalfToEven(7.0), 7.0);
|
||||
});
|
||||
});
|
||||
|
||||
group('aeicBuildIndexes', () {
|
||||
test('matches my_build_indexes on the probe scales', () {
|
||||
final scales = Float32List.fromList(_Golden.probeScales);
|
||||
expect(aeicBuildIndexes(scales), _Golden.probeIndexes);
|
||||
});
|
||||
|
||||
test('the 0.08 threshold is strict, and clamps do not leak', () {
|
||||
// 0.08 itself is NOT skipped; it clamps to row 0 because ln(0.08) is
|
||||
// below ln(0.11).
|
||||
expect(aeicBuildIndexes(Float32List.fromList(<double>[0.08])).first, 0);
|
||||
expect(
|
||||
aeicBuildIndexes(Float32List.fromList(<double>[0.0799])).first,
|
||||
-1,
|
||||
);
|
||||
expect(
|
||||
aeicBuildIndexes(Float32List.fromList(<double>[1e9])).first,
|
||||
kAeicScalesLevels - 1,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
group('aeicZIndexes', () {
|
||||
test('is the channel arange broadcast over H*W', () {
|
||||
final geometry = AeicEntropyGeometry.forResolution(512);
|
||||
final indexes = aeicZIndexes(geometry);
|
||||
expect(indexes.length, 2048);
|
||||
expect(indexes.first, 0);
|
||||
expect(indexes[15], 0);
|
||||
expect(indexes[16], 1);
|
||||
expect(indexes.last, 127);
|
||||
// Verified against results/golden/vectors/kodim01.gv, which stores the
|
||||
// exact int16 array the C++ coder was given.
|
||||
for (var i = 0; i < indexes.length; i++) {
|
||||
expect(indexes[i], i ~/ 16);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
group('aeicToSymbols', () {
|
||||
test('rejects a value the int16 wire format cannot carry', () {
|
||||
expect(
|
||||
() => aeicToSymbols(Float32List.fromList(<double>[40000.0])),
|
||||
throwsStateError,
|
||||
);
|
||||
expect(
|
||||
() => aeicToSymbols(Float32List.fromList(<double>[1.5])),
|
||||
throwsStateError,
|
||||
);
|
||||
expect(aeicToSymbols(Float32List.fromList(<double>[-3.0])), <int>[-3]);
|
||||
});
|
||||
});
|
||||
|
||||
group('aeicRgbToChw', () {
|
||||
test('reproduces ToTensor + Normalize([0.5], [0.5])', () {
|
||||
final chw = aeicRgbToChw(Uint8List.fromList(<int>[0, 128, 255]), 1);
|
||||
expect(chw.length, 3);
|
||||
expect(chw[0], -1.0);
|
||||
expect(chw[1], closeTo(0.00392, 1e-4));
|
||||
expect(chw[2], 1.0);
|
||||
});
|
||||
|
||||
test('is channel-planar, not interleaved', () {
|
||||
final rgb = Uint8List(4 * 3);
|
||||
for (var i = 0; i < 4; i++) {
|
||||
rgb[i * 3] = 255; // R
|
||||
rgb[i * 3 + 1] = 0; // G
|
||||
rgb[i * 3 + 2] = 0; // B
|
||||
}
|
||||
final chw = aeicRgbToChw(rgb, 2);
|
||||
expect(chw.sublist(0, 4), <double>[1.0, 1.0, 1.0, 1.0]);
|
||||
expect(chw.sublist(4, 8), <double>[-1.0, -1.0, -1.0, -1.0]);
|
||||
});
|
||||
|
||||
test('rejects a byte count that is not the stated square', () {
|
||||
expect(() => aeicRgbToChw(Uint8List(11), 2), throwsArgumentError);
|
||||
});
|
||||
});
|
||||
|
||||
group('four-stage symbol packing (golden)', () {
|
||||
test('C=8 H=4 W=4: symbols and indexes match torch exactly', () {
|
||||
final geometry = AeicEntropyGeometry.forResolution(128, yChannels: 8);
|
||||
final masks = AeicMaskSet(geometry);
|
||||
final recipe = _recipe(8, 4, 4);
|
||||
for (var stage = 0; stage < 4; stage++) {
|
||||
final out = _runStage(
|
||||
masks,
|
||||
recipe.y,
|
||||
recipe.means,
|
||||
recipe.scales,
|
||||
stage,
|
||||
);
|
||||
expect(out.symbols, _Golden.symbols[stage], reason: 'symbols $stage');
|
||||
expect(out.indexes, _Golden.indexes[stage], reason: 'indexes $stage');
|
||||
}
|
||||
});
|
||||
|
||||
test('C=256 H=W=16 (shipping geometry): fingerprint matches torch', () {
|
||||
final geometry = AeicEntropyGeometry.forResolution(512);
|
||||
final masks = AeicMaskSet(geometry);
|
||||
final recipe = _recipe(256, 16, 16);
|
||||
for (var stage = 0; stage < 4; stage++) {
|
||||
final out = _runStage(
|
||||
masks,
|
||||
recipe.y,
|
||||
recipe.means,
|
||||
recipe.scales,
|
||||
stage,
|
||||
);
|
||||
expect(out.symbols.length, 16384);
|
||||
expect(
|
||||
out.symbols.fold<int>(0, (a, b) => a + b),
|
||||
_Golden.fullSymbolSum[stage],
|
||||
reason: 'symbol sum $stage',
|
||||
);
|
||||
expect(out.symbols.sublist(0, 12), _Golden.fullSymbolHead[stage]);
|
||||
expect(out.symbols.sublist(16372), _Golden.fullSymbolTail[stage]);
|
||||
expect(
|
||||
out.indexes.fold<int>(0, (a, b) => a + b),
|
||||
_Golden.fullIndexSum[stage],
|
||||
reason: 'index sum $stage',
|
||||
);
|
||||
expect(out.indexes.sublist(0, 12), _Golden.fullIndexHead[stage]);
|
||||
expect(out.indexes.sublist(16372), _Golden.fullIndexTail[stage]);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
group('AeicEntropyCodec', () {
|
||||
test('encode pushes z, y0, y1, y2, y3 in that order and no other', () async {
|
||||
final geometry = AeicEntropyGeometry.forResolution(512);
|
||||
final network = _FakeNetwork(geometry);
|
||||
final coders = _RecordingCoders();
|
||||
final codec = AeicEntropyCodec(
|
||||
geometry: geometry,
|
||||
network: network,
|
||||
coders: coders,
|
||||
);
|
||||
final progress = <double>[];
|
||||
final stream = await codec.encode(
|
||||
Uint8List(512 * 512 * 3),
|
||||
onProgress: progress.add,
|
||||
);
|
||||
|
||||
expect(coders.encoder.groups, <int>[
|
||||
kAeicZCdfGroup,
|
||||
kAeicYCdfGroup,
|
||||
kAeicYCdfGroup,
|
||||
kAeicYCdfGroup,
|
||||
kAeicYCdfGroup,
|
||||
]);
|
||||
expect(coders.encoder.lengths, <int>[2048, 16384, 16384, 16384, 16384]);
|
||||
expect(stream, isNotEmpty);
|
||||
expect(progress.last, 1.0);
|
||||
// The image really was normalized before the graph saw it: 0 -> -1.
|
||||
expect(network.lastInput!.first, -1.0);
|
||||
});
|
||||
|
||||
test('encode refuses a graph that returns the wrong z size', () async {
|
||||
final geometry = AeicEntropyGeometry.forResolution(512);
|
||||
final codec = AeicEntropyCodec(
|
||||
geometry: geometry,
|
||||
network: _FakeNetwork(geometry, zElements: 7),
|
||||
coders: _RecordingCoders(),
|
||||
);
|
||||
await expectLater(
|
||||
codec.encode(Uint8List(512 * 512 * 3)),
|
||||
throwsStateError,
|
||||
);
|
||||
});
|
||||
|
||||
test('encode honours shouldCancel between stages', () async {
|
||||
final geometry = AeicEntropyGeometry.forResolution(512);
|
||||
final codec = AeicEntropyCodec(
|
||||
geometry: geometry,
|
||||
network: _FakeNetwork(geometry),
|
||||
coders: _RecordingCoders(),
|
||||
);
|
||||
await expectLater(
|
||||
codec.encode(Uint8List(512 * 512 * 3), shouldCancel: () => true),
|
||||
throwsA(isA<AeicEntropyCancelled>()),
|
||||
);
|
||||
});
|
||||
|
||||
test('decode reports a send-side-only graph instead of guessing', () async {
|
||||
final geometry = AeicEntropyGeometry.forResolution(512);
|
||||
final codec = AeicEntropyCodec(
|
||||
geometry: geometry,
|
||||
network: _FakeNetwork(geometry, decodeSide: false),
|
||||
coders: _RecordingCoders(),
|
||||
);
|
||||
await expectLater(
|
||||
codec.decodeToLatent(Uint8List(16)),
|
||||
throwsA(isA<AeicEntropyUnavailable>()),
|
||||
);
|
||||
});
|
||||
|
||||
test('decode walks the stages in order and rebuilds y_hat', () async {
|
||||
final geometry = AeicEntropyGeometry.forResolution(512);
|
||||
final network = _FakeNetwork(geometry);
|
||||
final coders = _RecordingCoders();
|
||||
final codec = AeicEntropyCodec(
|
||||
geometry: geometry,
|
||||
network: network,
|
||||
coders: coders,
|
||||
);
|
||||
final yHat = await codec.decodeToLatent(Uint8List(16));
|
||||
|
||||
expect(network.stageCalls, <int>[0, 1, 2, 3]);
|
||||
expect(coders.decoder.groups, <int>[
|
||||
kAeicZCdfGroup,
|
||||
kAeicYCdfGroup,
|
||||
kAeicYCdfGroup,
|
||||
kAeicYCdfGroup,
|
||||
kAeicYCdfGroup,
|
||||
]);
|
||||
expect(yHat.length, geometry.yElements);
|
||||
// The fake decoder returns symbol 1 everywhere and the fake network
|
||||
// returns means 0, so every position of y_hat must be exactly 1 — which
|
||||
// only holds if the four masks tile the tensor and each stage's squeezed
|
||||
// symbols were unsqueezed back into the right channel group.
|
||||
expect(yHat.every((v) => v == 1.0), isTrue);
|
||||
});
|
||||
});
|
||||
// The end-to-end proof: the entropy layer plus the pure-Dart range coder
|
||||
// reproduce, byte for byte, the bitstreams the C++ coder produced for real
|
||||
// images — and decode them back to the same symbols.
|
||||
group('bitstream round trip against the C++ golden vectors', () {
|
||||
final Directory goldenDir = _resolveGoldenDir();
|
||||
final EntropyTables tables = EntropyTables.parse(
|
||||
File('${goldenDir.path}/aeic_cdf_ft32.bin').readAsBytesSync(),
|
||||
);
|
||||
final geometry = AeicEntropyGeometry.forResolution(512);
|
||||
final masks = AeicMaskSet(geometry);
|
||||
|
||||
// kodim01 is the plain case; kodim23 and image2 both carry index -1
|
||||
// (scales below 0.08), which the coder must skip on encode and read back as
|
||||
// a literal 0.
|
||||
for (final name in <String>['kodim01', 'kodim23', 'image2']) {
|
||||
test('$name: symbols -> bitstream -> symbols', () async {
|
||||
final vector = _readGoldenVector(
|
||||
File('${goldenDir.path}/vectors/$name.gv').readAsBytesSync(),
|
||||
);
|
||||
final expectedStream = File(
|
||||
'${goldenDir.path}/vectors/$name.bin',
|
||||
).readAsBytesSync();
|
||||
|
||||
// Rebuild the pre-fold tensors the graph would have produced. `squeeze`
|
||||
// is a bijection on masked tensors, so unsqueezing the golden arrays
|
||||
// recovers a legitimate input and the loop's own fold has to invert it.
|
||||
final yQ = <Float32List>[];
|
||||
final scales = <Float32List>[];
|
||||
for (var stage = 0; stage < 4; stage++) {
|
||||
final symbols = vector['y_q$stage']!;
|
||||
final indexes = vector['y_indexes$stage']!;
|
||||
final squeezedY = Float32List(symbols.length);
|
||||
final squeezedS = Float32List(indexes.length);
|
||||
for (var i = 0; i < symbols.length; i++) {
|
||||
squeezedY[i] = symbols[i].toDouble();
|
||||
squeezedS[i] = _scaleForIndex(indexes[i]);
|
||||
}
|
||||
// Self-check: the synthesized scales must land back on the exact
|
||||
// golden indexes, or this test is measuring the wrong thing.
|
||||
expect(aeicBuildIndexes(squeezedS), indexes, reason: 'stage $stage');
|
||||
yQ.add(masks.unsqueeze(squeezedY, stage));
|
||||
scales.add(masks.unsqueeze(squeezedS, stage));
|
||||
}
|
||||
final zQ = Float32List(vector['z_q']!.length);
|
||||
for (var i = 0; i < zQ.length; i++) {
|
||||
zQ[i] = vector['z_q']![i].toDouble();
|
||||
}
|
||||
|
||||
final network = _ReplayNetwork(
|
||||
geometry: geometry,
|
||||
tensors: AeicEncodeSideTensors(zQ: zQ, yQ: yQ, scales: scales),
|
||||
);
|
||||
final codec = AeicEntropyCodec(
|
||||
geometry: geometry,
|
||||
network: network,
|
||||
coders: AeicRansCoders(tables),
|
||||
);
|
||||
|
||||
// ENCODE: byte-for-byte against the C++ coder's output.
|
||||
final stream = await codec.encode(Uint8List(512 * 512 * 3));
|
||||
expect(stream, expectedStream, reason: 'bitstream for $name');
|
||||
|
||||
// DECODE: the same bytes back to the same symbols. means are zero, so
|
||||
// y_hat is exactly the four stages' symbols tiled back into place —
|
||||
// which only holds if every mask, fold and unfold agrees with encode.
|
||||
final yHat = await codec.decodeToLatent(stream);
|
||||
var expected = Float32List(geometry.yElements);
|
||||
for (var stage = 0; stage < 4; stage++) {
|
||||
expected = masks.mergeContext(expected, yQ[stage], stage);
|
||||
}
|
||||
expect(yHat, expected, reason: 'y_hat for $name');
|
||||
expect(network.stageCalls, <int>[0, 1, 2, 3]);
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
Directory _resolveGoldenDir() {
|
||||
for (final candidate in <String>[
|
||||
'test/services/golden',
|
||||
'../test/services/golden',
|
||||
'golden',
|
||||
]) {
|
||||
final dir = Directory(candidate);
|
||||
if (dir.existsSync()) {
|
||||
return dir;
|
||||
}
|
||||
}
|
||||
throw StateError(
|
||||
'golden vectors not found; expected test/services/golden relative to the '
|
||||
'package root',
|
||||
);
|
||||
}
|
||||
|
||||
/// Parses the `.gv` container documented in `aic/results/rans_port_spec.md` §10.
|
||||
Map<String, Int16List> _readGoldenVector(Uint8List bytes) {
|
||||
final data = ByteData.sublistView(bytes);
|
||||
const magic = <int>[0x41, 0x45, 0x49, 0x43, 0x47, 0x56, 0x00, 0x01];
|
||||
for (var i = 0; i < magic.length; i++) {
|
||||
if (bytes[i] != magic[i]) {
|
||||
throw StateError('not a .gv container');
|
||||
}
|
||||
}
|
||||
final count = data.getUint32(12, Endian.little);
|
||||
final names = <String>[];
|
||||
final counts = <int>[];
|
||||
final dtypes = <int>[];
|
||||
var off = 16;
|
||||
for (var i = 0; i < count; i++) {
|
||||
final raw = bytes.sublist(off, off + 16);
|
||||
final end = raw.indexOf(0);
|
||||
names.add(String.fromCharCodes(raw.sublist(0, end < 0 ? 16 : end)));
|
||||
dtypes.add(data.getUint32(off + 16, Endian.little));
|
||||
counts.add(data.getUint32(off + 20, Endian.little));
|
||||
off += 24;
|
||||
}
|
||||
final out = <String, Int16List>{};
|
||||
for (var i = 0; i < count; i++) {
|
||||
if (dtypes[i] != 0) {
|
||||
throw StateError('${names[i]} is not int16');
|
||||
}
|
||||
final values = Int16List(counts[i]);
|
||||
for (var j = 0; j < counts[i]; j++) {
|
||||
values[j] = data.getInt16(off + j * 2, Endian.little);
|
||||
}
|
||||
off += counts[i] * 2;
|
||||
out[names[i]] = values;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/// A scale that `my_build_indexes` maps back to exactly [index].
|
||||
///
|
||||
/// Row centres, so float32 rounding cannot push one over a boundary; index -1
|
||||
/// means "skipped", which any scale below 0.08 produces.
|
||||
double _scaleForIndex(int index) {
|
||||
if (index < 0) {
|
||||
return 0.0;
|
||||
}
|
||||
return f32(math.exp(kAeicLogScaleMin + (index + 0.5) * kAeicLogScaleStep));
|
||||
}
|
||||
|
||||
/// Replays fixed tensors as if they came from the graph, for both directions.
|
||||
class _ReplayNetwork implements AeicEntropyNetwork {
|
||||
final AeicEntropyGeometry geometry;
|
||||
final AeicEncodeSideTensors tensors;
|
||||
final List<int> stageCalls = <int>[];
|
||||
|
||||
_ReplayNetwork({required this.geometry, required this.tensors});
|
||||
|
||||
@override
|
||||
bool get supportsDecodeSide => true;
|
||||
|
||||
@override
|
||||
Future<AeicEncodeSideTensors> runEncodeSide(Float32List imageChw) async =>
|
||||
tensors;
|
||||
|
||||
@override
|
||||
Future<Float32List> runHyperSynthesis(Float32List zQ) async {
|
||||
// The real h_s consumes z_hat; here the only thing under test is that the
|
||||
// decoded z symbols reach it. Assert that and hand back a zero context.
|
||||
expect(zQ.length, geometry.zElements);
|
||||
for (var i = 0; i < zQ.length; i++) {
|
||||
expect(zQ[i], tensors.zQ[i], reason: 'z symbol $i');
|
||||
}
|
||||
return Float32List(geometry.yElements);
|
||||
}
|
||||
|
||||
@override
|
||||
Future<AeicStageParams> runStage(int stage, Float32List base) async {
|
||||
stageCalls.add(stage);
|
||||
return AeicStageParams(
|
||||
meansSupp: Float32List(geometry.yElements),
|
||||
scalesSupp: tensors.scales[stage],
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// A stand-in for the ONNX graph: shapes and call order, no arithmetic.
|
||||
class _FakeNetwork implements AeicEntropyNetwork {
|
||||
final AeicEntropyGeometry geometry;
|
||||
final int? zElements;
|
||||
final bool decodeSide;
|
||||
final List<int> stageCalls = <int>[];
|
||||
Float32List? lastInput;
|
||||
|
||||
_FakeNetwork(this.geometry, {this.zElements, this.decodeSide = true});
|
||||
|
||||
@override
|
||||
bool get supportsDecodeSide => decodeSide;
|
||||
|
||||
@override
|
||||
Future<AeicEncodeSideTensors> runEncodeSide(Float32List imageChw) async {
|
||||
lastInput = imageChw;
|
||||
return AeicEncodeSideTensors(
|
||||
zQ: Float32List(zElements ?? geometry.zElements),
|
||||
yQ: <Float32List>[
|
||||
for (var i = 0; i < 4; i++) Float32List(geometry.yElements),
|
||||
],
|
||||
scales: <Float32List>[
|
||||
for (var i = 0; i < 4; i++)
|
||||
Float32List(geometry.yElements)..fillRange(0, geometry.yElements, 1.0),
|
||||
],
|
||||
);
|
||||
}
|
||||
|
||||
@override
|
||||
Future<Float32List> runHyperSynthesis(Float32List zQ) async =>
|
||||
Float32List(geometry.yElements);
|
||||
|
||||
@override
|
||||
Future<AeicStageParams> runStage(int stage, Float32List base) async {
|
||||
stageCalls.add(stage);
|
||||
return AeicStageParams(
|
||||
meansSupp: Float32List(geometry.yElements),
|
||||
scalesSupp: Float32List(geometry.yElements)
|
||||
..fillRange(0, geometry.yElements, 1.0),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
class _RecordingEncoder implements AeicRansEncoder {
|
||||
final List<int> groups = <int>[];
|
||||
final List<int> lengths = <int>[];
|
||||
|
||||
@override
|
||||
void pushSymbols(Int16List symbols, Int16List indexes, int cdfGroup) {
|
||||
expect(symbols.length, indexes.length);
|
||||
groups.add(cdfGroup);
|
||||
lengths.add(symbols.length);
|
||||
}
|
||||
|
||||
@override
|
||||
Uint8List finish() => Uint8List.fromList(<int>[0x11, 4, 0]);
|
||||
}
|
||||
|
||||
class _RecordingDecoder implements AeicRansDecoder {
|
||||
final List<int> groups = <int>[];
|
||||
|
||||
@override
|
||||
Int16List decodeStream(Int16List indexes, int cdfGroup) {
|
||||
groups.add(cdfGroup);
|
||||
return Int16List(indexes.length)..fillRange(0, indexes.length, 1);
|
||||
}
|
||||
}
|
||||
|
||||
class _RecordingCoders implements AeicRansCoderFactory {
|
||||
final _RecordingEncoder encoder = _RecordingEncoder();
|
||||
final _RecordingDecoder decoder = _RecordingDecoder();
|
||||
|
||||
@override
|
||||
AeicRansEncoder createEncoder() => encoder;
|
||||
|
||||
@override
|
||||
AeicRansDecoder createDecoder(Uint8List bitstream) => decoder;
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
import 'dart:typed_data';
|
||||
|
||||
import 'package:flutter_test/flutter_test.dart';
|
||||
import 'package:meshcore_open/services/image_codec_service.dart';
|
||||
|
||||
/// Covers the one piece of the decode path that can be executed without a model
|
||||
/// file, a device or ONNX Runtime: turning the backend's packed RGB output into
|
||||
/// PNG bytes a widget can render.
|
||||
///
|
||||
/// It needs the engine's image codecs, hence the binding.
|
||||
void main() {
|
||||
TestWidgetsFlutterBinding.ensureInitialized();
|
||||
|
||||
group('ImageCodecService.rgbToPng', () {
|
||||
test('encodes a PNG with the right magic bytes and dimensions', () async {
|
||||
const side = 8;
|
||||
final rgb = Uint8List(side * side * 3);
|
||||
for (var i = 0; i < rgb.length; i++) {
|
||||
rgb[i] = i & 0xFF;
|
||||
}
|
||||
|
||||
final png = await ImageCodecService.rgbToPng(rgb, side);
|
||||
|
||||
expect(png.sublist(0, 8), [
|
||||
0x89,
|
||||
0x50,
|
||||
0x4E,
|
||||
0x47,
|
||||
0x0D,
|
||||
0x0A,
|
||||
0x1A,
|
||||
0x0A,
|
||||
]);
|
||||
// IHDR width/height, big-endian at offsets 16 and 20.
|
||||
final header = ByteData.sublistView(png);
|
||||
expect(header.getUint32(16), side);
|
||||
expect(header.getUint32(20), side);
|
||||
});
|
||||
|
||||
test('rejects a buffer that is not RGB at the stated size', () async {
|
||||
await expectLater(
|
||||
ImageCodecService.rgbToPng(Uint8List(10), 8),
|
||||
throwsA(isA<ArgumentError>()),
|
||||
);
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,263 @@
|
||||
import 'dart:convert';
|
||||
import 'dart:io';
|
||||
import 'dart:typed_data';
|
||||
|
||||
import 'package:flutter_test/flutter_test.dart';
|
||||
import 'package:meshcore_open/services/entropy_tables.dart';
|
||||
import 'package:meshcore_open/services/rans_coder.dart';
|
||||
|
||||
/// Golden-vector conformance for the pure-Dart rANS port.
|
||||
///
|
||||
/// The bar is byte-identical: encoding the golden symbol/index arrays must
|
||||
/// reproduce the exact bitstream the C++ coder produced, and decoding that
|
||||
/// bitstream must reproduce the exact symbols. A single differing byte
|
||||
/// desynchronises rANS and silently corrupts most of an image.
|
||||
void main() {
|
||||
final Directory goldenDir = _resolveGoldenDir();
|
||||
final EntropyTables tables = EntropyTables.parse(
|
||||
File('${goldenDir.path}/aeic_cdf_ft32.bin').readAsBytesSync(),
|
||||
);
|
||||
final Map<String, dynamic> manifest =
|
||||
jsonDecode(File('${goldenDir.path}/manifest.json').readAsStringSync())
|
||||
as Map<String, dynamic>;
|
||||
final List<Map<String, dynamic>> images = (manifest['images'] as List<dynamic>)
|
||||
.cast<Map<String, dynamic>>();
|
||||
final List<Map<String, dynamic>> synthetic =
|
||||
(manifest['synthetic'] as List<dynamic>).cast<Map<String, dynamic>>();
|
||||
|
||||
test('golden corpus is complete', () {
|
||||
expect(images.length, 10);
|
||||
expect(synthetic.length, 7);
|
||||
expect(manifest['stream_parts'], 2);
|
||||
expect(manifest['reference_port_selfcheck'], <String, dynamic>{
|
||||
'ok': 17,
|
||||
'total': 17,
|
||||
});
|
||||
});
|
||||
|
||||
group('image vectors', () {
|
||||
for (final Map<String, dynamic> rec in images) {
|
||||
final String stem = rec['stem'] as String;
|
||||
test('$stem encodes and decodes byte-identically', () {
|
||||
final Map<String, List<int>> arrays = _readGoldenVector(
|
||||
File(
|
||||
'${goldenDir.path}/vectors/${rec['vector_file']}',
|
||||
).readAsBytesSync(),
|
||||
);
|
||||
final Uint8List want = File(
|
||||
'${goldenDir.path}/vectors/${rec['bitstream_file']}',
|
||||
).readAsBytesSync();
|
||||
|
||||
// Call order is part of the format: z, then y0..y3.
|
||||
final List<_Call> calls = <_Call>[
|
||||
_Call(arrays['z_q']!, arrays['z_indexes']!, 0),
|
||||
for (var i = 0; i < 4; i++)
|
||||
_Call(arrays['y_q$i']!, arrays['y_indexes$i']!, 1),
|
||||
];
|
||||
|
||||
final RansEncoder encoder = RansEncoder(tables);
|
||||
for (final _Call c in calls) {
|
||||
encoder.encodeWithIndexes(c.symbols, c.indexes, c.group);
|
||||
}
|
||||
_expectSameBytes(encoder.finish(), want, stem);
|
||||
|
||||
// One decoder, five incremental calls sharing the sub-stream states.
|
||||
final RansDecoder decoder = RansDecoder(tables, want);
|
||||
for (final _Call c in calls) {
|
||||
_expectSameSymbols(decoder.decodeStream(c.indexes, c.group), c, stem);
|
||||
}
|
||||
|
||||
final List<Uint8List> parts = parseRansContainer(want);
|
||||
expect(
|
||||
parts.map((Uint8List p) => p.length).toList(),
|
||||
(rec['substream_sizes'] as List<dynamic>).cast<int>(),
|
||||
);
|
||||
expect(want[0], rec['container_flag']);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
group('synthetic vectors', () {
|
||||
for (final Map<String, dynamic> rec in synthetic) {
|
||||
final String name = rec['name'] as String;
|
||||
test('$name encodes and decodes byte-identically', () {
|
||||
final int group = rec['cdf_group'] as int;
|
||||
final Map<String, List<int>> arrays = _readGoldenVector(
|
||||
File(
|
||||
'${goldenDir.path}/vectors/${rec['vector_file']}',
|
||||
).readAsBytesSync(),
|
||||
);
|
||||
final Uint8List want = File(
|
||||
'${goldenDir.path}/vectors/${rec['bitstream_file']}',
|
||||
).readAsBytesSync();
|
||||
final _Call call = _Call(arrays['symbols']!, arrays['indexes']!, group);
|
||||
|
||||
final RansEncoder encoder = RansEncoder(tables);
|
||||
encoder.encodeWithIndexes(call.symbols, call.indexes, call.group);
|
||||
_expectSameBytes(encoder.finish(), want, name);
|
||||
|
||||
final RansDecoder decoder = RansDecoder(tables, want);
|
||||
_expectSameSymbols(
|
||||
decoder.decodeStream(call.indexes, call.group),
|
||||
call,
|
||||
name,
|
||||
);
|
||||
|
||||
final List<Uint8List> parts = parseRansContainer(want);
|
||||
expect(
|
||||
parts.map((Uint8List p) => p.length).toList(),
|
||||
(rec['substream_sizes'] as List<dynamic>).cast<int>(),
|
||||
);
|
||||
expect(want[0], rec['container_flag']);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
test('container round-trips through build/parse', () {
|
||||
final List<Uint8List> parts = <Uint8List>[
|
||||
Uint8List.fromList(<int>[1, 2, 3, 4, 5]),
|
||||
Uint8List.fromList(<int>[9, 8, 7]),
|
||||
];
|
||||
final Uint8List packed = buildRansContainer(parts);
|
||||
expect(packed[0], 0x11);
|
||||
final List<Uint8List> back = parseRansContainer(packed);
|
||||
expect(back.length, 2);
|
||||
expect(back[0], parts[0]);
|
||||
expect(back[1], parts[1]);
|
||||
});
|
||||
|
||||
test('a flipped bitstream byte would be caught', () {
|
||||
// Guards the comparison itself against being vacuous.
|
||||
final Map<String, List<int>> arrays = _readGoldenVector(
|
||||
File('${goldenDir.path}/vectors/synth_y_tiny.gv').readAsBytesSync(),
|
||||
);
|
||||
final Uint8List want = File(
|
||||
'${goldenDir.path}/vectors/synth_y_tiny.bin',
|
||||
).readAsBytesSync();
|
||||
final Uint8List mutated = Uint8List.fromList(want);
|
||||
mutated[mutated.length - 1] ^= 0x01;
|
||||
final RansEncoder encoder = RansEncoder(tables);
|
||||
encoder.encodeWithIndexes(arrays['symbols']!, arrays['indexes']!, 1);
|
||||
final Uint8List got = encoder.finish();
|
||||
expect(got, equals(want));
|
||||
expect(got, isNot(equals(mutated)));
|
||||
});
|
||||
|
||||
test('encoder rejects a second finish()', () {
|
||||
final RansEncoder encoder = RansEncoder(tables);
|
||||
encoder.finish();
|
||||
expect(encoder.finish, throwsStateError);
|
||||
});
|
||||
}
|
||||
|
||||
class _Call {
|
||||
_Call(this.symbols, this.indexes, this.group);
|
||||
|
||||
final List<int> symbols;
|
||||
final List<int> indexes;
|
||||
final int group;
|
||||
}
|
||||
|
||||
void _expectSameBytes(Uint8List got, Uint8List want, String label) {
|
||||
final int n = got.length < want.length ? got.length : want.length;
|
||||
for (var i = 0; i < n; i++) {
|
||||
if (got[i] != want[i]) {
|
||||
fail(
|
||||
'$label: first byte divergence at offset $i of ${want.length} '
|
||||
'(got 0x${got[i].toRadixString(16)}, '
|
||||
'want 0x${want[i].toRadixString(16)})',
|
||||
);
|
||||
}
|
||||
}
|
||||
expect(
|
||||
got.length,
|
||||
want.length,
|
||||
reason: '$label: length differs (prefix matched)',
|
||||
);
|
||||
}
|
||||
|
||||
void _expectSameSymbols(Int16List got, _Call call, String label) {
|
||||
expect(got.length, call.symbols.length, reason: '$label: length');
|
||||
for (var i = 0; i < got.length; i++) {
|
||||
// idx < 0 is asymmetric: it emits nothing on encode, decodes as literal 0.
|
||||
final int want = call.indexes[i] < 0 ? 0 : call.symbols[i];
|
||||
if (got[i] != want) {
|
||||
fail(
|
||||
'$label: first symbol divergence at $i '
|
||||
'(got ${got[i]}, want $want, index ${call.indexes[i]})',
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Directory _resolveGoldenDir() {
|
||||
for (final String candidate in <String>[
|
||||
'test/services/golden',
|
||||
'../test/services/golden',
|
||||
'golden',
|
||||
]) {
|
||||
final Directory d = Directory(candidate);
|
||||
if (d.existsSync()) return d;
|
||||
}
|
||||
return Directory('test/services/golden');
|
||||
}
|
||||
|
||||
/// Reads a `.gv` golden-vector container.
|
||||
///
|
||||
/// char[8] magic "AEICGV\0\x01", u32 version, u32 nArrays,
|
||||
/// nArrays x { char[16] name, u32 dtype (0=int16, 1=int32), u32 count },
|
||||
/// then the payloads back to back, little-endian.
|
||||
Map<String, List<int>> _readGoldenVector(Uint8List raw) {
|
||||
const List<int> magic = <int>[0x41, 0x45, 0x49, 0x43, 0x47, 0x56, 0x00, 0x01];
|
||||
for (var i = 0; i < magic.length; i++) {
|
||||
if (raw[i] != magic[i]) {
|
||||
throw FormatException('bad .gv magic at byte $i');
|
||||
}
|
||||
}
|
||||
final ByteData bd = ByteData.view(
|
||||
raw.buffer,
|
||||
raw.offsetInBytes,
|
||||
raw.lengthInBytes,
|
||||
);
|
||||
final int version = bd.getUint32(8, Endian.little);
|
||||
if (version != 1) {
|
||||
throw FormatException('unsupported .gv version $version');
|
||||
}
|
||||
final int n = bd.getUint32(12, Endian.little);
|
||||
var off = 16;
|
||||
final List<String> names = <String>[];
|
||||
final List<int> dtypes = <int>[];
|
||||
final List<int> counts = <int>[];
|
||||
for (var i = 0; i < n; i++) {
|
||||
final List<int> nameBytes = raw.sublist(off, off + 16);
|
||||
var end = nameBytes.indexOf(0);
|
||||
if (end < 0) end = nameBytes.length;
|
||||
names.add(ascii.decode(nameBytes.sublist(0, end)));
|
||||
dtypes.add(bd.getUint32(off + 16, Endian.little));
|
||||
counts.add(bd.getUint32(off + 20, Endian.little));
|
||||
off += 24;
|
||||
}
|
||||
final Map<String, List<int>> out = <String, List<int>>{};
|
||||
for (var k = 0; k < n; k++) {
|
||||
final int count = counts[k];
|
||||
if (dtypes[k] == 0) {
|
||||
final Int16List a = Int16List(count);
|
||||
for (var i = 0; i < count; i++) {
|
||||
a[i] = bd.getInt16(off + i * 2, Endian.little);
|
||||
}
|
||||
off += count * 2;
|
||||
out[names[k]] = a;
|
||||
} else {
|
||||
final Int32List a = Int32List(count);
|
||||
for (var i = 0; i < count; i++) {
|
||||
a[i] = bd.getInt32(off + i * 4, Endian.little);
|
||||
}
|
||||
off += count * 4;
|
||||
out[names[k]] = a;
|
||||
}
|
||||
}
|
||||
if (off != raw.length) {
|
||||
throw FormatException('.gv trailing data: $off of ${raw.length}');
|
||||
}
|
||||
return out;
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user