zjs81/meshcore-open
1
0
Fork 0
mirror of https://github.com/zjs81/meshcore-open.git synced 2026-08-11 18:26:27 +10:00
Code Issues Packages Projects Releases 2 Wiki Activity

rebuild translations

Browse Source
This commit is contained in:
zjs81
2026-08-11 00:15:45 -07:00
parent eee72b4c65
commit cf6c93c50a
63 changed files with 6584 additions and 4040 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/services/entropy_tables_test.dart
+2 -1
View File
@@ -116,7 +116,8 @@ void main() {
});
test('rejects trailing garbage', () {
final Uint8List extra = Uint8List(raw.length + 1)..setRange(0, raw.length, raw);
final Uint8List extra = Uint8List(raw.length + 1)
..setRange(0, raw.length, raw);
expect(
() => EntropyTables.parse(extra),
throwsA(isA<EntropyTableFormatException>()),
test/services/image_codec_boot_payload_test.dart
+23 -23
View File
@@ -16,21 +16,19 @@ import 'package:meshcore_open/services/image_codec_session_io.dart';
/// needs to rebuild a bundle that can decode.
void main() {
ImageCodecBundle fiveAssetBundle() => const ImageCodecBundle(
decoderGraphPath: '/models/aeic_decoder_qdq_conv_pct.onnx',
entropyGraphPath: '/models/aeic_entropy_side_fp32_op17.onnx',
entropyDecodeGraphPath: '/models/aeic_entropy_decode_fp32_op17.onnx',
tablesPath: '/models/aeic_cdf_ft32.bin',
ratePoint: AeicRatePoint.ft32,
);
decoderGraphPath: '/models/aeic_decoder_qdq_conv_pct.onnx',
entropyGraphPath: '/models/aeic_entropy_side_fp32_op17.onnx',
entropyDecodeGraphPath: '/models/aeic_entropy_decode_fp32_op17.onnx',
tablesPath: '/models/aeic_cdf_ft32.bin',
ratePoint: AeicRatePoint.ft32,
);
group('codec worker boot payload', () {
test('a five-asset bundle survives the round trip and can still decode', () {
final sent = fiveAssetBundle();
expect(sent.supportsDecode, isTrue, reason: 'precondition');
final rebuilt = debugBundleFromBootPayload(
debugBootPayloadFor(sent),
);
final rebuilt = debugBundleFromBootPayload(debugBootPayloadFor(sent));
expect(rebuilt.decoderGraphPath, sent.decoderGraphPath);
expect(rebuilt.entropyGraphPath, sent.entropyGraphPath);
@@ -56,20 +54,22 @@ void main() {
expect(payload[6], '/models/aeic_entropy_decode_fp32_op17.onnx');
});
test('a send-only bundle rebuilds as send-only rather than half-decoding',
() {
const sendOnly = ImageCodecBundle(
decoderGraphPath: '/models/decoder.onnx',
entropyGraphPath: '/models/entropy.onnx',
tablesPath: '/models/tables.bin',
ratePoint: AeicRatePoint.ft32,
);
final rebuilt = debugBundleFromBootPayload(
debugBootPayloadFor(sendOnly),
);
expect(rebuilt.entropyDecodeGraphPath, isNull);
expect(rebuilt.supportsDecode, isFalse);
});
test(
'a send-only bundle rebuilds as send-only rather than half-decoding',
() {
const sendOnly = ImageCodecBundle(
decoderGraphPath: '/models/decoder.onnx',
entropyGraphPath: '/models/entropy.onnx',
tablesPath: '/models/tables.bin',
ratePoint: AeicRatePoint.ft32,
);
final rebuilt = debugBundleFromBootPayload(
debugBootPayloadFor(sendOnly),
);
expect(rebuilt.entropyDecodeGraphPath, isNull);
expect(rebuilt.supportsDecode, isFalse);
},
);
test('a short payload does not crash the worker', () {
// An older sender, or a truncated message, must degrade to "cannot
test/services/image_codec_download_test.dart
+1 -4
View File
@@ -250,10 +250,7 @@ void main() {
expect(await graph.readAsBytes(), small);
expect(await weights.length(), large.length);
expect(_sha256(await weights.readAsBytes()), _sha256(large));
expect(
await File('${tempDir.path}/entropy.onnx').readAsBytes(),
entropy,
);
expect(await File('${tempDir.path}/entropy.onnx').readAsBytes(), entropy);
// Two different entropy exports land side by side; neither may overwrite
// or be mistaken for the other.
expect(
test/services/image_codec_e2e_test.dart
+54 -47
View File
@@ -121,45 +121,49 @@ void main() {
}
});
test('ENCODE: Dart bitstream is byte-identical to the C++ bitstream',
() async {
final _ReplayNetwork network = _ReplayNetwork(r);
final AeicEntropyCodec codec = AeicEntropyCodec(
geometry: geometry,
network: network,
coders: coders,
);
final Uint8List got = await codec.encode(
Uint8List(geometry.resolution * geometry.resolution * 3),
);
expect(network.encodeCalls, 1);
_expectSameBytes(got, r.u8('enc/bitstream'), name);
expect(
sha256.convert(got).toString(),
r.meta['bitstream_sha256'],
reason: '$name: bitstream sha256',
);
});
test(
'ENCODE: Dart bitstream is byte-identical to the C++ bitstream',
() async {
final _ReplayNetwork network = _ReplayNetwork(r);
final AeicEntropyCodec codec = AeicEntropyCodec(
geometry: geometry,
network: network,
coders: coders,
);
final Uint8List got = await codec.encode(
Uint8List(geometry.resolution * geometry.resolution * 3),
);
expect(network.encodeCalls, 1);
_expectSameBytes(got, r.u8('enc/bitstream'), name);
expect(
sha256.convert(got).toString(),
r.meta['bitstream_sha256'],
reason: '$name: bitstream sha256',
);
},
);
test('DECODE: y_hat from the C++ bitstream is exactly the recorded y_hat',
() async {
final _ReplayNetwork network = _ReplayNetwork(r);
final AeicEntropyCodec codec = AeicEntropyCodec(
geometry: geometry,
network: network,
coders: coders,
);
final Float32List got = await codec.decodeToLatent(
r.u8('enc/bitstream'),
);
expect(network.hyperCalls, 1);
expect(network.stageCalls, <int>[0, 1, 2, 3]);
_expectSameFloats(got, r.f32('dec/y_hat'), '$name: y_hat');
// The recording asserts the decoder's latent equals the encoder's; if
// that holds in Python it must hold here too.
expect(r.meta['decoded_y_hat_equals_encoder_y_hat'], isTrue);
_expectSameFloats(got, r.f32('enc/y_hat'), '$name: y_hat vs encoder');
});
test(
'DECODE: y_hat from the C++ bitstream is exactly the recorded y_hat',
() async {
final _ReplayNetwork network = _ReplayNetwork(r);
final AeicEntropyCodec codec = AeicEntropyCodec(
geometry: geometry,
network: network,
coders: coders,
);
final Float32List got = await codec.decodeToLatent(
r.u8('enc/bitstream'),
);
expect(network.hyperCalls, 1);
expect(network.stageCalls, <int>[0, 1, 2, 3]);
_expectSameFloats(got, r.f32('dec/y_hat'), '$name: y_hat');
// The recording asserts the decoder's latent equals the encoder's; if
// that holds in Python it must hold here too.
expect(r.meta['decoded_y_hat_equals_encoder_y_hat'], isTrue);
_expectSameFloats(got, r.f32('enc/y_hat'), '$name: y_hat vs encoder');
},
);
test('a single flipped bitstream byte does not still pass', () async {
// Byte 3 is the first payload byte of sub-stream 0 (1-byte flag +
@@ -185,7 +189,8 @@ void main() {
expect(
_sameFloats(got, r.f32('dec/y_hat')),
isFalse,
reason: '$name: corrupting the stream changed nothing — the '
reason:
'$name: corrupting the stream changed nothing — the '
'comparison is vacuous',
);
});
@@ -253,7 +258,8 @@ class _ReplayNetwork implements AeicEntropyNetwork {
'stage $stage input base',
);
}
final Map<String, dynamic> outputs = call['outputs'] as Map<String, dynamic>;
final Map<String, dynamic> outputs =
call['outputs'] as Map<String, dynamic>;
return AeicStageParams(
meansSupp: r.f32(outputs['means'] as String),
scalesSupp: r.f32(outputs['scales'] as String),
@@ -265,12 +271,11 @@ class _ReplayNetwork implements AeicEntropyNetwork {
/// 32-byte header, an 8-byte-aligned tensor blob, then a UTF-8 JSON index.
class _Recording {
_Recording(this.index, this.bytes)
: _entries = <String, Map<String, dynamic>>{
for (final Map<String, dynamic> e
in (index['entries'] as List<dynamic>)
.cast<Map<String, dynamic>>())
e['name'] as String: e,
};
: _entries = <String, Map<String, dynamic>>{
for (final Map<String, dynamic> e
in (index['entries'] as List<dynamic>).cast<Map<String, dynamic>>())
e['name'] as String: e,
};
final Map<String, dynamic> index;
final Uint8List bytes;
@@ -291,7 +296,9 @@ class _Recording {
final int indexLength = bd.getUint32(24, Endian.little);
final Map<String, dynamic> index =
jsonDecode(
utf8.decode(bytes.sublist(indexOffset, indexOffset + indexLength)),
utf8.decode(
bytes.sublist(indexOffset, indexOffset + indexLength),
),
)
as Map<String, dynamic>;
return _Recording(index, bytes);
test/services/image_codec_entropy_test.dart
+320 -49
View File
@@ -40,25 +40,281 @@ class _Golden {
];
static const List<List<int>> symbols = <List<int>>[
<int>[-1, -1, -2, 4, 0, 0, 6, -3, 2, 1, 1, -1, -5, 1, 0, 0, -1, -1, -2, 4,
0, 0, 5, -3, 2, 1, 1, -1, -5, 1, 0, 0],
<int>[-1, -1, 4, 4, 0, 0, 0, 5, 2, 1, -1, -1, 3, -5, 1, 0, -1, -2, 4, 4,
0, 0, 0, 5, 1, 1, -1, -1, 3, -5, 1, 0],
<int>[-1, -2, -2, 4, 1, 0, 0, 5, 2, 0, -1, -1, 3, -5, 1, 0, -1, -2, -2, 4,
0, 0, 0, 5, 2, -1, -1, -1, 3, -5, 1, 0],
<int>[-1, -1, -2, -2, 0, 0, 5, -3, 2, 1, -1, -1, -5, -5, 0, 0, -1, -1, -2,
-2, 0, 0, 5, -3, 2, 1, -1, -1, -5, 1, 0, 0],
<int>[
-1,
-1,
-2,
4,
0,
0,
6,
-3,
2,
1,
1,
-1,
-5,
1,
0,
0,
-1,
-1,
-2,
4,
0,
0,
5,
-3,
2,
1,
1,
-1,
-5,
1,
0,
0,
],
<int>[
-1,
-1,
4,
4,
0,
0,
0,
5,
2,
1,
-1,
-1,
3,
-5,
1,
0,
-1,
-2,
4,
4,
0,
0,
0,
5,
1,
1,
-1,
-1,
3,
-5,
1,
0,
],
<int>[
-1,
-2,
-2,
4,
1,
0,
0,
5,
2,
0,
-1,
-1,
3,
-5,
1,
0,
-1,
-2,
-2,
4,
0,
0,
0,
5,
2,
-1,
-1,
-1,
3,
-5,
1,
0,
],
<int>[
-1,
-1,
-2,
-2,
0,
0,
5,
-3,
2,
1,
-1,
-1,
-5,
-5,
0,
0,
-1,
-1,
-2,
-2,
0,
0,
5,
-3,
2,
1,
-1,
-1,
-5,
1,
0,
0,
],
];
static const List<List<int>> indexes = <List<int>>[
<int>[-1, 6, -1, 8, 12, 15, 13, 16, 0, 9, 3, 10, 13, 16, 14, 17, 0, 9, 3,
10, 13, 16, 14, 17, 5, 11, 7, 12, 15, 17, 15, 18],
<int>[14, 11, 15, 12, 7, 0, 8, 2, 15, 13, 16, 14, 9, 4, 11, 6, 15, 13, 16,
14, 10, 4, 11, 6, 16, 14, 17, 15, 11, 7, 12, 9],
<int>[11, 14, 12, 15, -1, 8, 0, 9, 12, 16, 13, 16, 2, 10, 5, 11, 13, 16,
13, 16, 3, 10, 5, 11, 14, 17, 15, 17, 7, 12, 8, 13],
<int>[5, -1, 7, 0, 15, 12, 15, 13, 8, 1, 10, 4, 16, 14, 16, 14, 8, 2, 10,
4, 16, 14, 16, 14, 11, 6, 12, 8, 17, 15, 17, 16],
<int>[
-1,
6,
-1,
8,
12,
15,
13,
16,
0,
9,
3,
10,
13,
16,
14,
17,
0,
9,
3,
10,
13,
16,
14,
17,
5,
11,
7,
12,
15,
17,
15,
18,
],
<int>[
14,
11,
15,
12,
7,
0,
8,
2,
15,
13,
16,
14,
9,
4,
11,
6,
15,
13,
16,
14,
10,
4,
11,
6,
16,
14,
17,
15,
11,
7,
12,
9,
],
<int>[
11,
14,
12,
15,
-1,
8,
0,
9,
12,
16,
13,
16,
2,
10,
5,
11,
13,
16,
13,
16,
3,
10,
5,
11,
14,
17,
15,
17,
7,
12,
8,
13,
],
<int>[
5,
-1,
7,
0,
15,
12,
15,
13,
8,
1,
10,
4,
16,
14,
16,
14,
8,
2,
10,
4,
16,
14,
16,
14,
11,
6,
12,
8,
17,
15,
17,
16,
],
];
// --- full case: C = 256, H = W = 16 (the shipping geometry) ---
@@ -113,7 +369,21 @@ class _Golden {
10000.0,
];
static const List<int> probeIndexes = <int>[
-1, -1, -1, -1, 0, 0, 0, 12, 17, 23, 34, 51, 62, 63, 63,
-1,
-1,
-1,
-1,
0,
0,
0,
12,
17,
23,
34,
51,
62,
63,
63,
];
}
@@ -220,10 +490,7 @@ void main() {
final geometry = AeicEntropyGeometry.forResolution(512);
final masks = AeicMaskSet(geometry);
for (var stage = 0; stage < 4; stage++) {
final live = masks
.maskTensor(stage)
.where((v) => v == 1.0)
.length;
final live = masks.maskTensor(stage).where((v) => v == 1.0).length;
expect(live, geometry.symbolsPerStage);
}
});
@@ -401,34 +668,37 @@ void main() {
});
group('AeicEntropyCodec', () {
test('encode pushes z, y0, y1, y2, y3 in that order and no other', () async {
final geometry = AeicEntropyGeometry.forResolution(512);
final network = _FakeNetwork(geometry);
final coders = _RecordingCoders();
final codec = AeicEntropyCodec(
geometry: geometry,
network: network,
coders: coders,
);
final progress = <double>[];
final stream = await codec.encode(
Uint8List(512 * 512 * 3),
onProgress: progress.add,
);
test(
'encode pushes z, y0, y1, y2, y3 in that order and no other',
() async {
final geometry = AeicEntropyGeometry.forResolution(512);
final network = _FakeNetwork(geometry);
final coders = _RecordingCoders();
final codec = AeicEntropyCodec(
geometry: geometry,
network: network,
coders: coders,
);
final progress = <double>[];
final stream = await codec.encode(
Uint8List(512 * 512 * 3),
onProgress: progress.add,
);
expect(coders.encoder.groups, <int>[
kAeicZCdfGroup,
kAeicYCdfGroup,
kAeicYCdfGroup,
kAeicYCdfGroup,
kAeicYCdfGroup,
]);
expect(coders.encoder.lengths, <int>[2048, 16384, 16384, 16384, 16384]);
expect(stream, isNotEmpty);
expect(progress.last, 1.0);
// The image really was normalized before the graph saw it: 0 -> -1.
expect(network.lastInput!.first, -1.0);
});
expect(coders.encoder.groups, <int>[
kAeicZCdfGroup,
kAeicYCdfGroup,
kAeicYCdfGroup,
kAeicYCdfGroup,
kAeicYCdfGroup,
]);
expect(coders.encoder.lengths, <int>[2048, 16384, 16384, 16384, 16384]);
expect(stream, isNotEmpty);
expect(progress.last, 1.0);
// The image really was normalized before the graph saw it: 0 -> -1.
expect(network.lastInput!.first, -1.0);
},
);
test('encode refuses a graph that returns the wrong z size', () async {
final geometry = AeicEntropyGeometry.forResolution(512);
@@ -697,7 +967,8 @@ class _FakeNetwork implements AeicEntropyNetwork {
],
scales: <Float32List>[
for (var i = 0; i < 4; i++)
Float32List(geometry.yElements)..fillRange(0, geometry.yElements, 1.0),
Float32List(geometry.yElements)
..fillRange(0, geometry.yElements, 1.0),
],
);
}
test/services/rans_coder_test.dart
+2 -2
View File
@@ -20,8 +20,8 @@ void main() {
final Map<String, dynamic> manifest =
jsonDecode(File('${goldenDir.path}/manifest.json').readAsStringSync())
as Map<String, dynamic>;
final List<Map<String, dynamic>> images = (manifest['images'] as List<dynamic>)
.cast<Map<String, dynamic>>();
final List<Map<String, dynamic>> images =
(manifest['images'] as List<dynamic>).cast<Map<String, dynamic>>();
final List<Map<String, dynamic>> synthetic =
(manifest['synthetic'] as List<dynamic>).cast<Map<String, dynamic>>();
test/services/received_image_store_test.dart
+184 -139
View File
@@ -59,7 +59,10 @@ class _FakeDecoder implements ReceivedImageDecoder {
}
}
Future<ImageCodecResult?> _run(AeicRatePoint ratePoint, int resolution) async {
Future<ImageCodecResult?> _run(
AeicRatePoint ratePoint,
int resolution,
) async {
if (delay > Duration.zero) await Future<void>.delayed(delay);
switch (mode) {
case 'null':
@@ -149,7 +152,11 @@ void main() {
imgId: 7,
firstSeen: DateTime.fromMillisecondsSinceEpoch(0x693F21 * 1000),
);
expect(id, '1a2b07' '00693f21');
expect(
id,
'1a2b07'
'00693f21',
);
expect(id.length, 14);
expect(ReceivedImageRef.parse(ReceivedImageRef.encode(id)), id);
});
@@ -161,8 +168,10 @@ void main() {
expect(ReceivedImageRef.parse('aeic:1:1A2B0700693F21'), isNull);
expect(ReceivedImageRef.parse('aeic:2:1a2b0700693f21'), isNull);
expect(ReceivedImageRef.parse('@[Bob] hi'), isNull);
expect(ReceivedImageRef.parse(' aeic:1:1a2b0700693f21 '),
'1a2b0700693f21');
expect(
ReceivedImageRef.parse(' aeic:1:1a2b0700693f21 '),
'1a2b0700693f21',
);
});
});
@@ -184,8 +193,11 @@ void main() {
expect(entry!.state, ReceivedImageState.reassembled);
expect(entry.rate, AeicRatePoint.ft32);
expect(entry.metadataAssumed, isFalse);
expect(h.blobs.hasBitstream(entry.streamId), isTrue,
reason: 'bitstream must be written before the state changes');
expect(
h.blobs.hasBitstream(entry.streamId),
isTrue,
reason: 'bitstream must be written before the state changes',
);
await h.store.settle();
final decoded = h.store.entryFor(entry.streamId)!;
@@ -218,8 +230,11 @@ void main() {
h.reassembler.addChunk(set.blobs[1], channelIndex: 3),
channelIndex: 3,
);
expect(second!.streamId, first.streamId,
reason: 'the sentinel must not change as chunks arrive');
expect(
second!.streamId,
first.streamId,
reason: 'the sentinel must not change as chunks arrive',
);
expect(second.state, ReceivedImageState.reassembled);
await h.store.settle();
expect(
@@ -300,8 +315,10 @@ void main() {
expect(entry!.state, ReceivedImageState.reassembled);
expect(entry.recoveredWithParity, isTrue);
await h.store.settle();
expect(h.store.entryFor(entry.streamId)!.state,
ReceivedImageState.decoded);
expect(
h.store.entryFor(entry.streamId)!.state,
ReceivedImageState.decoded,
);
});
test('loopback and malformed blobs create nothing', () async {
@@ -574,37 +591,42 @@ void main() {
});
group('eviction', () {
test('byte budget drops the oldest PNG first and keeps the bitstream',
() async {
final decoder = _FakeDecoder(
png: Uint8List.fromList(List<int>.filled(1000, 3)),
);
final h = _build(decoder: decoder, maxBytes: 2500);
final ids = <String>[];
for (var i = 0; i < 3; i++) {
final entry = await _completeOne(h, imgId: 40 + i, seed: i);
await h.store.settle();
ids.add(entry.streamId);
h.advance(const Duration(seconds: 5));
}
// 3 x 1000 B PNG > 2500 B, so the oldest PNG must have gone.
expect(h.store.entryFor(ids[0])!.state, ReceivedImageState.evicted);
expect(h.store.entryFor(ids[0])!.pngStored, isFalse);
expect(h.blobs.hasPng(ids[0]), isFalse);
expect(h.blobs.hasBitstream(ids[0]), isTrue,
reason: 'a ~156 B bitstream is cheap; keep it so we can re-decode');
expect(h.store.entryFor(ids[0])!.canRetryDecode, isTrue);
expect(h.store.entryFor(ids[2])!.state, ReceivedImageState.decoded);
expect(h.store.totalBytes, lessThanOrEqualTo(2500));
test(
'byte budget drops the oldest PNG first and keeps the bitstream',
() async {
final decoder = _FakeDecoder(
png: Uint8List.fromList(List<int>.filled(1000, 3)),
);
final h = _build(decoder: decoder, maxBytes: 2500);
final ids = <String>[];
for (var i = 0; i < 3; i++) {
final entry = await _completeOne(h, imgId: 40 + i, seed: i);
await h.store.settle();
ids.add(entry.streamId);
h.advance(const Duration(seconds: 5));
}
// 3 x 1000 B PNG > 2500 B, so the oldest PNG must have gone.
expect(h.store.entryFor(ids[0])!.state, ReceivedImageState.evicted);
expect(h.store.entryFor(ids[0])!.pngStored, isFalse);
expect(h.blobs.hasPng(ids[0]), isFalse);
expect(
h.blobs.hasBitstream(ids[0]),
isTrue,
reason: 'a ~156 B bitstream is cheap; keep it so we can re-decode',
);
expect(h.store.entryFor(ids[0])!.canRetryDecode, isTrue);
expect(h.store.entryFor(ids[2])!.state, ReceivedImageState.decoded);
expect(h.store.totalBytes, lessThanOrEqualTo(2500));
// "Decode again" works off the surviving bitstream, and the image we just
// decoded is never the budget's own victim (that would thrash forever).
await h.store.requestDecode(ids[0]);
await h.store.settle();
expect(h.store.entryFor(ids[0])!.state, ReceivedImageState.decoded);
expect(h.store.entryFor(ids[1])!.state, ReceivedImageState.evicted);
expect(h.store.totalBytes, lessThanOrEqualTo(2500));
});
// "Decode again" works off the surviving bitstream, and the image we just
// decoded is never the budget's own victim (that would thrash forever).
await h.store.requestDecode(ids[0]);
await h.store.settle();
expect(h.store.entryFor(ids[0])!.state, ReceivedImageState.decoded);
expect(h.store.entryFor(ids[1])!.state, ReceivedImageState.evicted);
expect(h.store.totalBytes, lessThanOrEqualTo(2500));
},
);
test('image-count budget evicts oldest first', () async {
final h = _build(maxImages: 2);
@@ -625,8 +647,10 @@ void main() {
final h = _build(maxAge: const Duration(minutes: 10));
final entry = await _completeOne(h);
await h.store.settle();
expect(h.store.entryFor(entry.streamId)!.state,
ReceivedImageState.decoded);
expect(
h.store.entryFor(entry.streamId)!.state,
ReceivedImageState.decoded,
);
h.advance(const Duration(minutes: 11));
final evicted = await h.store.evictToBudget();
expect(evicted, contains(entry.streamId));
@@ -762,8 +786,11 @@ void main() {
expect(entry.state, ReceivedImageState.reassembled);
expect(entry.needsManualDecode, isTrue);
expect(h.store.decodeQueue, isEmpty);
expect(h.decoder.calls, 0,
reason: 'a radio packet must never start a 2.16 GiB decode by itself');
expect(
h.decoder.calls,
0,
reason: 'a radio packet must never start a 2.16 GiB decode by itself',
);
// Everything the placeholder card needs is already on the entry.
expect(entry.senderPrefix, kSender);
expect(entry.bitstreamByteCount, greaterThan(0));
@@ -779,28 +806,30 @@ void main() {
expect(h.decoder.calls, 1);
});
test('a finished model download does not decode the whole backlog',
() async {
final decoder = _FakeDecoder(
availability: ImageCodecAvailability.disabled,
);
final h = _build(decoder: decoder, processAutomatically: false);
final ids = <String>[];
for (var i = 0; i < 4; i++) {
ids.add((await _completeOne(h, imgId: 80 + i, seed: i)).streamId);
h.advance(const Duration(seconds: 2));
}
// The model lands.
decoder.availability = ImageCodecAvailability.ready;
h.store.notifyDecoderChanged();
await h.store.settle();
expect(decoder.calls, 0);
expect(h.store.decodeQueue, isEmpty);
for (final id in ids) {
expect(h.store.entryFor(id)!.state, ReceivedImageState.reassembled);
expect(h.store.entryFor(id)!.needsManualDecode, isTrue);
}
});
test(
'a finished model download does not decode the whole backlog',
() async {
final decoder = _FakeDecoder(
availability: ImageCodecAvailability.disabled,
);
final h = _build(decoder: decoder, processAutomatically: false);
final ids = <String>[];
for (var i = 0; i < 4; i++) {
ids.add((await _completeOne(h, imgId: 80 + i, seed: i)).streamId);
h.advance(const Duration(seconds: 2));
}
// The model lands.
decoder.availability = ImageCodecAvailability.ready;
h.store.notifyDecoderChanged();
await h.store.settle();
expect(decoder.calls, 0);
expect(h.store.decodeQueue, isEmpty);
for (final id in ids) {
expect(h.store.entryFor(id)!.state, ReceivedImageState.reassembled);
expect(h.store.entryFor(id)!.needsManualDecode, isTrue);
}
},
);
test('turning the setting on affects future arrivals only', () async {
final h = _build(processAutomatically: false);
@@ -810,15 +839,21 @@ void main() {
h.store.processAutomatically = true;
await h.store.settle();
expect(h.decoder.calls, 0, reason: 'no retro-decode of the backlog');
expect(h.store.entryFor(parked.streamId)!.state,
ReceivedImageState.reassembled);
expect(
h.store.entryFor(parked.streamId)!.state,
ReceivedImageState.reassembled,
);
final fresh = await _completeOne(h, imgId: 91, seed: 5);
await h.store.settle();
expect(h.store.entryFor(fresh.streamId)!.state,
ReceivedImageState.decoded);
expect(h.store.entryFor(parked.streamId)!.state,
ReceivedImageState.reassembled);
expect(
h.store.entryFor(fresh.streamId)!.state,
ReceivedImageState.decoded,
);
expect(
h.store.entryFor(parked.streamId)!.state,
ReceivedImageState.reassembled,
);
});
test('a reassembled entry restored from disk is always tappable', () async {
@@ -849,10 +884,7 @@ void main() {
h.decoder.availability = ImageCodecAvailability.disabled;
expect(h.store.decoderAvailability, ImageCodecAvailability.disabled);
final noDecoder = ReceivedImageStore(decoder: null);
expect(
noDecoder.decoderAvailability,
ImageCodecAvailability.unavailable,
);
expect(noDecoder.decoderAvailability, ImageCodecAvailability.unavailable);
noDecoder.dispose();
});
});
@@ -874,8 +906,11 @@ void main() {
await h.store.settle();
expect(decoder.calls, 10);
expect(decoder.maxConcurrent, 1,
reason: 'two concurrent decodes is ~4.3 GiB and an OOM kill');
expect(
decoder.maxConcurrent,
1,
reason: 'two concurrent decodes is ~4.3 GiB and an OOM kill',
);
for (final id in ids) {
expect(h.store.entryFor(id)!.state, ReceivedImageState.decoded);
}
@@ -895,33 +930,35 @@ void main() {
});
group('deletion hooks', () {
test('deleteImagesForChannel reclaims every image of one conversation',
() async {
final h = _build();
final a = await _completeOne(h, imgId: 130);
h.advance(const Duration(seconds: 2));
final b = await _completeOne(h, imgId: 131, seed: 1);
await h.store.settle();
// A third image on another channel must survive.
final other = await h.store.registerOutgoing(
channelIndex: 9,
senderPrefix: kSelf,
imgId: 5,
previewPng: Uint8List.fromList(List<int>.filled(32, 2)),
rate: AeicRatePoint.ft32,
chunkCount: 1,
);
test(
'deleteImagesForChannel reclaims every image of one conversation',
() async {
final h = _build();
final a = await _completeOne(h, imgId: 130);
h.advance(const Duration(seconds: 2));
final b = await _completeOne(h, imgId: 131, seed: 1);
await h.store.settle();
// A third image on another channel must survive.
final other = await h.store.registerOutgoing(
channelIndex: 9,
senderPrefix: kSelf,
imgId: 5,
previewPng: Uint8List.fromList(List<int>.filled(32, 2)),
rate: AeicRatePoint.ft32,
chunkCount: 1,
);
final removed = await h.store.deleteImagesForChannel(3);
expect(removed, containsAll(<String>[a.streamId, b.streamId]));
expect(h.store.entryFor(a.streamId), isNull);
expect(h.store.entryFor(b.streamId), isNull);
expect(h.blobs.hasPng(a.streamId), isFalse);
expect(h.blobs.hasBitstream(a.streamId), isFalse);
expect(h.blobs.hasSidecar(a.streamId), isFalse);
expect(h.store.entryFor(other.streamId), isNotNull);
expect(h.store.totalBytes, other.storedBytes);
});
final removed = await h.store.deleteImagesForChannel(3);
expect(removed, containsAll(<String>[a.streamId, b.streamId]));
expect(h.store.entryFor(a.streamId), isNull);
expect(h.store.entryFor(b.streamId), isNull);
expect(h.blobs.hasPng(a.streamId), isFalse);
expect(h.blobs.hasBitstream(a.streamId), isFalse);
expect(h.blobs.hasSidecar(a.streamId), isFalse);
expect(h.store.entryFor(other.streamId), isNotNull);
expect(h.store.totalBytes, other.storedBytes);
},
);
test('deleteImageForSentinel takes the message text', () async {
final h = _build();
@@ -950,9 +987,8 @@ void main() {
}
});
FileReceivedImageBlobStore newStore() => FileReceivedImageBlobStore(
baseDirectory: () async => tempDir,
);
FileReceivedImageBlobStore newStore() =>
FileReceivedImageBlobStore(baseDirectory: () async => tempDir);
test('round-trips bytes and sidecars through real files', () async {
final blobs = newStore();
@@ -964,7 +1000,9 @@ void main() {
expect(await blobs.readPng('1a2b0700693f21'), _payload(4096, 3));
expect(await blobs.bitstreamSize('1a2b0700693f21'), 155);
expect(await blobs.pngSize('1a2b0700693f21'), 4096);
expect(await blobs.readSidecars(), {'1a2b0700693f21': '{"streamId":"x"}'});
expect(await blobs.readSidecars(), {
'1a2b0700693f21': '{"streamId":"x"}',
});
expect(blobs.pngPath('1a2b0700693f21'), endsWith('1a2b0700693f21.png'));
expect(File(blobs.pngPath('1a2b0700693f21')!).existsSync(), isTrue);
@@ -1044,8 +1082,10 @@ void main() {
at: now,
);
await store.settle();
expect(store.entryFor(entry!.streamId)!.state,
ReceivedImageState.decoded);
expect(
store.entryFor(entry!.streamId)!.state,
ReceivedImageState.decoded,
);
store.dispose();
final reborn = ReceivedImageStore(
@@ -1055,8 +1095,11 @@ void main() {
);
await reborn.load();
final loaded = reborn.entryFor(entry.streamId);
expect(loaded, isNotNull,
reason: 'the whole point: images outlive the process');
expect(
loaded,
isNotNull,
reason: 'the whole point: images outlive the process',
);
expect(loaded!.state, ReceivedImageState.decoded);
expect(loaded.pngStored, isTrue);
expect(loaded.pngBytes, isNull, reason: 'pixels are read lazily');
@@ -1098,33 +1141,35 @@ void main() {
});
});
test('listeners fire for the message list and for the single bubble',
() async {
final h = _build();
var storeNotifications = 0;
h.store.addListener(() => storeNotifications++);
final set = buildImageChunks(
payload: _payload(kImageChunkFirstCapacity + 1),
metadata: const ImageStreamMetadata(rate: ImageCodecRatePoint.standard),
senderPrefix: kSender,
imgId: 77,
);
final first = await h.store.handleOutcome(
h.reassembler.addChunk(set.blobs[0], channelIndex: 3),
channelIndex: 3,
);
final listenable = h.store.listenableFor(first!.streamId);
var bubbleNotifications = 0;
listenable.addListener(() => bubbleNotifications++);
await h.store.handleOutcome(
h.reassembler.addChunk(set.blobs[1], channelIndex: 3),
channelIndex: 3,
);
await h.store.settle();
expect(storeNotifications, greaterThan(1));
expect(bubbleNotifications, greaterThan(1));
expect(listenable.value!.state, ReceivedImageState.decoded);
});
test(
'listeners fire for the message list and for the single bubble',
() async {
final h = _build();
var storeNotifications = 0;
h.store.addListener(() => storeNotifications++);
final set = buildImageChunks(
payload: _payload(kImageChunkFirstCapacity + 1),
metadata: const ImageStreamMetadata(rate: ImageCodecRatePoint.standard),
senderPrefix: kSender,
imgId: 77,
);
final first = await h.store.handleOutcome(
h.reassembler.addChunk(set.blobs[0], channelIndex: 3),
channelIndex: 3,
);
final listenable = h.store.listenableFor(first!.streamId);
var bubbleNotifications = 0;
listenable.addListener(() => bubbleNotifications++);
await h.store.handleOutcome(
h.reassembler.addChunk(set.blobs[1], channelIndex: 3),
channelIndex: 3,
);
await h.store.settle();
expect(storeNotifications, greaterThan(1));
expect(bubbleNotifications, greaterThan(1));
expect(listenable.value!.state, ReceivedImageState.decoded);
},
);
}
/// Delivers a whole single-chunk image and returns its entry (state