mirror of
https://github.com/zjs81/meshcore-open.git
synced 2026-08-11 18:26:27 +10:00
Zach
499 lines
16 KiB
Dart
499 lines
16 KiB
Dart
// Pure-Dart port of the AEIC rANS entropy coder.
|
|
//
|
|
// Ported from the C++ reference in `aeic/src/cpp/rans/` (rans.h, rans_byte.h,
|
|
// rans.cpp), which is itself ryg_rans + CompressAI's unbounded-index range
|
|
// coding. The port must be BYTE-IDENTICAL to the C++ coder: a single differing
|
|
// byte desynchronises rANS and silently corrupts most of the image. The golden
|
|
// vectors under `test/services/golden/` pin that down.
|
|
//
|
|
// Notes on the port:
|
|
// * Dart `int` is 64-bit and signed; the C++ state is `uint32_t`. Every place
|
|
// the C++ relies on 32-bit behaviour is either provably in-range (the state
|
|
// invariant keeps `x < 2^31`) or masked explicitly with `& 0xFFFFFFFF`.
|
|
// * `>>` on a negative Dart int is arithmetic. All shifted values here are
|
|
// non-negative by construction; `>>>` is used where the shift consumes the
|
|
// state so the intent is unambiguous.
|
|
//
|
|
// Format recap (see `results/rans_port_spec.md`):
|
|
// * precision 16, bypassPrecision 2, RANS_L = 1 << 23, streamParts 2.
|
|
// * Each `encodeWithIndexes` call is split evenly across the sub-streams:
|
|
// part p covers `[p * (n ~/ parts), ...)`, the last part taking the
|
|
// remainder. Every part accumulates across all calls; flush happens once.
|
|
// * A sub-stream's first 4 bytes are the final rANS state, little-endian.
|
|
// * Container: `flag = ((nParts - 1) << 4) | (hdrLen == 2 ? 1 : 0)`, then the
|
|
// lengths of the first `nParts - 1` sub-streams (hdrLen bytes each, LE),
|
|
// then the sub-streams back to back.
|
|
library;
|
|
|
|
import 'dart:typed_data';
|
|
|
|
import 'entropy_tables.dart';
|
|
|
|
/// Lower bound of the rANS normalisation interval (`RANS_BYTE_L`).
|
|
const int kRansLowerBound = 1 << 23;
|
|
|
|
/// Thrown when a bitstream cannot be interpreted.
|
|
class RansFormatException implements Exception {
|
|
RansFormatException(this.message);
|
|
|
|
final String message;
|
|
|
|
@override
|
|
String toString() => 'RansFormatException: $message';
|
|
}
|
|
|
|
/// Splits the shipped bytes into rANS sub-streams. Mirror of
|
|
/// `RansDecoder::set_stream`.
|
|
List<Uint8List> parseRansContainer(Uint8List stream) {
|
|
if (stream.isEmpty) {
|
|
throw RansFormatException('empty stream');
|
|
}
|
|
final int flag = stream[0];
|
|
final int nStreams = (flag >> 4) + 1;
|
|
final int hdr = (flag & 0x0F) == 1 ? 2 : 4;
|
|
var off = 1;
|
|
final List<int> sizes = <int>[];
|
|
var total = 0;
|
|
for (var i = 0; i < nStreams - 1; i++) {
|
|
if (off + hdr > stream.length) {
|
|
throw RansFormatException('truncated sub-stream size table');
|
|
}
|
|
var sz = 0;
|
|
for (var b = 0; b < hdr; b++) {
|
|
sz |= stream[off + b] << (8 * b);
|
|
}
|
|
off += hdr;
|
|
sizes.add(sz);
|
|
total += sz;
|
|
}
|
|
final int last = stream.length - off - total;
|
|
if (last < 0) {
|
|
throw RansFormatException('sub-stream sizes exceed the stream length');
|
|
}
|
|
sizes.add(last);
|
|
final List<Uint8List> parts = <Uint8List>[];
|
|
var p = off;
|
|
for (final int sz in sizes) {
|
|
if (p + sz > stream.length) {
|
|
throw RansFormatException('truncated sub-stream');
|
|
}
|
|
parts.add(Uint8List.sublistView(stream, p, p + sz));
|
|
p += sz;
|
|
}
|
|
return parts;
|
|
}
|
|
|
|
/// Assembles sub-streams into the bytes that go on the air.
|
|
Uint8List buildRansContainer(List<Uint8List> parts) {
|
|
if (parts.isEmpty) {
|
|
throw RansFormatException('no sub-streams');
|
|
}
|
|
if (parts.length > 16) {
|
|
throw RansFormatException('too many sub-streams (${parts.length})');
|
|
}
|
|
var maximum = 0;
|
|
for (var i = 0; i < parts.length - 1; i++) {
|
|
if (parts[i].length > maximum) maximum = parts[i].length;
|
|
}
|
|
final int hdr = maximum > 65535 ? 4 : 2;
|
|
final int flag = ((parts.length - 1) << 4) | (hdr == 2 ? 1 : 0);
|
|
var total = 1 + hdr * (parts.length - 1);
|
|
for (final Uint8List p in parts) {
|
|
total += p.length;
|
|
}
|
|
final Uint8List out = Uint8List(total);
|
|
out[0] = flag;
|
|
var off = 1;
|
|
for (var i = 0; i < parts.length - 1; i++) {
|
|
final int n = parts[i].length;
|
|
for (var b = 0; b < hdr; b++) {
|
|
out[off + b] = (n >> (8 * b)) & 0xFF;
|
|
}
|
|
off += hdr;
|
|
}
|
|
for (final Uint8List p in parts) {
|
|
out.setRange(off, off + p.length, p);
|
|
off += p.length;
|
|
}
|
|
return out;
|
|
}
|
|
|
|
/// Growable (start, range) entry list, stored interleaved in one Int32List.
|
|
/// `range == 0` is the bypass sentinel: `start` then carries the raw bits.
|
|
class _EntryBuffer {
|
|
Int32List _data = Int32List(2048);
|
|
int _length = 0;
|
|
|
|
int get length => _length;
|
|
|
|
void add(int start, int range) {
|
|
final int need = (_length + 1) * 2;
|
|
if (need > _data.length) {
|
|
final Int32List bigger = Int32List(_data.length * 2);
|
|
bigger.setRange(0, _data.length, _data);
|
|
_data = bigger;
|
|
}
|
|
_data[_length * 2] = start;
|
|
_data[_length * 2 + 1] = range;
|
|
_length++;
|
|
}
|
|
|
|
int startAt(int i) => _data[i * 2];
|
|
|
|
int rangeAt(int i) => _data[i * 2 + 1];
|
|
|
|
void clear() => _length = 0;
|
|
}
|
|
|
|
/// Growable byte sink used for the reversed encoder output.
|
|
class _ByteSink {
|
|
Uint8List _data = Uint8List(4096);
|
|
int _length = 0;
|
|
|
|
void add(int byte) {
|
|
if (_length == _data.length) {
|
|
final Uint8List bigger = Uint8List(_data.length * 2);
|
|
bigger.setRange(0, _data.length, _data);
|
|
_data = bigger;
|
|
}
|
|
_data[_length++] = byte & 0xFF;
|
|
}
|
|
|
|
/// Returns the bytes in reverse of the order they were added, which is the
|
|
/// order they appear in the sub-stream.
|
|
Uint8List reversedBytes() {
|
|
final Uint8List out = Uint8List(_length);
|
|
for (var i = 0; i < _length; i++) {
|
|
out[i] = _data[_length - 1 - i];
|
|
}
|
|
return out;
|
|
}
|
|
}
|
|
|
|
/// rANS encoder. Call [encodeWithIndexes] once per stage in format order
|
|
/// (z, y0, y1, y2, y3), then [finish] exactly once.
|
|
class RansEncoder {
|
|
factory RansEncoder(EntropyTables tables, {int? streamParts}) =>
|
|
RansEncoder._(tables, streamParts ?? tables.streamParts);
|
|
|
|
RansEncoder._(EntropyTables tables, int streamParts)
|
|
: tables = tables,
|
|
streamParts = streamParts,
|
|
_precision = tables.precision,
|
|
_bypassPrecision = tables.bypassPrecision,
|
|
_entries = List<_EntryBuffer>.generate(
|
|
streamParts,
|
|
(_) => _EntryBuffer(),
|
|
);
|
|
|
|
final EntropyTables tables;
|
|
final int streamParts;
|
|
final int _precision;
|
|
final int _bypassPrecision;
|
|
final List<_EntryBuffer> _entries;
|
|
bool _finished = false;
|
|
|
|
int get _maxBypassVal => (1 << _bypassPrecision) - 1;
|
|
|
|
/// Accumulates one encode call. [symbols] and [indexes] must be the same
|
|
/// length; an index `< 0` emits nothing at all.
|
|
///
|
|
/// Never pass an odd-length array: the reference splitter (and therefore the
|
|
/// on-air format) mis-sizes the last part's index vector in that case.
|
|
void encodeWithIndexes(
|
|
List<int> symbols,
|
|
List<int> indexes,
|
|
int cdfGroupIndex,
|
|
) {
|
|
if (_finished) {
|
|
throw StateError('RansEncoder.finish() has already been called');
|
|
}
|
|
if (symbols.length != indexes.length) {
|
|
throw ArgumentError(
|
|
'symbols (${symbols.length}) and indexes (${indexes.length}) differ',
|
|
);
|
|
}
|
|
if (cdfGroupIndex < 0 || cdfGroupIndex >= tables.groups.length) {
|
|
throw ArgumentError('no CDF group $cdfGroupIndex');
|
|
}
|
|
final CdfGroup group = tables.groups[cdfGroupIndex];
|
|
final int total = symbols.length;
|
|
final int each = total ~/ streamParts;
|
|
for (var p = 0; p < streamParts; p++) {
|
|
final int lo = p * each;
|
|
final int hi = p == streamParts - 1 ? total : lo + each;
|
|
_push(_entries[p], symbols, indexes, lo, hi, group);
|
|
}
|
|
}
|
|
|
|
void _push(
|
|
_EntryBuffer out,
|
|
List<int> symbols,
|
|
List<int> indexes,
|
|
int lo,
|
|
int hi,
|
|
CdfGroup group,
|
|
) {
|
|
final Int32List cdf = group.quantizedCdf;
|
|
final Int32List cdfLength = group.cdfLength;
|
|
final Int32List offsets = group.offset;
|
|
final int width = group.cdfWidth;
|
|
final int maxBypassVal = _maxBypassVal;
|
|
final int bypassPrecision = _bypassPrecision;
|
|
|
|
for (var i = lo; i < hi; i++) {
|
|
final int cdfIdx = indexes[i];
|
|
if (cdfIdx < 0) {
|
|
continue;
|
|
}
|
|
if (cdfIdx >= group.numCdfs) {
|
|
throw RansFormatException(
|
|
'index $cdfIdx out of range (${group.numCdfs} CDF rows)',
|
|
);
|
|
}
|
|
final int maxValue = cdfLength[cdfIdx] - 2;
|
|
var value = symbols[i] - offsets[cdfIdx];
|
|
var rawVal = 0;
|
|
if (value < 0) {
|
|
rawVal = -2 * value - 1;
|
|
value = maxValue;
|
|
} else if (value >= maxValue) {
|
|
rawVal = 2 * (value - maxValue);
|
|
value = maxValue;
|
|
}
|
|
|
|
final int base = cdfIdx * width;
|
|
final int start = cdf[base + value];
|
|
out.add(start, cdf[base + value + 1] - start);
|
|
|
|
if (value == maxValue) {
|
|
// Bypass mode: raw bits, `bypassPrecision` at a time.
|
|
var nBypass = 0;
|
|
while ((rawVal >> (nBypass * bypassPrecision)) != 0) {
|
|
nBypass++;
|
|
}
|
|
var val = nBypass;
|
|
while (val >= maxBypassVal) {
|
|
out.add(maxBypassVal, 0);
|
|
val -= maxBypassVal;
|
|
}
|
|
out.add(val, 0);
|
|
for (var j = 0; j < nBypass; j++) {
|
|
out.add((rawVal >> (j * bypassPrecision)) & maxBypassVal, 0);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Flushes every sub-stream and returns the container bytes.
|
|
Uint8List finish() {
|
|
if (_finished) {
|
|
throw StateError('RansEncoder.finish() has already been called');
|
|
}
|
|
_finished = true;
|
|
final List<Uint8List> parts = <Uint8List>[
|
|
for (final _EntryBuffer e in _entries) _flush(e),
|
|
];
|
|
return buildRansContainer(parts);
|
|
}
|
|
|
|
/// Discards accumulated entries so the encoder can be reused.
|
|
void reset() {
|
|
for (final _EntryBuffer e in _entries) {
|
|
e.clear();
|
|
}
|
|
_finished = false;
|
|
}
|
|
|
|
Uint8List _flush(_EntryBuffer entries) {
|
|
final _ByteSink sink = _ByteSink();
|
|
// The state is always in [2^23, 2^31); native Dart ints need no masking,
|
|
// but the emission path is written to stay explicitly byte-wise anyway.
|
|
var x = kRansLowerBound;
|
|
final int bypassXMax = (1 << (_precision - _bypassPrecision)) << 15;
|
|
for (var k = entries.length - 1; k >= 0; k--) {
|
|
final int range = entries.rangeAt(k);
|
|
final int start = entries.startAt(k);
|
|
if (range != 0) {
|
|
final int xMax = range << 15;
|
|
while (x >= xMax) {
|
|
sink.add(x & 0xFF);
|
|
x = x >>> 8;
|
|
}
|
|
x = ((x ~/ range) << _precision) + (x % range) + start;
|
|
} else {
|
|
while (x >= bypassXMax) {
|
|
sink.add(x & 0xFF);
|
|
x = x >>> 8;
|
|
}
|
|
x = ((x << _bypassPrecision) | start) & 0xFFFFFFFF;
|
|
}
|
|
}
|
|
// RansEncFlush writes the 32-bit state little-endian at the front of the
|
|
// stream, i.e. in emission order it is the high byte first.
|
|
sink.add((x >>> 24) & 0xFF);
|
|
sink.add((x >>> 16) & 0xFF);
|
|
sink.add((x >>> 8) & 0xFF);
|
|
sink.add(x & 0xFF);
|
|
return sink.reversedBytes();
|
|
}
|
|
}
|
|
|
|
/// rANS decoder. Decoding is INCREMENTAL: construct once over the whole
|
|
/// container, then call [decodeStream] per stage in the same order the encoder
|
|
/// used (z, y0, y1, y2, y3). Each call resumes the sub-stream states where the
|
|
/// previous one left off, because stage i's indexes are unknown until the
|
|
/// earlier stages have been decoded and run back through the network.
|
|
class RansDecoder {
|
|
factory RansDecoder(
|
|
EntropyTables tables,
|
|
Uint8List stream, {
|
|
int? streamParts,
|
|
}) => RansDecoder._(tables, stream, streamParts ?? tables.streamParts);
|
|
|
|
RansDecoder._(EntropyTables tables, Uint8List stream, int expected)
|
|
: tables = tables,
|
|
_precision = tables.precision,
|
|
_bypassPrecision = tables.bypassPrecision,
|
|
_parts = parseRansContainer(stream) {
|
|
if (_parts.length != expected) {
|
|
throw RansFormatException(
|
|
'container has ${_parts.length} sub-streams, expected $expected',
|
|
);
|
|
}
|
|
_states = List<int>.filled(_parts.length, 0);
|
|
_ptrs = List<int>.filled(_parts.length, 0);
|
|
for (var i = 0; i < _parts.length; i++) {
|
|
final Uint8List p = _parts[i];
|
|
if (p.length < 4) {
|
|
throw RansFormatException('sub-stream $i is shorter than 4 bytes');
|
|
}
|
|
_states[i] = p[0] | (p[1] << 8) | (p[2] << 16) | (p[3] << 24);
|
|
_ptrs[i] = 4;
|
|
}
|
|
}
|
|
|
|
final EntropyTables tables;
|
|
final int _precision;
|
|
final int _bypassPrecision;
|
|
final List<Uint8List> _parts;
|
|
late final List<int> _states;
|
|
late final List<int> _ptrs;
|
|
|
|
int get streamParts => _parts.length;
|
|
|
|
/// Decodes one stage. Returns one symbol per entry of [indexes]; positions
|
|
/// whose index is `< 0` yield a literal 0 and consume nothing.
|
|
Int16List decodeStream(List<int> indexes, int cdfGroupIndex) {
|
|
if (cdfGroupIndex < 0 || cdfGroupIndex >= tables.groups.length) {
|
|
throw ArgumentError('no CDF group $cdfGroupIndex');
|
|
}
|
|
final CdfGroup group = tables.groups[cdfGroupIndex];
|
|
final Int32List cdf = group.quantizedCdf;
|
|
final Int32List cdfLength = group.cdfLength;
|
|
final Int32List offsets = group.offset;
|
|
final int width = group.cdfWidth;
|
|
final int mask = (1 << _precision) - 1;
|
|
final int bypassPrecision = _bypassPrecision;
|
|
final int maxBypassVal = (1 << bypassPrecision) - 1;
|
|
final int bypassMask = maxBypassVal;
|
|
|
|
final int total = indexes.length;
|
|
final int nParts = _parts.length;
|
|
final int each = total ~/ nParts;
|
|
final Int16List out = Int16List(total);
|
|
|
|
for (var pi = 0; pi < nParts; pi++) {
|
|
final int lo = pi * each;
|
|
final int hi = pi == nParts - 1 ? total : lo + each;
|
|
final Uint8List buf = _parts[pi];
|
|
var x = _states[pi];
|
|
var ptr = _ptrs[pi];
|
|
|
|
for (var i = lo; i < hi; i++) {
|
|
final int cdfIdx = indexes[i];
|
|
if (cdfIdx < 0) {
|
|
out[i] = 0;
|
|
continue;
|
|
}
|
|
if (cdfIdx >= group.numCdfs) {
|
|
throw RansFormatException(
|
|
'index $cdfIdx out of range (${group.numCdfs} CDF rows)',
|
|
);
|
|
}
|
|
final int n = cdfLength[cdfIdx];
|
|
final int maxValue = n - 2;
|
|
final int base = cdfIdx * width;
|
|
final int cum = x & mask;
|
|
|
|
// upper_bound(row[0:n], cum) - 1
|
|
var loo = 0;
|
|
var hii = n;
|
|
while (loo < hii) {
|
|
final int mid = (loo + hii) >> 1;
|
|
if (cdf[base + mid] > cum) {
|
|
hii = mid;
|
|
} else {
|
|
loo = mid + 1;
|
|
}
|
|
}
|
|
final int s = loo - 1;
|
|
if (s < 0 || s >= n - 1) {
|
|
throw RansFormatException('corrupt stream: symbol $s out of range');
|
|
}
|
|
final int start = cdf[base + s];
|
|
final int range = cdf[base + s + 1] - start;
|
|
|
|
x = (range * (x >>> _precision) + (x & mask) - start) & 0xFFFFFFFF;
|
|
while (x < kRansLowerBound) {
|
|
if (ptr >= buf.length) {
|
|
throw RansFormatException('sub-stream $pi exhausted');
|
|
}
|
|
x = ((x << 8) | buf[ptr]) & 0xFFFFFFFF;
|
|
ptr++;
|
|
}
|
|
|
|
var value = s;
|
|
if (value == maxValue) {
|
|
// Bypass mode. Note the renormalisation here is a single `if`, not a
|
|
// loop -- that asymmetry with the symbol path is part of the format.
|
|
int getBits() {
|
|
final int v = x & bypassMask;
|
|
x = x >>> bypassPrecision;
|
|
if (x < kRansLowerBound) {
|
|
if (ptr >= buf.length) {
|
|
throw RansFormatException('sub-stream $pi exhausted');
|
|
}
|
|
x = ((x << 8) | buf[ptr]) & 0xFFFFFFFF;
|
|
ptr++;
|
|
}
|
|
return v;
|
|
}
|
|
|
|
var val = getBits();
|
|
var nBypass = val;
|
|
while (val == maxBypassVal) {
|
|
val = getBits();
|
|
nBypass += val;
|
|
}
|
|
var rawVal = 0;
|
|
for (var j = 0; j < nBypass; j++) {
|
|
rawVal |= getBits() << (j * bypassPrecision);
|
|
}
|
|
value = rawVal >> 1;
|
|
if ((rawVal & 1) != 0) {
|
|
value = -value - 1;
|
|
} else {
|
|
value += maxValue;
|
|
}
|
|
}
|
|
out[i] = value + offsets[cdfIdx];
|
|
}
|
|
|
|
_states[pi] = x;
|
|
_ptrs[pi] = ptr;
|
|
}
|
|
return out;
|
|
}
|
|
}
|