Shrink numeric types

This commit is contained in:
Pyrex 2024-04-07 20:30:35 -07:00
parent f76be1b112
commit 4edf6831d0
5 changed files with 48 additions and 45 deletions

View File

@ -5,14 +5,14 @@ use crate::{constants::{MAGIC, TILE_SZ, TILE_SZ2}, protocol::{self, ProtocolWrit
struct PixelTile {
// i32: representation that supports Walsh-Hadamard
pixels: [i32; TILE_SZ2]
pixels: [i16; TILE_SZ2]
}
struct CoefTile {
coefs: [i32; TILE_SZ2]
coefs: [i16; TILE_SZ2]
}
struct QuantTile {
quants: [i32; TILE_SZ2]
quants: [i16; TILE_SZ2]
}
@ -75,7 +75,7 @@ impl PixelTile {
let src_y = y0 + y;
pixels[y * TILE_SZ + x] =
if src_x < width && src_y < height {
layer[src_y * width + src_x] as i32
layer[src_y * width + src_x] as i16
} else {
0
};
@ -112,13 +112,13 @@ impl QuantTile {
}
fn write_zero<W: Write>(&self, writer: &mut ProtocolWriter<W>) -> ProtocolWriterResult<()> {
writer.write_i32_packed(self.quants[0])?;
writer.write_i16_wide(self.quants[0])?;
Ok(())
}
fn write_rest<W: Write>(&self, writer: &mut ProtocolWriter<W>) -> ProtocolWriterResult<()> {
for i in 1..self.quants.len() {
writer.write_i32_packed(self.quants[i])?;
writer.write_i16_packed(self.quants[i])?;
}
Ok(())
}

View File

@ -4,17 +4,17 @@ use crate::{constants::{MAGIC, TILE_SZ, TILE_SZ2}, protocol::{ProtocolReader, Pr
struct PixelTile {
// i32: representation that supports Walsh-Hadamard
pixels: [i32; TILE_SZ2]
pixels: [i16; TILE_SZ2]
}
#[derive(Clone)]
struct CoefTile {
coefs: [i32; TILE_SZ2]
coefs: [i16; TILE_SZ2]
}
#[derive(Clone)]
struct QuantTile {
quants: [i32; TILE_SZ2]
quants: [i16; TILE_SZ2]
}
pub fn decompress<R: Read>(
@ -93,11 +93,11 @@ impl PixelTile {
}
}
fn clip(&self, x: i32) -> u8 {
if x > u8::MAX as i32 {
fn clip(&self, x: i16) -> u8 {
if x > u8::MAX as i16 {
return u8::MAX
}
if x < u8::MIN as i32 {
if x < u8::MIN as i16 {
return u8::MIN;
}
return x as u8;
@ -116,13 +116,13 @@ impl QuantTile {
}
fn load_zero<R: Read>(&mut self, reader: &mut ProtocolReader<R>) -> ProtocolReaderResult<()> {
self.quants[0] = reader.read_i32_packed()?;
self.quants[0] = reader.read_i16_wide()?;
Ok(())
}
fn load_rest<R: Read>(&mut self, reader: &mut ProtocolReader<R>) -> ProtocolReaderResult<()> {
for i in 1..self.quants.len() {
self.quants[i] = reader.read_i32_packed()?;
self.quants[i] = reader.read_i16_packed()?;
}
Ok(())
}

View File

@ -61,21 +61,19 @@ impl<W: Write> ProtocolWriter<W> {
self.writer.write_all(&value.to_le_bytes())?;
Ok(())
}
pub fn write_i32_packed(&mut self, value: i32) -> ProtocolWriterResult<()> {
pub fn write_i16_wide(&mut self, value: i16) -> ProtocolWriterResult<()> {
self.writer.write_all(&value.to_le_bytes())?;
Ok(())
}
pub fn write_i16_packed(&mut self, value: i16) -> ProtocolWriterResult<()> {
// We reserve i8::MAX and i16::MAX as signal values that `value` is too big
// for each respective type
if (i8::MIN as i32..i8::MAX as i32).contains(&value) {
if (i8::MIN as i16..i8::MAX as i16).contains(&value) {
self.writer.write_all(&(value as i8).to_le_bytes())?;
return Ok(())
}
self.writer.write_all(&(i8::MAX).to_le_bytes())?;
if (i16::MIN as i32..i16::MAX as i32).contains(&value) {
self.writer.write_all(&(value as i16).to_le_bytes())?;
return Ok(())
}
self.writer.write_all(&(i16::MAX).to_le_bytes())?;
self.writer.write_all(&value.to_le_bytes())?;
Ok(())
}
}
@ -105,21 +103,23 @@ impl<R: Read> ProtocolReader<R> {
Ok(u32::from_le_bytes(u32_buf))
}
pub fn read_i32_packed(&mut self) -> ProtocolReaderResult<i32> {
pub fn read_i16_wide(&mut self) -> ProtocolReaderResult<i16> {
let mut i16_buf = [0; 2];
self.read_exact(&mut i16_buf)?;
Ok(i16::from_le_bytes(i16_buf))
}
pub fn read_i16_packed(&mut self) -> ProtocolReaderResult<i16> {
let mut i8_buf = [0; 1];
self.read_exact(&mut i8_buf)?;
let i8_value = i8::from_le_bytes(i8_buf);
if i8_value != i8::MAX { return Ok(i8_value as i32) }
if i8_value != i8::MAX { return Ok(i8_value as i16) }
let mut i16_buf = [0; 2];
self.read_exact(&mut i16_buf)?;
let i16_value = i16::from_le_bytes(i16_buf);
if i16_value != i16::MAX { return Ok(i16_value as i32) }
let mut i32_buf = [0; 4];
self.read_exact(&mut i32_buf)?;
let i32_value = i32::from_le_bytes(i32_buf);
Ok(i32_value)
Ok(i16_value as i16)
}
// wrap UnexpectedEof, since that's an error for other Io clients,

View File

@ -12,9 +12,9 @@ const ZIGZAG: [u8; TILE_SZ2] = [
];
pub fn to_quantized(
coefs: [i32; TILE_SZ2]
) -> [i32; TILE_SZ2] {
let mut quant: [i32; TILE_SZ2] = [0; TILE_SZ2];
coefs: [i16; TILE_SZ2]
) -> [i16; TILE_SZ2] {
let mut quant: [i16; TILE_SZ2] = [0; TILE_SZ2];
for cf_ix in 0..TILE_SZ2 {
let div = divisor(cf_ix);
@ -35,21 +35,21 @@ pub fn to_quantized(
}
pub fn from_quantized(
quant: [i32; TILE_SZ2]
) -> [i32; TILE_SZ2] {
let mut coefs: [i32; TILE_SZ2] = [0; TILE_SZ2];
quant: [i16; TILE_SZ2]
) -> [i16; TILE_SZ2] {
let mut coefs: [i16; TILE_SZ2] = [0; TILE_SZ2];
for cf_ix in 0..TILE_SZ2 {
let div = divisor(cf_ix);
coefs[cf_ix] = quant[ZIGZAG[cf_ix] as usize] * div;
coefs[cf_ix] = quant[ZIGZAG[cf_ix] as usize].wrapping_mul(div);
}
coefs
}
pub fn divisor(cf_ix: usize) -> i32 {
pub fn divisor(cf_ix: usize) -> i16 {
if cf_ix == 0 { return 1; }
let x = cf_ix % 8;
let y = cf_ix / 8;
let div = 32 + (x as i32 + y as i32) * 12;
let div = 32 + (x as i16 + y as i16) * 12;
if div==32 && cf_ix != 0 {
dbg!(cf_ix, x, y, div);
}

View File

@ -3,15 +3,15 @@ use crate::constants::TILE_SZ;
// This is the Walsh-Hadamard transform, specialized to 8px
// Ported from:
// - https://colab.research.google.com/drive/1WjtKwUcqxWafAumFO9ET74RsckZSsw6n#scrollTo=e3r9-RBpcgwH
pub fn encode(data: &mut [i32], zero: usize, stride: usize) {
pub fn encode(data: &mut [i16], zero: usize, stride: usize) {
transform(data, zero, stride, 0, 0, 1)
}
pub fn decode(data: &mut [i32], zero: usize, stride: usize) {
pub fn decode(data: &mut [i16], zero: usize, stride: usize) {
transform(data, zero, stride, 0, 0, 8)
}
fn transform(data: &mut [i32], zero: usize, stride: usize, offset_in: i32, offset_out: i32, divisor: i32) {
fn transform(data: &mut [i16], zero: usize, stride: usize, offset_in: i16, offset_out: i16, divisor: i16) {
let ix = |t| zero + t * stride;
let mut row = [0; TILE_SZ];
@ -47,13 +47,16 @@ mod tests {
use crate::{constants::TILE_SZ, transform::{decode, encode}};
#[derive(Clone, Debug)]
struct Encodable { args: [i32; TILE_SZ] }
struct Encodable { args: [i16; TILE_SZ] }
impl Arbitrary for Encodable {
fn arbitrary(g: &mut quickcheck::Gen) -> Self {
let mut args = [0; TILE_SZ];
for i in 0..TILE_SZ {
args[i] = i16::arbitrary(g) as i32;
args[i] = (
i8::arbitrary(g).signum() as i32 *
(i32::arbitrary(g) % 255 * 16)
) as i16;
}
return Encodable { args };
}