From 31def7216cf8c78c062c2cb54469752f83bc90b9 Mon Sep 17 00:00:00 2001 From: RetroGameDeveloper <40120498+RetroGameDeveloper@users.noreply.github.com> Date: Sat, 14 Nov 2026 17:03:55 +0000 Subject: [PATCH] decomp-assets 0.4.0: textures in a chunked-RNC filesystem and in a deflated model asset table Two N64 games had empty texture browsers because their assets sit in containers nothing read: - An off-road RC racer packs its whole filesystem as one RNC stream with method byte 0x81: 4 KB chunks, each a method-1 stream of its own, listed in a size table. rnc.chunked/unpack_chunked read any range from only the chunks holding it. The stream unpacks to an FFS filesystem (id, size, offset records) whose I64 images (CI4, CI8, I4, I8) are taken when header and file size agree exactly: 220 images. - A platformer sequel keeps its assets as raw deflate behind a table of word offsets. Models (0x0000000B) carry a texture list; a model is taken when its first texture starts the data, the others follow and all fit: 5,013 textures in 861 models. Checked on the user's cartridges locally (nothing committed): samples of 40-48 rendered correctly, and the other 49 N64 cartridges gain nothing (no false positives). Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- pyproject.toml | 2 +- src/decompassets/__init__.py | 2 +- src/decompassets/n64/__init__.py | 9 ++- src/decompassets/n64/ffs.py | 118 ++++++++++++++++++++++++++++ src/decompassets/n64/rare_models.py | 116 +++++++++++++++++++++++++++ src/decompassets/rnc.py | 73 ++++++++++++++--- tests/test_packed_filesystems.py | 111 ++++++++++++++++++++++++++ uv.lock | 2 +- 9 files changed, 419 insertions(+), 16 deletions(-) create mode 100644 src/decompassets/n64/ffs.py create mode 100644 src/decompassets/n64/rare_models.py create mode 100644 tests/test_packed_filesystems.py diff --git a/README.md b/README.md index e65f351..066049a 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ the RNC unpacker work for any console. The interface may change before 1.0. ## Use ```bash -uv add "decompassets @ https://github.com/RetroReversing/decomp-assets/archive/refs/tags/v0.3.0.tar.gz" +uv add "decompassets @ https://github.com/RetroReversing/decomp-assets/archive/refs/tags/v0.4.0.tar.gz" ``` ```python diff --git a/pyproject.toml b/pyproject.toml index 15b20fb..3ccc0c5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "decompassets" -version = "0.3.0" +version = "0.4.0" description = "Find and preview game assets (textures, audio banks, models, fonts) inside cartridge and disc images by their structure, with no per-game offsets." readme = "README.md" license = "AGPL-3.0-only" diff --git a/src/decompassets/__init__.py b/src/decompassets/__init__.py index 7f3801e..1aad5ab 100644 --- a/src/decompassets/__init__.py +++ b/src/decompassets/__init__.py @@ -10,4 +10,4 @@ Everything here reads bytes and returns plain dicts. It never writes a game file, runs a tool or downloads. """ -__version__ = "0.3.0" +__version__ = "0.4.0" diff --git a/src/decompassets/n64/__init__.py b/src/decompassets/n64/__init__.py index 1e5481c..729049a 100644 --- a/src/decompassets/n64/__init__.py +++ b/src/decompassets/n64/__init__.py @@ -3,6 +3,8 @@ - textures.py: indexed texture banks (CI4/CI8, RGBA32, the six-level RGBA16 mipmap layout), raw or RNC-packed. - texture_packs.py: texture packs with a trailing count, inside archives. - display_lists.py: textures that uncompressed display lists load, in files placed by their vertices and pointers. +- ffs.py: I64 images in an FFS filesystem packed as one chunked RNC stream. +- rare_models.py: model textures in a deflated asset table. - audio.py: libultra B1 sound banks, their sample tables and mono VADPCM previews. - ptr_audio.py: libmus PtrTablesV2 banks, marked wave tables and sequences. - models.py: indexed geometry with byte-plane vertices and cache-relative polygons. @@ -29,15 +31,18 @@ def regions(image, *, context=None, **kwargs): archived_models, audio, display_lists, + ffs, fonts, models, native_scene, ptr_audio, + rare_models, resources, texture_packs, textures, ) known=(textures.regions(image,**kwargs)+texture_packs.regions(image,**kwargs)+display_lists.regions(image,**kwargs) + +ffs.regions(image,**kwargs)+rare_models.regions(image,**kwargs) +audio.regions(image,context=context,**kwargs) +ptr_audio.regions(image,**kwargs)+ptr_audio.sequences(image,**kwargs)+models.regions(image,**kwargs) +archived_models.regions(image,**kwargs)) @@ -48,7 +53,9 @@ def regions(image, *, context=None, **kwargs): def preview(data, asset, selection): """A preview of one asset `regions` found: an image, a sound, a model or a table, chosen by `selection`.""" - from . import archived_models, audio, fonts, models, ptr_audio, resources, texture_packs, textures + from . import archived_models, audio, ffs, fonts, models, ptr_audio, rare_models, resources, texture_packs, textures + if asset.get('layout')=='ffs-i64':return ffs.preview(data,asset,selection) + if asset.get('layout')=='rare-model':return rare_models.preview(data,asset,selection) if asset.get('font_layout')=='native-i4-pointer-glyphs':return fonts.preview(data,asset,selection) if asset.get('audio_kind') in ('libmus sample bank','libmus sequence'):return ptr_audio.preview(data,asset,selection) if asset.get('layout')=='archived-pd3v-model':return archived_models.preview(data,asset,selection) diff --git a/src/decompassets/n64/ffs.py b/src/decompassets/n64/ffs.py new file mode 100644 index 0000000..8d4d6ce --- /dev/null +++ b/src/decompassets/n64/ffs.py @@ -0,0 +1,118 @@ +"""Recognize I64 images in an FFS filesystem packed as one chunked RNC stream. + +The stream (method byte 0x81) holds the whole filesystem in 4 KB chunks, each packed on its own +(`rnc.chunked`). Unpacked, it starts with "FFS\\0", a file count and one 24-byte record per file: an id, +a size and an offset, with no names. An I64 image is "I64\\0", width, height and a format word, then the +palette (if any) and the pixels; its file size must be exactly that. The formats seen are 0 CI4, 1 CI8, +4 I4 and 5 I8, palettes RGBA 5551. Only the chunks holding a file's first bytes are unpacked to find it. +""" +from __future__ import annotations + +import re +import struct + +from .. import formats as asset_formats +from .. import rnc +from ..formats import AssetError + +MAX_FILES = 16384 +FORMATS = {0: ('ci4', 16), 1: ('ci8', 256), 4: ('i4', 0), 5: ('i8', 0)} +BITS = {'ci4': 4, 'ci8': 8, 'i4': 4, 'i8': 8} + + +class _Reader: + """Output bytes of a chunked stream, keeping the chunks already unpacked.""" + + def __init__(self, image, layout, check): + self.image, self.layout, self.check, self.chunks = image, layout, check, {} + + def read(self, start, end): + out = bytearray() + for index in range(start // rnc.CHUNK_BYTES, (max(start, end-1)) // rnc.CHUNK_BYTES + 1): + if index not in self.chunks: + self.check() + at = index * rnc.CHUNK_BYTES + self.chunks[index] = rnc.unpack_chunked(self.image, self.layout, at, min(self.layout['size'], at + rnc.CHUNK_BYTES)) + out += self.chunks[index] + base = start // rnc.CHUNK_BYTES * rnc.CHUNK_BYTES + return bytes(out[start-base:end-base]) + + +def files(head, size): + """The filesystem's records (id, size, offset), each inside the stream, or None when it isn't one.""" + if head[:4] != b'FFS\0':return None + count = int.from_bytes(head[4:8], 'big') + table = 12 + 24*count + if not 0 < count <= MAX_FILES or table > size:return None + return count, table + + +def image_layout(header, file_size): + """An I64 image's preview settings, relative to its file, when its size is exactly what it says.""" + if header[:4] != b'I64\0':return None + width, height, code = struct.unpack_from('>3I', header, 4) + if code not in FORMATS or not 1 <= width <= 1024 or not 1 <= height <= 1024:return None + fmt, entries = FORMATS[code] + pixels = (width*BITS[fmt] + 7)//8*height + if 16 + 2*entries + pixels != file_size:return None + settings = dict(format=fmt, width=width, height=height, offset=16 + 2*entries) + item = dict(preview_settings=settings, frames=1, palettes=1 if entries else 0, frame_offsets=[settings['offset']], + palette_offsets=[16] if entries else [], palette_entries=[entries] if entries else [], + levels=[dict(width=width, height=height, offset=0)]) + if entries:settings.update(palette_offset=16, palette_format='rgba5551-be') + return item + + +def regions(image, *, check=lambda: None, progress=lambda _: None): + """One region per packed chunk that starts an I64 image, holding every image that starts in it.""" + results = [] + for match in re.finditer(rb'RNC\x81', image): + check() + layout = rnc.chunked(image, match.start()) + if not layout:continue + reader = _Reader(image, layout, check) + try: + found = files(reader.read(0, 12), layout['size']) + if not found:continue + count, table = found + records = [struct.unpack_from('>3I', reader.read(12 + 24*i, 24 + 24*i)) for i in range(count)] + except rnc.RncError:continue + groups = {} + for index, (ident, size, offset) in enumerate(records): + if index % 64 == 0:progress(dict(phase='Reading the packed filesystem', done=index, total=count)) + if size < 16 or offset < table or offset + size > layout['size']:continue + try:item = image_layout(reader.read(offset, offset + 16), size) + except rnc.RncError:continue + if not item:continue + item['file'] = dict(id=ident, offset=offset, size=size) + groups.setdefault(offset // rnc.CHUNK_BYTES, []).append(item) + for chunk, items in sorted(groups.items()): + at, packed = layout['chunks'][chunk] + first = items[0]['preview_settings'] + label = (f"FFS file {items[0]['file']['id']:#x} · {first['format'].upper()} {first['width']}×{first['height']}" + if len(items) == 1 else f'FFS images ({len(items)})') + results.append(dict(offset=at, size=packed + (packed & 1), kind='image', label=label, confidence='validated', + layout='ffs-i64', stream=match.start(), textures=items, + **{k: v for k, v in items[0].items() if k not in ('file', 'preview_settings')}, + preview_settings={**first, 'source': 'rom', 'offset': at}, + reason='Every chunk read unpacked to its full 4 KB; the FFS record and the I64 header agree ' + 'with the file size exactly. Runtime use is not established.')) + return results + + +def preview(image, asset, selection): + """Unpack only the chosen image's file and show it with its own palette.""" + layout = rnc.chunked(image, asset['stream']) + texture = asset_formats.integer(selection.get('texture', 0), 'Texture', 0, len(asset['textures'])-1) + file = asset['textures'][texture]['file'] + try:raw = rnc.unpack_chunked(image, layout, file['offset'], file['offset'] + file['size']) if layout else None + except rnc.RncError:raw = None + item = raw and image_layout(raw[:16], len(raw)) + if not item:raise AssetError('The packed filesystem changed. Rescan the ROM.') + asset_formats.integer(selection.get('frame', 0), 'Frame', 0, 0) + asset_formats.integer(selection.get('mip', 0), 'Mip level', 0, 0) + result = asset_formats.preview(raw, item['preview_settings']) + result['details'].update(texture=texture, frame=0, palette=0 if item['palettes'] else 'none', note=asset['reason']) + result['decoded'] = dict(codec='RNC chunked (method 0x81)', bytes=len(raw), block_offset=asset['offset'], member=f"FFS file {file['id']:#x}") + result['settings'] = dict(source='rom', format='auto', offset=asset['offset'], texture=texture, frame=0, palette=0, mip=0) + return result diff --git a/src/decompassets/n64/rare_models.py b/src/decompassets/n64/rare_models.py new file mode 100644 index 0000000..6a33aad --- /dev/null +++ b/src/decompassets/n64/rare_models.py @@ -0,0 +1,116 @@ +"""Recognize model textures in a deflated asset table of the kind a Rare platformer sequel uses. + +The table is a count, 0xFFFFFFFF, then one word per asset whose top 24 bits are its offset in words from +the table's end; an asset is a two-byte header and a raw deflate stream that must end inside its slot. +A model starts with 0x0000000B; the halfword at 8 points to its texture list: a byte size, a count, then +eight bytes per texture (data offset, type, width, height) and the texture data, a palette before its +pixels. Types: 1 CI4, 2 CI8, 4 RGBA16, 8 RGBA32, 16 IA8; 0x8000 marks smaller mip levels after the first, +which is the one shown. A model is taken only when its first texture starts the data, every later one starts after it, and all fit. +""" +from __future__ import annotations + +import re +import struct +import zlib + +from .. import formats as asset_formats +from ..formats import AssetError + +MAX_ASSETS = 65535 +MAX_MODEL = 4 * 1024 * 1024 +MAX_EXPANDED = 256 * 1024 * 1024 +MODEL = b'\0\0\0\x0b' +TYPES = {1: ('ci4', 16, 4), 2: ('ci8', 256, 8), 4: ('rgba5551-be', 0, 16), 8: ('rgba8888', 0, 32), 16: ('ia8', 0, 8)} + + +def tables(image, check=lambda: None): + """Each asset table: (where its entries start, the base their offsets count from, the entries).""" + found = [] + for match in re.finditer(rb'\x00\x00..\xff\xff\xff\xff\x00\x00\x00', image, re.S): + check() + at = match.start() + if at % 4:continue + count = int.from_bytes(image[at:at+4], 'big') + start = at + 8 + if not 16 <= count <= MAX_ASSETS or start + 4*count > len(image):continue + entries = struct.unpack_from(f'>{count}I', image, start) + words = [e >> 8 for e in entries] + base = start + 4*count + if words[0] or any(b < a for a, b in zip(words, words[1:])) or base + 4*words[-1] > len(image):continue + found.append((start, base, entries)) + return found + + +def _inflate(data, limit): + stream = zlib.decompressobj(-15) + try:out = stream.decompress(data[2:], limit) + except zlib.error:return None + return out if stream.eof else None + + +def texture_list(model): + """The model's textures as preview items relative to the model, or None when the list doesn't hold.""" + if model[:4] != MODEL or len(model) < 16:return None + setup = int.from_bytes(model[8:10], 'big') + if not setup or setup + 8 > len(model):return None + count = int.from_bytes(model[setup+4:setup+6], 'big') + data = setup + 8 + 8*count + if not 0 < count <= 256 or data > len(model):return None + items, last = [], -1 + for index in range(count): + offset, kind, width, height = struct.unpack_from('>IHBB', model, setup + 8 + 8*index) + if kind & 0x7FFF not in TYPES or not width or not height or offset <= last or not index and offset:return None + fmt, entries, bits = TYPES[kind & 0x7FFF] + start, pixels = data + offset, (width*bits + 7)//8*height + if start + 2*entries + pixels > len(model):return None + last = offset + settings = dict(format=fmt, width=width, height=height, offset=start + 2*entries) + if entries:settings.update(palette_offset=start, palette_format='rgba5551-be') + items.append(dict(preview_settings=settings, frames=1, palettes=1 if entries else 0, frame_offsets=[settings['offset']], + palette_offsets=[start] if entries else [], palette_entries=[entries] if entries else [], + levels=[dict(width=width, height=height, offset=0)], mipmapped=bool(kind & 0x8000))) + return items + + +def regions(image, *, check=lambda: None, progress=lambda _: None): + """One region per model asset whose texture list checks out, holding all its textures.""" + results, expanded = [], 0 + for start, base, entries in tables(image, check): + for index, (entry, following) in enumerate(zip(entries, entries[1:])): + if index % 256 == 0: + check();progress(dict(phase='Reading the deflated asset table', done=index, total=len(entries))) + offset, end = base + 4*(entry >> 8), base + 4*(following >> 8) + if end - offset < 4:continue + head = zlib.decompressobj(-15) + try: + if head.decompress(image[offset+2:end], 4) != MODEL:continue + except zlib.error:continue + model = _inflate(image[offset:end], MAX_MODEL) + items = model and texture_list(model) + if not items:continue + expanded += len(model) + if expanded > MAX_EXPANDED:return results + first = items[0]['preview_settings'] + results.append(dict(offset=offset, size=end-offset, kind='image', confidence='validated', layout='rare-model', + label=f'Model {index} · {len(items)} texture(s)', asset_index=index, textures=items, + **{k: v for k, v in items[0].items() if k != 'preview_settings'}, + preview_settings={**first, 'source': 'rom', 'offset': offset}, + reason='The asset table entry and its deflate stream end together; the model\'s texture list ' + 'has rising offsets and every texture fits. Runtime use is not established.')) + return results + + +def preview(image, asset, selection): + """Inflate the model again and show the chosen texture's first level.""" + model = _inflate(image[asset['offset']:asset['offset']+asset['size']], MAX_MODEL) + items = model and texture_list(model) + if not items:raise AssetError('The model asset changed. Rescan the ROM.') + texture = asset_formats.integer(selection.get('texture', 0), 'Texture', 0, len(items)-1) + asset_formats.integer(selection.get('frame', 0), 'Frame', 0, 0) + asset_formats.integer(selection.get('mip', 0), 'Mip level', 0, 0) + item = items[texture] + result = asset_formats.preview(model, item['preview_settings']) + result['details'].update(texture=texture, frame=0, palette=0 if item['palettes'] else 'none', note=asset['reason']) + result['decoded'] = dict(codec='deflate', bytes=len(model), block_offset=asset['offset'], member=f"asset {asset['asset_index']}") + result['settings'] = dict(source='rom', format='auto', offset=asset['offset'], texture=texture, frame=0, palette=0, mip=0) + return result diff --git a/src/decompassets/rnc.py b/src/decompassets/rnc.py index 6680019..6ff69ff 100644 --- a/src/decompassets/rnc.py +++ b/src/decompassets/rnc.py @@ -114,16 +114,8 @@ def _value(bits: _Bits, table: list[tuple[int, int, int, int]]) -> int: raise RncError("RNC stream has no matching Huffman code") -def unpack(data: bytes, *, max_output: int = MAX_OUTPUT, literals: list[Literal] | None = None) -> bytes: - """Unpack a checksummed method-1 block. Optional literal locations support a verified compression template.""" - if len(data) < HEADER_BYTES or data[:4] != b"RNC\x01": - raise RncError("not an RNC method-1 container") - size, packed, unpacked_crc, packed_crc = struct.unpack_from(">IIHH", data, 4) - if not 0 < size <= max_output or packed < 2 or HEADER_BYTES + packed > len(data): - raise RncError("invalid RNC sizes") - payload = data[HEADER_BYTES:HEADER_BYTES + packed] - if crc16(payload) != packed_crc: - raise RncError("RNC packed checksum differs") +def _method1(payload: bytes, size: int, *, literals: list[Literal] | None = None, max_chunks: int = 255) -> bytearray: + """Decode one method-1 bit stream to exactly `size` bytes; the container's checks are the caller's.""" bits = _Bits(payload) if bits.take(2): raise RncError("encrypted or locked RNC streams are unsupported") @@ -149,8 +141,22 @@ def unpack(data: bytes, *, max_output: int = MAX_OUTPUT, literals: list[Literal] for _ in range(n): out.append(out[-offset]) # a run can overlap its own already-decoded bytes chunks += 1 - if chunks > 255: + if chunks > max_chunks: raise RncError("too many RNC chunks") + return out + + +def unpack(data: bytes, *, max_output: int = MAX_OUTPUT, literals: list[Literal] | None = None) -> bytes: + """Unpack a checksummed method-1 block. Optional literal locations support a verified compression template.""" + if len(data) < HEADER_BYTES or data[:4] != b"RNC\x01": + raise RncError("not an RNC method-1 container") + size, packed, unpacked_crc, packed_crc = struct.unpack_from(">IIHH", data, 4) + if not 0 < size <= max_output or packed < 2 or HEADER_BYTES + packed > len(data): + raise RncError("invalid RNC sizes") + payload = data[HEADER_BYTES:HEADER_BYTES + packed] + if crc16(payload) != packed_crc: + raise RncError("RNC packed checksum differs") + out = _method1(payload, size, literals=literals) if crc16(out) != unpacked_crc: raise RncError("RNC unpacked checksum differs") return bytes(out) @@ -257,3 +263,48 @@ def literal(count): for _ in range(length):out.append(out[-offset]) if len(out)!=size or crc16(out)!=out_crc:raise RncError('RNC unpacked size or checksum differs') return bytes(out) + + +CHUNK_BYTES = 4096 +MAX_CHUNKED = 64 << 20 + + +def chunked(image: bytes, offset: int) -> dict | None: + """A chunked stream's layout: method byte 0x81, then a table of each 4 KB chunk's packed size. + + Each chunk is a method-1 stream of its own, so any part of the output can be read without the rest; a + chunk starts on an even offset. Its checksum fields don't use the method-1 CRC, so a stream is accepted + only when every chunk table entry is plausible and all of them fit inside the image.""" + if image[offset:offset+4] != b"RNC\x81" or offset + HEADER_BYTES > len(image): + return None + size, packed = struct.unpack_from(">II", image, offset + 4) + count = -(-size // CHUNK_BYTES) + table = offset + HEADER_BYTES + if not 0 < size <= MAX_CHUNKED or table + 2*count > len(image): + return None + sizes = struct.unpack_from(f">{count}H", image, table) + starts, at = [], table + 2*count + for packed_size in sizes: + if not 2 <= packed_size <= 2*CHUNK_BYTES: + return None + starts.append(at) + at += packed_size + (packed_size & 1) + if at > len(image): + return None + return dict(offset=offset, size=size, chunks=list(zip(starts, sizes)), end=at) + + +def unpack_chunked(image: bytes, layout: dict, start: int = 0, end: int | None = None, *, check=lambda: None) -> bytes: + """Output bytes `start` to `end` of a chunked stream, decoding only the chunks that hold them.""" + end = layout["size"] if end is None else end + if not 0 <= start <= end <= layout["size"]: + raise RncError("range is outside the chunked stream") + first, last = start // CHUNK_BYTES, max(start, end - 1) // CHUNK_BYTES + out = bytearray() + for index in range(first, last + 1): + check() + at, packed = layout["chunks"][index] + want = min(CHUNK_BYTES, layout["size"] - index*CHUNK_BYTES) + out += _method1(image[at:at + packed], want) + base = first * CHUNK_BYTES + return bytes(out[start - base:end - base]) diff --git a/tests/test_packed_filesystems.py b/tests/test_packed_filesystems.py new file mode 100644 index 0000000..3f75e83 --- /dev/null +++ b/tests/test_packed_filesystems.py @@ -0,0 +1,111 @@ +"""Chunked RNC streams with an FFS filesystem, and a deflated asset table with models: homemade data only.""" +import base64 +import struct +import zlib + +import pytest + +from decompassets import n64, rnc +from decompassets.n64 import ffs, rare_models + +from .test_rnc import literal_block +from .test_rom_assets import pixels + +RED, GREEN = bytes.fromhex('f801'), bytes.fromhex('07c1') + + +def chunked(output): + """A method-0x81 stream: each 4 KB chunk a literal-only method-1 stream, padded to an even length.""" + payloads = [literal_block(output[at:at+4096])[rnc.HEADER_BYTES:] for at in range(0, len(output), 4096)] + body = b''.join(p + bytes(len(p) & 1) for p in payloads) + table = b''.join(struct.pack('>H', len(p)) for p in payloads) + return b'RNC\x81' + struct.pack('>IIHHBB', len(output), len(table) + len(body), 0, 0, 0, 0) + table + body + + +def i64(code, width, height, palette=b'', pixels_=b''): + return b'I64\0' + struct.pack('>3I', width, height, code) + palette + pixels_ + + +def filesystem(files): + table = 12 + 24*len(files) + records, data, at = b'', b'', table + for index, body in enumerate(files): + records += struct.pack('>3I', 0x100 + index, len(body), at) + bytes(12) + data += body;at += len(body) + return b'FFS\0' + struct.pack('>II', len(files), 0) + records + data + + +def test_chunked_stream_reads_any_range_from_its_own_chunks(): + output = bytes(range(256)) * 40 + image = bytes(10) + chunked(output) + layout = rnc.chunked(image, 10) + assert layout['size'] == len(output) and len(layout['chunks']) == 3 + assert rnc.unpack_chunked(image, layout, 4000, 8300) == output[4000:8300] + assert rnc.unpack_chunked(image, layout) == output + with pytest.raises(rnc.RncError):rnc.unpack_chunked(image, layout, 0, len(output) + 1) + assert rnc.chunked(image[:-1], 10) is None and rnc.chunked(image, 11) is None + + +def test_ffs_images_are_found_grouped_by_chunk_and_previewed(): + ci4 = i64(0, 8, 2, RED + GREEN + RED*14, bytes([0x01]*8)) + grey = i64(5, 4, 4, b'', bytes([255]*16)) + wrong = i64(0, 8, 2, RED*16, bytes(7)) # one pixel byte short + stream = filesystem([ci4, b'not an image', grey, wrong, bytes(5000), ci4]) + image = bytes(64) + chunked(stream) + found = ffs.regions(image) + assert [len(a['textures']) for a in found] == [2, 1] # two images start in chunk 0, one in chunk 1 + assert [t['preview_settings']['format'] for a in found for t in a['textures']] == ['ci4', 'i8', 'ci4'] + assert found[0]['offset'] == rnc.chunked(image, 64)['chunks'][0][0] + shown = pixels(base64.b64decode(ffs.preview(image, found[0], {'texture': 0})['data_url'].split(',')[1])) + assert shown[:8] == bytes([255, 0, 0, 255, 0, 255, 0, 255]) + assert n64.preview(image, found[0], {'texture': 1})['details']['format'] == 'i8' + assert any(a.get('layout') == 'ffs-i64' for a in n64.regions(image)) + + +def deflated(body, header=b'\x00\x82'): + stream = zlib.compressobj(9, zlib.DEFLATED, -15) + return header + stream.compress(body) + stream.flush() + + +def model(textures, first_offset=0): + """A model: magic, the texture list's offset at 8, then the list (size, count, entries) and the data.""" + setup = 0x50 + entries, data, at = b'', b'', first_offset + for kind, width, height, body in textures: + entries += struct.pack('>IHBB', at, kind, width, height);data += body;at += len(body) + data = bytes(first_offset) + data + head = bytearray(setup);head[:4] = b'\0\0\0\x0b';head[8:10] = setup.to_bytes(2, 'big') + return bytes(head) + struct.pack('>IHH', 8 + len(entries) + len(data), len(textures), 0) + entries + data + + +def asset_table(assets): + blobs = [deflated(a) for a in assets] + blobs = [b + bytes(-len(b) % 4) for b in blobs] + entries, at = [], 0 + for blob in blobs + [b'']: + entries.append((at//4) << 8 | 0x13);at += len(blob) + return struct.pack('>II', len(entries), 0xFFFFFFFF) + b''.join(struct.pack('>I', e) for e in entries) + b''.join(blobs) + + +def test_model_textures_in_a_deflated_asset_table(): + ci4 = RED + GREEN + RED*14 + bytes([0x01]*32) + rgba = (GREEN*16) + bytes(8) # a mip-flagged level, smaller levels after it + assets = [model([(1, 8, 8, ci4), (0x8004, 4, 4, rgba)]), b'\0\0\0\x05 not a model', + model([(1, 8, 8, ci4)], first_offset=8)] + [bytes(64)]*14 + image = bytes(32) + asset_table(assets) + found = rare_models.regions(image) + assert [(a['asset_index'], len(a['textures'])) for a in found] == [(0, 2)] + asset = found[0] + assert [t['preview_settings']['format'] for t in asset['textures']] == ['ci4', 'rgba5551-be'] + assert asset['textures'][1]['mipmapped'] + shown = pixels(base64.b64decode(n64.preview(image, asset, {'texture': 0})['data_url'].split(',')[1])) + assert shown[:8] == bytes([255, 0, 0, 255, 0, 255, 0, 255]) + assert rare_models.preview(image, asset, {'texture': 1})['details']['width'] == 4 + broken = bytearray(image);broken[asset['offset'] + 4] ^= 0xFF + with pytest.raises(n64.AssetError, match='Rescan'):rare_models.preview(bytes(broken), asset, {}) + + +def test_a_table_needs_rising_entries(): + table = bytearray(asset_table([model([(1, 8, 8, RED*16 + bytes(32))])] + [bytes(64)]*15)) + table[12:16] = (0xFFFF00).to_bytes(4, 'big') # the second entry before the first + assert rare_models.tables(bytes(table)) == [] diff --git a/uv.lock b/uv.lock index cb250f3..d21ee7a 100644 --- a/uv.lock +++ b/uv.lock @@ -51,7 +51,7 @@ wheels = [ [[package]] name = "decompassets" -version = "0.3.0" +version = "0.4.0" source = { editable = "." } [package.dev-dependencies]