Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ the RNC unpacker work for any console. The interface may change before 1.0.
## Use

```bash
uv add "decompassets @ https://github.com/RetroReversing/decomp-assets/archive/refs/tags/v0.3.0.tar.gz"
uv add "decompassets @ https://github.com/RetroReversing/decomp-assets/archive/refs/tags/v0.4.0.tar.gz"
```

```python
Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[project]
name = "decompassets"
version = "0.3.0"
version = "0.4.0"
description = "Find and preview game assets (textures, audio banks, models, fonts) inside cartridge and disc images by their structure, with no per-game offsets."
readme = "README.md"
license = "AGPL-3.0-only"
Expand Down
2 changes: 1 addition & 1 deletion src/decompassets/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,4 @@
Everything here reads bytes and returns plain dicts. It never writes a game file, runs a tool or downloads.
"""

__version__ = "0.3.0"
__version__ = "0.4.0"
9 changes: 8 additions & 1 deletion src/decompassets/n64/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
- textures.py: indexed texture banks (CI4/CI8, RGBA32, the six-level RGBA16 mipmap layout), raw or RNC-packed.
- texture_packs.py: texture packs with a trailing count, inside archives.
- display_lists.py: textures that uncompressed display lists load, in files placed by their vertices and pointers.
- ffs.py: I64 images in an FFS filesystem packed as one chunked RNC stream.
- rare_models.py: model textures in a deflated asset table.
- audio.py: libultra B1 sound banks, their sample tables and mono VADPCM previews.
- ptr_audio.py: libmus PtrTablesV2 banks, marked wave tables and sequences.
- models.py: indexed geometry with byte-plane vertices and cache-relative polygons.
Expand All @@ -29,15 +31,18 @@ def regions(image, *, context=None, **kwargs):
archived_models,
audio,
display_lists,
ffs,
fonts,
models,
native_scene,
ptr_audio,
rare_models,
resources,
texture_packs,
textures,
)
known=(textures.regions(image,**kwargs)+texture_packs.regions(image,**kwargs)+display_lists.regions(image,**kwargs)
+ffs.regions(image,**kwargs)+rare_models.regions(image,**kwargs)
+audio.regions(image,context=context,**kwargs)
+ptr_audio.regions(image,**kwargs)+ptr_audio.sequences(image,**kwargs)+models.regions(image,**kwargs)
+archived_models.regions(image,**kwargs))
Expand All @@ -48,7 +53,9 @@ def regions(image, *, context=None, **kwargs):

def preview(data, asset, selection):
"""A preview of one asset `regions` found: an image, a sound, a model or a table, chosen by `selection`."""
from . import archived_models, audio, fonts, models, ptr_audio, resources, texture_packs, textures
from . import archived_models, audio, ffs, fonts, models, ptr_audio, rare_models, resources, texture_packs, textures
if asset.get('layout')=='ffs-i64':return ffs.preview(data,asset,selection)
if asset.get('layout')=='rare-model':return rare_models.preview(data,asset,selection)
if asset.get('font_layout')=='native-i4-pointer-glyphs':return fonts.preview(data,asset,selection)
if asset.get('audio_kind') in ('libmus sample bank','libmus sequence'):return ptr_audio.preview(data,asset,selection)
if asset.get('layout')=='archived-pd3v-model':return archived_models.preview(data,asset,selection)
Expand Down
118 changes: 118 additions & 0 deletions src/decompassets/n64/ffs.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
"""Recognize I64 images in an FFS filesystem packed as one chunked RNC stream.

The stream (method byte 0x81) holds the whole filesystem in 4 KB chunks, each packed on its own
(`rnc.chunked`). Unpacked, it starts with "FFS\\0", a file count and one 24-byte record per file: an id,
a size and an offset, with no names. An I64 image is "I64\\0", width, height and a format word, then the
palette (if any) and the pixels; its file size must be exactly that. The formats seen are 0 CI4, 1 CI8,
4 I4 and 5 I8, palettes RGBA 5551. Only the chunks holding a file's first bytes are unpacked to find it.
"""
from __future__ import annotations

import re
import struct

from .. import formats as asset_formats
from .. import rnc
from ..formats import AssetError

MAX_FILES = 16384
FORMATS = {0: ('ci4', 16), 1: ('ci8', 256), 4: ('i4', 0), 5: ('i8', 0)}
BITS = {'ci4': 4, 'ci8': 8, 'i4': 4, 'i8': 8}


class _Reader:
"""Output bytes of a chunked stream, keeping the chunks already unpacked."""

def __init__(self, image, layout, check):
self.image, self.layout, self.check, self.chunks = image, layout, check, {}

def read(self, start, end):
out = bytearray()
for index in range(start // rnc.CHUNK_BYTES, (max(start, end-1)) // rnc.CHUNK_BYTES + 1):
if index not in self.chunks:
self.check()
at = index * rnc.CHUNK_BYTES
self.chunks[index] = rnc.unpack_chunked(self.image, self.layout, at, min(self.layout['size'], at + rnc.CHUNK_BYTES))
out += self.chunks[index]
base = start // rnc.CHUNK_BYTES * rnc.CHUNK_BYTES
return bytes(out[start-base:end-base])


def files(head, size):
"""The filesystem's records (id, size, offset), each inside the stream, or None when it isn't one."""
if head[:4] != b'FFS\0':return None
count = int.from_bytes(head[4:8], 'big')
table = 12 + 24*count
if not 0 < count <= MAX_FILES or table > size:return None
return count, table


def image_layout(header, file_size):
"""An I64 image's preview settings, relative to its file, when its size is exactly what it says."""
if header[:4] != b'I64\0':return None
width, height, code = struct.unpack_from('>3I', header, 4)
if code not in FORMATS or not 1 <= width <= 1024 or not 1 <= height <= 1024:return None
fmt, entries = FORMATS[code]
pixels = (width*BITS[fmt] + 7)//8*height
if 16 + 2*entries + pixels != file_size:return None
settings = dict(format=fmt, width=width, height=height, offset=16 + 2*entries)
item = dict(preview_settings=settings, frames=1, palettes=1 if entries else 0, frame_offsets=[settings['offset']],
palette_offsets=[16] if entries else [], palette_entries=[entries] if entries else [],
levels=[dict(width=width, height=height, offset=0)])
if entries:settings.update(palette_offset=16, palette_format='rgba5551-be')
return item


def regions(image, *, check=lambda: None, progress=lambda _: None):
"""One region per packed chunk that starts an I64 image, holding every image that starts in it."""
results = []
for match in re.finditer(rb'RNC\x81', image):
check()
layout = rnc.chunked(image, match.start())
if not layout:continue
reader = _Reader(image, layout, check)
try:
found = files(reader.read(0, 12), layout['size'])
if not found:continue
count, table = found
records = [struct.unpack_from('>3I', reader.read(12 + 24*i, 24 + 24*i)) for i in range(count)]
except rnc.RncError:continue
groups = {}
for index, (ident, size, offset) in enumerate(records):
if index % 64 == 0:progress(dict(phase='Reading the packed filesystem', done=index, total=count))
if size < 16 or offset < table or offset + size > layout['size']:continue
try:item = image_layout(reader.read(offset, offset + 16), size)
except rnc.RncError:continue
if not item:continue
item['file'] = dict(id=ident, offset=offset, size=size)
groups.setdefault(offset // rnc.CHUNK_BYTES, []).append(item)
for chunk, items in sorted(groups.items()):
at, packed = layout['chunks'][chunk]
first = items[0]['preview_settings']
label = (f"FFS file {items[0]['file']['id']:#x} · {first['format'].upper()} {first['width']}×{first['height']}"
if len(items) == 1 else f'FFS images ({len(items)})')
results.append(dict(offset=at, size=packed + (packed & 1), kind='image', label=label, confidence='validated',
layout='ffs-i64', stream=match.start(), textures=items,
**{k: v for k, v in items[0].items() if k not in ('file', 'preview_settings')},
preview_settings={**first, 'source': 'rom', 'offset': at},
reason='Every chunk read unpacked to its full 4 KB; the FFS record and the I64 header agree '
'with the file size exactly. Runtime use is not established.'))
return results


def preview(image, asset, selection):
"""Unpack only the chosen image's file and show it with its own palette."""
layout = rnc.chunked(image, asset['stream'])
texture = asset_formats.integer(selection.get('texture', 0), 'Texture', 0, len(asset['textures'])-1)
file = asset['textures'][texture]['file']
try:raw = rnc.unpack_chunked(image, layout, file['offset'], file['offset'] + file['size']) if layout else None
except rnc.RncError:raw = None
item = raw and image_layout(raw[:16], len(raw))
if not item:raise AssetError('The packed filesystem changed. Rescan the ROM.')
asset_formats.integer(selection.get('frame', 0), 'Frame', 0, 0)
asset_formats.integer(selection.get('mip', 0), 'Mip level', 0, 0)
result = asset_formats.preview(raw, item['preview_settings'])
result['details'].update(texture=texture, frame=0, palette=0 if item['palettes'] else 'none', note=asset['reason'])
result['decoded'] = dict(codec='RNC chunked (method 0x81)', bytes=len(raw), block_offset=asset['offset'], member=f"FFS file {file['id']:#x}")
result['settings'] = dict(source='rom', format='auto', offset=asset['offset'], texture=texture, frame=0, palette=0, mip=0)
return result
116 changes: 116 additions & 0 deletions src/decompassets/n64/rare_models.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
"""Recognize model textures in a deflated asset table of the kind a Rare platformer sequel uses.

The table is a count, 0xFFFFFFFF, then one word per asset whose top 24 bits are its offset in words from
the table's end; an asset is a two-byte header and a raw deflate stream that must end inside its slot.
A model starts with 0x0000000B; the halfword at 8 points to its texture list: a byte size, a count, then
eight bytes per texture (data offset, type, width, height) and the texture data, a palette before its
pixels. Types: 1 CI4, 2 CI8, 4 RGBA16, 8 RGBA32, 16 IA8; 0x8000 marks smaller mip levels after the first,
which is the one shown. A model is taken only when its first texture starts the data, every later one starts after it, and all fit.
"""
from __future__ import annotations

import re
import struct
import zlib

from .. import formats as asset_formats
from ..formats import AssetError

MAX_ASSETS = 65535
MAX_MODEL = 4 * 1024 * 1024
MAX_EXPANDED = 256 * 1024 * 1024
MODEL = b'\0\0\0\x0b'
TYPES = {1: ('ci4', 16, 4), 2: ('ci8', 256, 8), 4: ('rgba5551-be', 0, 16), 8: ('rgba8888', 0, 32), 16: ('ia8', 0, 8)}


def tables(image, check=lambda: None):
"""Each asset table: (where its entries start, the base their offsets count from, the entries)."""
found = []
for match in re.finditer(rb'\x00\x00..\xff\xff\xff\xff\x00\x00\x00', image, re.S):
check()
at = match.start()
if at % 4:continue
count = int.from_bytes(image[at:at+4], 'big')
start = at + 8
if not 16 <= count <= MAX_ASSETS or start + 4*count > len(image):continue
entries = struct.unpack_from(f'>{count}I', image, start)
words = [e >> 8 for e in entries]
base = start + 4*count
if words[0] or any(b < a for a, b in zip(words, words[1:])) or base + 4*words[-1] > len(image):continue
found.append((start, base, entries))
return found


def _inflate(data, limit):
stream = zlib.decompressobj(-15)
try:out = stream.decompress(data[2:], limit)
except zlib.error:return None
return out if stream.eof else None


def texture_list(model):
"""The model's textures as preview items relative to the model, or None when the list doesn't hold."""
if model[:4] != MODEL or len(model) < 16:return None
setup = int.from_bytes(model[8:10], 'big')
if not setup or setup + 8 > len(model):return None
count = int.from_bytes(model[setup+4:setup+6], 'big')
data = setup + 8 + 8*count
if not 0 < count <= 256 or data > len(model):return None
items, last = [], -1
for index in range(count):
offset, kind, width, height = struct.unpack_from('>IHBB', model, setup + 8 + 8*index)
if kind & 0x7FFF not in TYPES or not width or not height or offset <= last or not index and offset:return None
fmt, entries, bits = TYPES[kind & 0x7FFF]
start, pixels = data + offset, (width*bits + 7)//8*height
if start + 2*entries + pixels > len(model):return None
last = offset
settings = dict(format=fmt, width=width, height=height, offset=start + 2*entries)
if entries:settings.update(palette_offset=start, palette_format='rgba5551-be')
items.append(dict(preview_settings=settings, frames=1, palettes=1 if entries else 0, frame_offsets=[settings['offset']],
palette_offsets=[start] if entries else [], palette_entries=[entries] if entries else [],
levels=[dict(width=width, height=height, offset=0)], mipmapped=bool(kind & 0x8000)))
return items


def regions(image, *, check=lambda: None, progress=lambda _: None):
"""One region per model asset whose texture list checks out, holding all its textures."""
results, expanded = [], 0
for start, base, entries in tables(image, check):
for index, (entry, following) in enumerate(zip(entries, entries[1:])):
if index % 256 == 0:
check();progress(dict(phase='Reading the deflated asset table', done=index, total=len(entries)))
offset, end = base + 4*(entry >> 8), base + 4*(following >> 8)
if end - offset < 4:continue
head = zlib.decompressobj(-15)
try:
if head.decompress(image[offset+2:end], 4) != MODEL:continue
except zlib.error:continue
model = _inflate(image[offset:end], MAX_MODEL)
items = model and texture_list(model)
if not items:continue
expanded += len(model)
if expanded > MAX_EXPANDED:return results
first = items[0]['preview_settings']
results.append(dict(offset=offset, size=end-offset, kind='image', confidence='validated', layout='rare-model',
label=f'Model {index} · {len(items)} texture(s)', asset_index=index, textures=items,
**{k: v for k, v in items[0].items() if k != 'preview_settings'},
preview_settings={**first, 'source': 'rom', 'offset': offset},
reason='The asset table entry and its deflate stream end together; the model\'s texture list '
'has rising offsets and every texture fits. Runtime use is not established.'))
return results


def preview(image, asset, selection):
"""Inflate the model again and show the chosen texture's first level."""
model = _inflate(image[asset['offset']:asset['offset']+asset['size']], MAX_MODEL)
items = model and texture_list(model)
if not items:raise AssetError('The model asset changed. Rescan the ROM.')
texture = asset_formats.integer(selection.get('texture', 0), 'Texture', 0, len(items)-1)
asset_formats.integer(selection.get('frame', 0), 'Frame', 0, 0)
asset_formats.integer(selection.get('mip', 0), 'Mip level', 0, 0)
item = items[texture]
result = asset_formats.preview(model, item['preview_settings'])
result['details'].update(texture=texture, frame=0, palette=0 if item['palettes'] else 'none', note=asset['reason'])
result['decoded'] = dict(codec='deflate', bytes=len(model), block_offset=asset['offset'], member=f"asset {asset['asset_index']}")
result['settings'] = dict(source='rom', format='auto', offset=asset['offset'], texture=texture, frame=0, palette=0, mip=0)
return result
Loading
Loading