Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ the RNC unpacker work for any console. The interface may change before 1.0.
## Use

```bash
uv add "decompassets @ https://github.com/RetroReversing/decomp-assets/archive/refs/tags/v0.5.0.tar.gz"
uv add "decompassets @ https://github.com/RetroReversing/decomp-assets/archive/refs/tags/v0.6.0.tar.gz"
```

```python
Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[project]
name = "decompassets"
version = "0.5.0"
version = "0.6.0"
description = "Find and preview game assets (textures, audio banks, models, fonts) inside cartridge and disc images by their structure, with no per-game offsets."
readme = "README.md"
license = "AGPL-3.0-only"
Expand Down
2 changes: 1 addition & 1 deletion src/decompassets/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,4 @@
Everything here reads bytes and returns plain dicts. It never writes a game file, runs a tool or downloads.
"""

__version__ = "0.5.0"
__version__ = "0.6.0"
6 changes: 5 additions & 1 deletion src/decompassets/n64/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
- display_lists.py: textures that uncompressed display lists load, in files placed by their vertices and pointers.
- ffs.py: I64 images in an FFS filesystem packed as one chunked RNC stream.
- rare_models.py: model textures in a deflated asset table.
- blast_textures.py: textures packed with a demolition game's own codecs, listed in its texture table.
- uv_textures.py: textures in a UV filesystem of FORM files, drawn by each file's own display list.
- audio.py: libultra B1 sound banks, their sample tables and mono VADPCM previews.
- ptr_audio.py: libmus PtrTablesV2 banks, marked wave tables and sequences.
Expand All @@ -31,6 +32,7 @@ def regions(image, *, context=None, **kwargs):
from . import (
archived_models,
audio,
blast_textures,
display_lists,
ffs,
fonts,
Expand All @@ -44,7 +46,7 @@ def regions(image, *, context=None, **kwargs):
uv_textures,
)
known=(textures.regions(image,**kwargs)+texture_packs.regions(image,**kwargs)+display_lists.regions(image,**kwargs)
+ffs.regions(image,**kwargs)+rare_models.regions(image,**kwargs)+uv_textures.regions(image,**kwargs)
+ffs.regions(image,**kwargs)+rare_models.regions(image,**kwargs)+uv_textures.regions(image,**kwargs)+blast_textures.regions(image,**kwargs)
+audio.regions(image,context=context,**kwargs)
+ptr_audio.regions(image,**kwargs)+ptr_audio.sequences(image,**kwargs)+models.regions(image,**kwargs)
+archived_models.regions(image,**kwargs))
Expand All @@ -58,6 +60,7 @@ def preview(data, asset, selection):
from . import (
archived_models,
audio,
blast_textures,
ffs,
fonts,
models,
Expand All @@ -68,6 +71,7 @@ def preview(data, asset, selection):
textures,
uv_textures,
)
if asset.get('layout')=='blast':return blast_textures.preview(data,asset,selection)
if asset.get('layout')=='uv-texture':return uv_textures.preview(data,asset,selection)
if asset.get('layout')=='ffs-i64':return ffs.preview(data,asset,selection)
if asset.get('layout')=='rare-model':return rare_models.preview(data,asset,selection)
Expand Down
165 changes: 165 additions & 0 deletions src/decompassets/n64/blast_textures.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,165 @@
"""Recognize textures packed with a demolition game's own back-reference codecs, from its texture table.

The table is one eight-byte entry per resource (offset from the table, packed size, codec), the first resource
starting right after the table and nearly every next one at the previous one's end, rounded up to eight bytes. Codec 0 is
stored data (among it the look-up tables codecs 4 and 5 read: 256 bytes for codec 5, 128 for codec 4, each
used by the textures after it until the next). The others unpack halfwords: the top bit set copies an earlier
run (length in the low five bits), clear it is one or two texels in the codec's own packing:

1 RGBA16, 2 RGBA32 (four bits a channel), 3 and 6 IA8 (two texels), 4 IA16 (two texels through a 128-byte
table), 5 RGBA32 (a 256-byte colour table and four bits of alpha).

The table names no dimensions; the game's code passes them. The width is inferred from the texels: at the true
width each texel is closest to the one below it, a dip that strides one shorter or longer don't share. Square
sizes are preferred when they score nearly as well. Checked against the dimensions a decompilation lists for one
release: 96% agree. The preview lets the width be changed.
"""
from __future__ import annotations

import math
import struct
from operator import sub

from .. import formats as asset_formats
from ..formats import AssetError

MAX_ENTRIES = 8192
MAX_PACKED = 64 * 1024
CODECS = {1: ('rgba5551-be', 2), 2: ('rgba8888', 4), 3: ('ia8', 1), 4: ('ia16', 2), 5: ('rgba8888', 4), 6: ('ia8', 1)}
TABLES = {4: 128, 5: 256}
NAMES = {'rgba5551-be': 'RGBA16', 'rgba8888': 'RGBA32', 'ia8': 'IA8', 'ia16': 'IA16'}
SQUARE_MARGIN = 1.3


def table(image, at):
"""The table's entries as (ROM offset, packed size, codec), or None when no table starts at `at`."""
if at + 16 > len(image):return None
first = int.from_bytes(image[at:at+4], 'big')
count = first // 8
if first % 8 or not 64 <= count <= MAX_ENTRIES or at + first > len(image):return None
entries, expected, unchained = [], first, 0
for offset, size, codec in struct.iter_unpack('>IHH', image[at:at+first]):
if codec > 6 or at + offset + size > len(image):return None
# A few placeholder entries point back at one shared resource; the rest must chain.
if offset != expected:unchained += 1
entries.append((at + offset, size, codec))
expected = max(expected, offset + size + (-size) % 8)
return entries if unchained * 20 <= count and sum(c for _, _, c in entries) else None


def _single(codec, word, lut):
if codec == 1:return struct.pack('>H', (word & 0xFFC0) << 1 | word & 0x3F)
if codec == 2:
return struct.pack('>I', (word & 0x7800) << 17 | (word & 0x0780) << 13 | (word & 0x78) << 9 | (word & 7) << 5)
if codec == 3:return bytes((((word >> 8) << 1) & 0xFF, ((word & 0xFF) << 1) & 0xFF))
if codec == 6:
high, low = word >> 8, word & 0xFF
return bytes(((high & 0x38) << 2 | (high & 7) << 1, (low & 0x38) << 2 | (low & 7) << 1))
if codec == 4:
high, low = word >> 8, word & 0xFF
a = int.from_bytes(lut[high & 0xFE:(high & 0xFE)+2], 'big') << 1 | high & 1
b = int.from_bytes(lut[low & 0xFE:(low & 0xFE)+2], 'big') << 1 | low & 1
return struct.pack('>HH', a & 0xFFFF, b & 0xFFFF)
colour = int.from_bytes(lut[(word >> 4) << 1:((word >> 4) << 1)+2], 'big')
return struct.pack('>I', ((colour & 0x7C00) << 17 | (colour & 0x03E0) << 14 | (colour & 0x1F) << 11 | (word & 0xF) << 4)
& 0xFFFFFFFF)


def unpack(codec, packed, lut=b''):
"""A texture's texels, unpacked; AssetError when a back-reference reaches outside them."""
if codec in TABLES and len(lut) != TABLES[codec]:raise AssetError('This texture\'s colour table is missing. Rescan the ROM.')
element = 4 if codec in (2, 4, 5) else 2
mask, shift = (0x7FE0, 4) if element == 4 else (0x7FFF, 5)
out = bytearray()
for (word,) in struct.iter_unpack('>H', packed[:len(packed) & ~1]):
if word & 0x8000:
start = len(out) - ((word & mask) >> shift)
length = (word & 0x1F) * element
if start < 0 or start + length > len(out):raise AssetError('The packed texture is not valid here. Rescan the ROM.')
out += out[start:start+length]
else:out += _single(codec, word, lut)
return bytes(out)


def _levels(texels, fmt, count):
"""One brightness value per texel (the sum of its RGBA), for comparing texels."""
pixels = asset_formats._rgba(texels, fmt, count)
return [sum(pixels[i:i+4]) for i in range(0, 4*count, 4)]


def width(texels, fmt, bytes_per_texel):
"""The inferred width, and the candidate widths with their scores (lower is better)."""
count = len(texels) // bytes_per_texel
candidates = [w for w in range(4, 257, 4) if count % w == 0 and 2 <= count // w <= 256 and w <= 8*(count//w) and count//w <= 8*w]
if not candidates:return None, {}
levels = _levels(texels, fmt, count)

def mean(stride):
n = count - stride
return sum(map(abs, map(sub, levels[stride:], levels[:n]))) / n if n > 0 else 0.0
scores = {}
for w in candidates:
here = mean(w)
# How well rows line up (the difference to the texel below), times how much better than a stride off by one.
scores[w] = here * (here + 1) / (min(mean(w-1), mean(w+1)) + 1)
best = min(scores, key=scores.get)
side = math.isqrt(count)
if side*side == count and side in scores and scores[side] <= SQUARE_MARGIN * scores[best]:best = side
return best, scores


def regions(image, *, check=lambda: None, progress=lambda _: None):
"""One region per packed texture in a texture table, its width inferred from its texels."""
for at in range(0, len(image) - 16, 8):
if not at & 0xFFFF:check()
if image[at] or image[at+1] or image[at+6] or image[at+7] > 6:continue
entries = table(image, at)
if entries:break
else:return []
results, luts = [], {}
for index, (offset, size, codec) in enumerate(entries):
if index % 128 == 0:
check();progress(dict(phase='Reading the texture table', done=index, total=len(entries)))
if codec == 0:
for wanted, length in TABLES.items():
if size == length:luts[wanted] = offset
continue
if not size or size > MAX_PACKED or (codec in TABLES and codec not in luts):continue
lut = image[luts[codec]:luts[codec]+TABLES[codec]] if codec in TABLES else b''
try:texels = unpack(codec, image[offset:offset+size], lut)
except AssetError:continue
fmt, per = CODECS[codec]
best, _scores = width(texels, fmt, per)
if not best:continue
height = len(texels) // per // best
settings = dict(format=fmt, width=best, height=height, offset=0)
results.append(dict(offset=offset, size=size, kind='image', confidence='inferred', layout='blast',
label=f'Texture {index} · {NAMES[fmt]} {best}×{height}', table=at, entry=index, codec=codec,
lut=luts.get(codec), preview_settings={**settings, 'source': 'rom', 'offset': offset},
frames=1, palettes=0, frame_offsets=[0], palette_offsets=[], palette_entries=[],
levels=[dict(width=best, height=height, offset=0)],
reason=f'Entry {index} of the texture table at {at:#x}, unpacked with codec {codec} to exactly its '
'texels. The table names no dimensions: the width is inferred from where rows line up. '
'Runtime use is not established.'))
return results


def preview(image, asset, selection):
"""Unpack the texture again and show it at the inferred width, or the width chosen."""
entries = table(image, asset['table'])
if not entries or asset['entry'] >= len(entries) or entries[asset['entry']][0] != asset['offset']:
raise AssetError('The texture table changed. Rescan the ROM.')
_, size, codec = entries[asset['entry']]
lut = image[asset['lut']:asset['lut']+TABLES[codec]] if codec in TABLES and asset.get('lut') is not None else b''
texels = unpack(codec, image[asset['offset']:asset['offset']+size], lut)
fmt, per = CODECS[codec]
count = len(texels) // per
chosen = asset_formats.integer(selection.get('width', asset['preview_settings']['width']), 'Width', 1, count)
if count % chosen:raise AssetError(f'Choose a width that divides the texture\'s {count} texels.')
asset_formats.integer(selection.get('frame', 0), 'Frame', 0, 0)
asset_formats.integer(selection.get('mip', 0), 'Mip level', 0, 0)
result = asset_formats.preview(texels, dict(format=fmt, width=chosen, height=count // chosen, offset=0))
result['details'].update(texture=0, frame=0, palette='none', note=asset['reason'])
result['decoded'] = dict(codec=f'texture codec {codec}', bytes=len(texels), block_offset=asset['offset'], member=f"entry {asset['entry']}")
result['settings'] = dict(source='rom', format='auto', offset=asset['offset'], texture=0, frame=0, palette=0, mip=0)
return result
63 changes: 63 additions & 0 deletions tests/test_blast_textures.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
"""A texture table of back-reference-packed textures: homemade data only."""
import base64
import struct

import pytest

from decompassets import n64
from decompassets.formats import AssetError
from decompassets.n64 import blast_textures

from .test_rom_assets import pixels


def rgba16_words(columns, rows):
"""Codec-1 halfwords for an image whose texels differ along a row and repeat down it, then a back-reference."""
words = [x << 6 | x for x in range(columns)]
# Each later row copies the one before it: back 2*columns bytes, `columns` texels.
words += [0x8000 | (columns*2) << 5 | columns] * (rows - 1)
return struct.pack(f'>{len(words)}H', *words)


def build(resources, count=64):
"""A table of `count` entries (the rest empty, pointing at the end) followed by the resources it lists."""
head, body, offset = b'', b'', 8*count
for codec, packed in resources:
head += struct.pack('>IHH', offset, len(packed), codec)
packed += bytes(-len(packed) % 8)
body += packed;offset += len(packed)
head += struct.pack('>IHH', offset, 0, 0) * (count - len(resources))
return head + body


def test_texture_table_textures_unpack_with_their_width_inferred():
lut = bytes(range(256)) # a colour table, then a codec-5 texture that reads it
image = bytes(24) + build([(1, rgba16_words(16, 8)), (0, lut), (5, struct.pack('>16H', *range(0x10, 0x110, 0x10))),
(1, rgba16_words(16, 16))])
found = blast_textures.regions(image)
assert [(a['codec'], a['preview_settings']['width'], a['preview_settings']['height']) for a in found] == [(1, 16, 8), (5, 4, 4), (1, 16, 16)]
assert found[1]['codec'] == 5 and found[1]['lut'] == 24 + 64*8 + 8*((len(rgba16_words(16, 8)) + 7)//8)
assert all(a['layout'] == 'blast' and a['confidence'] == 'inferred' for a in found)
assert len([a for a in n64.regions(image) if a.get('layout') == 'blast']) == 3
shown = pixels(base64.b64decode(n64.preview(image, found[0], {})['data_url'].split(',')[1]))
assert shown[:16*4] == shown[16*4:32*4] and shown[:4] != shown[4:8] # rows repeat, texels along a row differ
assert n64.preview(image, found[0], {'width': 32})['details']['width'] == 32
with pytest.raises(AssetError):n64.preview(image, found[0], {'width': 7})


def test_codecs_unpack_their_own_packings():
assert blast_textures.unpack(1, struct.pack('>H', 0x0041)) == struct.pack('>H', 0x0081)
assert blast_textures.unpack(3, struct.pack('>H', 0x7F01)) == bytes((0xFE, 0x02))
assert blast_textures.unpack(6, struct.pack('>H', 0x3F09)) == bytes((0xEE, 0x22))
assert blast_textures.unpack(1, struct.pack('>HH', 0x0001, 0x8041)) == struct.pack('>HH', 1, 1)
with pytest.raises(AssetError):blast_textures.unpack(1, struct.pack('>H', 0x8041)) # reaches before the start
with pytest.raises(AssetError):blast_textures.unpack(5, struct.pack('>H', 1)) # no colour table


def test_tables_that_do_not_chain_are_not_taken():
good = build([(1, rgba16_words(16, 8))])
assert blast_textures.table(good, 0)
broken = bytearray(good)
for k in range(8, 8*64, 8):struct.pack_into('>I', broken, k, 8*k) # offsets that skip ahead every entry
assert blast_textures.table(bytes(broken), 0) is None
assert blast_textures.regions(bytes(64) + struct.pack('>IHH', 0x40, 4, 9) + bytes(64)) == []
2 changes: 1 addition & 1 deletion uv.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading