Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ the RNC unpacker work for any console. The interface may change before 1.0.
## Use

```bash
uv add "decompassets @ https://github.com/RetroReversing/decomp-assets/archive/refs/tags/v0.4.0.tar.gz"
uv add "decompassets @ https://github.com/RetroReversing/decomp-assets/archive/refs/tags/v0.5.0.tar.gz"
```

```python
Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[project]
name = "decompassets"
version = "0.4.0"
version = "0.5.0"
description = "Find and preview game assets (textures, audio banks, models, fonts) inside cartridge and disc images by their structure, with no per-game offsets."
readme = "README.md"
license = "AGPL-3.0-only"
Expand Down
2 changes: 1 addition & 1 deletion src/decompassets/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,4 @@
Everything here reads bytes and returns plain dicts. It never writes a game file, runs a tool or downloads.
"""

__version__ = "0.4.0"
__version__ = "0.5.0"
19 changes: 17 additions & 2 deletions src/decompassets/n64/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
- display_lists.py: textures that uncompressed display lists load, in files placed by their vertices and pointers.
- ffs.py: I64 images in an FFS filesystem packed as one chunked RNC stream.
- rare_models.py: model textures in a deflated asset table.
- uv_textures.py: textures in a UV filesystem of FORM files, drawn by each file's own display list.
- audio.py: libultra B1 sound banks, their sample tables and mono VADPCM previews.
- ptr_audio.py: libmus PtrTablesV2 banks, marked wave tables and sequences.
- models.py: indexed geometry with byte-plane vertices and cache-relative polygons.
Expand Down Expand Up @@ -40,9 +41,10 @@ def regions(image, *, context=None, **kwargs):
resources,
texture_packs,
textures,
uv_textures,
)
known=(textures.regions(image,**kwargs)+texture_packs.regions(image,**kwargs)+display_lists.regions(image,**kwargs)
+ffs.regions(image,**kwargs)+rare_models.regions(image,**kwargs)
+ffs.regions(image,**kwargs)+rare_models.regions(image,**kwargs)+uv_textures.regions(image,**kwargs)
+audio.regions(image,context=context,**kwargs)
+ptr_audio.regions(image,**kwargs)+ptr_audio.sequences(image,**kwargs)+models.regions(image,**kwargs)
+archived_models.regions(image,**kwargs))
Expand All @@ -53,7 +55,20 @@ def regions(image, *, context=None, **kwargs):

def preview(data, asset, selection):
"""A preview of one asset `regions` found: an image, a sound, a model or a table, chosen by `selection`."""
from . import archived_models, audio, ffs, fonts, models, ptr_audio, rare_models, resources, texture_packs, textures
from . import (
archived_models,
audio,
ffs,
fonts,
models,
ptr_audio,
rare_models,
resources,
texture_packs,
textures,
uv_textures,
)
if asset.get('layout')=='uv-texture':return uv_textures.preview(data,asset,selection)
if asset.get('layout')=='ffs-i64':return ffs.preview(data,asset,selection)
if asset.get('layout')=='rare-model':return rare_models.preview(data,asset,selection)
if asset.get('font_layout')=='native-i4-pointer-glyphs':return fonts.preview(data,asset,selection)
Expand Down
159 changes: 159 additions & 0 deletions src/decompassets/n64/uv_textures.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,159 @@
"""Recognize textures in the "UV" filesystem of IFF-style FORM files that a flight-simulation studio's games use.

Each file is "FORM", a length and a type; inside are tagged chunks: "PAD " fillers, and "COMM", either raw or
wrapped in a "GZIP" chunk (the inner tag, the unpacked size, then a MIO0 stream). A texture file (UVTX) has a
COMM of: the texel bytes' size and the display list's command count (two halfwords), a header of floats (20 bytes
in all in one game, 28 in another), the texels, the display list, then the width and height. The display list is
what the game draws the texture with, so its tile commands give the format and size exactly; its first texture
image is the file's own texels, at the address it names (the game adds the texels' RAM address to it).

A file is taken when its display list ends where the counts say and its first drawn tile reads only its own
texels. Colour-indexed tiles whose palette is another file are left out: which file that is comes from the level.
"""
from __future__ import annotations

import re
import struct

from .. import formats as asset_formats
from ..formats import AssetError
from . import display_lists

HEADERS = (20, 28)
MAX_FILE = 1024 * 1024
END = b'\xdf' + bytes(7)


def comm(image, at):
"""The file's COMM bytes and how they were stored ('raw' or 'MIO0'), or None when this isn't a UVTX file."""
if image[at:at+4] != b'FORM' or image[at+8:at+12] != b'UVTX':return None
end = at + 8 + int.from_bytes(image[at+4:at+8], 'big')
if end > len(image) or end - at > MAX_FILE:return None
p = at + 12
while p + 8 <= end:
tag, size = image[p:p+4], int.from_bytes(image[p+4:p+8], 'big')
body = image[p+8:p+8+size]
if p + 8 + size > end:return None
if tag == b'COMM':return body, 'raw'
if tag == b'GZIP' and body[:4] == b'COMM' and body[8:12] == b'MIO0':
expected = int.from_bytes(body[4:8], 'big')
if not 0 < expected <= MAX_FILE:return None
try:data, _codec = asset_formats.unpack(body[8:])
except AssetError:return None
return (data, 'MIO0') if len(data) == expected else None
if tag != b'PAD ':return None
p += 8 + size
return None


def _tile(commands):
"""The first tile drawn from TMEM a block or tile load filled: (format, width, height, address, palette).

The tile's width is its TMEM row (`line`, in 64-bit words) and its height what the load put there, so a
tile whose size is drawn half a texel in (as one game clamps them) is still read whole; the height is
as many rows as the load filled. Last comes the row size when odd rows are stored swapped, else False. The palette is the
load-table's address, or None, or False when it came from another image than the texels'."""
timg = None;images = 0;tiles = {};loaded = {};palette = None
for k in range(0, len(commands), 8):
w0, w1 = struct.unpack_from('>II', commands, k)
cmd = w0 >> 24
if cmd == display_lists.G_SETTIMG:
images += 1;timg = dict(siz=(w0 >> 19) & 3, width=(w0 & 0xFFF)+1, address=w1 & 0xFFFFFF, own=images == 1)
elif cmd == display_lists.G_SETTILE:
tiles[(w1 >> 24) & 7] = dict(fmt=(w0 >> 21) & 7, siz=(w0 >> 19) & 3, line=(w0 >> 9) & 0x1FF, tmem=w0 & 0x1FF)
elif cmd in (display_lists.G_LOADBLOCK, display_lists.G_LOADTILE, display_lists.G_LOADTLUT) and timg:
tile = tiles.get((w1 >> 24) & 7)
if not tile:continue
if cmd == display_lists.G_LOADTLUT:
palette = timg['address'] if timg['own'] else False
elif timg['own']:
bits = display_lists.BITS[timg['siz']]
if cmd == display_lists.G_LOADBLOCK:size = (((w1 >> 12) & 0xFFF)+1)*bits//8
else:size = (((w1 >> 12) & 0xFFF)//4 - ((w0 >> 12) & 0xFFF)//4 + 1)*((w1 & 0xFFF)//4 - (w0 & 0xFFF)//4 + 1)*bits//8
# A block load with no row step (dxt 0) copies bytes as they are: the game stored them as TMEM
# wants them, each odd row's 32-bit words swapped in pairs.
loaded[tile['tmem']] = dict(timg, bytes=size, swapped=cmd == display_lists.G_LOADBLOCK and not w1 & 0xFFF)
elif cmd == display_lists.G_SETTILESIZE:
tile = tiles.get((w1 >> 24) & 7);source = tile and loaded.get(tile['tmem'])
fmt = tile and display_lists.FORMATS.get((tile['fmt'], tile['siz']))
if not source or not fmt or not tile['line']:continue
row = 8*tile['line'];bits = display_lists.BITS[tile['siz']]
width, height = row*8//bits, -(-source['bytes']//row)
if not height or (w1 >> 12 & 0xFFF)//4 >= width or (w1 & 0xFFF)//4 >= height + 1:continue
return fmt, width, height, source['address'], palette, source['swapped'] and row
return None


def texture(data):
"""The texture a COMM block draws, as preview settings relative to the block, or None."""
if len(data) < 24:return None
size, count = struct.unpack_from('>HH', data)
for header in HEADERS:
start = header + size
if not count or start + 8*count + 4 > len(data):continue
last = data[start+8*count-8:start+8*count]
if last[0] not in display_lists.END_LISTS or any(last[1:]):continue
found = _tile(data[start:start+8*count])
if not found:return None
fmt, width, height, address, palette, swapped = found
# The trailer's height is the image's own; the tile may be drawn a texel short of it (clamped).
shown = int.from_bytes(data[start+8*count+2:start+8*count+4], 'big')
if 0 < shown <= height:height = shown
bits = {'rgba8888': 32, 'rgba5551-be': 16, 'ia16': 16, 'ci8': 8, 'ia8': 8, 'i8': 8}.get(fmt, 4)
entries = {'ci4': 16, 'ci8': 256}.get(fmt, 0)
if address + width*height*bits//8 > size:return None
settings = dict(format=fmt, width=width, height=height, offset=header + address)
palettes = []
if entries:
if palette is None or palette is False or palette + 2*entries > size:return None
palettes = [header + palette]
settings.update(palette_offset=palettes[0], palette_format='rgba5551-be')
return dict(preview_settings=settings, swapped_rows=swapped if fmt != 'rgba8888' else False, frames=1, palettes=len(palettes), frame_offsets=[settings['offset']],
palette_offsets=palettes, palette_entries=[entries] if palettes else [],
levels=[dict(width=width, height=height, offset=0)])
return None


def regions(image, *, check=lambda: None, progress=lambda _: None):
"""One region per UVTX file whose display list names a texture in its own texels."""
results = []
matches = [m.start() for m in re.finditer(rb'FORM.{4}UVTX', image, re.S) if not m.start() & 3]
for index, at in enumerate(matches):
if index % 64 == 0:
check();progress(dict(phase='Reading UV texture files', done=index, total=len(matches)))
stored = comm(image, at)
item = stored and texture(stored[0])
if not item:continue
settings = item['preview_settings']
names = display_lists.NAMES[settings['format']]
results.append(dict(offset=at, size=8 + int.from_bytes(image[at+4:at+8], 'big'), kind='image',
confidence='validated', layout='uv-texture', codec=stored[1],
label=f"UV texture {len(results)} · {names} {settings['width']}×{settings['height']}",
**{k: v for k, v in item.items() if k not in ('preview_settings', 'swapped_rows')},
preview_settings={**settings, 'source': 'rom', 'offset': at},
reason='The UVTX file\'s chunks fit it exactly and its own display list, which ends where its '
'counts say, loads and draws this tile from the file\'s texels. Runtime use is not established.'))
return results


def preview(image, asset, selection):
"""Unpack the file again and draw its texture as its display list sets it up."""
stored = comm(image, asset['offset'])
item = stored and texture(stored[0])
if not item:raise AssetError('The UV texture file changed. Rescan the ROM.')
asset_formats.integer(selection.get('texture', 0), 'Texture', 0, 0)
asset_formats.integer(selection.get('frame', 0), 'Frame', 0, 0)
asset_formats.integer(selection.get('mip', 0), 'Mip level', 0, 0)
data, row = stored[0], item['swapped_rows']
if row:
settings = item['preview_settings']
data = bytearray(data)
for at in range(settings['offset'] + row, settings['offset'] + row*settings['height'], 2*row):
for word in range(at, min(at + row, len(data)) - 7, 8):
data[word:word+8] = data[word+4:word+8] + data[word:word+4]
data = bytes(data)
result = asset_formats.preview(data, item['preview_settings'])
result['details'].update(texture=0, frame=0, palette=0 if item['palettes'] else 'none', note=asset['reason'])
result['decoded'] = dict(codec=stored[1], bytes=len(data), block_offset=asset['offset'], member='COMM')
result['settings'] = dict(source='rom', format='auto', offset=asset['offset'], texture=0, frame=0, palette=0, mip=0)
return result
76 changes: 76 additions & 0 deletions tests/test_uv_textures.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
"""UVTX texture files in a UV filesystem of FORM files: homemade data only."""
import base64
import struct

import pytest

from decompassets import n64
from decompassets.formats import AssetError
from decompassets.n64 import uv_textures

from .test_rom_assets import pixels

RED, GREEN = bytes.fromhex('f801'), bytes.fromhex('07c1')


def mio0(raw):
"""A literal-only MIO0 stream: every mask bit set, no back-references."""
mask = bytes([0xFF]*((len(raw) + 7)//8))
mask += bytes(-len(mask) % 4)
data = 16 + len(mask)
return b'MIO0' + struct.pack('>III', len(raw), data, data) + mask + raw


def chunk(tag, body):
return tag + struct.pack('>I', len(body)) + body


def uvtx(comm, packed=False):
body = chunk(b'GZIP', b'COMM' + struct.pack('>I', len(comm)) + mio0(comm)) if packed else chunk(b'COMM', comm)
inner = b'UVTX' + chunk(b'PAD ', bytes(4)) + body
return b'FORM' + struct.pack('>I', len(inner)) + inner


def comm(header, texels, commands, width, height):
words = b''.join(struct.pack('>II', *c) for c in commands)
return (struct.pack('>HH', len(texels), len(commands)) + bytes(header - 4) + texels + words
+ struct.pack('>HH', width, height) + bytes(24))


# An RGBA16 4×2 tile: block load with a row step, tile 0 one 64-bit word per row, Fast3D end.
RGBA16 = [(0xFD100000, 0), (0xF5100000, 0x07000000), (0xF3000000, 0x07007800), (0xF5100200, 0),
(0xF2000000, 0x0000C004), (0xB8000000, 0)]
# An I4 16×2 tile loaded with no row step (dxt 0), so its odd row is stored swapped; F3DEX2-style end.
I4 = [(0xFD100000, 0), (0xF5100000, 0x07000000), (0xF3000000, 0x07007000), (0xF5800200, 0),
(0xF2000000, 0x0003C004), (0xDF000000, 0)]


def test_uvtx_files_are_found_raw_and_packed_and_previewed():
rgba = RED*2 + GREEN*2 + GREEN*4
row0, row1 = bytes([0xF0]*4 + [0x00]*4), bytes([0x0F]*4 + [0xFF]*4)
stored = row0 + row1[4:] + row1[:4]
image = bytes(16) + uvtx(comm(20, rgba, RGBA16, 4, 2)) + uvtx(comm(28, stored, I4, 16, 2), packed=True)
found = uv_textures.regions(image)
assert [(a['offset'], a['codec'], a['preview_settings']['format'], a['preview_settings']['width'],
a['preview_settings']['height']) for a in found] == [(16, 'raw', 'rgba5551-be', 4, 2),
(found[1]['offset'], 'MIO0', 'i4', 16, 2)]
assert found[1]['offset'] == 16 + found[0]['size']
assert [a['layout'] for a in n64.regions(image) if a.get('layout') == 'uv-texture'] == ['uv-texture']*2
shown = pixels(base64.b64decode(n64.preview(image, found[0], {})['data_url'].split(',')[1]))
assert shown[:4] == bytes([255, 0, 0, 255]) and shown[16:20] == bytes([0, 255, 0, 255])
shown = pixels(base64.b64decode(n64.preview(image, found[1], {})['data_url'].split(',')[1]))
assert shown[16*4:16*4+4] == bytes([0, 0, 0, 255]) # row 1 starts with the 0x0F bytes unswapped
assert shown[24*4:24*4+4] == bytes([255, 255, 255, 255])


def test_uvtx_files_that_do_not_hold_together_are_left_out():
rgba = RED*8
unended = uvtx(comm(20, rgba, RGBA16[:-1] + [(0xE7000000, 0)], 4, 2))
short = uvtx(comm(20, rgba[:8], RGBA16, 4, 2)) # the tile reads past the texels
ci4 = [(0xFD100000, 0), (0xF5100000, 0x07000000), (0xF3000000, 0x07007800), (0xF5400200, 0),
(0xFD100000, 0), (0xF0000000, 0x0703C000), (0xF2000000, 0x0001C004), (0xDF000000, 0)]
borrowed = uvtx(comm(20, rgba, ci4, 8, 2)) # its palette is another file's
assert uv_textures.regions(unended + short + borrowed) == []
image = uvtx(comm(20, rgba, RGBA16, 4, 2))
asset = uv_textures.regions(image)[0]
with pytest.raises(AssetError):uv_textures.preview(image[:-30] + bytes(30), dict(asset, offset=1), {})
2 changes: 1 addition & 1 deletion uv.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading