From cd08c5a535fb191c5ee3d2bc6f4bbe0cf5d34211 Mon Sep 17 00:00:00 2001 From: RetroGameDeveloper <40120498+RetroGameDeveloper@users.noreply.github.com> Date: Sat, 14 Nov 2026 20:36:58 +0000 Subject: [PATCH] decomp-assets 0.5.0: textures in a UV filesystem of FORM files Two N64 games from one studio keep their textures as UVTX files in an IFF-style filesystem, most packed with MIO0 inside a GZIP chunk. Each file carries the display list the game draws it with, so the tile's format and size are read from it exactly. Tiles are sized from their TMEM row and what the load filled (one game draws them half a texel in), the height from the file's own trailer, and odd rows stored swapped for a block load with no row step are swapped back. On the two games: 453 of 463 and 1,175 of 1,437 files show. Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- pyproject.toml | 2 +- src/decompassets/__init__.py | 2 +- src/decompassets/n64/__init__.py | 19 +++- src/decompassets/n64/uv_textures.py | 159 ++++++++++++++++++++++++++++ tests/test_uv_textures.py | 76 +++++++++++++ uv.lock | 2 +- 7 files changed, 256 insertions(+), 6 deletions(-) create mode 100644 src/decompassets/n64/uv_textures.py create mode 100644 tests/test_uv_textures.py diff --git a/README.md b/README.md index 066049a..7dfc7a0 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ the RNC unpacker work for any console. The interface may change before 1.0. ## Use ```bash -uv add "decompassets @ https://github.com/RetroReversing/decomp-assets/archive/refs/tags/v0.4.0.tar.gz" +uv add "decompassets @ https://github.com/RetroReversing/decomp-assets/archive/refs/tags/v0.5.0.tar.gz" ``` ```python diff --git a/pyproject.toml b/pyproject.toml index 3ccc0c5..746f78c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "decompassets" -version = "0.4.0" +version = "0.5.0" description = "Find and preview game assets (textures, audio banks, models, fonts) inside cartridge and disc images by their structure, with no per-game offsets." readme = "README.md" license = "AGPL-3.0-only" diff --git a/src/decompassets/__init__.py b/src/decompassets/__init__.py index 1aad5ab..f755d5f 100644 --- a/src/decompassets/__init__.py +++ b/src/decompassets/__init__.py @@ -10,4 +10,4 @@ Everything here reads bytes and returns plain dicts. It never writes a game file, runs a tool or downloads. """ -__version__ = "0.4.0" +__version__ = "0.5.0" diff --git a/src/decompassets/n64/__init__.py b/src/decompassets/n64/__init__.py index 729049a..b1c06e0 100644 --- a/src/decompassets/n64/__init__.py +++ b/src/decompassets/n64/__init__.py @@ -5,6 +5,7 @@ - display_lists.py: textures that uncompressed display lists load, in files placed by their vertices and pointers. - ffs.py: I64 images in an FFS filesystem packed as one chunked RNC stream. - rare_models.py: model textures in a deflated asset table. +- uv_textures.py: textures in a UV filesystem of FORM files, drawn by each file's own display list. - audio.py: libultra B1 sound banks, their sample tables and mono VADPCM previews. - ptr_audio.py: libmus PtrTablesV2 banks, marked wave tables and sequences. - models.py: indexed geometry with byte-plane vertices and cache-relative polygons. @@ -40,9 +41,10 @@ def regions(image, *, context=None, **kwargs): resources, texture_packs, textures, + uv_textures, ) known=(textures.regions(image,**kwargs)+texture_packs.regions(image,**kwargs)+display_lists.regions(image,**kwargs) - +ffs.regions(image,**kwargs)+rare_models.regions(image,**kwargs) + +ffs.regions(image,**kwargs)+rare_models.regions(image,**kwargs)+uv_textures.regions(image,**kwargs) +audio.regions(image,context=context,**kwargs) +ptr_audio.regions(image,**kwargs)+ptr_audio.sequences(image,**kwargs)+models.regions(image,**kwargs) +archived_models.regions(image,**kwargs)) @@ -53,7 +55,20 @@ def regions(image, *, context=None, **kwargs): def preview(data, asset, selection): """A preview of one asset `regions` found: an image, a sound, a model or a table, chosen by `selection`.""" - from . import archived_models, audio, ffs, fonts, models, ptr_audio, rare_models, resources, texture_packs, textures + from . import ( + archived_models, + audio, + ffs, + fonts, + models, + ptr_audio, + rare_models, + resources, + texture_packs, + textures, + uv_textures, + ) + if asset.get('layout')=='uv-texture':return uv_textures.preview(data,asset,selection) if asset.get('layout')=='ffs-i64':return ffs.preview(data,asset,selection) if asset.get('layout')=='rare-model':return rare_models.preview(data,asset,selection) if asset.get('font_layout')=='native-i4-pointer-glyphs':return fonts.preview(data,asset,selection) diff --git a/src/decompassets/n64/uv_textures.py b/src/decompassets/n64/uv_textures.py new file mode 100644 index 0000000..d01c2b3 --- /dev/null +++ b/src/decompassets/n64/uv_textures.py @@ -0,0 +1,159 @@ +"""Recognize textures in the "UV" filesystem of IFF-style FORM files that a flight-simulation studio's games use. + +Each file is "FORM", a length and a type; inside are tagged chunks: "PAD " fillers, and "COMM", either raw or +wrapped in a "GZIP" chunk (the inner tag, the unpacked size, then a MIO0 stream). A texture file (UVTX) has a +COMM of: the texel bytes' size and the display list's command count (two halfwords), a header of floats (20 bytes +in all in one game, 28 in another), the texels, the display list, then the width and height. The display list is +what the game draws the texture with, so its tile commands give the format and size exactly; its first texture +image is the file's own texels, at the address it names (the game adds the texels' RAM address to it). + +A file is taken when its display list ends where the counts say and its first drawn tile reads only its own +texels. Colour-indexed tiles whose palette is another file are left out: which file that is comes from the level. +""" +from __future__ import annotations + +import re +import struct + +from .. import formats as asset_formats +from ..formats import AssetError +from . import display_lists + +HEADERS = (20, 28) +MAX_FILE = 1024 * 1024 +END = b'\xdf' + bytes(7) + + +def comm(image, at): + """The file's COMM bytes and how they were stored ('raw' or 'MIO0'), or None when this isn't a UVTX file.""" + if image[at:at+4] != b'FORM' or image[at+8:at+12] != b'UVTX':return None + end = at + 8 + int.from_bytes(image[at+4:at+8], 'big') + if end > len(image) or end - at > MAX_FILE:return None + p = at + 12 + while p + 8 <= end: + tag, size = image[p:p+4], int.from_bytes(image[p+4:p+8], 'big') + body = image[p+8:p+8+size] + if p + 8 + size > end:return None + if tag == b'COMM':return body, 'raw' + if tag == b'GZIP' and body[:4] == b'COMM' and body[8:12] == b'MIO0': + expected = int.from_bytes(body[4:8], 'big') + if not 0 < expected <= MAX_FILE:return None + try:data, _codec = asset_formats.unpack(body[8:]) + except AssetError:return None + return (data, 'MIO0') if len(data) == expected else None + if tag != b'PAD ':return None + p += 8 + size + return None + + +def _tile(commands): + """The first tile drawn from TMEM a block or tile load filled: (format, width, height, address, palette). + + The tile's width is its TMEM row (`line`, in 64-bit words) and its height what the load put there, so a + tile whose size is drawn half a texel in (as one game clamps them) is still read whole; the height is + as many rows as the load filled. Last comes the row size when odd rows are stored swapped, else False. The palette is the + load-table's address, or None, or False when it came from another image than the texels'.""" + timg = None;images = 0;tiles = {};loaded = {};palette = None + for k in range(0, len(commands), 8): + w0, w1 = struct.unpack_from('>II', commands, k) + cmd = w0 >> 24 + if cmd == display_lists.G_SETTIMG: + images += 1;timg = dict(siz=(w0 >> 19) & 3, width=(w0 & 0xFFF)+1, address=w1 & 0xFFFFFF, own=images == 1) + elif cmd == display_lists.G_SETTILE: + tiles[(w1 >> 24) & 7] = dict(fmt=(w0 >> 21) & 7, siz=(w0 >> 19) & 3, line=(w0 >> 9) & 0x1FF, tmem=w0 & 0x1FF) + elif cmd in (display_lists.G_LOADBLOCK, display_lists.G_LOADTILE, display_lists.G_LOADTLUT) and timg: + tile = tiles.get((w1 >> 24) & 7) + if not tile:continue + if cmd == display_lists.G_LOADTLUT: + palette = timg['address'] if timg['own'] else False + elif timg['own']: + bits = display_lists.BITS[timg['siz']] + if cmd == display_lists.G_LOADBLOCK:size = (((w1 >> 12) & 0xFFF)+1)*bits//8 + else:size = (((w1 >> 12) & 0xFFF)//4 - ((w0 >> 12) & 0xFFF)//4 + 1)*((w1 & 0xFFF)//4 - (w0 & 0xFFF)//4 + 1)*bits//8 + # A block load with no row step (dxt 0) copies bytes as they are: the game stored them as TMEM + # wants them, each odd row's 32-bit words swapped in pairs. + loaded[tile['tmem']] = dict(timg, bytes=size, swapped=cmd == display_lists.G_LOADBLOCK and not w1 & 0xFFF) + elif cmd == display_lists.G_SETTILESIZE: + tile = tiles.get((w1 >> 24) & 7);source = tile and loaded.get(tile['tmem']) + fmt = tile and display_lists.FORMATS.get((tile['fmt'], tile['siz'])) + if not source or not fmt or not tile['line']:continue + row = 8*tile['line'];bits = display_lists.BITS[tile['siz']] + width, height = row*8//bits, -(-source['bytes']//row) + if not height or (w1 >> 12 & 0xFFF)//4 >= width or (w1 & 0xFFF)//4 >= height + 1:continue + return fmt, width, height, source['address'], palette, source['swapped'] and row + return None + + +def texture(data): + """The texture a COMM block draws, as preview settings relative to the block, or None.""" + if len(data) < 24:return None + size, count = struct.unpack_from('>HH', data) + for header in HEADERS: + start = header + size + if not count or start + 8*count + 4 > len(data):continue + last = data[start+8*count-8:start+8*count] + if last[0] not in display_lists.END_LISTS or any(last[1:]):continue + found = _tile(data[start:start+8*count]) + if not found:return None + fmt, width, height, address, palette, swapped = found + # The trailer's height is the image's own; the tile may be drawn a texel short of it (clamped). + shown = int.from_bytes(data[start+8*count+2:start+8*count+4], 'big') + if 0 < shown <= height:height = shown + bits = {'rgba8888': 32, 'rgba5551-be': 16, 'ia16': 16, 'ci8': 8, 'ia8': 8, 'i8': 8}.get(fmt, 4) + entries = {'ci4': 16, 'ci8': 256}.get(fmt, 0) + if address + width*height*bits//8 > size:return None + settings = dict(format=fmt, width=width, height=height, offset=header + address) + palettes = [] + if entries: + if palette is None or palette is False or palette + 2*entries > size:return None + palettes = [header + palette] + settings.update(palette_offset=palettes[0], palette_format='rgba5551-be') + return dict(preview_settings=settings, swapped_rows=swapped if fmt != 'rgba8888' else False, frames=1, palettes=len(palettes), frame_offsets=[settings['offset']], + palette_offsets=palettes, palette_entries=[entries] if palettes else [], + levels=[dict(width=width, height=height, offset=0)]) + return None + + +def regions(image, *, check=lambda: None, progress=lambda _: None): + """One region per UVTX file whose display list names a texture in its own texels.""" + results = [] + matches = [m.start() for m in re.finditer(rb'FORM.{4}UVTX', image, re.S) if not m.start() & 3] + for index, at in enumerate(matches): + if index % 64 == 0: + check();progress(dict(phase='Reading UV texture files', done=index, total=len(matches))) + stored = comm(image, at) + item = stored and texture(stored[0]) + if not item:continue + settings = item['preview_settings'] + names = display_lists.NAMES[settings['format']] + results.append(dict(offset=at, size=8 + int.from_bytes(image[at+4:at+8], 'big'), kind='image', + confidence='validated', layout='uv-texture', codec=stored[1], + label=f"UV texture {len(results)} · {names} {settings['width']}×{settings['height']}", + **{k: v for k, v in item.items() if k not in ('preview_settings', 'swapped_rows')}, + preview_settings={**settings, 'source': 'rom', 'offset': at}, + reason='The UVTX file\'s chunks fit it exactly and its own display list, which ends where its ' + 'counts say, loads and draws this tile from the file\'s texels. Runtime use is not established.')) + return results + + +def preview(image, asset, selection): + """Unpack the file again and draw its texture as its display list sets it up.""" + stored = comm(image, asset['offset']) + item = stored and texture(stored[0]) + if not item:raise AssetError('The UV texture file changed. Rescan the ROM.') + asset_formats.integer(selection.get('texture', 0), 'Texture', 0, 0) + asset_formats.integer(selection.get('frame', 0), 'Frame', 0, 0) + asset_formats.integer(selection.get('mip', 0), 'Mip level', 0, 0) + data, row = stored[0], item['swapped_rows'] + if row: + settings = item['preview_settings'] + data = bytearray(data) + for at in range(settings['offset'] + row, settings['offset'] + row*settings['height'], 2*row): + for word in range(at, min(at + row, len(data)) - 7, 8): + data[word:word+8] = data[word+4:word+8] + data[word:word+4] + data = bytes(data) + result = asset_formats.preview(data, item['preview_settings']) + result['details'].update(texture=0, frame=0, palette=0 if item['palettes'] else 'none', note=asset['reason']) + result['decoded'] = dict(codec=stored[1], bytes=len(data), block_offset=asset['offset'], member='COMM') + result['settings'] = dict(source='rom', format='auto', offset=asset['offset'], texture=0, frame=0, palette=0, mip=0) + return result diff --git a/tests/test_uv_textures.py b/tests/test_uv_textures.py new file mode 100644 index 0000000..65cf825 --- /dev/null +++ b/tests/test_uv_textures.py @@ -0,0 +1,76 @@ +"""UVTX texture files in a UV filesystem of FORM files: homemade data only.""" +import base64 +import struct + +import pytest + +from decompassets import n64 +from decompassets.formats import AssetError +from decompassets.n64 import uv_textures + +from .test_rom_assets import pixels + +RED, GREEN = bytes.fromhex('f801'), bytes.fromhex('07c1') + + +def mio0(raw): + """A literal-only MIO0 stream: every mask bit set, no back-references.""" + mask = bytes([0xFF]*((len(raw) + 7)//8)) + mask += bytes(-len(mask) % 4) + data = 16 + len(mask) + return b'MIO0' + struct.pack('>III', len(raw), data, data) + mask + raw + + +def chunk(tag, body): + return tag + struct.pack('>I', len(body)) + body + + +def uvtx(comm, packed=False): + body = chunk(b'GZIP', b'COMM' + struct.pack('>I', len(comm)) + mio0(comm)) if packed else chunk(b'COMM', comm) + inner = b'UVTX' + chunk(b'PAD ', bytes(4)) + body + return b'FORM' + struct.pack('>I', len(inner)) + inner + + +def comm(header, texels, commands, width, height): + words = b''.join(struct.pack('>II', *c) for c in commands) + return (struct.pack('>HH', len(texels), len(commands)) + bytes(header - 4) + texels + words + + struct.pack('>HH', width, height) + bytes(24)) + + +# An RGBA16 4×2 tile: block load with a row step, tile 0 one 64-bit word per row, Fast3D end. +RGBA16 = [(0xFD100000, 0), (0xF5100000, 0x07000000), (0xF3000000, 0x07007800), (0xF5100200, 0), + (0xF2000000, 0x0000C004), (0xB8000000, 0)] +# An I4 16×2 tile loaded with no row step (dxt 0), so its odd row is stored swapped; F3DEX2-style end. +I4 = [(0xFD100000, 0), (0xF5100000, 0x07000000), (0xF3000000, 0x07007000), (0xF5800200, 0), + (0xF2000000, 0x0003C004), (0xDF000000, 0)] + + +def test_uvtx_files_are_found_raw_and_packed_and_previewed(): + rgba = RED*2 + GREEN*2 + GREEN*4 + row0, row1 = bytes([0xF0]*4 + [0x00]*4), bytes([0x0F]*4 + [0xFF]*4) + stored = row0 + row1[4:] + row1[:4] + image = bytes(16) + uvtx(comm(20, rgba, RGBA16, 4, 2)) + uvtx(comm(28, stored, I4, 16, 2), packed=True) + found = uv_textures.regions(image) + assert [(a['offset'], a['codec'], a['preview_settings']['format'], a['preview_settings']['width'], + a['preview_settings']['height']) for a in found] == [(16, 'raw', 'rgba5551-be', 4, 2), + (found[1]['offset'], 'MIO0', 'i4', 16, 2)] + assert found[1]['offset'] == 16 + found[0]['size'] + assert [a['layout'] for a in n64.regions(image) if a.get('layout') == 'uv-texture'] == ['uv-texture']*2 + shown = pixels(base64.b64decode(n64.preview(image, found[0], {})['data_url'].split(',')[1])) + assert shown[:4] == bytes([255, 0, 0, 255]) and shown[16:20] == bytes([0, 255, 0, 255]) + shown = pixels(base64.b64decode(n64.preview(image, found[1], {})['data_url'].split(',')[1])) + assert shown[16*4:16*4+4] == bytes([0, 0, 0, 255]) # row 1 starts with the 0x0F bytes unswapped + assert shown[24*4:24*4+4] == bytes([255, 255, 255, 255]) + + +def test_uvtx_files_that_do_not_hold_together_are_left_out(): + rgba = RED*8 + unended = uvtx(comm(20, rgba, RGBA16[:-1] + [(0xE7000000, 0)], 4, 2)) + short = uvtx(comm(20, rgba[:8], RGBA16, 4, 2)) # the tile reads past the texels + ci4 = [(0xFD100000, 0), (0xF5100000, 0x07000000), (0xF3000000, 0x07007800), (0xF5400200, 0), + (0xFD100000, 0), (0xF0000000, 0x0703C000), (0xF2000000, 0x0001C004), (0xDF000000, 0)] + borrowed = uvtx(comm(20, rgba, ci4, 8, 2)) # its palette is another file's + assert uv_textures.regions(unended + short + borrowed) == [] + image = uvtx(comm(20, rgba, RGBA16, 4, 2)) + asset = uv_textures.regions(image)[0] + with pytest.raises(AssetError):uv_textures.preview(image[:-30] + bytes(30), dict(asset, offset=1), {}) diff --git a/uv.lock b/uv.lock index d21ee7a..dacf0e9 100644 --- a/uv.lock +++ b/uv.lock @@ -51,7 +51,7 @@ wheels = [ [[package]] name = "decompassets" -version = "0.4.0" +version = "0.5.0" source = { editable = "." } [package.dev-dependencies]