diff --git a/README.md b/README.md index 066049a..7dfc7a0 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ the RNC unpacker work for any console. The interface may change before 1.0. ## Use ```bash -uv add "decompassets @ https://github.com/RetroReversing/decomp-assets/archive/refs/tags/v0.4.0.tar.gz" +uv add "decompassets @ https://github.com/RetroReversing/decomp-assets/archive/refs/tags/v0.5.0.tar.gz" ``` ```python diff --git a/pyproject.toml b/pyproject.toml index 3ccc0c5..746f78c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "decompassets" -version = "0.4.0" +version = "0.5.0" description = "Find and preview game assets (textures, audio banks, models, fonts) inside cartridge and disc images by their structure, with no per-game offsets." readme = "README.md" license = "AGPL-3.0-only" diff --git a/src/decompassets/__init__.py b/src/decompassets/__init__.py index 1aad5ab..f755d5f 100644 --- a/src/decompassets/__init__.py +++ b/src/decompassets/__init__.py @@ -10,4 +10,4 @@ Everything here reads bytes and returns plain dicts. It never writes a game file, runs a tool or downloads. """ -__version__ = "0.4.0" +__version__ = "0.5.0" diff --git a/src/decompassets/n64/__init__.py b/src/decompassets/n64/__init__.py index 729049a..b1c06e0 100644 --- a/src/decompassets/n64/__init__.py +++ b/src/decompassets/n64/__init__.py @@ -5,6 +5,7 @@ - display_lists.py: textures that uncompressed display lists load, in files placed by their vertices and pointers. - ffs.py: I64 images in an FFS filesystem packed as one chunked RNC stream. - rare_models.py: model textures in a deflated asset table. +- uv_textures.py: textures in a UV filesystem of FORM files, drawn by each file's own display list. - audio.py: libultra B1 sound banks, their sample tables and mono VADPCM previews. - ptr_audio.py: libmus PtrTablesV2 banks, marked wave tables and sequences. - models.py: indexed geometry with byte-plane vertices and cache-relative polygons. @@ -40,9 +41,10 @@ def regions(image, *, context=None, **kwargs): resources, texture_packs, textures, + uv_textures, ) known=(textures.regions(image,**kwargs)+texture_packs.regions(image,**kwargs)+display_lists.regions(image,**kwargs) - +ffs.regions(image,**kwargs)+rare_models.regions(image,**kwargs) + +ffs.regions(image,**kwargs)+rare_models.regions(image,**kwargs)+uv_textures.regions(image,**kwargs) +audio.regions(image,context=context,**kwargs) +ptr_audio.regions(image,**kwargs)+ptr_audio.sequences(image,**kwargs)+models.regions(image,**kwargs) +archived_models.regions(image,**kwargs)) @@ -53,7 +55,20 @@ def regions(image, *, context=None, **kwargs): def preview(data, asset, selection): """A preview of one asset `regions` found: an image, a sound, a model or a table, chosen by `selection`.""" - from . import archived_models, audio, ffs, fonts, models, ptr_audio, rare_models, resources, texture_packs, textures + from . import ( + archived_models, + audio, + ffs, + fonts, + models, + ptr_audio, + rare_models, + resources, + texture_packs, + textures, + uv_textures, + ) + if asset.get('layout')=='uv-texture':return uv_textures.preview(data,asset,selection) if asset.get('layout')=='ffs-i64':return ffs.preview(data,asset,selection) if asset.get('layout')=='rare-model':return rare_models.preview(data,asset,selection) if asset.get('font_layout')=='native-i4-pointer-glyphs':return fonts.preview(data,asset,selection) diff --git a/src/decompassets/n64/uv_textures.py b/src/decompassets/n64/uv_textures.py new file mode 100644 index 0000000..d01c2b3 --- /dev/null +++ b/src/decompassets/n64/uv_textures.py @@ -0,0 +1,159 @@ +"""Recognize textures in the "UV" filesystem of IFF-style FORM files that a flight-simulation studio's games use. + +Each file is "FORM", a length and a type; inside are tagged chunks: "PAD " fillers, and "COMM", either raw or +wrapped in a "GZIP" chunk (the inner tag, the unpacked size, then a MIO0 stream). A texture file (UVTX) has a +COMM of: the texel bytes' size and the display list's command count (two halfwords), a header of floats (20 bytes +in all in one game, 28 in another), the texels, the display list, then the width and height. The display list is +what the game draws the texture with, so its tile commands give the format and size exactly; its first texture +image is the file's own texels, at the address it names (the game adds the texels' RAM address to it). + +A file is taken when its display list ends where the counts say and its first drawn tile reads only its own +texels. Colour-indexed tiles whose palette is another file are left out: which file that is comes from the level. +""" +from __future__ import annotations + +import re +import struct + +from .. import formats as asset_formats +from ..formats import AssetError +from . import display_lists + +HEADERS = (20, 28) +MAX_FILE = 1024 * 1024 +END = b'\xdf' + bytes(7) + + +def comm(image, at): + """The file's COMM bytes and how they were stored ('raw' or 'MIO0'), or None when this isn't a UVTX file.""" + if image[at:at+4] != b'FORM' or image[at+8:at+12] != b'UVTX':return None + end = at + 8 + int.from_bytes(image[at+4:at+8], 'big') + if end > len(image) or end - at > MAX_FILE:return None + p = at + 12 + while p + 8 <= end: + tag, size = image[p:p+4], int.from_bytes(image[p+4:p+8], 'big') + body = image[p+8:p+8+size] + if p + 8 + size > end:return None + if tag == b'COMM':return body, 'raw' + if tag == b'GZIP' and body[:4] == b'COMM' and body[8:12] == b'MIO0': + expected = int.from_bytes(body[4:8], 'big') + if not 0 < expected <= MAX_FILE:return None + try:data, _codec = asset_formats.unpack(body[8:]) + except AssetError:return None + return (data, 'MIO0') if len(data) == expected else None + if tag != b'PAD ':return None + p += 8 + size + return None + + +def _tile(commands): + """The first tile drawn from TMEM a block or tile load filled: (format, width, height, address, palette). + + The tile's width is its TMEM row (`line`, in 64-bit words) and its height what the load put there, so a + tile whose size is drawn half a texel in (as one game clamps them) is still read whole; the height is + as many rows as the load filled. Last comes the row size when odd rows are stored swapped, else False. The palette is the + load-table's address, or None, or False when it came from another image than the texels'.""" + timg = None;images = 0;tiles = {};loaded = {};palette = None + for k in range(0, len(commands), 8): + w0, w1 = struct.unpack_from('>II', commands, k) + cmd = w0 >> 24 + if cmd == display_lists.G_SETTIMG: + images += 1;timg = dict(siz=(w0 >> 19) & 3, width=(w0 & 0xFFF)+1, address=w1 & 0xFFFFFF, own=images == 1) + elif cmd == display_lists.G_SETTILE: + tiles[(w1 >> 24) & 7] = dict(fmt=(w0 >> 21) & 7, siz=(w0 >> 19) & 3, line=(w0 >> 9) & 0x1FF, tmem=w0 & 0x1FF) + elif cmd in (display_lists.G_LOADBLOCK, display_lists.G_LOADTILE, display_lists.G_LOADTLUT) and timg: + tile = tiles.get((w1 >> 24) & 7) + if not tile:continue + if cmd == display_lists.G_LOADTLUT: + palette = timg['address'] if timg['own'] else False + elif timg['own']: + bits = display_lists.BITS[timg['siz']] + if cmd == display_lists.G_LOADBLOCK:size = (((w1 >> 12) & 0xFFF)+1)*bits//8 + else:size = (((w1 >> 12) & 0xFFF)//4 - ((w0 >> 12) & 0xFFF)//4 + 1)*((w1 & 0xFFF)//4 - (w0 & 0xFFF)//4 + 1)*bits//8 + # A block load with no row step (dxt 0) copies bytes as they are: the game stored them as TMEM + # wants them, each odd row's 32-bit words swapped in pairs. + loaded[tile['tmem']] = dict(timg, bytes=size, swapped=cmd == display_lists.G_LOADBLOCK and not w1 & 0xFFF) + elif cmd == display_lists.G_SETTILESIZE: + tile = tiles.get((w1 >> 24) & 7);source = tile and loaded.get(tile['tmem']) + fmt = tile and display_lists.FORMATS.get((tile['fmt'], tile['siz'])) + if not source or not fmt or not tile['line']:continue + row = 8*tile['line'];bits = display_lists.BITS[tile['siz']] + width, height = row*8//bits, -(-source['bytes']//row) + if not height or (w1 >> 12 & 0xFFF)//4 >= width or (w1 & 0xFFF)//4 >= height + 1:continue + return fmt, width, height, source['address'], palette, source['swapped'] and row + return None + + +def texture(data): + """The texture a COMM block draws, as preview settings relative to the block, or None.""" + if len(data) < 24:return None + size, count = struct.unpack_from('>HH', data) + for header in HEADERS: + start = header + size + if not count or start + 8*count + 4 > len(data):continue + last = data[start+8*count-8:start+8*count] + if last[0] not in display_lists.END_LISTS or any(last[1:]):continue + found = _tile(data[start:start+8*count]) + if not found:return None + fmt, width, height, address, palette, swapped = found + # The trailer's height is the image's own; the tile may be drawn a texel short of it (clamped). + shown = int.from_bytes(data[start+8*count+2:start+8*count+4], 'big') + if 0 < shown <= height:height = shown + bits = {'rgba8888': 32, 'rgba5551-be': 16, 'ia16': 16, 'ci8': 8, 'ia8': 8, 'i8': 8}.get(fmt, 4) + entries = {'ci4': 16, 'ci8': 256}.get(fmt, 0) + if address + width*height*bits//8 > size:return None + settings = dict(format=fmt, width=width, height=height, offset=header + address) + palettes = [] + if entries: + if palette is None or palette is False or palette + 2*entries > size:return None + palettes = [header + palette] + settings.update(palette_offset=palettes[0], palette_format='rgba5551-be') + return dict(preview_settings=settings, swapped_rows=swapped if fmt != 'rgba8888' else False, frames=1, palettes=len(palettes), frame_offsets=[settings['offset']], + palette_offsets=palettes, palette_entries=[entries] if palettes else [], + levels=[dict(width=width, height=height, offset=0)]) + return None + + +def regions(image, *, check=lambda: None, progress=lambda _: None): + """One region per UVTX file whose display list names a texture in its own texels.""" + results = [] + matches = [m.start() for m in re.finditer(rb'FORM.{4}UVTX', image, re.S) if not m.start() & 3] + for index, at in enumerate(matches): + if index % 64 == 0: + check();progress(dict(phase='Reading UV texture files', done=index, total=len(matches))) + stored = comm(image, at) + item = stored and texture(stored[0]) + if not item:continue + settings = item['preview_settings'] + names = display_lists.NAMES[settings['format']] + results.append(dict(offset=at, size=8 + int.from_bytes(image[at+4:at+8], 'big'), kind='image', + confidence='validated', layout='uv-texture', codec=stored[1], + label=f"UV texture {len(results)} · {names} {settings['width']}×{settings['height']}", + **{k: v for k, v in item.items() if k not in ('preview_settings', 'swapped_rows')}, + preview_settings={**settings, 'source': 'rom', 'offset': at}, + reason='The UVTX file\'s chunks fit it exactly and its own display list, which ends where its ' + 'counts say, loads and draws this tile from the file\'s texels. Runtime use is not established.')) + return results + + +def preview(image, asset, selection): + """Unpack the file again and draw its texture as its display list sets it up.""" + stored = comm(image, asset['offset']) + item = stored and texture(stored[0]) + if not item:raise AssetError('The UV texture file changed. Rescan the ROM.') + asset_formats.integer(selection.get('texture', 0), 'Texture', 0, 0) + asset_formats.integer(selection.get('frame', 0), 'Frame', 0, 0) + asset_formats.integer(selection.get('mip', 0), 'Mip level', 0, 0) + data, row = stored[0], item['swapped_rows'] + if row: + settings = item['preview_settings'] + data = bytearray(data) + for at in range(settings['offset'] + row, settings['offset'] + row*settings['height'], 2*row): + for word in range(at, min(at + row, len(data)) - 7, 8): + data[word:word+8] = data[word+4:word+8] + data[word:word+4] + data = bytes(data) + result = asset_formats.preview(data, item['preview_settings']) + result['details'].update(texture=0, frame=0, palette=0 if item['palettes'] else 'none', note=asset['reason']) + result['decoded'] = dict(codec=stored[1], bytes=len(data), block_offset=asset['offset'], member='COMM') + result['settings'] = dict(source='rom', format='auto', offset=asset['offset'], texture=0, frame=0, palette=0, mip=0) + return result diff --git a/tests/test_uv_textures.py b/tests/test_uv_textures.py new file mode 100644 index 0000000..65cf825 --- /dev/null +++ b/tests/test_uv_textures.py @@ -0,0 +1,76 @@ +"""UVTX texture files in a UV filesystem of FORM files: homemade data only.""" +import base64 +import struct + +import pytest + +from decompassets import n64 +from decompassets.formats import AssetError +from decompassets.n64 import uv_textures + +from .test_rom_assets import pixels + +RED, GREEN = bytes.fromhex('f801'), bytes.fromhex('07c1') + + +def mio0(raw): + """A literal-only MIO0 stream: every mask bit set, no back-references.""" + mask = bytes([0xFF]*((len(raw) + 7)//8)) + mask += bytes(-len(mask) % 4) + data = 16 + len(mask) + return b'MIO0' + struct.pack('>III', len(raw), data, data) + mask + raw + + +def chunk(tag, body): + return tag + struct.pack('>I', len(body)) + body + + +def uvtx(comm, packed=False): + body = chunk(b'GZIP', b'COMM' + struct.pack('>I', len(comm)) + mio0(comm)) if packed else chunk(b'COMM', comm) + inner = b'UVTX' + chunk(b'PAD ', bytes(4)) + body + return b'FORM' + struct.pack('>I', len(inner)) + inner + + +def comm(header, texels, commands, width, height): + words = b''.join(struct.pack('>II', *c) for c in commands) + return (struct.pack('>HH', len(texels), len(commands)) + bytes(header - 4) + texels + words + + struct.pack('>HH', width, height) + bytes(24)) + + +# An RGBA16 4×2 tile: block load with a row step, tile 0 one 64-bit word per row, Fast3D end. +RGBA16 = [(0xFD100000, 0), (0xF5100000, 0x07000000), (0xF3000000, 0x07007800), (0xF5100200, 0), + (0xF2000000, 0x0000C004), (0xB8000000, 0)] +# An I4 16×2 tile loaded with no row step (dxt 0), so its odd row is stored swapped; F3DEX2-style end. +I4 = [(0xFD100000, 0), (0xF5100000, 0x07000000), (0xF3000000, 0x07007000), (0xF5800200, 0), + (0xF2000000, 0x0003C004), (0xDF000000, 0)] + + +def test_uvtx_files_are_found_raw_and_packed_and_previewed(): + rgba = RED*2 + GREEN*2 + GREEN*4 + row0, row1 = bytes([0xF0]*4 + [0x00]*4), bytes([0x0F]*4 + [0xFF]*4) + stored = row0 + row1[4:] + row1[:4] + image = bytes(16) + uvtx(comm(20, rgba, RGBA16, 4, 2)) + uvtx(comm(28, stored, I4, 16, 2), packed=True) + found = uv_textures.regions(image) + assert [(a['offset'], a['codec'], a['preview_settings']['format'], a['preview_settings']['width'], + a['preview_settings']['height']) for a in found] == [(16, 'raw', 'rgba5551-be', 4, 2), + (found[1]['offset'], 'MIO0', 'i4', 16, 2)] + assert found[1]['offset'] == 16 + found[0]['size'] + assert [a['layout'] for a in n64.regions(image) if a.get('layout') == 'uv-texture'] == ['uv-texture']*2 + shown = pixels(base64.b64decode(n64.preview(image, found[0], {})['data_url'].split(',')[1])) + assert shown[:4] == bytes([255, 0, 0, 255]) and shown[16:20] == bytes([0, 255, 0, 255]) + shown = pixels(base64.b64decode(n64.preview(image, found[1], {})['data_url'].split(',')[1])) + assert shown[16*4:16*4+4] == bytes([0, 0, 0, 255]) # row 1 starts with the 0x0F bytes unswapped + assert shown[24*4:24*4+4] == bytes([255, 255, 255, 255]) + + +def test_uvtx_files_that_do_not_hold_together_are_left_out(): + rgba = RED*8 + unended = uvtx(comm(20, rgba, RGBA16[:-1] + [(0xE7000000, 0)], 4, 2)) + short = uvtx(comm(20, rgba[:8], RGBA16, 4, 2)) # the tile reads past the texels + ci4 = [(0xFD100000, 0), (0xF5100000, 0x07000000), (0xF3000000, 0x07007800), (0xF5400200, 0), + (0xFD100000, 0), (0xF0000000, 0x0703C000), (0xF2000000, 0x0001C004), (0xDF000000, 0)] + borrowed = uvtx(comm(20, rgba, ci4, 8, 2)) # its palette is another file's + assert uv_textures.regions(unended + short + borrowed) == [] + image = uvtx(comm(20, rgba, RGBA16, 4, 2)) + asset = uv_textures.regions(image)[0] + with pytest.raises(AssetError):uv_textures.preview(image[:-30] + bytes(30), dict(asset, offset=1), {}) diff --git a/uv.lock b/uv.lock index d21ee7a..dacf0e9 100644 --- a/uv.lock +++ b/uv.lock @@ -51,7 +51,7 @@ wheels = [ [[package]] name = "decompassets" -version = "0.4.0" +version = "0.5.0" source = { editable = "." } [package.dev-dependencies]