diff --git a/README.md b/README.md index c16f974..fb75359 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ the RNC unpacker work for any console. The interface may change before 1.0. ## Use ```bash -uv add "decompassets @ https://github.com/RetroReversing/decomp-assets/archive/refs/tags/v0.6.0.tar.gz" +uv add "decompassets @ https://github.com/RetroReversing/decomp-assets/archive/refs/tags/v0.7.0.tar.gz" ``` ```python diff --git a/pyproject.toml b/pyproject.toml index f48ffc3..3161a3d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "decompassets" -version = "0.6.0" +version = "0.7.0" description = "Find and preview game assets (textures, audio banks, models, fonts) inside cartridge and disc images by their structure, with no per-game offsets." readme = "README.md" license = "AGPL-3.0-only" diff --git a/src/decompassets/__init__.py b/src/decompassets/__init__.py index e84e047..b37b6e6 100644 --- a/src/decompassets/__init__.py +++ b/src/decompassets/__init__.py @@ -10,4 +10,4 @@ Everything here reads bytes and returns plain dicts. It never writes a game file, runs a tool or downloads. """ -__version__ = "0.6.0" +__version__ = "0.7.0" diff --git a/src/decompassets/n64/__init__.py b/src/decompassets/n64/__init__.py index b4e8804..572b2fc 100644 --- a/src/decompassets/n64/__init__.py +++ b/src/decompassets/n64/__init__.py @@ -6,6 +6,7 @@ - ffs.py: I64 images in an FFS filesystem packed as one chunked RNC stream. - rare_models.py: model textures in a deflated asset table. - blast_textures.py: textures packed with a demolition game's own codecs, listed in its texture table. +- reloc_files.py: textures and sprites in a table of relocatable files, some VPK0-packed (vpk0.py). - uv_textures.py: textures in a UV filesystem of FORM files, drawn by each file's own display list. - audio.py: libultra B1 sound banks, their sample tables and mono VADPCM previews. - ptr_audio.py: libmus PtrTablesV2 banks, marked wave tables and sequences. @@ -40,13 +41,14 @@ def regions(image, *, context=None, **kwargs): native_scene, ptr_audio, rare_models, + reloc_files, resources, texture_packs, textures, uv_textures, ) known=(textures.regions(image,**kwargs)+texture_packs.regions(image,**kwargs)+display_lists.regions(image,**kwargs) - +ffs.regions(image,**kwargs)+rare_models.regions(image,**kwargs)+uv_textures.regions(image,**kwargs)+blast_textures.regions(image,**kwargs) + +ffs.regions(image,**kwargs)+rare_models.regions(image,**kwargs)+uv_textures.regions(image,**kwargs)+blast_textures.regions(image,**kwargs)+reloc_files.regions(image,**kwargs) +audio.regions(image,context=context,**kwargs) +ptr_audio.regions(image,**kwargs)+ptr_audio.sequences(image,**kwargs)+models.regions(image,**kwargs) +archived_models.regions(image,**kwargs)) @@ -66,11 +68,13 @@ def preview(data, asset, selection): models, ptr_audio, rare_models, + reloc_files, resources, texture_packs, textures, uv_textures, ) + if asset.get('layout')=='reloc-file':return reloc_files.preview(data,asset,selection) if asset.get('layout')=='blast':return blast_textures.preview(data,asset,selection) if asset.get('layout')=='uv-texture':return uv_textures.preview(data,asset,selection) if asset.get('layout')=='ffs-i64':return ffs.preview(data,asset,selection) diff --git a/src/decompassets/n64/display_lists.py b/src/decompassets/n64/display_lists.py index 11766dc..04efea7 100644 --- a/src/decompassets/n64/display_lists.py +++ b/src/decompassets/n64/display_lists.py @@ -184,21 +184,43 @@ def _palette(found, palettes, palette_type): palette_format='ia16' if palette_type == 3 else 'rgba5551-be') +def _tile(tiles, sizes, loaded, number): + """The texture tile `number` reads, from its settings, its size and what TMEM holds there; None if it can't.""" + tile = tiles.get(number);size = sizes.get(number);source = tile and loaded.get(tile['tmem']) + if not source or not size:return None + uls, ult, lrs, lrt = size + if uls or ult or lrs & 3 or lrt & 3:return None + fmt = FORMATS.get((tile['fmt'], tile['siz'])) + width, height = lrs//4+1, lrt//4+1 + if not fmt or width*height > 1024*1024:return None + row = (width*BITS[tile['siz']]+7)//8 + if source['block']: + if row*height > source['texels']*BITS[source['siz']]//8 + 8:return None + elif source['width']*BITS[source['siz']] != width*BITS[tile['siz']]:return None + return dict(fmt=fmt, width=width, height=height, bytes=row*height, space=source['space'], + address=source['address'], at=source['at'], palette=tile['palette'], + swapped=source['swapped'] and height > 1) + + def _textures(image, group): """Walk the file's commands as the RDP would, and yield each tile that is drawn from loaded texels. - A tile is taken when something is drawn with it, so a palette loaded after the tile's size (as some games - order it) is the one used. TMEM's palette half is tracked entry by entry: a smaller palette load replaces - only the entries it covers.""" - timg = None;tiles = {};loaded = {};palettes = {};palette_type = 2;pending = [];loads = 0 + A tile is read when something is drawn, from what TMEM holds then: some games set a tile's size before + loading its texels, others load a palette after it. The tiles read are those sized since the last draw and + the one the draw uses (G_TEXTURE's tile, or a texture rectangle's). TMEM's palette half is tracked entry by + entry: a smaller palette load replaces only the entries it covers.""" + timg = None;tiles = {};sizes = {};loaded = {};palettes = {};palette_type = 2;touched = set();current = 0;loads = 0 for at in range(group['start'], min(group['end'] + 0x100, len(image) - 8), 8): w0, w1 = struct.unpack_from('>II', image, at) cmd = w0 >> 24 - if cmd in DRAWS and pending: - for found in pending: - found = _palette(found, palettes, palette_type) - if found:yield found - pending = [] + if cmd in DRAWS: + used = (w1 >> 24) & 7 if cmd in (0xE4, 0xE5) else current + if touched or cmd not in END_LISTS: + for number in sorted(touched | ({used} if cmd not in END_LISTS else set())): + found = _tile(tiles, sizes, loaded, number) + found = found and _palette(found, palettes, palette_type) + if found:yield found + touched = set() if cmd == G_SETTIMG: space = _space(w1) timg = None if space is None else dict(fmt=(w0 >> 21) & 7, siz=(w0 >> 19) & 3, width=(w0 & 0xFFF)+1, @@ -206,6 +228,8 @@ def _textures(image, group): elif cmd == G_SETTILE: tiles[(w1 >> 24) & 7] = dict(fmt=(w0 >> 21) & 7, siz=(w0 >> 19) & 3, line=(w0 >> 9) & 0x1FF, tmem=w0 & 0x1FF, palette=(w1 >> 20) & 0xF) + elif cmd in (0xBB, 0xD7): + current = (w0 >> 8) & 7 elif cmd in (G_LOADBLOCK, G_LOADTILE, G_LOADTLUT) and timg: tile = tiles.get((w1 >> 24) & 7) if not tile:continue @@ -214,27 +238,20 @@ def _textures(image, group): for index in range(((w1 >> 14) & 0x3FF)+1): palettes[tile['tmem']+index] = (loads, timg['space'], timg['address']+2*index) elif cmd == G_LOADBLOCK: - if not w0 & 0xFFFFFF:loaded[tile['tmem']] = dict(timg, texels=((w1 >> 12) & 0xFFF)+1, block=True) + # No row step (dxt 0): the bytes were stored as TMEM holds them, odd rows' words swapped. + if not w0 & 0xFFFFFF:loaded[tile['tmem']] = dict(timg, texels=((w1 >> 12) & 0xFFF)+1, block=True, + swapped=not w1 & 0xFFF) elif not w0 & 0xFFFFFF: - loaded[tile['tmem']] = dict(timg, texels=(((w1 >> 12) & 0xFFF)//4+1)*((w1 & 0xFFF)//4+1), block=False) + loaded[tile['tmem']] = dict(timg, texels=(((w1 >> 12) & 0xFFF)//4+1)*((w1 & 0xFFF)//4+1), block=False, + swapped=False) elif cmd in (0xBA, 0xE3): # Set other mode H: only its texture-lookup-table type (bits 14-15) matters here. shift, length = ((w0 >> 8) & 0xFF, (w0 & 0xFF)) if cmd == 0xBA else (32-((w0 >> 8) & 0xFF)-((w0 & 0xFF)+1), (w0 & 0xFF)+1) if shift <= 14 and 16 <= shift+length:palette_type = (w1 >> 14) & 3 elif cmd == G_SETTILESIZE: - tile = tiles.get((w1 >> 24) & 7);source = tile and loaded.get(tile['tmem']) - if not source:continue - uls, ult, lrs, lrt = (w0 >> 12) & 0xFFF, w0 & 0xFFF, (w1 >> 12) & 0xFFF, w1 & 0xFFF - if uls or ult or lrs & 3 or lrt & 3:continue - fmt = FORMATS.get((tile['fmt'], tile['siz'])) - width, height = lrs//4+1, lrt//4+1 - if not fmt or width*height > 1024*1024:continue - row = (width*BITS[tile['siz']]+7)//8 - if source['block']: - if row*height > source['texels']*BITS[source['siz']]//8 + 8:continue - elif source['width']*BITS[source['siz']] != width*BITS[tile['siz']]:continue - pending.append(dict(fmt=fmt, width=width, height=height, bytes=row*height, space=source['space'], - address=source['address'], at=source['at'], palette=tile['palette'])) + number = (w1 >> 24) & 7 + sizes[number] = ((w0 >> 12) & 0xFFF, w0 & 0xFFF, (w1 >> 12) & 0xFFF, w1 & 0xFFF) + touched.add(number) def regions(image, *, check=lambda: None, progress=lambda _: None): diff --git a/src/decompassets/n64/reloc_files.py b/src/decompassets/n64/reloc_files.py new file mode 100644 index 0000000..5130933 --- /dev/null +++ b/src/decompassets/n64/reloc_files.py @@ -0,0 +1,258 @@ +"""Recognize textures in a table of relocatable files, the kind a fighting game keeps its assets in. + +The table is one twelve-byte entry per file, and one more that ends it: the file's data offset from the table's +end (top bit set when the file is VPK0-packed, `vpk0.py`), the word where its internal pointer chain starts, +its packed size in words, the word where its external chain starts (0xFFFF: none) and its unpacked size in +words. The first offset is 0 and each next one starts at or after the previous file's end. + +A file's pointers aren't addresses on the cartridge: each pointer word holds the next chain link (high half) +and the word it points to (low half). Internal pointers point into the file; external ones into another file +whose number follows the packed data, one halfword per link. With the chains followed, the file's display lists +name each texture's file and offset exactly, and the RDP's commands give its format and size. Sprites (libultra's +`Sprite`, 68 bytes, with its `Bitmap` strips) are found the same way: a pointer at a sprite's bitmap field (+0x34) +leads to its strips, each pointing at its texels (+8); a sprite's strips are stacked into one image, their odd +rows swapped back from the order TMEM holds them in. +""" +from __future__ import annotations + +import struct + +from .. import formats as asset_formats +from ..formats import AssetError +from . import display_lists, vpk0 + +MIN_FILES = 32 +MAX_FILES = 8192 +MAX_FILE = 1024 * 1024 +END = 0xFFFF + + +def table(image, at): + """The table's files as dicts (offset, packed, size, compressed, intern, extern), or None.""" + files, base = [], None + for index in range(MAX_FILES + 1): + if at + 12*index + 12 > len(image):return None + word, intern, packed, extern, size = struct.unpack_from('>IHHHH', image, at + 12*index) + offset = word & 0x7FFFFFFF + if index == 0 and word & 0x7FFFFFFF:return None + if files and offset < files[-1]['offset'] + 4*files[-1]['packed']:return None + if not packed and not size and not intern and not extern: + if index < MIN_FILES:return None + base = at + 12*(index + 1) + break + compressed = bool(word & 0x80000000) + if not packed or not size or (not compressed and packed != size) or 4*size > MAX_FILE:return None + if intern != END and intern >= size or extern != END and extern >= size:return None + files.append(dict(offset=offset, packed=packed, size=size, compressed=compressed, intern=intern, extern=extern)) + if base is None or base + files[-1]['offset'] + 4*files[-1]['packed'] > len(image):return None + for item in files:item['offset'] += base + return files + + +def find(image, check=lambda: None): + """Where the table starts, and its files, or (None, None).""" + for at in range(0, len(image) - 64, 4): + if not at & 0xFFFF:check() + # The first entry: offset 0, then a second entry whose offset is the first file's packed length. + if image[at+1] or image[at+2] or image[at+3] or image[at] not in (0, 0x80):continue + second = int.from_bytes(image[at+12:at+16], 'big') & 0x7FFFFFFF + if second != 4*int.from_bytes(image[at+6:at+8], 'big'):continue + files = table(image, at) + if files:return at, files + return None, None + + +def unpack(image, item): + data = bytes(image[item['offset']:item['offset'] + 4*item['packed']]) + out = vpk0.unpack(data, MAX_FILE) if item['compressed'] else data + if len(out) != 4*item['size']:raise AssetError('A file in the table unpacks to another size. Rescan the ROM.') + return out + + +def links(image, files, index, data): + """The file's pointers, each as (word, file it points into, byte offset there).""" + item, out = files[index], [] + at, seen = item['intern'], set() + while at != END and at not in seen and 4*at + 4 <= len(data): + seen.add(at) + following, words = struct.unpack_from('>HH', data, 4*at) + out.append((at, index, 4*words));at = following + ids = item['offset'] + 4*item['packed'] + at, seen = item['extern'], set() + while at != END and at not in seen and 4*at + 4 <= len(data): + seen.add(at) + following, words = struct.unpack_from('>HH', data, 4*at) + target = int.from_bytes(image[ids:ids+2], 'big');ids += 2 + if target < len(files):out.append((at, target, 4*words)) + at = following + return out + + +def _resolved(data, pointers): + """The file with its pointers made addresses: segment 1 points into the file itself, segment 2 at another + file (pointer number in the upper bits, the offset there in the low twelve), with where each one goes.""" + patched, targets = bytearray(data), [] + for word, file, offset in pointers[:4095]: + targets.append((file, offset)) + struct.pack_into('>I', patched, 4*word, 0x02000000 | (len(targets) - 1) << 12) + return bytes(patched) + bytes(0x108), targets + + +SPRITE = 68 +FORMATS = {(0, 2): 'rgba5551-be', (0, 3): 'rgba8888', (2, 0): 'ci4', (2, 1): 'ci8', (3, 0): 'ia4', (3, 1): 'ia8', + (3, 2): 'ia16', (4, 0): 'i4', (4, 1): 'i8'} +BITS = {0: 4, 1: 8, 2: 16, 3: 32} +TEXEL_BITS = {fmt: BITS[siz] for (_, siz), fmt in FORMATS.items()} + + +def sprites(data, pointers): + """The textures of each libultra sprite in a file: (file, strips, format, width, palette) per sprite, a strip + being (offset, rows): its strips are stacked into one image when they share a file and a row width (each strip + starts on eight bytes, so they are rarely back to back). Places come from `pointers`, {byte offset of a + pointer: (file, offset)}, a palette being one such place.""" + found = [] + for at, (file, bitmaps) in pointers.items(): + start = at - 0x34 + if start < 0 or start + SPRITE > len(data):continue + x, y, width, height = struct.unpack_from('>4h', data, start) + scale_x, scale_y = struct.unpack_from('>2f', data, start + 8) + count, _lists = struct.unpack_from('>2h', data, start + 40) + strip, _real = struct.unpack_from('>2h', data, start + 44) + fmt = FORMATS.get((data[start+48], data[start+49])) + if x or y or scale_x != 1.0 or scale_y != 1.0 or not fmt or not 0 < width <= 2048 or not 0 < height <= 2048:continue + if not 0 < count <= 256 or not 0 < strip <= 2048 or file != pointers.get(start + 0x34, (None,))[0]:continue + palette = pointers.get(start + 0x20) if fmt in ('ci4', 'ci8') else None + if fmt in ('ci4', 'ci8') and palette is None:continue + strips = [] + for k in range(count): + bm = bitmaps + 16*k + place = pointers.get(bm + 8) if file is not None else None + if bm + 16 > len(data) or place is None:break + _w, stride, _s, _t = struct.unpack_from('>4h', data, bm) + rows = struct.unpack_from('>h', data, bm + 12)[0] + if not 0 < stride <= 2048 or not 0 < rows <= 2048:break + strips.append((place, stride, rows)) + if len(strips) != count:continue + (first_file, _), stride, _rows = strips[0] + if all(f == first_file and w == stride for (f, _), w, _ in strips): + found.append((first_file, [(at_, rows) for (_, at_), _, rows in strips], fmt, stride, palette)) + else:found += [(f, [(at_, rows)], fmt, w, palette) for (f, at_), w, rows in strips] + return found + + +def regions(image, *, check=lambda: None, progress=lambda _: None): + """One region per file holding textures its own or other files' display lists draw.""" + at, files = find(image, check) + if files is None:return [] + cache, found = {}, {} + + def data(index): + if index not in cache: + try:cache[index] = unpack(image, files[index]) + except AssetError:cache[index] = None + return cache[index] + + for index in range(len(files)): + if index % 32 == 0: + check();progress(dict(phase='Reading the relocatable file table', done=index, total=len(files))) + own = data(index) + if own is None:continue + pointers = links(image, files, index, own) + place = {4*word: (file, offset) for word, file, offset in pointers} + for file, strips, fmt, width, palette in sprites(own, place): + texels = data(file) + row = (width*TEXEL_BITS[fmt] + 7)//8 + if texels is None or any(at_ + row*rows > len(texels) for at_, rows in strips):continue + offset, height = strips[0][0], sum(rows for _, rows in strips) + if palette is not None: + pal = data(palette[0]) + entries = 16 if fmt == 'ci4' else 256 + if pal is None or palette[1] + 2*entries > len(pal):continue + palette = (palette[0], palette[1], entries, 'rgba5551-be') + found.setdefault(file, {}).setdefault((offset, fmt, width, height), dict( + file=file, offset=offset, format=fmt, width=width, height=height, palette=palette, strips=strips)) + if display_lists.G_SETTIMG not in own[::8]:continue + patched, targets = _resolved(own, pointers) + slots = {4*word for word, _, _ in pointers} + for tile in display_lists._textures(patched, dict(start=0, end=len(own))): + # Only an image the chain points at: a segment address the file holds itself is the game's own. + if tile['space'] != 2 or tile['at'] + 4 not in slots or tile['address'] >> 12 >= len(targets):continue + file, offset = targets[tile['address'] >> 12];offset += tile['address'] & 0xFFF + texels = data(file) + if texels is None or offset + tile['bytes'] > len(texels):continue + palette = None + if 'palette_address' in tile: + if tile['palette_space'] != 2 or tile['palette_address'] >> 12 >= len(targets):continue + pfile, poffset = targets[tile['palette_address'] >> 12];poffset += tile['palette_address'] & 0xFFF + pal = data(pfile) + if pal is None or poffset + 2*tile['palette_entries'] > len(pal):continue + # A palette inside the texels themselves is a load this walk misread. + if pfile == file and poffset < offset + tile['bytes'] and offset < poffset + 2*tile['palette_entries']:continue + palette = (pfile, poffset, tile['palette_entries'], tile['palette_format']) + elif tile['fmt'] in ('ci4', 'ci8'):continue + key = (offset, tile['fmt'], tile['width'], tile['height']) + texture = dict(file=file, offset=offset, format=tile['fmt'], width=tile['width'], height=tile['height'], palette=palette) + if tile['swapped']:texture['strips'] = [(offset, tile['height'])] + found.setdefault(file, {}).setdefault(key, texture) + results = [] + for file, textures in sorted(found.items()): + items = [] + for texture in sorted(textures.values(), key=lambda t: t['offset']): + settings = dict(format=texture['format'], width=texture['width'], height=texture['height'], offset=texture['offset']) + items.append(dict(texture, preview_settings=settings, frames=1, palettes=1 if texture['palette'] else 0, + frame_offsets=[texture['offset']], palette_offsets=[texture['palette'][1]] if texture['palette'] else [], + palette_entries=[texture['palette'][2]] if texture['palette'] else [], + levels=[dict(width=texture['width'], height=texture['height'], offset=0)])) + item = files[file] + first = items[0]['preview_settings'] + results.append(dict(offset=item['offset'], size=4*item['packed'], kind='image', confidence='validated', + layout='reloc-file', table=at, file=file, textures=items, + label=f'File {file} · {len(items)} texture(s)', + **{k: v for k, v in items[0].items() if k in ('frames', 'palettes', 'frame_offsets', 'palette_offsets', 'palette_entries', 'levels')}, + preview_settings={**first, 'source': 'rom', 'offset': item['offset']}, + reason='The file table checks out entry by entry; following each file\'s pointer chains, a display ' + 'list loads and draws these textures from this file. Runtime use is not established.')) + return results + + +def _sprite(data, item): + """A sprite's strips stacked into one image. The game loads each strip into TMEM as a block with no row step, + so it stores them as TMEM holds them: each strip's odd rows have their words swapped in pairs (32-bit words, + or 64-bit for RGBA32), and they are swapped back.""" + row = (item['width']*TEXEL_BITS[item['format']] + 7)//8 + word = 8 if item['format'] == 'rgba8888' else 4 + out = bytearray() + for at, rows in item['strips']: + strip = bytearray(data[at:at+row*rows]) + for y in range(1, rows, 2): + for k in range(y*row, y*row + row - 2*word + 1, 2*word): + strip[k:k+2*word] = strip[k+word:k+2*word] + strip[k:k+word] + out += strip + return bytes(out) + + +def preview(image, asset, selection): + """Unpack the file (and the one holding the palette) and show the chosen texture.""" + files = table(image, asset['table']) + if not files or asset['file'] >= len(files) or files[asset['file']]['offset'] != asset['offset']: + raise AssetError('The file table changed. Rescan the ROM.') + texture = asset_formats.integer(selection.get('texture', 0), 'Texture', 0, len(asset['textures'])-1) + asset_formats.integer(selection.get('frame', 0), 'Frame', 0, 0) + asset_formats.integer(selection.get('mip', 0), 'Mip level', 0, 0) + item = asset['textures'][texture] + data = whole = unpack(image, files[asset['file']]) + settings = dict(item['preview_settings']) + if item.get('strips'): + data = _sprite(data, item) + settings['offset'] = 0 + if item['palette']: + pfile, poffset, _entries, palette_format = item['palette'] + palette = whole if pfile == asset['file'] else unpack(image, files[pfile]) + settings.update(palette_offset=len(data) + poffset, palette_format=palette_format) + data = data + palette + result = asset_formats.preview(data, settings) + result['details'].update(texture=texture, frame=0, palette=0 if item['palette'] else 'none', note=asset['reason']) + result['decoded'] = dict(codec='VPK0' if files[asset['file']]['compressed'] else 'stored', bytes=4*files[asset['file']]['size'], + block_offset=asset['offset'], member=f"file {asset['file']}") + result['settings'] = dict(source='rom', format='auto', offset=asset['offset'], texture=texture, frame=0, palette=0, mip=0) + return result diff --git a/src/decompassets/n64/vpk0.py b/src/decompassets/n64/vpk0.py new file mode 100644 index 0000000..62216aa --- /dev/null +++ b/src/decompassets/n64/vpk0.py @@ -0,0 +1,79 @@ +"""VPK0, the Huffman-coded LZ packing a fighting game's files use, unpacked as its own decoder does. + +The stream is "vpk0", the unpacked size (32 bits), a sample method (8 bits), then a bit stream read most +significant bit first: two Huffman trees (bit 0 then eight bits: a leaf; bit 1: join the two nodes on top of the +stack, or end the tree when fewer than two are there), whose leaves are bit counts for back-reference distances +and lengths, then the data: bit 0 then eight bits is a literal byte, bit 1 a back-reference. With method 1 a +distance is read in two samples: a value of 0-2 adds 1-3 bytes to a distance read again, counted in words. +""" +from __future__ import annotations + +from ..formats import AssetError + +MAX_OUTPUT = 4 * 1024 * 1024 +MAX_NODES = 64 + + +class _Bits: + """Bits most significant first, refilled four bytes at a time.""" + + def __init__(self, data): + self.data, self.at, self.acc, self.n = data, 0, 0, 0 + + def read(self, n): + while self.n < n: + chunk = self.data[self.at:self.at+4] + if not chunk:raise AssetError('The VPK0 stream ends early. Rescan the ROM.') + self.acc = (self.acc << 8*len(chunk)) | int.from_bytes(chunk, 'big') + self.n += 8*len(chunk);self.at += 4 + self.n -= n + value = self.acc >> self.n + self.acc &= (1 << self.n) - 1 + return value + + +def _tree(bits): + """A Huffman tree as nested (left, right) tuples with int leaves (bit counts).""" + stack, nodes = [], 0 + while True: + if bits.read(1): + if len(stack) < 2:break + right = stack.pop();stack[-1] = (stack[-1], right) + else:stack.append(bits.read(8)) + nodes += 1 + if nodes > MAX_NODES:raise AssetError('The VPK0 stream\'s code tree is not valid here.') + if not stack:raise AssetError('The VPK0 stream has an empty code tree.') + return stack[0] + + +def _value(bits, tree): + while isinstance(tree, tuple):tree = tree[bits.read(1)] + return bits.read(tree) + + +def size(data): + """The unpacked size a VPK0 header gives, or None when `data` doesn't start with one.""" + return int.from_bytes(data[4:8], 'big') if data[:4] == b'vpk0' and len(data) >= 9 else None + + +def unpack(data, limit=MAX_OUTPUT): + total = size(data) + if total is None or not 0 < total <= limit:raise AssetError('No VPK0 stream starts here, or it unpacks beyond the limit.') + method = data[8] + bits = _Bits(bytes(data[9:])) + offsets, lengths = _tree(bits), _tree(bits) + out = bytearray() + while len(out) < total: + if not bits.read(1): + out.append(bits.read(8));continue + if method: + value, extra = _value(bits, offsets), 0 + if value <= 2:extra, value = value + 1, _value(bits, offsets) + start = len(out) - value*4 - extra + 8 + else:start = len(out) - _value(bits, offsets) + length = _value(bits, lengths) + if start < 0 or start >= len(out) and length:raise AssetError('The VPK0 stream refers back before its start.') + if start + length <= len(out):out += out[start:start+length] + else: + for k in range(length):out.append(out[start + k]) + return bytes(out[:total]) diff --git a/tests/test_display_list_textures.py b/tests/test_display_list_textures.py index 1bf8a46..a0c2e73 100644 --- a/tests/test_display_list_textures.py +++ b/tests/test_display_list_textures.py @@ -92,3 +92,15 @@ def test_scan_stops_and_cartridges_without_lists_are_quick(): def stop():raise RuntimeError('stopped') with pytest.raises(RuntimeError, match='stopped'):display_lists.regions(image, check=stop) assert display_lists.regions(bytes(0x10000)) == [] + + +def test_a_tile_sized_before_its_texels_load_reads_the_new_texels(): + """The tile is read when it is drawn: sized first, then loaded from another image, the new image is the one.""" + stale, fresh = 0x100, 0x200 + commands = [cmd(0xFD100000, 0x01000000 | stale), cmd(0xF5100000, 0x07000000), cmd(0xF3000000, 0x07007800), + cmd(0xF5100200, 0), cmd(0xF2000000, 0x0000C004), # sized while `stale` is loaded + cmd(0xFD100000, 0x01000000 | fresh), cmd(0xF5100000, 0x07000000), cmd(0xF3000000, 0x07007800), + cmd(0xBF000000, 0x00020406), cmd(0xB8000000)] + data = b''.join(commands) + bytes(0x108) + found = list(display_lists._textures(data, dict(start=0, end=8*len(commands)))) + assert [(t['address'], t['width'], t['height']) for t in found] == [(fresh, 4, 2)] diff --git a/tests/test_reloc_files.py b/tests/test_reloc_files.py new file mode 100644 index 0000000..1cdacd9 --- /dev/null +++ b/tests/test_reloc_files.py @@ -0,0 +1,118 @@ +"""A table of relocatable files with display lists and sprites, one file VPK0-packed: homemade data only.""" +import base64 +import struct + +import pytest + +from decompassets import n64 +from decompassets.formats import AssetError +from decompassets.n64 import reloc_files, vpk0 + +from .test_rom_assets import pixels + +END = 0xFFFF + + +class Bits: + def __init__(self): + self.bits = '' + + def put(self, value, count): + self.bits += format(value, f'0{count}b') if count else '' + + def bytes(self): + padded = self.bits + '0' * (-len(self.bits) % 8) + return bytes(int(padded[i:i+8], 2) for i in range(0, len(padded), 8)) + + +def pack(raw, back=None): + """A method-0 VPK0 stream: 8-bit distances, 5-bit lengths, literals, and one back-reference (at, distance, length).""" + bits = Bits() + for leaf in (8, 5): + bits.put(0, 1);bits.put(leaf, 8);bits.put(1, 1) + at = 0 + while at < len(raw): + if back and at == back[0]: + bits.put(1, 1);bits.put(back[1], 8);bits.put(back[2], 5);at += back[2] + else: + bits.put(0, 1);bits.put(raw[at], 8);at += 1 + stream = b'vpk0' + struct.pack('>I', len(raw)) + b'\0' + bits.bytes() + return stream + bytes(-len(stream) % 4) + + +def link(following, target): + return struct.pack('>HH', following, target // 4) + + +def first_file(): + """A display list loading an RGBA16 4×2 image, and an I8 8×2 sprite stored with its odd row swapped.""" + data = bytearray(0xA8) + commands = [(0xFD100000, None), (0xF5100000, 0x07000000), (0xF3000000, 0x07007800), (0xF5100200, 0), + (0xF2000000, 0x0000C004), (0xDF000000, 0)] + for k, (w0, w1) in enumerate(commands):struct.pack_into('>I', data, 8*k, w0);struct.pack_into('>I', data, 8*k+4, w1 or 0) + data[0x30:0x40] = bytes.fromhex('f801') * 8 # red texels + struct.pack_into('>4h2f', data, 0x40, 0, 0, 8, 2, 1.0, 1.0) # sprite: x, y, width, height, scale + struct.pack_into('>2h', data, 0x40 + 40, 1, 0) # one bitmap + struct.pack_into('>2h2B', data, 0x40 + 44, 2, 2, 4, 1) # strip height, I8 + struct.pack_into('>4h', data, 0x88, 8, 8, 0, 0) # bitmap: width, row width + struct.pack_into('>h', data, 0x88 + 12, 2) # two rows + data[0x98:0xA8] = bytes([0x00]*8) + bytes([0xFF]*4 + [0x80]*4) # row 1 stored swapped + # Internal chain: the image's address (word 1) -> texels, the sprite's bitmap pointer -> bitmap, its buffer -> texels. + data[4:8] = link(29, 0x30);data[0x74:0x78] = link(36, 0x88);data[0x90:0x94] = link(END, 0x98) + return bytes(data), 1 + + +def table(files): + """files: (data, packed bytes or None, intern, extern, extern ids).""" + head, body = b'', b'' + for data, packed, intern, extern, ids in files: + stored = packed or data + head += struct.pack('>IHHHH', len(body) | (0x80000000 if packed else 0), intern, len(stored)//4, extern, len(data)//4) + body += stored + ids + bytes(-len(ids) % 4) + return head + struct.pack('>IHHHH', len(body), 0, 0, 0, 0) + body + + +def build(): + data, intern = first_file() + # File 1 points at file 0's texels from outside: an RGBA16 4×2 image through its external chain. + other = bytearray(0x30) + for k, (w0, w1) in enumerate([(0xFD100000, 0), (0xF5100000, 0x07000000), (0xF3000000, 0x07007800), + (0xF5100200, 0), (0xF2000000, 0x0000C004), (0xDF000000, 0)]): + struct.pack_into('>II', other, 8*k, w0, w1) + other[4:8] = link(END, 0x98) + raw = bytes(range(16)) * 2 + files = [(data, None, intern, END, b''), (bytes(other), None, END, 1, struct.pack('>H', 0)), + (raw, pack(raw, (16, 16, 16)), END, END, b'')] + files += [(bytes(4), None, END, END, b'')] * 32 + return bytes(64) + table(files) + + +def test_vpk0_unpacks_literals_and_back_references(): + raw = bytes(range(16)) * 2 + assert vpk0.unpack(pack(raw, (16, 16, 16))) == raw + with pytest.raises(AssetError):vpk0.unpack(pack(raw, (16, 17, 16))) # reaches before the start + with pytest.raises(AssetError):vpk0.unpack(b'vpk0' + bytes(8)) + + +def test_file_table_textures_are_found_through_pointer_chains(): + image = build() + at, files = reloc_files.find(image) + assert at == 64 and len(files) == 35 and files[2]['compressed'] + assert reloc_files.unpack(image, files[2]) == bytes(range(16)) * 2 + found = reloc_files.regions(image) + assert [a['file'] for a in found] == [0] + textures = [(t['offset'], t['format'], t['width'], t['height']) for t in found[0]['textures']] + assert sorted(textures) == [(0x30, 'rgba5551-be', 4, 2), (0x98, 'i8', 8, 2), (0x98, 'rgba5551-be', 4, 2)] + assert [a['layout'] for a in n64.regions(image) if a.get('layout') == 'reloc-file'] == ['reloc-file'] + red = pixels(base64.b64decode(n64.preview(image, found[0], {'texture': 0})['data_url'].split(',')[1])) + assert red[:4] == bytes([255, 0, 0, 255]) + sprite_index = next(k for k, t in enumerate(found[0]['textures']) if t['format'] == 'i8') + sprite = pixels(base64.b64decode(n64.preview(image, found[0], {'texture': sprite_index})['data_url'].split(',')[1])) + assert sprite[8*4:8*4+4] == bytes([0x80, 0x80, 0x80, 255]) and sprite[12*4:12*4+4] == bytes([255, 255, 255, 255]) + + +def test_tables_that_do_not_chain_are_not_taken(): + image = bytearray(build()) + struct.pack_into('>I', image, 64 + 12*5, 0x10) # a file starting inside an earlier one + assert reloc_files.find(bytes(image)) == (None, None) + assert reloc_files.regions(bytes(64) + table([(bytes(4), None, END, END, b'')] * 8)) == [] # too few files diff --git a/uv.lock b/uv.lock index 09afc52..991aabd 100644 --- a/uv.lock +++ b/uv.lock @@ -51,7 +51,7 @@ wheels = [ [[package]] name = "decompassets" -version = "0.6.0" +version = "0.7.0" source = { editable = "." } [package.dev-dependencies]