Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ the RNC unpacker work for any console. The interface may change before 1.0.
## Use

```bash
uv add "decompassets @ https://github.com/RetroReversing/decomp-assets/archive/refs/tags/v0.8.0.tar.gz"
uv add "decompassets @ https://github.com/RetroReversing/decomp-assets/archive/refs/tags/v0.9.0.tar.gz"
```

```python
Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[project]
name = "decompassets"
version = "0.8.0"
version = "0.9.0"
description = "Find and preview game assets (textures, audio banks, models, fonts) inside cartridge and disc images by their structure, with no per-game offsets."
readme = "README.md"
license = "AGPL-3.0-only"
Expand Down
2 changes: 1 addition & 1 deletion src/decompassets/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,4 @@
Everything here reads bytes and returns plain dicts. It never writes a game file, runs a tool or downloads.
"""

__version__ = "0.8.0"
__version__ = "0.9.0"
6 changes: 5 additions & 1 deletion src/decompassets/n64/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
- textures.py: indexed texture banks (CI4/CI8, RGBA32, the six-level RGBA16 mipmap layout), raw or RNC-packed.
- texture_packs.py: texture packs with a trailing count, inside archives.
- display_lists.py: textures that uncompressed display lists load, in files placed by their vertices and pointers.
- fla2.py: texture banks packed with FLA2, the LZ variant of a platformer about a magic glove.
- ffs.py: I64 images in an FFS filesystem packed as one chunked RNC stream.
- rare_models.py: model textures in a deflated asset table.
- blast_textures.py: textures packed with a demolition game's own codecs, listed in its texture table.
Expand Down Expand Up @@ -37,6 +38,7 @@ def regions(image, *, context=None, **kwargs):
blast_textures,
display_lists,
ffs,
fla2,
fonts,
mainfs,
models,
Expand All @@ -50,7 +52,7 @@ def regions(image, *, context=None, **kwargs):
uv_textures,
)
known=(textures.regions(image,**kwargs)+texture_packs.regions(image,**kwargs)+display_lists.regions(image,**kwargs)
+ffs.regions(image,**kwargs)+rare_models.regions(image,**kwargs)+uv_textures.regions(image,**kwargs)+blast_textures.regions(image,**kwargs)+reloc_files.regions(image,**kwargs)+mainfs.regions(image,**kwargs)
+ffs.regions(image,**kwargs)+rare_models.regions(image,**kwargs)+uv_textures.regions(image,**kwargs)+blast_textures.regions(image,**kwargs)+reloc_files.regions(image,**kwargs)+mainfs.regions(image,**kwargs)+fla2.regions(image,**kwargs)
+audio.regions(image,context=context,**kwargs)
+ptr_audio.regions(image,**kwargs)+ptr_audio.sequences(image,**kwargs)+models.regions(image,**kwargs)
+archived_models.regions(image,**kwargs))
Expand All @@ -66,6 +68,7 @@ def preview(data, asset, selection):
audio,
blast_textures,
ffs,
fla2,
fonts,
mainfs,
models,
Expand All @@ -77,6 +80,7 @@ def preview(data, asset, selection):
textures,
uv_textures,
)
if asset.get('layout')=='fla2-texbank':return fla2.preview(data,asset,selection)
if asset.get('layout')=='mainfs':return mainfs.preview(data,asset,selection)
if asset.get('layout')=='reloc-file':return reloc_files.preview(data,asset,selection)
if asset.get('layout')=='blast':return blast_textures.preview(data,asset,selection)
Expand Down
121 changes: 121 additions & 0 deletions src/decompassets/n64/fla2.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
"""Recognize texture banks, raw or packed with FLA2, the LZ variant a platformer about a magic glove uses.

FLA2 (formats as the MIT-licensed libgarib project documents them): "FLA2", the unpacked size (32 bits, little
endian), then groups of a flag byte read high bit first and eight items: bit 0 a literal byte, bit 1 two bytes
naming a run in a 4 KB window (length = low nibble of the first + 2, distance = its high nibble × 256 + the
second). Two zero bytes in place of a run end the stream.

A texture bank is a count, then per texture a 36-byte header (an id, palette animation, flags, width, height,
masks, its length with the header, a colour format, a texel format, a data pointer the game fills in, the
palette's offset from the header) and its data; the texture names follow. Texel formats: 0 CI4, 1 CI8 (palette
of RGBA16 colours at the palette offset, to the texture's end), 2 16-bit (RGBA16, or IA16 when the colour format
is 3), 3 RGBA32.
"""
from __future__ import annotations

import re
import struct

from .. import formats as asset_formats
from ..formats import AssetError

MAX_OUTPUT = 4 * 1024 * 1024
MAX_TEXTURES = 4096
HEADER = 36


def unpack(data, limit=MAX_OUTPUT):
return _unpack(data, limit)[0]


def _unpack(data, limit=MAX_OUTPUT):
"""The unpacked bytes and how many packed bytes the stream took."""
if data[:4] != b'FLA2':raise AssetError('No FLA2 stream starts here.')
size = int.from_bytes(data[4:8], 'little')
if not 0 < size <= limit:raise AssetError('The FLA2 stream unpacks beyond the limit.')
window, cursor, out, i = bytearray(0x1000), 0, bytearray(), 8
while True:
if i >= len(data):raise AssetError('The FLA2 stream ends early. Rescan the ROM.')
flags = data[i];i += 1
for bit in range(7, -1, -1):
if not flags >> bit & 1:
if i >= len(data):raise AssetError('The FLA2 stream ends early. Rescan the ROM.')
value = data[i];i += 1
out.append(value);window[cursor] = value;cursor = (cursor + 1) & 0xFFF
continue
if i + 1 >= len(data):raise AssetError('The FLA2 stream ends early. Rescan the ROM.')
first, second = data[i], data[i+1];i += 2
if not first and not second:
if len(out) != size:raise AssetError('The FLA2 stream ends before its size.')
return bytes(out), i
start = cursor - (((first & 0xF0) << 4) + second)
for k in range((first & 0x0F) + 2):
value = window[(start + k) & 0xFFF]
out.append(value);window[cursor] = value;cursor = (cursor + 1) & 0xFFF
if len(out) > size:raise AssetError('The FLA2 stream unpacks past its size.')


def textures(bank):
"""A texture bank's textures as preview items relative to the bank, or None when it isn't one."""
if len(bank) < 4 + HEADER:return None
count = int.from_bytes(bank[:4], 'big')
if not 0 < count <= MAX_TEXTURES:return None
items, at = [], 4
for _ in range(count):
if at + HEADER > len(bank):return None
_ident, _low, _high, _flags, _inc, _counter, width, height, _s, _t, length, colour, texel, _ptr, palette = \
struct.unpack_from('>IBBHhhHHHHIHHII', bank, at)
if length < HEADER or at + length > len(bank) or not 0 < width <= 1024 or not 0 < height <= 1024:return None
if texel > 3 or colour > 4:return None
bits = (4, 8, 16, 32)[texel]
data = at + HEADER
pixels = (width*bits + 7)//8*height
if HEADER + pixels > length:return None
fmt = ('ci4', 'ci8', 'ia16' if colour == 3 else 'rgba5551-be', 'rgba8888')[texel]
settings = dict(format=fmt, width=width, height=height, offset=data)
entries = 0
if texel < 2:
entries = (length - palette)//2
if palette < HEADER + pixels or palette >= length or entries < 1:return None
settings.update(palette_offset=at + palette, palette_format='rgba5551-be')
items.append(dict(preview_settings=settings, frames=1, palettes=1 if entries else 0, frame_offsets=[data],
palette_offsets=[at + palette] if entries else [], palette_entries=[entries] if entries else [],
levels=[dict(width=width, height=height, offset=0)]))
at += length
return items


def regions(image, *, check=lambda: None, progress=lambda _: None):
"""One region per FLA2 stream that unpacks to a texture bank."""
results = []
matches = [m.start() for m in re.finditer(rb'FLA2', image)]
for number, at in enumerate(matches):
check();progress(dict(phase='Unpacking FLA2 banks', done=number, total=len(matches)))
size = int.from_bytes(image[at+4:at+8], 'little')
if not 0 < size <= MAX_OUTPUT or int.from_bytes(image[at+8:at+12], 'big') == 0 and size < 4:continue
try:bank, used = _unpack(bytes(image[at:at + 2*size + 64]))
except AssetError:continue
items = textures(bank)
if not items:continue
first = items[0]['preview_settings']
results.append(dict(offset=at, size=used, kind='image', confidence='validated',
layout='fla2-texbank', textures=items, label=f'Texture bank · {len(items)} texture(s)',
**{k: v for k, v in items[0].items() if k != 'preview_settings'},
preview_settings={**first, 'source': 'rom', 'offset': at},
reason='The FLA2 stream unpacks to exactly its size, and every texture header in the bank fits it, '
'one after another. Runtime use is not established.'))
return results


def preview(image, asset, selection):
bank = unpack(bytes(image[asset['offset']:asset['offset'] + asset['size']]))
items = textures(bank)
if not items:raise AssetError('The texture bank changed. Rescan the ROM.')
texture = asset_formats.integer(selection.get('texture', 0), 'Texture', 0, len(items)-1)
asset_formats.integer(selection.get('frame', 0), 'Frame', 0, 0)
asset_formats.integer(selection.get('mip', 0), 'Mip level', 0, 0)
result = asset_formats.preview(bank, items[texture]['preview_settings'])
result['details'].update(texture=texture, frame=0, palette=0 if items[texture]['palettes'] else 'none', note=asset['reason'])
result['decoded'] = dict(codec='FLA2', bytes=len(bank), block_offset=asset['offset'], member=f'texture {texture}')
result['settings'] = dict(source='rom', format='auto', offset=asset['offset'], texture=texture, frame=0, palette=0, mip=0)
return result
64 changes: 64 additions & 0 deletions tests/test_fla2.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
"""FLA2-packed texture banks: homemade data only."""
import base64
import struct

import pytest

from decompassets import n64
from decompassets.formats import AssetError
from decompassets.n64 import fla2

from .test_rom_assets import pixels

RED, GREEN = bytes.fromhex('f801'), bytes.fromhex('07c1')


def pack(raw, back=None):
"""Literals in groups of eight, one run (at, distance, length) if given, then the end marker."""
items, at = [], 0
while at < len(raw):
if back and at == back[0]:
items.append((1, bytes([((back[1] >> 4) & 0xF0) | (back[2] - 2), back[1] & 0xFF])));at += back[2]
else:items.append((0, raw[at:at+1]));at += 1
items.append((1, b'\0\0'))
out = b'FLA2' + struct.pack('<I', len(raw))
for group in range(0, len(items), 8):
chunk = items[group:group+8]
out += bytes([sum(bit << (7 - k) for k, (bit, _) in enumerate(chunk))]) + b''.join(data for _, data in chunk)
return out


def texture(width, height, texel, colour, pixels_, palette=b''):
length = 36 + len(pixels_) + len(palette)
return struct.pack('>IBBHhhHHHHIHHII', 1, 0, 0, 0, 0, 0, width, height, 0, 0, length, colour, texel, 0,
36 + len(pixels_) if palette else 0) + pixels_ + palette


def bank(textures):
return struct.pack('>I', len(textures)) + b''.join(textures) + b'one\0two\0'


def test_fla2_unpacks_literals_and_runs():
raw = bytes(range(10)) * 3
assert fla2.unpack(pack(raw, (10, 10, 10))) == raw
with pytest.raises(AssetError):fla2.unpack(pack(raw)[:-3])
with pytest.raises(AssetError):fla2.unpack(b'FLA2' + struct.pack('<I', 5) + b'\x80\0\0') # ends before its size


def test_texture_banks_are_found_and_previewed():
raw = bank([texture(4, 2, 0, 0, bytes([0x01, 0x10] * 2), RED + GREEN),
texture(2, 1, 2, 0, GREEN + RED), texture(1, 1, 2, 3, bytes([0x80, 0xFF]))])
image = bytes(32) + pack(raw) + bytes(16) + pack(b'not a bank' * 4)
found = fla2.regions(image)
assert len(found) == 1 and found[0]['offset'] == 32 and found[0]['size'] == len(pack(raw))
assert [t['preview_settings']['format'] for t in found[0]['textures']] == ['ci4', 'rgba5551-be', 'ia16']
assert [a['layout'] for a in n64.regions(image) if a.get('layout') == 'fla2-texbank'] == ['fla2-texbank']
shown = pixels(base64.b64decode(n64.preview(image, found[0], {'texture': 0})['data_url'].split(',')[1]))
assert shown[:8] == bytes([255, 0, 0, 255, 0, 255, 0, 255])
grey = pixels(base64.b64decode(n64.preview(image, found[0], {'texture': 2})['data_url'].split(',')[1]))
assert grey == bytes([0x80, 0x80, 0x80, 0xFF])


def test_banks_whose_textures_do_not_fit_are_left_out():
assert fla2.textures(bank([texture(64, 64, 2, 0, bytes(8))])) is None # texels past its length
assert fla2.textures(bank([texture(4, 2, 0, 0, bytes(4))])) is None # indexed with no palette
2 changes: 1 addition & 1 deletion uv.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading