diff --git a/ROADMAP.md b/ROADMAP.md index d08802461..b2ca9eb86 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -702,7 +702,7 @@ folder, `src/decompnet/consoles//`; a new console adds a column (CONTRIBUTIN | Feature | N64 | PS1 | PS2 | PSP | GameCube · Wii | Windows PC | DS | Public decomps | | :-- | :-: | :-: | :-: | :-: | :-: | :-: | :-: | :-: | | Loader/buffer tracing, named contracts and small wrappers | ✓ MIPS / named PI DMA | ✓ MIPS static | ✓ MIPS EE EABI | ✓ shared MIPS reader | — original image adapter | — original image adapter | — original image adapter | — original image adapter | -| Automatic texture banks (CI4/CI8, mixed CI8 palettes, RGBA32, RGBA16 mipmaps, RNC 1/2, named four-block banks, embedded ZIP packs, textures loaded by uncompressed display lists, I64 images in a chunked-RNC filesystem, model textures in a deflated asset table, UVTX files in a UV FORM filesystem, a texture table of back-reference-packed textures with inferred widths, display-list textures and sprites in a table of relocatable VPK0 files, image packs and model bitmaps in a party-game series' main filesystem, FLA2-packed texture banks; a fork's own splat image segments and JSON asset lists) | ✓ ROM | — native formats planned | — native formats planned | — native formats planned | — image adapter | — image adapter | — image adapter | ✓ forks with the game file (N64 checked) | +| Automatic texture banks (CI4/CI8, mixed CI8 palettes, RGBA32, RGBA16 mipmaps, RNC 1/2, named four-block banks, embedded ZIP packs, textures loaded by uncompressed display lists, I64 images in a chunked-RNC filesystem, model textures in a deflated asset table, UVTX files in a UV FORM filesystem, a texture table of back-reference-packed textures with inferred widths, display-list textures and sprites in a table of relocatable VPK0 files, image packs and model bitmaps in a party-game series' main filesystem, FLA2-packed texture banks; a fork's own splat image segments, JSON asset lists and sno texture sheets) | ✓ ROM | — native formats planned | — native formats planned | — native formats planned | — image adapter | — image adapter | — image adapter | ✓ forks with the game file (N64 checked) | | Audio browser, filters, sortable samples and explicit WAV preparation | ✓ ROM | ✓ ROM | ✓ ROM | ✓ ROM | — original image adapter | — original image adapter | — original image adapter | — original image adapter | | Texture gallery, filters and distinct address / loader reference links | ✓ ROM | ✓ ROM | ✓ ROM | ✓ ROM | — original image adapter | — original image adapter | — original image adapter | ✓ forks with the game file (N64 checked) | | Native directory roles: animation, collision, placements, grid, sequences and demos | ✓ validated schema / linked records | — native formats planned | — native formats planned | — native formats planned | — original image adapter | — original image adapter | — original image adapter | — original image adapter | @@ -1985,7 +1985,10 @@ Phases: demolition game's texture table and its six codecs (widths inferred, 96% agree with its decompilation). Done in 0.7.0: a fighting game's relocatable file table (VPK0, pointer chains, display lists and sprites). Done in 0.8.0: a party-game series' main filesystem (three packings, image packs, FORM model bitmaps). -- That series' HVQ-compressed backgrounds are not decoded yet. +- That series' HVQ-compressed backgrounds are not decoded yet: the games' own HVQ2 decoder is still assembly in + their decompilations (only its setup is C), and the SDK's source can't be used. It needs that decoder decompiled + or its routines reverse-engineered. +- A snowboarding sequel's tiled backgrounds (3 segments) use another layout than its texture sheets. - Done in 0.9.0: a glove platformer's FLA2-packed texture banks. - A monster-battle game builds its display lists in code from its own tables, and keeps raw texels in Yay0 blocks with no format or size: its textures need those tables read. diff --git a/docs/ROM_EXPLORER.md b/docs/ROM_EXPLORER.md index 6c8b7cd26..ca799d975 100644 --- a/docs/ROM_EXPLORER.md +++ b/docs/ROM_EXPLORER.md @@ -97,7 +97,9 @@ splat's image segments (`rgba16`, `ci4` with its `palette`, `i8`...) and a proje types whose arguments are a format and a size (`[0x1BB280, yay0_image, Font/000, i4, 24, 24]`), unpacked first when a Yay0, MIO0 or Yaz0 block starts there (`rom/config_images.py`), and the images a JSON asset list in the project's root declares (`assets*.json` in the style of the first 3D platformer's -decompilation: the format in the path, `meta.dims`, `meta.pal`, and `offsets` for the project's release). +decompilation: the format in the path, `meta.dims`, `meta.pal`, and `offsets` for the project's release), +and a project's sno-packed texture sheets (`sprite_sheet`, `course_texture_table` segments with their +`decompressed_size`: sno has no header, so the config's start and size are what find them). Object-build projects whose original is a disc or a folder still need original-image adapters. Processor adapters plug into `Arch.asset_flow`; console-specific call contracts and argument counts belong to `Console`. The preview decoders and local UI/API are shared by every ROM console. diff --git a/pyproject.toml b/pyproject.toml index 72a7cb217..85fbe9d71 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ dependencies = [ # the repair families, RetroReversing/decomp-repair: a tagged archive, so installing needs no git "decomprepair @ https://github.com/RetroReversing/decomp-repair/archive/refs/tags/v0.4.0.tar.gz", # the asset recognisers (N64 textures, sound banks, models, fonts; RNC), RetroReversing/decomp-assets - "decompassets @ https://github.com/RetroReversing/decomp-assets/archive/refs/tags/v0.9.0.tar.gz", + "decompassets @ https://github.com/RetroReversing/decomp-assets/archive/refs/tags/v0.10.0.tar.gz", "httpx>=0.28", "uvicorn>=0.30", "ninja>=1.11", # builds GameCube/Wii (dtk) and other ninja-based decomp projects diff --git a/src/decompnet/app/rom_explorer.py b/src/decompnet/app/rom_explorer.py index e5d83a56b..6cd7b2f97 100644 --- a/src/decompnet/app/rom_explorer.py +++ b/src/decompnet/app/rom_explorer.py @@ -47,7 +47,8 @@ def _source(net, proj): from ..rom import config_images # The images the project's own config declares (its textures, where it has worked them out). images=(config_images.images(config) if config else [])+config_images.asset_list_images(fp.root,cfg.get('version')) - return dict(path=image,segments=segments,gp=gp,arch=console.arch,big_endian=console.big_endian,fork=True,images=images) + return dict(path=image,segments=segments,gp=gp,arch=console.arch,big_endian=console.big_endian,fork=True,images=images, + sheets=config_images.sheets(config) if config else []) def _project(net, slug): @@ -72,7 +73,7 @@ def _revision(net, proj): cfg=net.cfg(proj) stat=source['path'].stat() layout={key:cfg.get(key) for key in ('rom','segments','platform','big_endian','gp')} - if source['fork']:layout['source']=[str(source['path']),source['segments'],source['gp'],source['images']] + if source['fork']:layout['source']=[str(source['path']),source['segments'],source['gp'],source['images'],source['sheets']] return hashlib.sha256(json.dumps([ANALYSIS_REVISION,layout,proj['big_endian'],(stat.st_size,stat.st_mtime_ns,stat.st_ctime_ns),[tuple(r) for r in rows],[tuple(r) for r in syms],_contracts(net,proj)],sort_keys=True).encode()).hexdigest() @@ -162,7 +163,8 @@ def _run(net, slug, log): report = analyse(image,segments,functions,symbols,big_endian=source['big_endian'], progress=log.progress,check=log.check_cancel,ranges=[c['source'] for c in flow['chains']], assets=resource_catalog.regions(image,check=log.check_cancel)+ - config_images.regions(image,source.get('images',[]),'splat config')+console.asset_regions(image,context=dict(functions=functions,symbols=symbols,segments=segments,gp=gp),check=log.check_cancel,progress=log.progress)) + config_images.regions(image,source.get('images',[]),'splat config')+ + config_images.sheet_regions(image,source.get('sheets',[]),'splat config')+console.asset_regions(image,context=dict(functions=functions,symbols=symbols,segments=segments,gp=gp),check=log.check_cancel,progress=log.progress)) resource_catalog.attach(report) asset_flow.attach(report,flow) log.check_cancel() diff --git a/src/decompnet/rom/config_images.py b/src/decompnet/rom/config_images.py index 6b7e10771..ed0edaf3a 100644 --- a/src/decompnet/rom/config_images.py +++ b/src/decompnet/rom/config_images.py @@ -12,6 +12,9 @@ (`assets.json`, `assets_image.json`): a path ending in the format (`bank_0/2.ci4.png`), `meta.dims` and, for colour-indexed images, `meta.pal` naming the palette's entry, and `offsets` per release: the ROM offset and the offset inside the block there, which is packed when a Yay0, MIO0 or Yaz0 block starts at it (`asset_list_images`). + +A project's own segment types for sno-packed texture sheets (`sprite_sheet`, `course_texture_table`, with a +`decompressed_size`) are read too (`sheets`): sno has no header, so the project's start and size are what find it. """ from __future__ import annotations @@ -86,6 +89,47 @@ def images(config): return out +# A project's segment types that are sno-packed texture sheets, and whether every entry is CI4. +SHEETS = {'sprite_sheet': False, 'course_texture_table': True} + + +def sheets(config): + """The sno texture sheets a splat config declares: offset, end, unpacked size, name, CI4-only.""" + out = [] + for start, stop, kind, name, _args, item in _entries(config.get('segments') if isinstance(config, dict) else None): + size = item.get('decompressed_size') if isinstance(item, dict) else None + if kind in SHEETS and isinstance(size, int) and 0 < size <= asset_formats.MAX_BYTES: + out.append(dict(offset=start, end=stop, size=size, name=name if isinstance(name, str) else f'{start:X}', + indexed=SHEETS[kind], declared=kind)) + return out + + +def _sheet(image, item): + from decompassets.n64 import sno + end = item['end'] if item['end'] and item['end'] > item['offset'] else item['offset'] + 2*item['size'] + 2 + data = sno.unpack(bytes(image[item['offset']:end]), item['size']) + return data, sno.sheet(data, indexed_only=item['indexed']) + + +def sheet_regions(image, declared, source): + """ROM Explorer assets for the declared sno texture sheets that unpack and read whole.""" + out = [] + for item in declared: + try:_data, textures = _sheet(image, item) + except AssetError:continue + if not textures:continue + first = textures[0]['preview_settings'] + span = (item['end'] or item['offset'] + 1) - item['offset'] + out.append(dict(offset=item['offset'], size=max(1, min(span, len(image) - item['offset'])), kind='image', + provider='splat-config', layout='sno-sheet', label=f"{item['name']} · {len(textures)} texture(s)", + confidence='declared', resource_path=item['name'], sheet=item, textures=textures, + **{k: v for k, v in textures[0].items() if k != 'preview_settings'}, + preview_settings={**first, 'source': 'rom', 'offset': item['offset']}, + reason=f"The project's splat config ({source}) declares a {item['declared']} here, sno-packed to " + f"{item['size']:,} bytes; its entries and palettes fit them. Shown as the project declares it.")) + return out + + def asset_list_images(root, version): """The images a project's JSON asset lists declare for one release (`version`, or the only one they list).""" entries = {} @@ -165,6 +209,15 @@ def regions(image, declared, source): def preview(image, asset, selection): """The declared image, unpacked first when its bytes are a packed block, with its palette.""" + if asset.get('layout') == 'sno-sheet': + data, textures = _sheet(image, asset['sheet']) + if not textures:raise AssetError('The texture sheet changed. Check the project\'s splat config, then rescan.') + texture = asset_formats.integer(selection.get('texture', 0), 'Texture', 0, len(textures)-1) + result = asset_formats.preview(data, textures[texture]['preview_settings']) + result['details'].update(texture=texture, frame=0, palette=0, note=asset['reason']) + result['decoded'] = dict(codec='sno', bytes=len(data), block_offset=asset['offset'], member=asset['sheet']['name']) + result['settings'] = dict(source='rom', format='auto', offset=asset['offset'], texture=texture, frame=0, palette=0, mip=0) + return result item = asset['declared'] texels, codec = _bytes(image, item['offset'], _pixels(item), item.get('inner', 0)) if len(texels) < _pixels(item):raise AssetError('The declared image is shorter than its size. Check the project\'s splat config, then rescan.') diff --git a/tests/test_rom_explorer.py b/tests/test_rom_explorer.py index 16fc85221..7f5529100 100644 --- a/tests/test_rom_explorer.py +++ b/tests/test_rom_explorer.py @@ -395,3 +395,23 @@ def test_a_fork_shows_the_images_its_json_asset_list_declares(tmp_path): red=next(a for a in assets if a['resource_path']=='textures/red.rgba16.png') shown=config_images.preview(bytes(image),red,{}) assert (shown['details']['width'],shown['details']['height'])==(4,2) and 'asset list' in red['reason'] + + +def test_a_fork_shows_the_sno_texture_sheets_its_splat_config_declares(): + """A project's own sprite_sheet segment: sno-packed, found only by the start and size the config gives.""" + import struct + + from decompnet.rom import config_images + palette=bytes.fromhex('f801')+bytes.fromhex('07c1')*15 + base=8+16+len(palette) + sheet=struct.pack('>II',0,1)+struct.pack('>I6H',base,0,4,2,0,0,0)+palette+bytes([0x01,0x10]*2) + packed=b''.join(bytes([0,value]) for value in sheet) + image=bytearray(0x400);image[0x100:0x100+len(packed)]=packed + config={'segments':[{'name':'menu','type':'sprite_sheet','start':0x100,'decompressed_size':len(sheet)}, + {'name':'broken','type':'sprite_sheet','start':0x300,'decompressed_size':len(sheet)},[0x380]]} + declared=config_images.sheets(config) + assert [s['name'] for s in declared]==['menu','broken'] + assets=config_images.sheet_regions(bytes(image),declared,'x') + assert [a['resource_path'] for a in assets]==['menu'] # zeros don't unpack to a sheet + shown=config_images.preview(bytes(image),assets[0],{}) + assert shown['decoded']['codec']=='sno' and (shown['details']['width'],shown['details']['height'])==(4,2) diff --git a/uv.lock b/uv.lock index 5ab65cec2..41c9813c5 100644 --- a/uv.lock +++ b/uv.lock @@ -291,9 +291,9 @@ wheels = [ [[package]] name = "decompassets" -version = "0.9.0" -source = { url = "https://github.com/RetroReversing/decomp-assets/archive/refs/tags/v0.9.0.tar.gz" } -sdist = { hash = "sha256:4bfeda4c5a5aede48f5b7ce40de525ec8e7d899f5ca50343e55a90c305f88c44" } +version = "0.10.0" +source = { url = "https://github.com/RetroReversing/decomp-assets/archive/refs/tags/v0.10.0.tar.gz" } +sdist = { hash = "sha256:14303593ec1bfdaeda01f5cc1b4991d30757a60d126879d18794f90dd67a0acf" } [[package]] name = "decompnet" @@ -329,7 +329,7 @@ dev = [ [package.metadata] requires-dist = [ { name = "capstone", specifier = ">=5.0.9" }, - { name = "decompassets", url = "https://github.com/RetroReversing/decomp-assets/archive/refs/tags/v0.9.0.tar.gz" }, + { name = "decompassets", url = "https://github.com/RetroReversing/decomp-assets/archive/refs/tags/v0.10.0.tar.gz" }, { name = "decomprepair", url = "https://github.com/RetroReversing/decomp-repair/archive/refs/tags/v0.4.0.tar.gz" }, { name = "httpx", specifier = ">=0.28" }, { name = "mcp", specifier = ">=2.3.0" },