seedds
eb858ec51a
#203 as submitted stopped the shader split of Tears of the Kingdom with a TypeError: its 1,202 Cave/.../C.crbin.<hash>/*.quad files are a 4-byte hash (the one in their folder name), then a zstd frame, so they pass the gate, and on the length mismatch the error path took len() of None. The PR checked Wonder and Yoshi only; of the RomFS of 39 games on hand, only those files have a frame at +4. - unsized: a frame whose header records a size other than the prefix is not such a container and is read as it is, without decompressing it (frame_size). A frame that records none (Kirby's LZ4 frames) must decompress to exactly the prefix's size: the whole output is compared, where the PR's max_length cut a longer frame at the prefix and let it pass. - The dictionary handling is shared with unzs (zs_dict_id, unzs_dict). A dictionary ZDICTS does not have is still reported, not fatal, as in the PR: the ID may be any bytes after a magic at +4. - unityfs.text_assets: an object shorter than the two length fields, or one whose name padding leaves no room for the script's length, is left out instead of reading past its end (struct.error, which stopped the split). Checked: - Kirby and the Forgotten Land (base and update 1.1.0): 121 shaders before, 27,288 after (27,302 with the update), no [RED] line. 5,121 zstd frames record their size, equal to the prefix; 2,080 + 354 LZ4 frames record none and decompress to the prefix's size. The PR's "383 in lyt/" is 378. - Tears of the Kingdom: the full split exits 1 with #203 alone; with this commit its package and tables are byte-identical to main's, no [RED] line. - main's split and this branch's give byte-identical packages and tables on Monster Train 2, Ace Attorney Investigations Collection, AI: THE SOMNIUM FILES, nirvanA Initiative, BALL x PIT, Advance Wars 1+2, Brain Memory Training Academy (Unity 2017.4 to 6000) and Super Mario Bros. Wonder. - New test_size_prefixed_frames.py; test_unity_text_glslc.py covers short objects (fails on #238's unityfs.py). All of dm/tools/tests pass. - Not checked here: FINAL FANTASY CRYSTAL CHRONICLES (#238's game) is not on hand; Kirby has not been run.
70 lines
2.8 KiB
Python
70 lines
2.8 KiB
Python
"""shader_split.unsized: HAL's .cmp containers (Kirby and the Forgotten Land), a 4-byte little-endian uncompressed size, then a zstd or an LZ4 frame. The frame
|
|
is opened only when it decompresses to exactly the size the prefix gives; anything else is returned as it is, so a file that is not such a container is
|
|
scanned as before. No game files needed."""
|
|
import contextlib
|
|
import io
|
|
import os
|
|
import struct
|
|
import sys
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
REPO = Path(__file__).resolve().parents[3]
|
|
sys.path.insert(0, str(REPO / 'dm/gfx/tools'))
|
|
_had = os.environ.get('DM_GAME'); os.environ['DM_GAME'] = '01007BB017812000' # shader_split loads a profile at import; other tests expect no default game
|
|
import shader_split as ss
|
|
if _had is None: del os.environ['DM_GAME']
|
|
else: os.environ['DM_GAME'] = _had
|
|
|
|
BODY = b'BNSH\0\0\0\0' + bytes(range(256)) * 64
|
|
|
|
|
|
def unsized(d):
|
|
out = io.StringIO()
|
|
with contextlib.redirect_stdout(out): r = ss.unsized(d, '/test.cmp')
|
|
return r, out.getvalue()
|
|
|
|
|
|
class Unsized(unittest.TestCase):
|
|
def test_zstd_frame(self):
|
|
d = struct.pack('<I', len(BODY)) + ss.zstd_mod().compress(BODY)
|
|
self.assertEqual(d[4:8], ss.ZSTD_MAGIC)
|
|
self.assertEqual(unsized(d), (BODY, ''))
|
|
|
|
def test_lz4_frame(self):
|
|
import lz4.frame
|
|
d = struct.pack('<I', len(BODY)) + lz4.frame.compress(BODY)
|
|
self.assertEqual(d[4:8], ss.LZ4_FRAME_MAGIC)
|
|
self.assertEqual(unsized(d), (BODY, ''))
|
|
|
|
def test_frame_size(self):
|
|
import lz4.frame
|
|
zs = ss.zstd_mod()
|
|
self.assertEqual(ss.frame_size(zs.compress(BODY)), len(BODY))
|
|
self.assertEqual(ss.frame_size(zs.compress(b'x' * 300)), 300) # the 2-byte field, which counts from 256
|
|
self.assertEqual(ss.frame_size(lz4.frame.compress(BODY, store_size=True)), len(BODY))
|
|
self.assertIsNone(ss.frame_size(lz4.frame.compress(BODY, store_size=False)))
|
|
|
|
def test_word_before_a_frame_that_records_another_size(self):
|
|
# Tears of the Kingdom's Cave/…/C.crbin.<hash>/*.quad: the hash, then a zstd frame of its own size; not this container, and nothing to report
|
|
d = bytes.fromhex('eb157a51') + ss.zstd_mod().compress(BODY)
|
|
self.assertEqual(unsized(d), (d, ''))
|
|
|
|
def test_frame_longer_than_its_prefix(self):
|
|
# the prefix is the whole length, not a limit: a frame that records no size and holds more is not this container
|
|
import lz4.frame
|
|
d = struct.pack('<I', len(BODY) - 1) + lz4.frame.compress(BODY, store_size=False)
|
|
r, log = unsized(d)
|
|
self.assertIs(r, d)
|
|
self.assertIn('[RED] /test.cmp', log)
|
|
|
|
def test_bytes_that_do_not_decompress(self):
|
|
d = struct.pack('<I', 64) + ss.ZSTD_MAGIC + b'\x00' * 60
|
|
r, log = unsized(d)
|
|
self.assertIs(r, d)
|
|
self.assertIn('[RED] /test.cmp', log)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|