From fa69e20a280413d56e64edcb8736bf9329957f77 Mon Sep 17 00:00:00 2001 From: seedds Date: Sat, 24 Oct 2026 22:17:57 +0800 Subject: [PATCH] Compressed MonoGame XNB shaders and the Mac text box (general, from PR #120, found on Stardew Valley). shader_split.py: MonoGame's compressed XNB files, ContentCompressedLzx (flag 0x80: a CHMLib-style LZX decoder with a 2^16 window inside XNB block framing) and ContentCompressedLz4 (0x40: one raw LZ4 block), are decompressed before the readers (xnb_decompress); for LZX only the first block is decoded unless it names EffectReader. An MGFX shader whose length runs past the data is no effect (a red line in an XNB). Stardew Valley's 3,549 XNB files (3,535 LZ4, 8 LZX, 6 plain) all decode to their declared sizes, and its one content effect (Content/Effects/ShadowRemove.xnb, LZ4) is now split: 65 shaders, 64 before. Celeste's uncompressed XNBs give its profile's tables byte for byte. front_mac.m: while the text box is up the game window sees no key-up or mouse-up, so the button or click that opened it stayed pressed and was read the moment the box closed (Stardew Valley's character creation re-opened the keyboard after every OK); the keyboard buttons, the WASD stick and the touch are released when the box opens. The box also stays up, saying so, while the text is shorter than ShowKeyboardArg's minimum length (+0x3b0, hle_swkbd.c g_swkbd_min_chars), as the applet keeps OK disabled. Left out of the PR: writing the text's byte count into the String's +8. The SDK's ReadCloseResultAndString (Pikmin 4's sdk+0x39b7d0) reads String{pointer, capacity} and copies at most the capacity, and GetInteractiveOutStorageCallback restores +8 after changing it, so on hardware +8 stays the game's capacity; t_swkbd now checks it is unchanged on every path (17/17). Checked: Stardew Valley packs from its NSPs with a provisional profile and boots on the Mac to the title, and in character creation (DM_KBD) Name, Farm Name and Favorite Thing each take the text from one keyboard call and OK starts the intro, with +8 left as the capacity; the host unit tests and the 41 tool tests pass. Not checked: the Mac text box by hand (a scripted DM_KBD never shows it), and no game here asks for a minimum length. The NSP script's MonoGame survey note and docs/agents/04 say so --- dm/gfx/tools/shader_split.py | 274 +++++++++++++++++++++++++++++++++- dm/host/front_mac.m | 31 ++-- dm/host/hle_swkbd.c | 12 +- dm/host/tests/t_swkbd.c | 5 + docs/agents/04-graphics.md | 2 +- scripts/build_ipa_from_nsp.py | 2 +- 6 files changed, 308 insertions(+), 18 deletions(-) diff --git a/dm/gfx/tools/shader_split.py b/dm/gfx/tools/shader_split.py index 272e975..23bd73e 100644 --- a/dm/gfx/tools/shader_split.py +++ b/dm/gfx/tools/shader_split.py @@ -698,6 +698,267 @@ def bdt_shaders(path, rel, ents, out): for nm, b in bnd3_members(d, label): if b[:4] == b'\x91\x68\x86\x19': out.extend(glslc_shaders(b, f'{label}/{nm}')) +class LzxError(Exception): pass + +# MonoGame's compressed XNB (Stardew Valley: 3,535 of its 3,549 .xnb files): ContentManager.cs knows +# ContentCompressedLzx = 0x80 and ContentCompressedLz4 = 0x40 (XNA's LZX flag was 0x40, MonoGame's own writer +# uses these). The LZX is as XMemCompress writes it: a CHMLib-style LZX decoder (jedwing/CHMLib src/lzx.c, from +# cabextract) with a 2^16 window, inside XNB block framing (MonoGame's LzxDecoderStream): each block is two +# bytes, 0xff then the u16be output and input sizes, or the u16be input size with an 0x8000 output. The LZ4 is +# one raw LZ4 block (its Lz4DecoderStream) +LZX_EXTRA = (0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, + 15, 15, 16, 16, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17) +LZX_BASE = (0, 1, 2, 3, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128, 192, 256, 384, 512, 768, 1024, 1536, 2048, 3072, + 4096, 6144, 8192, 12288, 16384, 24576, 32768, 49152, 65536, 98304, 131072, 196608, 262144, 393216, + 524288, 655360, 786432, 917504, 1048576, 1179648, 1310720, 1441792, 1572864, 1703936, 1835008, 1966080, 2097152) + +def _lzx_table(nsyms, nbits, length): + """make_decode_table: a huffman lookup table out of canonical code lengths (David Tritscher's, as in cabextract)""" + table = [0] * ((1 << nbits) + nsyms * 2) + pos, bit_num, table_mask = 0, 1, 1 << nbits + bit_mask = table_mask >> 1 + next_symbol = bit_mask + while bit_num <= nbits: + for sym in range(nsyms): + if length[sym] == bit_num: + leaf = pos + pos += bit_mask + if pos > table_mask: raise LzxError('decode table overrun') + for i in range(leaf, pos): table[i] = sym + bit_mask >>= 1; bit_num += 1 + if pos != table_mask: + for sym in range(pos, table_mask): table[sym] = 0 + pos <<= 16; table_mask <<= 16; bit_mask = 1 << 15 + while bit_num <= 16: + for sym in range(nsyms): + if length[sym] == bit_num: + leaf = pos >> 16 + for fill in range(bit_num - nbits): + if table[leaf] == 0: + table[next_symbol << 1] = 0 + table[(next_symbol << 1) + 1] = 0 + table[leaf] = next_symbol; next_symbol += 1 + leaf = table[leaf] << 1 + if (pos >> (15 - fill)) & 1: leaf += 1 + table[leaf] = sym + pos += bit_mask + if pos > table_mask: raise LzxError('decode table overflow') + bit_mask >>= 1; bit_num += 1 + if pos != table_mask and any(length[:nsyms]): raise LzxError('a bad or incomplete huffman table') + return table + +class LzxState: + """The decoder state, which lives for the whole XNB: the ring window, the LRU offsets and the block in progress""" + def __init__(self, window_bits=16): + self.window_size = 1 << window_bits + self.window = bytearray(self.window_size) + self.posn = 0 + self.R0 = self.R1 = self.R2 = 1 + self.main_elements = 256 + ((window_bits << 1) << 3) # position slots: window<<1 + self.header_read = False + self.block_type = 0 + self.block_remaining = 0 + self.intel_filesize = 0 + self.intel_curpos = 0 + self.intel_started = False + self.frames_read = 0 + self.main_len = bytearray(256 + 50 * 8 + 64) # + LZX_LENTABLE_SAFETY: deltas are applied to them across blocks + self.length_len = bytearray(249 + 1 + 64) + self.main_tbl = self.length_tbl = self.al_tbl = None # an LZX block can span XNB frames, so the tables persist with the state + self.al_len = None + self.d = b'' + self.ip = 0 + self.bb = 0 + self.bl = 0 + + def _ensure(self, n): + while self.bl < n: + self.bb = (self.bb << 16) | (self.d[self.ip] | (self.d[self.ip + 1] << 8) if self.ip + 1 < len(self.d) else 0) # a 16-bit overread past the block end is allowed, as in the C code + self.bl += 16; self.ip += 2 + + def _remove(self, n): self.bl -= n; self.bb &= (1 << self.bl) - 1 + + def _read(self, n): + self._ensure(n) + v = self.bb >> (self.bl - n) + self._remove(n) + return v + + def _huffsym(self, table, nbits, maxsyms, lens): + self._ensure(16) + i = table[self.bb >> (self.bl - nbits)] + if i >= maxsyms: + k = 0 + while True: + if self.bl - nbits - 1 - k < 0: raise LzxError('a huffman code longer than the bitstream') + i = (i << 1) | ((self.bb >> (self.bl - nbits - 1 - k)) & 1) + k += 1 + i = table[i] + if i < maxsyms: break + self._remove(lens[i]) + return i + + def _read_lens(self, lens, first, last): + pre_len = [self._read(4) for _ in range(20)] + pre = _lzx_table(20, 6, pre_len) + x = first + while x < last: + z = self._huffsym(pre, 6, 20, pre_len) + if z == 17: + y = self._read(4) + 4 + while y: lens[x] = 0; x += 1; y -= 1 + elif z == 18: + y = self._read(5) + 20 + while y: lens[x] = 0; x += 1; y -= 1 + elif z == 19: + y = self._read(1) + 4 + z = self._huffsym(pre, 6, 20, pre_len) + z = lens[x] - z + if z < 0: z += 17 + while y: lens[x] = z; x += 1; y -= 1 + else: + z = lens[x] - z + if z < 0: z += 17 + lens[x] = z; x += 1 + + def decompress(self, src, outlen): + self.d, self.ip, self.bb, self.bl = src, 0, 0, 0 + togo = outlen + if not self.header_read: + if self._read(1): + self.intel_filesize = (self._read(16) << 16) | self._read(16) + self.header_read = True + while togo > 0: + if self.block_remaining == 0: + if self.block_type == 3: # after an uncompressed block the bitstream returns to a byte boundary + if self.block_length & 1: self.ip += 1 + self.bb = 0; self.bl = 0 + self.block_type = self._read(3) + self.block_remaining = self.block_length = (self._read(16) << 8) | self._read(8) + al_table = pre = None + if self.block_type == 2: + self.al_len = [self._read(3) for _ in range(8)] + self.al_tbl = _lzx_table(8, 7, self.al_len) + if self.block_type in (1, 2): + self._read_lens(self.main_len, 0, 256) # the lengths are deltas against the previous block's, so they persist + self._read_lens(self.main_len, 256, self.main_elements) + self.main_tbl = _lzx_table(256 + 50 * 8, 12, self.main_len) + if self.main_len[0xE8]: self.intel_started = True + self._read_lens(self.length_len, 0, 249) + self.length_tbl = _lzx_table(249 + 1, 12, self.length_len) + elif self.block_type == 3: + self.intel_started = True + self._ensure(16) + if self.bl > 16: self.ip -= 2 # align the bitstream, as the C code does (the bits stay in the buffer) + self.R0 = int.from_bytes(self.d[self.ip:self.ip + 4], 'little'); self.ip += 4 + self.R1 = int.from_bytes(self.d[self.ip:self.ip + 4], 'little'); self.ip += 4 + self.R2 = int.from_bytes(self.d[self.ip:self.ip + 4], 'little'); self.ip += 4 + else: raise LzxError(f'block type {self.block_type}') + if self.ip > len(self.d) and (self.ip > len(self.d) + 2 or self.bl < 16): raise LzxError('the block ended early') + this_run = min(self.block_remaining, togo) + togo -= this_run + self.block_remaining -= this_run + self.posn &= self.window_size - 1 + if self.posn + this_run > self.window_size: raise LzxError('a run straddles the window') + posn = self.posn + if self.block_type == 3: + if self.ip + this_run > len(self.d): raise LzxError('the uncompressed block runs past the block') + self.window[posn:posn + this_run] = self.d[self.ip:self.ip + this_run] + self.ip += this_run; posn += this_run + else: + main, length, al_table, al_len = self.main_tbl, self.length_tbl, self.al_tbl, self.al_len + while this_run > 0: + main_element = self._huffsym(main, 12, 256 + 50 * 8, self.main_len) + if main_element < 256: + self.window[posn] = main_element; posn += 1; this_run -= 1 + continue + main_element -= 256 + match_length = main_element & 7 + if match_length == 7: + match_length += self._huffsym(length, 12, 249 + 1, self.length_len) + match_length += 2 + match_offset = main_element >> 3 + if match_offset > 2: + if self.block_type == 2: + extra = LZX_EXTRA[match_offset] + match_offset = LZX_BASE[match_offset] - 2 + if extra > 3: + extra -= 3 + match_offset += (self._read(extra) << 3) + self._huffsym(al_table, 7, 8, al_len) + elif extra == 3: + match_offset += self._huffsym(al_table, 7, 8, al_len) + elif extra > 0: + match_offset += self._read(extra) + else: + match_offset = 1 + else: + if match_offset != 3: + match_offset = LZX_BASE[match_offset] - 2 + self._read(LZX_EXTRA[match_offset]) + else: + match_offset = 1 + self.R2, self.R1, self.R0 = self.R1, self.R0, match_offset + elif match_offset == 0: match_offset = self.R0 + elif match_offset == 1: match_offset = self.R1; self.R1, self.R0 = self.R0, match_offset + else: match_offset = self.R2; self.R2, self.R0 = self.R0, match_offset + posn += match_length + if posn > self.window_size: raise LzxError('a match overruns the window') + this_run -= match_length + src = posn - match_length - match_offset + if src < 0: src += self.window_size + if match_offset >= match_length and src + match_length <= self.window_size: + self.window[posn - match_length:posn] = self.window[src:src + match_length] + else: + w = self.window; dst = posn - match_length + for i in range(match_length): w[dst + i] = w[(src + i) % self.window_size] + self.posn = posn + start = self.posn - outlen if self.posn else self.window_size - outlen + out = bytes(self.window[start:self.posn or self.window_size]) + self.frames_read += 1 + if self.frames_read <= 32768 and self.intel_filesize != 0: # the Intel E8 transform + if outlen <= 6 or not self.intel_started: + self.intel_curpos += outlen + else: + data = bytearray(out) + curpos, filesize, pos, end = self.intel_curpos, self.intel_filesize, 0, outlen - 10 + self.intel_curpos = curpos + outlen + while pos < end: + b = data[pos]; pos += 1 + if b != 0xE8: curpos += 1; continue + abs_off = int.from_bytes(data[pos:pos + 4], 'little', signed=True) + if -curpos <= abs_off < filesize: + rel_off = abs_off - curpos if abs_off >= 0 else abs_off + filesize + data[pos:pos + 4] = (rel_off & 0xFFFFFFFF).to_bytes(4, 'little') + pos += 4 + curpos += 5 + out = bytes(data) + return out + +def xnb_decompress(d, first_block=False): + """The decompressed content of a compressed XNB (flags 0x80 LZX, 0x40 LZ4; the u32 at 6 is the file size, + the u32 at 10 the decompressed size). first_block stops after the first LZX block: the reader names sit + at the start of the content, so an effect file is recognized there without decompressing the rest""" + dec_size = struct.unpack_from(' len(d): # a shader whose length runs past the data: a stray 'MGFX' in other content, not an effect + if label.endswith('.xnb'): red(f'{label} MGFX @{p:#x}: shader {i} claims {n:#x} bytes past the end of the file') + return None + bc = r.raw(n) nctl = struct.unpack_from(' 1 or 8 + nctl + ncode != n: red(f'{label} MGFX @{p:#x} shader {i}: {n:#x} bytes, not a control section and a code area') ctl, code = bc[4:4 + nctl], bc[8 + nctl:] @@ -1270,6 +1535,13 @@ def scan_file(args): return rel, uniq_keys(found), var_pairs(found), trees d = unzs(path) if head[:4] == ZSTD_MAGIC else open(path, 'rb').read() # by the magic: Paper Mario: The Thousand-Year Door names its zstd files .zst (8,726, its shaders among them) if d[:4] == b'Yaz0': d = yaz0(d) + if d[:3] == b'XNB' and d[4] == 5 and d[5] & 0xC0 and len(d) > 14: # a compressed XNB (Stardew Valley: 3,535 LZ4 files, 8 LZX): only effects (EffectReader) carry MGFX shaders, and for LZX the reader names sit in the first block, so it decides whether the rest is worth decoding + if d[5] & 0x80: + head_d = xnb_decompress(d, first_block=True) + if b'EffectReader' not in head_d: return rel, {}, set(), {} + d = head_d if len(head_d) >= struct.unpack_from(' 0 ? g_swkbd_min_chars : 0; NSString *info = [NSString stringWithFormat:@"%s%s%s", sub ? sub : "", sub && *sub && guide && *guide ? "\n" : "", guide ? guide : ""]; - a.informativeText = info; - [a addButtonWithTitle:@"OK"]; [a addButtonWithTitle:@"Cancel"]; - NSTextField *f = [[NSTextField alloc] initWithFrame:NSMakeRect(0, 0, 300, 24)]; - f.stringValue = initial ? @(initial) : @""; - if (guide && *guide) f.placeholderString = @(guide); - a.accessoryView = f; a.window.initialFirstResponder = f; - r = [a runModal] == NSAlertFirstButtonReturn; - text = f.stringValue; + for (;;) { + NSAlert *a = [NSAlert new]; + a.messageText = title && *title ? @(title) : @"Input"; + a.informativeText = info; + [a addButtonWithTitle:@"OK"]; [a addButtonWithTitle:@"Cancel"]; + NSTextField *f = [[NSTextField alloc] initWithFrame:NSMakeRect(0, 0, 300, 24)]; + f.stringValue = text ?: (initial ? @(initial) : @""); + if (guide && *guide) f.placeholderString = @(guide); + a.accessoryView = f; a.window.initialFirstResponder = f; + r = [a runModal] == NSAlertFirstButtonReturn; + text = f.stringValue; + if (!r || (int)text.length >= min_chars) break; + // On hardware the applet would not have accepted the shorter text (OK stayed disabled); a game that rejects it re-opens the keyboard for ever, so the box comes back with the rule instead + info = [info stringByAppendingFormat:@"\nAt least %d characters are needed here.", min_chars]; + } }); if (!r) return 0; (void)max_chars; // Truncation is done uniformly in hle_swkbd.c in UTF-16 units diff --git a/dm/host/hle_swkbd.c b/dm/host/hle_swkbd.c index e4071ec..31e86e0 100644 --- a/dm/host/hle_swkbd.c +++ b/dm/host/hle_swkbd.c @@ -5,7 +5,7 @@ // +0x3ac maximum length, +0x3b0 minimum length, +0x3bd isUseUtf8, +0x3c0 offset of the initial text within the work buffer, +0x3c4 UTF-16 length of the initial text; // work buffer pointer / size at +0x4d0 / +0x4d8 (handed to the applet together with CreateLargeStorage) // SetInitialTextUtf8 0x3a74c0: UTF-8 → UTF-16 written to work buffer+offset (offset 0x14, at most 500 characters + terminating 0) -// ReadCloseResultAndString 0x3a63f0: output storage = u32 CloseResult + text area; the text area is copied as is into String{+0 pointer, +8 byte count} +// ReadCloseResultAndString 0x3a63f0: output storage = u32 CloseResult + text area; the text area is copied as is into String{+0 pointer, +8 capacity}, at most the capacity, which it only reads (Pikmin 4: sdk+0x39b7d0) // ShowKeyboard wrap-up 0x3a6acc: CloseResult == 0 (OK) → Result 0, otherwise (cancel) → 0x29F; GetRequiredStringBufferSize = 0x7D4 // A game that sets isUseUtf8 = 1 after MakePreset reads the result back as a C string; UTF-16 output is supported as well per the original convention. // No window and DM_KBD unset: return as cancel (no popup, no hang). DM_KBD="text": no popup, return that text directly as OK (windowless testing / lockstep comparison). @@ -56,6 +56,7 @@ size_t swkbd_u8_to_u16(const char *s, size_t max_units, uint16_t *out, size_t ca // hle_charname.c also lets bytes ≥0x80 through (Chinese names), so the rule becomes: English letters or Chinese characters, at most 15 UTF-8 bytes in total (3 bytes per Chinese character, at most 5). // While the naming panel is open the front end blocks on the spot by this rule and writes the rule in the dialog. diag.c maintains this flag from UI messages. int g_swkbd_name_panel; +int g_swkbd_min_chars = -1; // ≥0 = this input needs at least that many characters (ShowKeyboardArg's minimum length; the applet keeps OK disabled until the text is long enough, so the front end's box keeps itself up until then) int g_swkbd_numeric_max = -1; // ≥0 = this input is a quantity box, value = upper limit (set by dm_textinput.c, read by the front end input box, set back to -1 after use) int (*g_swkbd_unfilter)(void); // registered by the host (dm_charname_unfilter in hle_charname.c; NULL in unit tests): keyboard requests from the naming panel bypass the game's ASCII typing filter typedef int (*swkbd_input_fn)(const char *title, const char *sub, const char *guide, const char *initial, int max_chars, char *out, int out_cap); @@ -65,7 +66,7 @@ Result swkbd_show(void *out, const void *arg_, swkbd_input_fn input) { swkbd_u16_to_u8((const uint16_t *)(arg + ARG_HEADER), HEADER_N, title, sizeof title); swkbd_u16_to_u8((const uint16_t *)(arg + ARG_SUB), SUB_N, sub, sizeof sub); swkbd_u16_to_u8((const uint16_t *)(arg + ARG_GUIDE), GUIDE_N, guide, sizeof guide); - int32_t maxlen, ilen, ioff; memcpy(&maxlen, arg + ARG_MAXLEN, 4); memcpy(&ilen, arg + ARG_INIT_LEN, 4); memcpy(&ioff, arg + ARG_INIT_OFF, 4); + int32_t maxlen, minlen, ilen, ioff; memcpy(&maxlen, arg + ARG_MAXLEN, 4); memcpy(&minlen, arg + ARG_MINLEN, 4); memcpy(&ilen, arg + ARG_INIT_LEN, 4); memcpy(&ioff, arg + ARG_INIT_OFF, 4); const uint8_t *work; memcpy(&work, arg + ARG_WORKBUF, 8); if (ilen > 0 && work) swkbd_u16_to_u8((const uint16_t *)(work + ioff), (size_t)ilen, init, sizeof init); @@ -77,9 +78,10 @@ Result swkbd_show(void *out, const void *arg_, swkbd_input_fn input) { // Test switch DM_TEST_SWKBD_HOLD_MS= (with DM_KBD): the keyboard stays up that long before DM_KBD's text is returned, the way a player types into the input box (for reproducing on the Mac what the game's other threads do meanwhile) static int hold = -1; if (hold < 0) { const char *h = getenv("DM_TEST_SWKBD_HOLD_MS"); hold = h ? atoi(h) : 0; } if (script && hold > 0) usleep((useconds_t)hold * 1000); - int r = script ? (snprintf(text, sizeof text, "%s", script), 1) : input(title, sub, guide, init, maxlen > 0 ? maxlen : 0, text, sizeof text); + int r = script ? (snprintf(text, sizeof text, "%s", script), 1) : (g_swkbd_min_chars = minlen > 0 ? minlen : -1, input(title, sub, guide, init, maxlen > 0 ? maxlen : 0, text, sizeof text)); g_swkbd_numeric_max = -1; // the numeric box mark applies to this one time only (set by dm_textinput.c before bringing up the keyboard) - LOG("swkbd: title '%s' initial '%s' limit %d → %s%s%s", title, init, maxlen, + g_swkbd_min_chars = -1; // the minimum length applies to this one time only + LOG("swkbd: title '%s' initial '%s' limit %d min %d → %s%s%s", title, init, maxlen, minlen, r > 0 ? "OK '" : r == 0 ? "cancel" : "no window, treated as cancel", r > 0 ? text : "", r > 0 ? "'" : ""); if (r <= 0) return SWKBD_RESULT_CANCELED; if (g_swkbd_name_panel && g_swkbd_unfilter && g_swkbd_unfilter()) LOG("swkbd: naming panel -- turned off the game's ASCII input filter on the returned string (Chinese names allowed, see hle_charname.c)"); @@ -88,7 +90,7 @@ Result swkbd_show(void *out, const void *arg_, swkbd_input_fn input) { if (!buf || !cap) return RES_OK; // same as the SDK: if String has no buffer, the text is not read memset(buf, 0, cap); uint16_t u16[1003]; size_t n = swkbd_u8_to_u16(text, maxlen > 0 ? (size_t)maxlen : 1002, u16, 1003); // truncate to the character limit (counted in UTF-16 units) - if (arg[ARG_UTF8]) swkbd_u16_to_u8(u16, n, (char *)buf, cap); + if (arg[ARG_UTF8]) swkbd_u16_to_u8(u16, n, (char *)buf, cap); // String's +8 stays the game's capacity: ReadCloseResultAndString only reads it (Pikmin 4's sdk+0x39b81c) else memcpy(buf, u16, (n + 1) * 2 <= cap ? n * 2 : (cap / 2 - 1) * 2); { const char *lag = getenv("DM_TEST_SWKBD_LAG_MS"); if (lag) usleep((useconds_t)atoi(lag) * 1000); } // test: slow down "keyboard handed back → the game's thread gets the result" to verify that auto-accept does not jump the gun // front end path cleanup (dm_cursor.c): auto-accept after the name is typed on the naming page return RES_OK; diff --git a/dm/host/tests/t_swkbd.c b/dm/host/tests/t_swkbd.c index 8fce046..00b9d72 100644 --- a/dm/host/tests/t_swkbd.c +++ b/dm/host/tests/t_swkbd.c @@ -21,6 +21,7 @@ static uint8_t arg[0x500], work[0x1000], sbuf[0x7d4]; static struct { uint8_t *p; uint64_t n; } str = {sbuf, sizeof sbuf}; static void mkarg(int utf8, int maxlen, const uint16_t *init, int ilen) { memset(arg, 0, sizeof arg); memset(work, 0, sizeof work); memset(sbuf, 0xAA, sizeof sbuf); + str.n = sizeof sbuf; // a fresh String per call, its capacity at +8 (the SDK only reads it) static const uint16_t title[] = {0x8D77, 0x540D, 0}; // U+8D77 U+540D, the game's "Name your character" title memcpy(arg + 0x24, title, sizeof title); memcpy(arg + 0x3ac, &maxlen, 4); arg[0x3bd] = (uint8_t)utf8; @@ -46,6 +47,7 @@ int main(void) { static const uint8_t w1[] = {0xC3, 0x9C, 0x6E, 0xC3, 0xAF, 0xE5, 0x90, 0x8D, 0x00}; // U+00DC n U+00EF U+540D + terminating 0 CHECK(r == 0, "Result %#x expected 0", r); CHECK(bytes_eq(w1, sizeof w1), "UTF-8 write-back bytes differ"); + CHECK(str.n == sizeof sbuf, "String byte count %llu changed, the SDK leaves the capacity", (unsigned long long)str.n); CHECK(!strcmp(g_title, "\xE8\xB5\xB7\xE5\x90\x8D"), "title decoded as '%s', expected U+8D77 U+540D", g_title); CHECK(!strcmp(g_init, "Hero"), "initial text decoded as '%s', expected Hero", g_init); CHECK(g_max == 16, "limit %d expected 16", g_max); @@ -55,6 +57,7 @@ int main(void) { r = swkbd_show(&str, arg, stub); CHECK(r == 0x29F, "Result %#x expected 0x29F", r); CHECK(sbuf[0] == 0xAA && sbuf[sizeof sbuf - 1] == 0xAA, "cancel changed the buffer"); + CHECK(str.n == sizeof sbuf, "cancel changed the String byte count"); printf("3 no window (input returns -1) → treated as cancel 0x29F\n"); mkarg(1, 16, hero, 4); g_ret = -1; @@ -65,6 +68,7 @@ int main(void) { r = swkbd_show(&str, arg, stub); static const uint8_t w4[] = {0x41, 0x00, 0x0D, 0x54, 0x00, 0x00}; // 'A' U+540D terminating 0 (little endian) CHECK(r == 0 && bytes_eq(w4, sizeof w4), "UTF-16 write-back bytes differ"); + CHECK(str.n == sizeof sbuf, "String byte count %llu changed, the SDK leaves the capacity", (unsigned long long)str.n); CHECK(g_init[0] == 0, "initial length 0 should give no initial text"); printf("5 maximum length truncation (3 characters)\n"); @@ -72,6 +76,7 @@ int main(void) { swkbd_show(&str, arg, stub); static const uint8_t w5[] = {'A', 'B', 'C', 0}; CHECK(bytes_eq(w5, sizeof w5), "truncation result differs"); + CHECK(str.n == sizeof sbuf, "String byte count %llu changed, the SDK leaves the capacity", (unsigned long long)str.n); printf("6 truncation does not split a surrogate pair (limit 2 units, a + U+1F600 needs 3 units)\n"); mkarg(1, 2, hero, 4); g_ret = 1; g_reply = "a\xF0\x9F\x98\x80"; diff --git a/docs/agents/04-graphics.md b/docs/agents/04-graphics.md index 4c98929..9646da5 100644 --- a/docs/agents/04-graphics.md +++ b/docs/agents/04-graphics.md @@ -27,7 +27,7 @@ Do not stop at the obvious archive. In Mario the main `bfsha` held 4,888 of the - **BNSH in Bezel Engine archives** (Big Brain Academy: Brain vs. Brain, NintendoWare Bezel Engine 1.56): the shaders are BNSH in the `.bfsha` entries of the `.bea` archives (zstd entries, [01](01-extract.md)), most of them the engine's own in `System.nx.bea` (`_BezelSystemResources//shader/*.bfsha`: PBR shadow maps 846, post effects 770, …), the rest in the layouts' `__ArchiveShader.bnsh` and the per-archive shader packs (`.bnbshpk`), plus one BNSH in `main`. `shader_split.py` scans each archive entry as a file of its own (`bea_members`): 2,125 shaders in about 90 s. - **SPK in an MT Framework ARC** (The Great Ace Attorney Chronicles, Capcom's MT Framework): the engine's one shader package, `sc\NX_64\root` (`SPK\0`, type hash `0x02358e1a`) in `nativeNXx64/sa/NX_64/root.arc` ([01](01-extract.md)). After the package's tables every shader is a code area followed directly by its control section, back to back to the end of the file: 230 vertex shaders, then 230 pixel shaders. Each control section names its code length (+0x704), so the code is the bytes right before it, and every block must start where the previous one ended (`mtspk_shaders`). The tables that say which vertex and pixel shaders the engine combines are not read: the vertex shaders are paired with the single-stage cover (one pixel shader reads every input any of them reads). `CustomShaderPackage.mfx` (`MFX\0`, the effect descriptions) holds no shader code, and neither `main` nor any other ARC entry does. All 460 build, with Mario's binding layout; every binding decodes. - **Control sections loose in `main`** (Advance Wars 1+2: Re-Boot Camp, Unity 2020.2 with Bink 2's Unity plugin): the movie player builds its programs from 32 control sections and code areas in `main`'s read-only segment that are in no GLSLC output and no BNSH file. 26 are 0x20-byte records in two tables in `.data` (`{code area pointer, control section pointer, u32 code length, u32 control length}`, both pointers relative relocations); the code takes the other 6 by address (`adrp` + `add` to the control section and, a few instructions away, to the code area it copies with `memcpy`), the code area right after the control section's own length (+0x20, 16-byte aligned) or right before it, and two control sections share one code area. Both neighbours can pass the header and length checks, so the reader (`raw_shaders`) takes only a code area the code builds the address of, the nearest one to the control section's. The 8 compute shaders' code areas have no header (SDK 10). Without them the shaders were not in the map (two `[RED] shader … is not in the shader map` lines and `program has no vertex shader` as the opening movie starts) and every movie drew white. Of 25 `main`s on hand no other has such control sections. -- **MonoGame effects** (Celeste: MonoGame for the Switch, its C# compiled to C++ by Sickhead's BRUTE): the 13 XNB files in `Content/Effects` (XNB platform `S`, uncompressed, read by `EffectReader`) each hold one MGFX effect, and `main` holds SpriteBatch's built-in one; no other file and no GLSLC output. MGFX version 8 is MonoGame's own layout: `MGFX`, u8 version, u8 profile, u32 effect key, the constant buffers, then each shader as u8 vertex (1) / pixel (0), u32 length, the bytecode (u32 control length, the control section, u32 code length, the code area: GLSLC's output without its header), the samplers, the constant buffer indices and an attribute list (as MonoGame's OpenGL backend writes it), then the parameters and the techniques with their passes (`mgfx_effect`). The reader parses every effect to its end, which checks the layout; in `main`, four bytes `MGFX` count only when version 8 and a control section follow. A pass names its vertex and pixel shader by index, 255 for none, and 10 of the 13 effects have no vertex shader: SpriteBatch binds its own and the effect's pass only replaces the pixel shader. Vertex and pixel shaders therefore combine across effects, and they are paired as single-stage programs. 26 shaders (20 pixel, 6 vertex), all with Mario's binding layout; every binding decodes. +- **MonoGame effects** (Celeste: MonoGame for the Switch, its C# compiled to C++ by Sickhead's BRUTE): the 13 XNB files in `Content/Effects` (XNB platform `S`, uncompressed, read by `EffectReader`; a compressed XNB, MonoGame's LZX flag 0x80 or LZ4 flag 0x40 as in Stardew Valley, is decompressed first, `xnb_decompress`) each hold one MGFX effect, and `main` holds SpriteBatch's built-in one; no other file and no GLSLC output. MGFX version 8 is MonoGame's own layout: `MGFX`, u8 version, u8 profile, u32 effect key, the constant buffers, then each shader as u8 vertex (1) / pixel (0), u32 length, the bytecode (u32 control length, the control section, u32 code length, the code area: GLSLC's output without its header), the samplers, the constant buffer indices and an attribute list (as MonoGame's OpenGL backend writes it), then the parameters and the techniques with their passes (`mgfx_effect`). The reader parses every effect to its end, which checks the layout; in `main`, four bytes `MGFX` count only when version 8 and a control section follow. A pass names its vertex and pixel shader by index, 255 for none, and 10 of the 13 effects have no vertex shader: SpriteBatch binds its own and the effect's pass only replaces the pixel shader. Vertex and pixel shaders therefore combine across effects, and they are paired as single-stage programs. 26 shaders (20 pixel, 6 vertex), all with Mario's binding layout; every binding decodes. - **GLSLC outputs in FromSoftware's dvdbnd archives** (Dark Souls Remastered): RomFS is two archives, `dvdbnd0.bdt` and `dvdbnd1.bdt` (3.6 GB), each indexed by the `.bhd5` beside it: `BHD5`, 0xff (little-endian), u32 1, u32 file size, u32 bucket count, u32 0, u64 bucket table offset; each 16-byte bucket `{u32 count, u32, u64 offset}` points at 16-byte entries `{u32 path hash, u32 size, u64 offset}` (paths only hashed). 5,184 entries are DCX containers (`DCX\0`, big-endian header, `DFLT` = a zlib stream at +0x4c), most of them BND3 archives (28-byte entries `{u8 flags, i32 size, u64 offset, i32 ID, u32 name offset, i32 unpacked size}` in formats 0x5c / 0x7c). Six BND3 archives in `dvdbnd0.bdt` hold the shaders, one GLSLC output per entry (`.vpo` vertex, `.fpo` pixel, named `Shader/FRPG_Sfx/NVNShaderBinary/ps/…` or by base name in the two FLVER material archives, 2,602 + 486 entries): `bhd5_entries`, `undcx`, `bnd3_members`, `bdt_shaders` in `shader_split.py`, in 64 jobs by entry range, about 10 s. Only archives that contain a GLSLC magic are parsed (the others use format 0x78). SDK 4's GLSLC needs its section headers read (Dark Souls Remastered in section 6). - **BAHS** (agl `sharcb`, engine utility shaders): code region at header +0x1c (size +0x18). Binary entries in the table area are 16 bytes `{code offset, 0, control size (0x880/0x88c; 0x848 in BAHS version 9, Super Mario 3D World), signed offset from this entry to the control section}`. Control sections are shared between programs, so pair by entry, never by searching for code near a control. - **Shorter control sections (SDK 10, Super Mario 3D World + Bowser's Fury)**: the control section stores its own length at +0x20. Mario's, Engage's, Dread's and 13 Sentinels' are all at least 0x880 bytes; 3D World has 2,888 of 0x850 bytes (its ShaderData bfsha and six layouts) and 0x848 in BAHS version 9, each followed directly by other data. The key therefore hashes the first `min(0x880, control +0x20)` bytes (`ctl_len` in `shader_split.py`, `keys_hash` in `hle_nvn.c`); the other games' tables came out byte-identical. diff --git a/scripts/build_ipa_from_nsp.py b/scripts/build_ipa_from_nsp.py index d9c383c..2cf3391 100755 --- a/scripts/build_ipa_from_nsp.py +++ b/scripts/build_ipa_from_nsp.py @@ -528,7 +528,7 @@ def survey(nsp_dir, app, fam, keys, repo, py, env): elif rom['mags']['PlatinumGames PKZ']: notes.append('PlatinumGames PKZ archives (pkzl; Bayonetta 3): DAT archives inside, compressed per entry (OodleKraken, ZStandard or None); the shaders are shader objects (VSO / PSO / CSO …) in DATs, which shader_split.py reads (pkz_entries, pg_shader), Oodle Kraken through the game\'s own streaming decoder in the Mac host (dm/host/hle_oodle.c)') elif rom['mags']['MT Framework ARC']: notes.append('MT Framework ARC archives (Capcom, The Great Ace Attorney Chronicles): the shaders are in a shader package (SPK) inside one of them, which shader_split.py reads (mtarc_members, mtspk_shaders)') elif rom['mags']['Bezel Engine archive']: notes.append('NintendoWare Bezel Engine archives (.bea, magic SCNE; Big Brain Academy: Brain vs. Brain): every entry is a zstd frame, and the shaders are BNSH in their .bfsha entries, which shader_split.py reads (bea_members)') - elif rom['mags']['XNB'] and 'MonoGame' in F['markers']: notes.append('MonoGame (XNB content): the shaders are MonoGame effects (MGFX version 8, NVN control sections and code areas) in the XNB files EffectReader reads, and SpriteBatch\'s in main, which shader_split.py reads (mgfx_effect; Celeste). A compressed XNB (LZX / LZ4) is not read') + elif rom['mags']['XNB'] and 'MonoGame' in F['markers']: notes.append('MonoGame (XNB content): the shaders are MonoGame effects (MGFX version 8, NVN control sections and code areas) in the XNB files EffectReader reads, and SpriteBatch\'s in main, which shader_split.py reads (mgfx_effect; Celeste), a compressed XNB (MonoGame\'s LZX or LZ4, Stardew Valley) after decompressing it (xnb_decompress)') elif not (rom['mags']['BNSH'] or rom['mags']['BAHS']) and not rom['mags']['UnityFS'] and not vulkan and not gl and not F['glslc']: notes.append('no BNSH / BAHS at the top level: the shaders are inside compressed or other containers or in main, and must be located first') major = {v.split('_')[0] for v in F['sdkver']}