Fix GX_NRM_NBT and GX_NRM_NBT3 vertex attribute handling (#181)

Ahmad8864 · web-flow · commit 2c1e8ed840a5 · 2026-06-03T20:52:26.000-06:00
* Fix GX_NRM_NBT and GX_NRM_NBT3 vertex attribute handling

* Fix GX_NRM_NBT3 indexing, add GX_TG_BINRM/GX_TG_TANGENT support

* Add GX_TG_BUMP emboss bump mapping support
diff --git a/lib/gx/command_processor.cpp b/lib/gx/command_processor.cpp
@@ -106,13 +106,13 @@ static constexpr u8 CP_VAT_MASK = GX_VAT_MASK;
 
 // Read helpers for big/little endian
 #if _MSC_VER
-template<typename T>
+template <typename T>
 __forceinline // Yes, this was necessary.
-inline T unaligned_load(const T* ptr) {
+    inline T unaligned_load(const T* ptr) {
   return *static_cast<const __unaligned T*>(ptr);
 }
 #else
-template<typename T>
+template <typename T>
 inline T unaligned_load(const T* ptr) {
   T copy;
   memcpy(&copy, ptr, sizeof(T));
@@ -502,7 +502,8 @@ static void handle_bp(u32 value, bool bigEndian) {
     g_gxState.bpRegCache[0xFE] = 0x00FFFFFF;
     const u32 merged = (g_gxState.bpRegCache[regId] & ~ssMask) | (value & ssMask);
     value = (regId << 24) | (merged & 0x00FFFFFF);
-    if (g_gxState.bpRegCache[regId] == value) return;
+    if (g_gxState.bpRegCache[regId] == value)
+      return;
     g_gxState.bpRegCache[regId] = value;
   }
 
@@ -1179,7 +1180,9 @@ static void handle_cp(u8 addr, u32 value, bool bigEndian) {
       vf.attrs[GX_VA_POS].cnt = static_cast<GXCompCnt>(bp_get(value, 1, 0));
       vf.attrs[GX_VA_POS].type = static_cast<GXCompType>(bp_get(value, 3, 1));
       vf.attrs[GX_VA_POS].frac = static_cast<u8>(bp_get(value, 5, 4));
-      vf.attrs[GX_VA_NRM].cnt = static_cast<GXCompCnt>(bp_get(value, 1, 9));
+      const auto nrm_cnt = bp_get(value, 1, 9);
+      const auto nrm_nbt3 = bp_get(value, 1, 31);
+      vf.attrs[GX_VA_NRM].cnt = static_cast<GXCompCnt>(nrm_nbt3 ? GX_NRM_NBT3 : (nrm_cnt ? GX_NRM_NBT : GX_NRM_XYZ));
       vf.attrs[GX_VA_NRM].type = static_cast<GXCompType>(bp_get(value, 3, 10));
       if (vf.attrs[GX_VA_NRM].type == GX_U8 || vf.attrs[GX_VA_NRM].type == GX_S8) {
         vf.attrs[GX_VA_NRM].frac = 6;
@@ -1280,8 +1283,10 @@ static void handle_xf(const u8* data, u32& pos, u32 size, bool bigEndian) {
       u32 val = read_u32(xfData + i * 4, bigEndian);
 
       // Skip scalar register writes that haven't changed (viewport/projection handled below)
-      if (reg <= 0x19 && val == g_gxState.xfRegCache[reg]) continue;
-      if (reg <= 0x19) g_gxState.xfRegCache[reg] = val;
+      if (reg <= 0x19 && val == g_gxState.xfRegCache[reg])
+        continue;
+      if (reg <= 0x19)
+        g_gxState.xfRegCache[reg] = val;
 
       switch (reg) {
       case 0x08:
@@ -1450,11 +1455,13 @@ static void handle_xf(const u8* data, u32& pos, u32 size, bool bigEndian) {
             if (tgType == 0) {
               tcg.type = proj ? GX_TG_MTX3x4 : GX_TG_MTX2x4;
             } else if (tgType == 1) {
-              // Bump mapping
+              // Bump mapping: type encodes emboss light
               tcg.type = static_cast<GXTexGenType>(bp_get(val, 3, 15) + 2);
             } else if (tgType == 2 || tgType == 3) {
               tcg.type = GX_TG_SRTG;
             }
+            // Emboss source texcoord (bits 12-14); 0 for non-bump types
+            tcg.embossSrc = bp_get(val, 3, 12);
 
             // Decode source from row
             static const GXTexGenSrc rowToSrc[] = {GX_TG_POS,  GX_TG_NRM,  GX_TG_COLOR0, GX_TG_BINRM, GX_TG_TANGENT,
@@ -1516,10 +1523,10 @@ static u32 calculate_last_vtx_size(GXVtxFmt fmt) {
       break;
     }
     case GX_INDEX8:
-      vtxSize += 1;
+      vtxSize += (i == GX_VA_NRM && vtxFmt.attrs[i].cnt == GX_NRM_NBT3) ? 3 : 1;
       break;
     case GX_INDEX16:
-      vtxSize += 2;
+      vtxSize += (i == GX_VA_NRM && vtxFmt.attrs[i].cnt == GX_NRM_NBT3) ? 6 : 2;
       break;
     }
   }
@@ -1546,16 +1553,14 @@ static void handle_draw(u8 cmd, const u8* data, u32& pos, u32 size, bool bigEndi
   pos += 2;
 
   u32 vtxSize;
-  if (g_gxState.lastVtxFmt == fmt) LIKELY {
-    vtxSize = g_gxState.lastVtxSize;
-  } else UNLIKELY {
-    vtxSize = calculate_last_vtx_size(fmt);
-  }
+  if (g_gxState.lastVtxFmt == fmt)
+    LIKELY { vtxSize = g_gxState.lastVtxSize; }
+  else
+    UNLIKELY { vtxSize = calculate_last_vtx_size(fmt); }
 
   u32 totalVtxBytes = vtxCount * vtxSize;
-  if (pos + totalVtxBytes > size) UNLIKELY {
-    handle_draw_overrun(totalVtxBytes, data, pos, size);
-  }
+  if (pos + totalVtxBytes > size)
+    UNLIKELY { handle_draw_overrun(totalVtxBytes, data, pos, size); }
 
   // Push raw vertex data to buffer
   gfx::Range vertRange = gfx::push_verts(data + pos, totalVtxBytes);
diff --git a/lib/gx/gx.cpp b/lib/gx/gx.cpp
@@ -720,6 +720,9 @@ u8 comp_cnt_count(GXAttr attr, GXCompCnt cnt) noexcept {
     switch (cnt) {
     case GX_NRM_XYZ:
       return 3;
+    case GX_NRM_NBT:
+    case GX_NRM_NBT3:
+      return 9;
     default:
       break;
     }
@@ -766,6 +769,7 @@ void populate_pipeline_config(PipelineConfig& config, GXPrimitive primitive, GXV
     }
     const auto& attrFmt = vtxFmt.attrs[i];
     const auto cnt = comp_cnt_count(attr, attrFmt.cnt);
+    const bool nbt3 = attr == GX_VA_NRM && attrFmt.cnt == GX_NRM_NBT3;
     mapping = AttrConfig{
         .attrType = static_cast<u8>(type),
         .cnt = cnt,
@@ -774,6 +778,7 @@ void populate_pipeline_config(PipelineConfig& config, GXPrimitive primitive, GXV
         .stride = 0,
         .frac = attrFmt.frac,
         .le = false,
+        .nbt3 = nbt3,
     };
     switch (type) {
     case GX_DIRECT: {
@@ -783,12 +788,12 @@ void populate_pipeline_config(PipelineConfig& config, GXPrimitive primitive, GXV
     case GX_INDEX8:
       mapping.stride = g_gxState.arrays[i].stride;
       mapping.le = g_gxState.arrays[i].le;
-      vtxOffset += 1;
+      vtxOffset += nbt3 ? 3 : 1;
       break;
     case GX_INDEX16:
       mapping.stride = g_gxState.arrays[i].stride;
       mapping.le = g_gxState.arrays[i].le;
-      vtxOffset += 2;
+      vtxOffset += nbt3 ? 6 : 2;
       break;
     default:
       Log.fatal("populate_pipeline_config: Invalid vertex type {}", type);
diff --git a/lib/gx/gx.hpp b/lib/gx/gx.hpp
@@ -158,7 +158,7 @@ struct TcgConfig {
   GXTexMtx mtx = GX_IDENTITY;
   GXPTTexMtx postMtx = GX_PTIDENTITY;
   bool normalize = false;
-  u8 _p1 = 0;
+  u8 embossSrc = 0; // Emboss source texcoord (GX_TG_BUMP*)
   u8 _p2 = 0;
   u8 _p3 = 0;
 
@@ -441,7 +441,7 @@ struct AttrConfig {
   u8 stride = 0;         // Array stride
   u8 frac = 0;
   bool le = true;
-  u8 _p1 = 0;
+  bool nbt3 = false;     // GX_NRM_NBT3
 };
 struct ShaderConfig {
   u8 fogType = GX_FOG_NONE;
@@ -492,6 +492,7 @@ struct BindGroupRanges {
 void populate_pipeline_config(PipelineConfig& config, GXPrimitive primitive, GXVtxFmt fmt) noexcept;
 wgpu::RenderPipeline build_pipeline(const PipelineConfig& config, ArrayRef<wgpu::VertexBufferLayout> vtxBuffers,
                                     wgpu::ShaderModule shader, const char* label) noexcept;
+std::string build_shader_source(const ShaderConfig& config) noexcept;
 wgpu::ShaderModule build_shader(const ShaderConfig& config) noexcept;
 GXBindGroups build_bind_groups(const ShaderInfo& info) noexcept;
 
diff --git a/lib/gx/shader.cpp b/lib/gx/shader.cpp
@@ -629,25 +629,35 @@ auto fetch_color_attr(const AttrConfig& mapping, std::string_view buf, std::stri
   }
 }
 
+struct AttrAddress {
+  std::string offs;
+  std::string_view buf;
+  bool le;
+};
+
+auto attr_address(const AttrConfig& mapping, GXAttr attr, std::string_view vidx, u32 vtxStride, u32 dlExtra, u32 within)
+    -> AttrAddress {
+  const u32 dlOffset = mapping.offset + dlExtra;
+  if (mapping.attrType == GX_INDEX8) {
+    return {fmt::format("ubuf.array_start[{}] + raw_fetch_u8_1(&vbuf, ubuf.vtx_start + {} * {}u + {}u) * {}u + {}u",
+                        attr - GX_VA_POS, vidx, vtxStride, dlOffset, mapping.stride, within),
+            "abuf"sv, mapping.le};
+  }
+  if (mapping.attrType == GX_INDEX16) {
+    return {
+        fmt::format("ubuf.array_start[{}] + raw_fetch_u16_1(&vbuf, ubuf.vtx_start + {} * {}u + {}u, false) * {}u + {}u",
+                    attr - GX_VA_POS, vidx, vtxStride, dlOffset, mapping.stride, within),
+        "abuf"sv, mapping.le};
+  }
+  return {fmt::format("ubuf.vtx_start + {} * {}u + {}u", vidx, vtxStride, dlOffset + within), "vbuf"sv, false};
+}
+
 auto attr_load(const ShaderConfig& config, GXAttr attr, std::string_view vidx) -> std::string {
   const auto& mapping = config.attrs[attr];
   if (mapping.attrType == GX_NONE) {
     return vtx_attr(config, attr);
   }
-  auto buf = "vbuf"sv;
-  auto offs = fmt::format("ubuf.vtx_start + {} * {}u + {}u", vidx, config.vtxStride, mapping.offset);
-  auto le = false; // Vertex buffer is always big endian (for now)
-  if (mapping.attrType == GX_INDEX8) {
-    offs = fmt::format("ubuf.array_start[{}] + raw_fetch_u8_1(&{}, {}) * {}u", attr - GX_VA_POS, buf, offs,
-                       mapping.stride);
-    buf = "abuf"sv;
-    le = mapping.le;
-  } else if (mapping.attrType == GX_INDEX16) {
-    offs = fmt::format("ubuf.array_start[{}] + raw_fetch_u16_1(&{}, {}, {}) * {}u", attr - GX_VA_POS, buf, offs, le,
-                       mapping.stride);
-    buf = "abuf"sv;
-    le = mapping.le;
-  }
+  const auto [offs, buf, le] = attr_address(mapping, attr, vidx, config.vtxStride, 0u, 0u);
   switch (attr) {
   case GX_VA_PNMTXIDX:
     return fmt::format("(raw_fetch_u8_1(&{}, {}) / 3u)", buf, offs);
@@ -668,7 +678,12 @@ auto attr_load(const ShaderConfig& config, GXAttr attr, std::string_view vidx) -
     return posLoad;
   }
   case GX_VA_NRM:
-    // TODO check for NBT/NBT3
+    // NBT: normal only here; binormal/tangent loaded via attr_load_nbt_slice
+    if (mapping.cnt > 3) {
+      auto nrmMapping = mapping;
+      nrmMapping.cnt = 3;
+      return fetch_attr(nrmMapping, buf, offs, le);
+    }
     return fetch_attr(mapping, buf, offs, le);
   case GX_VA_CLR0:
   case GX_VA_CLR1:
@@ -692,6 +707,42 @@ auto attr_load(const ShaderConfig& config, GXAttr attr, std::string_view vidx) -
   }
 }
 
+enum class NbtSlice : u8 {
+  N,
+  B,
+  T,
+};
+
+auto attr_load_nbt_slice(const ShaderConfig& config, NbtSlice slice, std::string_view vidx) -> std::string {
+  const auto& mapping = config.attrs[GX_VA_NRM];
+  if (mapping.attrType == GX_NONE || mapping.cnt != 9) {
+    Log.fatal("attr_load_nbt_slice: GX_TG_BINRM/TANGENT requires GX_NRM_NBT or GX_NRM_NBT3");
+  }
+  const auto sliceIdx = static_cast<u32>(slice);
+  const auto compsize = comp_type_size(GX_VA_NRM, static_cast<GXCompType>(mapping.compType));
+  u32 dlExtra = 0;
+  if (mapping.nbt3) {
+    if (mapping.attrType == GX_INDEX8) {
+      dlExtra = sliceIdx;
+    } else if (mapping.attrType == GX_INDEX16) {
+      dlExtra = sliceIdx * 2u;
+    }
+  }
+  const u32 within = sliceIdx * 3u * compsize;
+  const auto [offs, buf, le] = attr_address(mapping, GX_VA_NRM, vidx, config.vtxStride, dlExtra, within);
+  auto sliceMapping = mapping;
+  sliceMapping.cnt = 3;
+  return fetch_attr(sliceMapping, buf, offs, le);
+}
+
+static constexpr std::string_view nbt_slice_local(NbtSlice slice) noexcept {
+  return slice == NbtSlice::B ? "in_binrm" : "in_tangent";
+}
+
+static constexpr bool is_emboss_texgen(GXTexGenType type) noexcept {
+  return type >= GX_TG_BUMP0 && type <= GX_TG_BUMP7;
+}
+
 auto lighting_func(const ShaderConfig& config, const ColorChannelConfig& cc, u8 i, bool alpha) -> std::string {
   std::string_view swizzle = alpha ? ".a"sv : ""sv;
   std::string outVar;
@@ -783,7 +834,7 @@ auto lighting_func(const ShaderConfig& config, const ColorChannelConfig& cc, u8
                      alpha ? "a"sv : ""sv);
 }
 
-wgpu::ShaderModule build_shader(const ShaderConfig& config) noexcept {
+std::string build_shader_source(const ShaderConfig& config) noexcept {
   ZoneScoped;
   const auto hash = xxh3_hash(config);
   const auto info = build_shader_info(config);
@@ -916,6 +967,24 @@ wgpu::ShaderModule build_shader(const ShaderConfig& config) noexcept {
       vtxXfrAttrsPre += fmt::format("\n    let {} = {};", vtx_attr(config, attr), attr_load(config, attr, vidxAttr));
     }
   }
+  bool needsBinrm = false;
+  bool needsTangent = false;
+  for (int i = 0; i < info.sampledTexCoords.size(); ++i) {
+    if (!info.sampledTexCoords.test(i)) {
+      continue;
+    }
+    const bool emboss = is_emboss_texgen(config.tcgs[i].type);
+    needsBinrm = needsBinrm || config.tcgs[i].src == GX_TG_BINRM || emboss;
+    needsTangent = needsTangent || config.tcgs[i].src == GX_TG_TANGENT || emboss;
+  }
+  if (needsBinrm) {
+    vtxXfrAttrsPre += fmt::format("\n    let {} = {};", nbt_slice_local(NbtSlice::B),
+                                  attr_load_nbt_slice(config, NbtSlice::B, vidxAttr));
+  }
+  if (needsTangent) {
+    vtxXfrAttrsPre += fmt::format("\n    let {} = {};", nbt_slice_local(NbtSlice::T),
+                                  attr_load_nbt_slice(config, NbtSlice::T, vidxAttr));
+  }
 
   if (config.lineMode == 0) {
     vtxXfrAttrsPre += fmt::format(
@@ -1097,6 +1166,19 @@ wgpu::ShaderModule build_shader(const ShaderConfig& config) noexcept {
     } else {
       vtxOutAttrs += fmt::format("\n    @location({}) tex{}_uv: vec2f,", vtxOutIdx++, i);
     }
+    if (is_emboss_texgen(tcg.type)) {
+      // Emboss bump: offset the source texcoord by the light projected onto tangent/binormal
+      const u32 lightIdx = tcg.type - GX_TG_BUMP0;
+      vtxXfrAttrs += fmt::format(
+          "\n    let bump_ldir{0} = normalize(ubuf.lights[{1}].pos - mv_pos);"
+          "\n    let bump_tan{0} = vec4f(in_tangent, 0.0) * ubuf.nrm_mtx[in_pnmtxidx];"
+          "\n    let bump_bin{0} = vec4f(in_binrm, 0.0) * ubuf.nrm_mtx[in_pnmtxidx];"
+          "\n    out.tex{0}_uv = tc{2}_proj.xy + vec2f(dot(bump_ldir{0}, bump_tan{0}), dot(bump_ldir{0}, "
+          "bump_bin{0}));",
+          i, lightIdx, tcg.embossSrc);
+      fragmentFnPre += fmt::format("\n    var tex{0}_uv = in.tex{0}_uv.xy;", i);
+      continue;
+    }
     if (tcg.src >= GX_TG_TEX0 && tcg.src <= GX_TG_TEX7) {
       vtxXfrAttrs += fmt::format("\n    var tc{} = vec4f({}, 1.0, 1.0);", i,
                                  vtx_attr(config, GXAttr(GX_VA_TEX0 + (tcg.src - GX_TG_TEX0))));
@@ -1108,6 +1190,10 @@ wgpu::ShaderModule build_shader(const ShaderConfig& config) noexcept {
       vtxXfrAttrs += fmt::format("\n    var tc{} = {};", i, vtx_attr(config, GX_VA_CLR0));
     } else if (tcg.src == GX_TG_COLOR1) {
       vtxXfrAttrs += fmt::format("\n    var tc{} = {};", i, vtx_attr(config, GX_VA_CLR1));
+    } else if (tcg.src == GX_TG_BINRM) {
+      vtxXfrAttrs += fmt::format("\n    var tc{} = vec4f({}, 1.0);", i, nbt_slice_local(NbtSlice::B));
+    } else if (tcg.src == GX_TG_TANGENT) {
+      vtxXfrAttrs += fmt::format("\n    var tc{} = vec4f({}, 1.0);", i, nbt_slice_local(NbtSlice::T));
     } else
       UNLIKELY FATAL("unhandled tcg src {}", underlying(tcg.src));
     if (tcg.type == GX_TG_MTX2x4 || tcg.type == GX_TG_MTX3x4) {
@@ -1468,8 +1554,9 @@ wgpu::ShaderModule build_shader(const ShaderConfig& config) noexcept {
     if (discard.constant == 1) {
       fragmentFn += "\n    // Alpha compare\n    discard;";
     } else if (discard.constant != 0) {
-      fragmentFn += "\n    // Alpha compare"
-                    "\n    let alphaCompare = u32(round(clamp(prev.a, 0.0, 1.0) * 255.0));";
+      fragmentFn +=
+          "\n    // Alpha compare"
+          "\n    let alphaCompare = u32(round(clamp(prev.a, 0.0, 1.0) * 255.0));";
       fragmentFn += fmt::format("\n    if ({}) {{ discard; }}", discard.expr);
     }
   }
@@ -1844,6 +1931,13 @@ fn fs_main(in: VertexOutput) -> @location(0) vec4f {{{6}{5}
     Log.info("Generated shader: {}", shaderSource);
   }
 
+  return shaderSource;
+}
+
+wgpu::ShaderModule build_shader(const ShaderConfig& config) noexcept {
+  ZoneScoped;
+  const auto shaderSource = build_shader_source(config);
+  const auto hash = xxh3_hash(config);
   wgpu::ShaderSourceWGSL wgslDescriptor{};
   wgslDescriptor.code = shaderSource.c_str();
   const auto label = fmt::format("GX Shader {:x}", hash);
diff --git a/lib/gx/shader_info.cpp b/lib/gx/shader_info.cpp
@@ -257,6 +257,18 @@ ShaderInfo build_shader_info(const ShaderConfig& config) noexcept {
     }
   }
 
+  // Emboss bump needs its source texcoord generated and a light enabled
+  for (int i = 0; i < info.sampledTexCoords.size(); ++i) {
+    if (!info.sampledTexCoords.test(i)) {
+      continue;
+    }
+    const auto& tcg = config.tcgs[i];
+    if (tcg.type >= GX_TG_BUMP0 && tcg.type <= GX_TG_BUMP7) {
+      info.sampledTexCoords.set(tcg.embossSrc);
+      info.lightingEnabled = true;
+    }
+  }
+
   info.uniformSize += info.loadsTevReg.count() * sizeof(Vec4<float>);
   for (int i = 0; i < info.sampledColorChannels.size(); ++i) {
     if (info.sampledColorChannels.test(i)) {
