Very big splat causes artifact in perspective

[WindyYam]

It looks like current shader code try to render splat on the world space splat center's screen projection
I try to align a box with the splat in center, now the splat covers the whole box, the splat ellipsoid should be at this position

But when I look close up, it is like this, the axes are at the object center

Apparently the shader try to render the on screen 2d splat at the 3d center position, but when splat is large and in oblique angle, the 3d center is not necessarily the 2d center of the area where the splat should cover
Moreover, when in extreme angle this artifact happens

This happens most for the gaussian scene generated by nerfstudio mcmc strategy, as it seems mcmc strategy prefer to generate very large and sharp gaussians
Those artifacts will not happen when running in render box debugging shader
Reference issue: nerfstudio-project/gsplat#785

[WindyYam]

Here is my current workaround (solution pulled out from chatGPT5):
In GaussianSplatting.hlsl, add the part where it outputs a screen pixel space center2D

float3 CalcCovariance2D(float3 worldPos, float3 cov3d0, float3 cov3d1, float4x4 matrixV, float4x4 matrixP, float4 screenParams, out float2 center2D)
{
    float4x4 viewMatrix = matrixV;
    float3 viewPos = mul(viewMatrix, float4(worldPos, 1)).xyz;

    // this is needed in order for splats that are visible in view but clipped "quite a lot" to work
    float aspect = matrixP._m00 / matrixP._m11;
    float tanFovX = rcp(matrixP._m00);
    float tanFovY = rcp(matrixP._m11 * aspect);
    float limX = 1.3 * tanFovX;
    float limY = 1.3 * tanFovY;
    float focal = screenParams.x * matrixP._m00 / 2;

    float z = viewPos.z;
    float2 screenPos = float2(focal * viewPos.x / z, focal * viewPos.y / z);

    viewPos.x = clamp(viewPos.x / viewPos.z, -limX, limX) * viewPos.z;
    viewPos.y = clamp(viewPos.y / viewPos.z, -limY, limY) * viewPos.z;

    float3x3 J = float3x3(
        focal / viewPos.z, 0, -(focal * viewPos.x) / (viewPos.z * viewPos.z),
        0, focal / viewPos.z, -(focal * viewPos.y) / (viewPos.z * viewPos.z),
        0, 0, 0
    );
    float3x3 W = (float3x3)viewMatrix;
    float3x3 T = mul(J, W);
    float3x3 V = float3x3(
        cov3d0.x, cov3d0.y, cov3d0.z,
        cov3d0.y, cov3d1.x, cov3d1.y,
        cov3d0.z, cov3d1.y, cov3d1.z
    );
    float3x3 cov = mul(T, mul(V, transpose(T)));

    // Mean-shift correction terms derived from Taylor expansion around the 3D mean.
    // IMPORTANT: use view-space covariance Σ_view = W * Σ_world * W^T
    float3x3 Sv = mul(W, mul(V, transpose(W)));
    float Vxz = Sv._m02;
    float Vyz = Sv._m12;
    float Vzz = Sv._m22;
    float invZ2 = rcp(z * z);
    float invZ3 = invZ2 * rcp(z);
    float2 meanShift = float2(
        focal * (-Vxz * invZ2 + viewPos.x * Vzz * invZ3),
        focal * (-Vyz * invZ2 + viewPos.y * Vzz * invZ3)
    );

    center2D = (screenPos + 2.0 /*3.0*/ * meanShift) * float2(-1, 1) + screenParams.xy * 0.5;

    // Low pass filter to make each splat at least 1px size.
    cov._m00 += 0.3;
    cov._m11 += 0.3;
    return float3(cov._m00, cov._m01, cov._m11);
}

In SplatUtilities.compute, offset the clip space splat pos:

void CSCalcViewData (uint3 id : SV_DispatchThreadID)
{
    uint idx = id.x;
    if (idx >= _SplatCount)
        return;

    SplatData splat = LoadSplatData(idx);
    SplatViewData view = (SplatViewData)0;
    
    float3 centerWorldPos = mul(_MatrixObjectToWorld, float4(splat.pos,1)).xyz;
    float4 centerClipPos = mul(UNITY_MATRIX_VP, float4(centerWorldPos, 1));
    half opacityScale = _SplatOpacityScale;
    float splatScale = _SplatScale;

    // deleted?
    if (_SplatBitsValid)
    {
        uint wordIdx = idx / 32;
        uint bitIdx = idx & 31;
        uint wordVal = _SplatDeletedBits.Load(wordIdx * 4);
        if (wordVal & (1 << bitIdx))
        {
            centerClipPos.w = 0;
        }
    }

    // cutouts
    if (IsSplatCut(splat.pos))
    {
        centerClipPos.w = 0;
    }

    view.pos = centerClipPos;
    bool behindCam = centerClipPos.w <= 0;
    if (!behindCam)
    {
        float4 boxRot = splat.rot;
        float3 boxSize = splat.scale;

        float3x3 splatRotScaleMat = CalcMatrixFromRotationScale(boxRot, boxSize);

        float3 cov3d0, cov3d1;
        CalcCovariance3D(splatRotScaleMat, cov3d0, cov3d1);
        float splatScale2 = splatScale * splatScale;
        cov3d0 *= splatScale2;
        cov3d1 *= splatScale2;
        
        // Calculate both covariance and screen 2D center in one call
        float2 screenCenter2D;
        float3 cov2d = CalcCovariance2D(splat.pos, cov3d0, cov3d1, _MatrixMV, UNITY_MATRIX_P, _VecScreenParams, screenCenter2D);
        view.pos.xy = (screenCenter2D * 2/_VecScreenParams.xy - 1) * view.pos.w;

        DecomposeCovariance(cov2d, view.axis1, view.axis2);

        float3 worldViewDir = _VecWorldSpaceCameraPos.xyz - centerWorldPos;
        float3 objViewDir = mul((float3x3)_MatrixWorldToObject, worldViewDir);
        objViewDir = normalize(objViewDir);

        half4 col;
        col.rgb = ShadeSH(splat.sh, objViewDir, _SHOrder, _SHOnly != 0);
        col.a = min(splat.opacity * opacityScale, 65000);
        view.color.x = (f32tof16(col.r) << 16) | f32tof16(col.g);
        view.color.y = (f32tof16(col.b) << 16) | f32tof16(col.a);
    }
    
    _SplatViewData[idx] = view;
}

This is not necessarily the best and most accurate solution, since it comes from chatGPT, but does ease the pain those giant spiky splats created

[WindyYam]

A bit of comparison if it helps to explain the problem
Before:
https://github.com/user-attachments/assets/134e3414-8b68-4b56-93c4-cba38d147fab
After:
https://github.com/user-attachments/assets/c6eec2f7-00f1-4b4c-80de-bc18b08a2abf

[trober]

This fixed issues I had with oversized nearby splats when camera fov increased (from OpenXR views or just manual main camera fov of 110 degrees). Saw the same thing where the debug boxes were fine.
Thank you!

[machinesarenotpeople]

I got projection errors on Quest 3 after implementing the fix (though it fixed the splat projection).

Here is a fix.

GaussianSplatting.hlsl

float3 CalcCovariance2D(float3 worldPos, float3 cov3d0, float3 cov3d1, float4x4 matrixV, float4x4 matrixP, float4 screenParams, out float2 centerClipSpaceShift)
{
    float4x4 viewMatrix = matrixV;
    float3 viewPos = mul(viewMatrix, float4(worldPos, 1)).xyz;

    // this is needed in order for splats that are visible in view but clipped "quite a lot" to work
    float aspect = matrixP._m00 / matrixP._m11;
    float tanFovX = rcp(matrixP._m00);
    float tanFovY = rcp(matrixP._m11 * aspect);
    float limX = 1.3 * tanFovX;
    float limY = 1.3 * tanFovY;
    viewPos.x = clamp(viewPos.x / viewPos.z, -limX, limX) * viewPos.z;
    viewPos.y = clamp(viewPos.y / viewPos.z, -limY, limY) * viewPos.z;

    float focal = screenParams.x * matrixP._m00 / 2;

    float3x3 J = float3x3(
        focal / viewPos.z, 0, -(focal * viewPos.x) / (viewPos.z * viewPos.z),
        0, focal / viewPos.z, -(focal * viewPos.y) / (viewPos.z * viewPos.z),
        0, 0, 0
    );
    float3x3 W = (float3x3)viewMatrix;
    float3x3 T = mul(J, W);
    float3x3 V = float3x3(
        cov3d0.x, cov3d0.y, cov3d0.z,
        cov3d0.y, cov3d1.x, cov3d1.y,
        cov3d0.z, cov3d1.y, cov3d1.z
    );
    float3x3 cov = mul(T, mul(V, transpose(T)));

    float3x3 Sv = mul(W, mul(V, transpose(W)));
    float Vxz = Sv._m02;
    float Vyz = Sv._m12;
    float Vzz = Sv._m22;
    float z = viewPos.z;
    float invZ2 = rcp(z * z);
    float invZ3 = invZ2 * rcp(z);
    centerClipSpaceShift = float2(
        matrixP._m00 * (Vxz * invZ2 - viewPos.x * Vzz * invZ3),
        matrixP._m11 * (Vyz * invZ2 - viewPos.y * Vzz * invZ3)
    );

    // Low pass filter to make each splat at least 1px size.
    cov._m00 += 0.3;
    cov._m11 += 0.3;
    return float3(cov._m00, cov._m01, cov._m11);
}

SplatUtilities.compute

void CSCalcViewData (uint3 id : SV_DispatchThreadID)
{
    uint idx = id.x;
    if (idx >= _SplatCount)
        return;

    SplatData splat = LoadSplatData(idx);
    SplatViewData view = (SplatViewData)0;
    
    float3 centerWorldPos = mul(_MatrixObjectToWorld, float4(splat.pos,1)).xyz;
    float4 centerClipPos = mul(UNITY_MATRIX_VP, float4(centerWorldPos, 1));
    half opacityScale = _SplatOpacityScale;
    float splatScale = _SplatScale;

    // deleted?
    if (_SplatBitsValid)
    {
        uint wordIdx = idx / 32;
        uint bitIdx = idx & 31;
        uint wordVal = _SplatDeletedBits.Load(wordIdx * 4);
        if (wordVal & (1 << bitIdx))
        {
            centerClipPos.w = 0;
        }
    }

    // cutouts
    if (IsSplatCut(splat.pos))
    {
        centerClipPos.w = 0;
    }

    view.pos = centerClipPos;
    bool behindCam = centerClipPos.w <= 0;
    if (!behindCam)
    {
        float4 boxRot = splat.rot;
        float3 boxSize = splat.scale;

        float3x3 splatRotScaleMat = CalcMatrixFromRotationScale(boxRot, boxSize);

        float3 cov3d0, cov3d1;
        CalcCovariance3D(splatRotScaleMat, cov3d0, cov3d1);
        float splatScale2 = splatScale * splatScale;
        cov3d0 *= splatScale2;
        cov3d1 *= splatScale2;
        float2 clipSpaceShift;
        float3 cov2d = CalcCovariance2D(splat.pos, cov3d0, cov3d1, _MatrixMV, UNITY_MATRIX_P, _VecScreenParams, clipSpaceShift);
        
        DecomposeCovariance(cov2d, view.axis1, view.axis2);
        view.pos.xy += clipSpaceShift * 2.0 * view.pos.w;

        float3 worldViewDir = _VecWorldSpaceCameraPos.xyz - centerWorldPos;
        float3 objViewDir = mul((float3x3)_MatrixWorldToObject, worldViewDir);
        objViewDir = normalize(objViewDir);

        half4 col;
        col.rgb = ShadeSH(splat.sh, objViewDir, _SHOrder, _SHOnly != 0);
        col.a = min(splat.opacity * opacityScale, 65000);
        view.color.x = (f32tof16(col.r) << 16) | f32tof16(col.g);
        view.color.y = (f32tof16(col.b) << 16) | f32tof16(col.a);
    }
    
    _SplatViewData[idx] = view;
}