Harden xtensa call_primitive_last and align encode_rrr with the ISA

call_primitive_last only supports args of the form [_, jit_state | _] (the
windowed-ABI tail call rewrites the second argument to jit_state_tail_call).
Reject every other shape in the function head, before any register allocation
or stream writes, so a scratch-exhausting argument list fails with a
descriptive error rather than an opaque first_avail/1 function_clause.

Also rename encode_rrr's op1/op2 parameters and align the RRR comments
(including SLLI) with the Xtensa ISA field order; emitted bytes are unchanged,
verified by the jit_xtensa_asm round-trip tests (73) and codegen tests (145).

Signed-off-by: Paul Guyot <pguyot@kallisys.net>

Author: pguyot

libs/jit/src/jit_xtensa.erl

@@ -671,7 +671,7 @@ call_primitive_last(
         stream = Stream0
     } = State0,
     Primitive,
-    Args
+    [_, jit_state | _] = Args
 ) ->
     %% Xtensa windowed ABI: CALLX8 to the primitive, move its return value
     %% from a10 (our view of callee's a2) into our a2, then RETW to C.
@@ -716,7 +716,12 @@ call_primitive_last(
                 used_regs = 0,
                 regs = jit_regs:unreachable(State2#state.regs)
             }
-    end.
+    end;
+%% The windowed-ABI tail-call sequence rewrites the second argument
+%% (jit_state -> jit_state_tail_call), so reject every other shape in the
+%% function head before allocating registers or appending to the stream.
+call_primitive_last(_State, _Primitive, Args) ->
+    error({unsupported_call_primitive_last_args, Args}).
 
 %%-----------------------------------------------------------------------------
 %% @doc Emit a return of a value if it's not equal to ctx.

libs/jit/src/jit_xtensa_asm.erl

@@ -144,10 +144,12 @@ reg_to_num(a15) -> 15.
 %% 24-bit Instruction Format Encoders
 %%=============================================================================
 
-%% RRR format: op1[23:20] | op2[19:16] | r[15:12] | s[11:8] | t[7:4] | op0[3:0]
+%% RRR format: op2[23:20] | op1[19:16] | r[15:12] | s[11:8] | t[7:4] | op0[3:0]
+%% Op2/Op1 are ordered high-to-low to match the ISA field layout above; the
+%% emitted bytes are unchanged from before this parameter rename.
 -spec encode_rrr(integer(), integer(), integer(), integer(), integer(), integer()) -> binary().
-encode_rrr(Op0, T, S, R, Op1, Op2) ->
-    Instr = (Op1 bsl 20) bor (Op2 bsl 16) bor (R bsl 12) bor (S bsl 8) bor (T bsl 4) bor Op0,
+encode_rrr(Op0, T, S, R, Op2, Op1) ->
+    Instr = (Op2 bsl 20) bor (Op1 bsl 16) bor (R bsl 12) bor (S bsl 8) bor (T bsl 4) bor Op0,
     <<Instr:24/little>>.
 
 %% RRI8 format: imm8[23:16] | r[15:12] | s[11:8] | t[7:4] | op0[3:0]
@@ -196,31 +198,31 @@ encode_rrrn(Op0, T, S, R) ->
 %%=============================================================================
 
 %% ADD: AR[r] = AR[s] + AR[t]
-%% op0=0, op1=8, op2=0
+%% op0=0, op1=0, op2=8
 -spec add(xtensa_register(), xtensa_register(), xtensa_register()) -> binary().
 add(Ar, As, At) ->
     encode_rrr(0, reg_to_num(At), reg_to_num(As), reg_to_num(Ar), 16#8, 0).
 
 %% SUB: AR[r] = AR[s] - AR[t]
-%% op0=0, op1=12, op2=0
+%% op0=0, op1=0, op2=12
 -spec sub(xtensa_register(), xtensa_register(), xtensa_register()) -> binary().
 sub(Ar, As, At) ->
     encode_rrr(0, reg_to_num(At), reg_to_num(As), reg_to_num(Ar), 16#C, 0).
 
 %% AND: AR[r] = AR[s] & AR[t]
-%% op0=0, op1=1, op2=0
+%% op0=0, op1=0, op2=1
 -spec and_(xtensa_register(), xtensa_register(), xtensa_register()) -> binary().
 and_(Ar, As, At) ->
     encode_rrr(0, reg_to_num(At), reg_to_num(As), reg_to_num(Ar), 16#1, 0).
 
 %% OR: AR[r] = AR[s] | AR[t]
-%% op0=0, op1=2, op2=0
+%% op0=0, op1=0, op2=2
 -spec or_(xtensa_register(), xtensa_register(), xtensa_register()) -> binary().
 or_(Ar, As, At) ->
     encode_rrr(0, reg_to_num(At), reg_to_num(As), reg_to_num(Ar), 16#2, 0).
 
 %% XOR: AR[r] = AR[s] ^ AR[t]
-%% op0=0, op1=3, op2=0
+%% op0=0, op1=0, op2=3
 -spec xor_(xtensa_register(), xtensa_register(), xtensa_register()) -> binary().
 xor_(Ar, As, At) ->
     encode_rrr(0, reg_to_num(At), reg_to_num(As), reg_to_num(Ar), 16#3, 0).
@@ -243,14 +245,14 @@ srl(Ar, _As, At) ->
     encode_rrr(0, reg_to_num(At), 0, reg_to_num(Ar), 16#9, 16#1).
 
 %% SSR: Set SAR for right shift. SAR = AR[s][4:0]
-%% op0=0, op1=4, op2=0, r=0, t=0
+%% op0=0, op1=0, op2=4, r=0, t=0
 -spec ssr(xtensa_register()) -> binary().
 ssr(As) ->
     encode_rrr(0, 0, reg_to_num(As), 0, 16#4, 0).
 
 %% SLLI: AR[r] = AR[s] << sa (1..31)
 %% op0=0, RRR format with shift amount encoded as (32 - sa).
-%% The encoded value split: sa_enc[4] at bits[23:20], op=1 at bits[19:16],
+%% The encoded value split: sa_enc[4] at bits[23:20], op1=1 at bits[19:16],
 %% r at bits[15:12], s at bits[11:8], sa_enc[3:0] at bits[7:4].
 -spec slli(xtensa_register(), xtensa_register(), 1..31) -> binary().
 slli(Ar, As, Sa) when Sa >= 1, Sa =< 31 ->
@@ -284,19 +286,19 @@ srai(Ar, At, Sa) when Sa >= 0, Sa =< 31 ->
 %%=============================================================================
 
 %% MULL: AR[r] = AR[s] * AR[t] (low 32 bits)
-%% op0=0, op1=8, op2=2
+%% op0=0, op1=2, op2=8
 -spec mull(xtensa_register(), xtensa_register(), xtensa_register()) -> binary().
 mull(Ar, As, At) ->
     encode_rrr(0, reg_to_num(At), reg_to_num(As), reg_to_num(Ar), 16#8, 16#2).
 
 %% QUOS: AR[r] = AR[s] / AR[t] (signed)
-%% op0=0, op1=13, op2=2
+%% op0=0, op1=2, op2=13
 -spec quos(xtensa_register(), xtensa_register(), xtensa_register()) -> binary().
 quos(Ar, As, At) ->
     encode_rrr(0, reg_to_num(At), reg_to_num(As), reg_to_num(Ar), 16#D, 16#2).
 
 %% REMS: AR[r] = AR[s] % AR[t] (signed)
-%% op0=0, op1=15, op2=2
+%% op0=0, op1=2, op2=15
 -spec rems(xtensa_register(), xtensa_register(), xtensa_register()) -> binary().
 rems(Ar, As, At) ->
     encode_rrr(0, reg_to_num(At), reg_to_num(As), reg_to_num(Ar), 16#F, 16#2).

tests/libs/jit/jit_xtensa_tests.erl

@@ -290,6 +290,16 @@ call_primitive_last_test() ->
     >>,
     ?assertStream(xtensa, Dump, Stream).
 
+% An argument list that exhausts the scratch registers must still be rejected
+% with a descriptive error, not an opaque first_avail/1 function_clause.
+call_primitive_last_unsupported_args_test() ->
+    State = ?BACKEND:new(?JIT_VARIANT_PIC, jit_stream_binary, jit_stream_binary:new(0)),
+    Args = [ctx, a9, a7, a6, a5, 0],
+    ?assertError(
+        {unsupported_call_primitive_last_args, Args},
+        ?BACKEND:call_primitive_last(State, 0, Args)
+    ).
+
 shift_right_test() ->
     State0 = ?BACKEND:new(?JIT_VARIANT_PIC, jit_stream_binary, jit_stream_binary:new(0)),
     {State1, RegA} = ?BACKEND:move_to_native_register(State0, {x_reg, 0}),