Fix RSA-OAEP to allow zero-length plaintext per RFC 8017

wolfcrypt/src/rsa.c

@@ -1773,8 +1773,10 @@ static int RsaUnPad_OAEP(byte *pkcsBlock, unsigned int pkcsBlockLen,
         c = c + (pkcsBlock[idx++] ^ 0x01); /* separator value is 0x01 */
         c = c + (pkcsBlock[0]     ^ 0x00); /* Y, the first value, should be 0 */
 
-        /* Return 0 data length on error. */
-        idx = ctMaskSelWord32(ctMaskEq(c, 0), idx, pkcsBlockLen);
+        /* On error, set idx past the end so the return value is negative.
+         * This distinguishes padding errors (ret == -1) from a valid
+         * zero-length message (ret == 0), which RFC 8017 permits. */
+        idx = ctMaskSelWord32(ctMaskEq(c, 0), idx, pkcsBlockLen + 1);
     }
 
     /* adjust pointer to correct location in array and return size of M */
@@ -3437,7 +3439,14 @@ static int RsaPublicEncryptEx(const byte* in, word32 inLen, byte* out,
     RsaPadding padding;
 #endif
 
-    if (in == NULL || inLen == 0 || out == NULL || key == NULL) {
+    if (out == NULL || key == NULL) {
+        return BAD_FUNC_ARG;
+    }
+
+    /* RFC 8017 Section 7.1.1 step 1b permits mLen == 0 for OAEP because
+     * the message-length bound (mLen <= k - 2*hLen - 2) is satisfied.
+     * Other padding types require non-empty input. */
+    if (in == NULL || (inLen == 0 && pad_type != WC_RSA_OAEP_PAD)) {
         return BAD_FUNC_ARG;
     }
 
@@ -3837,8 +3846,14 @@ static int RsaPrivateDecryptEx(const byte* in, word32 inLen, byte* out,
             ret = ctMaskSelInt(ctMaskLTE(ret, (int)outLen), ret,
                                WC_NO_ERR_TRACE(RSA_BUFFER_E));
     #ifndef WOLFSSL_RSA_DECRYPT_TO_0_LEN
-            ret = ctMaskSelInt(ctMaskNotEq(ret, 0), ret,
-                               WC_NO_ERR_TRACE(RSA_BUFFER_E));
+            /* RFC 8017 Section 7.1.2 step 3g: OAEP decryption may yield a
+             * valid zero-length message (mLen == 0). Only reject ret == 0
+             * for non-OAEP padding types. */
+            {
+                byte zeroOkMask = ctMaskEq(pad_type, WC_RSA_OAEP_PAD);
+                ret = ctMaskSelInt(ctMaskNotEq(ret, 0) | zeroOkMask, ret,
+                                   WC_NO_ERR_TRACE(RSA_BUFFER_E));
+            }
     #endif
 #else
             if (outLen < (word32)ret)

wolfcrypt/test/test.c

@@ -25796,6 +25796,65 @@ static wc_test_ret_t rsa_oaep_padding_test(RsaKey* key, WC_RNG* rng)
     TEST_SLEEP();
 #endif /* WOLFSSL_RSA_PUBLIC_ONLY */
 
+    /* RFC 8017 Section 7.1: OAEP permits zero-length messages. Verify
+     * encrypt/decrypt round-trip succeeds with inLen == 0 and that
+     * corrupted ciphertext still produces a negative error code. */
+    #ifndef NO_SHA256
+    #ifndef WOLFSSL_RSA_PUBLIC_ONLY
+    #ifndef WOLFSSL_SE050
+    {
+        byte emptyBuf = 0;
+        XMEMSET(out, 0, outSz);
+        XMEMSET(plain, 0, plainSz);
+        do {
+    #if defined(WOLFSSL_ASYNC_CRYPT)
+            ret = wc_AsyncWait(ret, &key->asyncDev, WC_ASYNC_FLAG_CALL_AGAIN);
+    #endif
+            if (ret >= 0) {
+                ret = wc_RsaPublicEncrypt_ex(&emptyBuf, 0, out, outSz, key, rng,
+                    WC_RSA_OAEP_PAD, WC_HASH_TYPE_SHA256, WC_MGF1SHA256,
+                    NULL, 0);
+            }
+        } while (ret == WC_NO_ERR_TRACE(WC_PENDING_E));
+        if (ret < 0)
+            ERROR_OUT(WC_TEST_RET_ENC_EC(ret), exit_rsa);
+
+        idx = (word32)ret;
+        do {
+    #if defined(WOLFSSL_ASYNC_CRYPT)
+            ret = wc_AsyncWait(ret, &key->asyncDev, WC_ASYNC_FLAG_CALL_AGAIN);
+    #endif
+            if (ret >= 0) {
+                ret = wc_RsaPrivateDecrypt_ex(out, idx, plain, plainSz, key,
+                    WC_RSA_OAEP_PAD, WC_HASH_TYPE_SHA256, WC_MGF1SHA256,
+                    NULL, 0);
+            }
+        } while (ret == WC_NO_ERR_TRACE(WC_PENDING_E));
+        if (ret != 0)
+            ERROR_OUT(WC_TEST_RET_ENC_EC(ret), exit_rsa);
+        TEST_SLEEP();
+
+        /* Corrupt the ciphertext and verify decryption returns an error,
+         * not a zero-length success. */
+        out[idx / 2] ^= 0x01;
+        do {
+    #if defined(WOLFSSL_ASYNC_CRYPT)
+            ret = wc_AsyncWait(ret, &key->asyncDev, WC_ASYNC_FLAG_CALL_AGAIN);
+    #endif
+            if (ret >= 0) {
+                ret = wc_RsaPrivateDecrypt_ex(out, idx, plain, plainSz, key,
+                    WC_RSA_OAEP_PAD, WC_HASH_TYPE_SHA256, WC_MGF1SHA256,
+                    NULL, 0);
+            }
+        } while (ret == WC_NO_ERR_TRACE(WC_PENDING_E));
+        if (ret >= 0)
+            ERROR_OUT(WC_TEST_RET_ENC_NC, exit_rsa);
+        ret = 0;
+    }
+    #endif /* WOLFSSL_SE050 */
+    #endif /* WOLFSSL_RSA_PUBLIC_ONLY */
+    #endif /* NO_SHA256 */
+
 exit_rsa:
     WC_FREE_VAR(in, HEAP_HINT);
     WC_FREE_VAR(out, HEAP_HINT);