[Qemu-devel] [PATCH v2 2/3] crypto: avoid undefined behavior in nettle calls

[Radim Krčmář]

Calling a function pointer that was cast from an incompatible function
results in undefined behavior.  'void *' isn't compatible with 'struct
XXX *', so we can't cast to nettle_cipher_func, but have to provide a
wrapper.  (Conversion from 'void *' to 'struct XXX *' might require
computation, which won't be done if we drop argument's true type, and
pointers can have different sizes so passing arguments on stack would
bug.)

Having two different prototypes based on nettle version doesn't make
this solution any nicer.

Reported-by: Peter Maydell <address@hidden>
Signed-off-by: Radim Krčmář <address@hidden>
---
 crypto/cipher-nettle.c | 43 +++++++++++++++++++++++++++++++++++++++----
 1 file changed, 39 insertions(+), 4 deletions(-)

diff --git a/crypto/cipher-nettle.c b/crypto/cipher-nettle.c
index e61aaa29f049..61f1cd3417d3 100644
--- a/crypto/cipher-nettle.c
+++ b/crypto/cipher-nettle.c
@@ -25,8 +25,43 @@
 
 #if CONFIG_NETTLE_VERSION_MAJOR < 3
 typedef nettle_crypt_func nettle_cipher_func;
+
+typedef void *       cipher_ctx_t;
+typedef unsigned     cipher_length_t;
+#else
+typedef const void * cipher_ctx_t;
+typedef size_t       cipher_length_t;
 #endif
 
+nettle_cipher_func aes_encrypt_wrapper;
+nettle_cipher_func aes_decrypt_wrapper;
+nettle_cipher_func des_encrypt_wrapper;
+nettle_cipher_func des_decrypt_wrapper;
+
+void aes_encrypt_wrapper(cipher_ctx_t ctx, cipher_length_t length,
+                         uint8_t *dst, const uint8_t *src)
+{
+    aes_encrypt(ctx, length, dst, src);
+}
+
+void aes_decrypt_wrapper(cipher_ctx_t ctx, cipher_length_t length,
+                         uint8_t *dst, const uint8_t *src)
+{
+    aes_encrypt(ctx, length, dst, src);
+}
+
+void des_encrypt_wrapper(cipher_ctx_t ctx, cipher_length_t length,
+                         uint8_t *dst, const uint8_t *src)
+{
+    des_encrypt(ctx, length, dst, src);
+}
+
+void des_decrypt_wrapper(cipher_ctx_t ctx, cipher_length_t length,
+                         uint8_t *dst, const uint8_t *src)
+{
+    des_decrypt(ctx, length, dst, src);
+}
+
 typedef struct QCryptoCipherNettle QCryptoCipherNettle;
 struct QCryptoCipherNettle {
     void *ctx_encrypt;
@@ -87,8 +122,8 @@ QCryptoCipher *qcrypto_cipher_new(QCryptoCipherAlgorithm alg,
         des_set_key(ctx->ctx_encrypt, rfbkey);
         g_free(rfbkey);
 
-        ctx->alg_encrypt = (nettle_cipher_func *)des_encrypt;
-        ctx->alg_decrypt = (nettle_cipher_func *)des_decrypt;
+        ctx->alg_encrypt = des_encrypt_wrapper;
+        ctx->alg_decrypt = des_decrypt_wrapper;
 
         ctx->niv = DES_BLOCK_SIZE;
         break;
@@ -102,8 +137,8 @@ QCryptoCipher *qcrypto_cipher_new(QCryptoCipherAlgorithm 
alg,
         aes_set_encrypt_key(ctx->ctx_encrypt, nkey, key);
         aes_set_decrypt_key(ctx->ctx_decrypt, nkey, key);
 
-        ctx->alg_encrypt = (nettle_cipher_func *)aes_encrypt;
-        ctx->alg_decrypt = (nettle_cipher_func *)aes_decrypt;
+        ctx->alg_encrypt = aes_encrypt_wrapper;
+        ctx->alg_decrypt = aes_decrypt_wrapper;
 
         ctx->niv = AES_BLOCK_SIZE;
         break;
-- 
2.4.5