From 7f14e0a637461f95fdd4cf0a094604bebc121c73 Mon Sep 17 00:00:00 2001
From: Jon Shallow <supjps-libcoap@jpshallow.com>
Date: Fri, 18 Oct 2024 13:03:42 +0100
Subject: [PATCH] coap_mbedtls.c: Breakout of permanent loop for (D)TLS setup

Stop MbedTLS continuing to try to establish a (D)TLS session where the
server keeps on sending back a Server-Hello when it fails to decrypt
a client handhake (i.e. mismatch on PSK).
---
 src/coap_mbedtls.c | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/src/coap_mbedtls.c b/src/coap_mbedtls.c
index 23c34bf8..a4f55659 100644
--- a/src/coap_mbedtls.c
+++ b/src/coap_mbedtls.c
@@ -137,6 +137,7 @@ typedef struct coap_mbedtls_env_t {
   coap_tick_t last_timeout;
   unsigned int retry_scalar;
   coap_ssl_t coap_ssl_data;
+  uint32_t server_hello_cnt;
 } coap_mbedtls_env_t;
 
 typedef struct pki_sni_entry {
@@ -1495,6 +1496,16 @@ do_mbedtls_handshake(coap_session_t *c_session,
     break;
   case MBEDTLS_ERR_SSL_WANT_READ:
   case MBEDTLS_ERR_SSL_WANT_WRITE:
+    if (m_env->ssl.state == MBEDTLS_SSL_SERVER_HELLO
+#if MBEDTLS_VERSION_NUMBER >= 0x03030000
+        || m_env->ssl.state == MBEDTLS_SSL_NEW_SESSION_TICKET
+#endif /* MBEDTLS_VERSION_NUMBER >= 0x03030000 */
+       ) {
+      if (++m_env->server_hello_cnt > 10) {
+        /* retried this too many times */
+        goto fail;
+      }
+    }
     errno = EAGAIN;
     ret = 0;
     break;