tls_openssl: Readd support for nonblocking operation

The rtmp protocol uses nonblocking reads, to poll for incoming
messages from the server while publishing a stream.

Prior to 94599a6de3 and
d13b124eaf, the tls protocol
handled the nonblocking flag, mostly as a side effect from not
using custom IO callbacks for reading from the socket. When custom
IO callbacks were taken into use in
d15eec4d6b, the handling of a nonblocking
socket wasn't necessary for the default blocking mode any longer.

The code was simplified, since it was overlooked that other code
within libavformat actually used the tls protocol in nonblocking mode.

This fixes publishing over rtmps, with the openssl backend.

Signed-off-by: Martin Storsjö <martin@martin.st>

libavformat/tls_openssl.c

@@ -109,6 +109,12 @@ void ff_openssl_deinit(void)
 
 static int print_tls_error(URLContext *h, int ret)
 {
+    TLSContext *c = h->priv_data;
+    if (h->flags & AVIO_FLAG_NONBLOCK) {
+        int err = SSL_get_error(c->ssl, ret);
+        if (err == SSL_ERROR_WANT_READ || err == SSL_ERROR_WANT_READ)
+            return AVERROR(EAGAIN);
+    }
     av_log(h, AV_LOG_ERROR, "%s\n", ERR_error_string(ERR_get_error(), NULL));
     return AVERROR(EIO);
 }
@@ -164,6 +170,8 @@ static int url_bio_bread(BIO *b, char *buf, int len)
     if (ret >= 0)
         return ret;
     BIO_clear_retry_flags(b);
+    if (ret == AVERROR(EAGAIN))
+        BIO_set_retry_read(b);
     if (ret == AVERROR_EXIT)
         return 0;
     return -1;
@@ -176,6 +184,8 @@ static int url_bio_bwrite(BIO *b, const char *buf, int len)
     if (ret >= 0)
         return ret;
     BIO_clear_retry_flags(b);
+    if (ret == AVERROR(EAGAIN))
+        BIO_set_retry_write(b);
     if (ret == AVERROR_EXIT)
         return 0;
     return -1;
@@ -292,7 +302,11 @@ fail:
 static int tls_read(URLContext *h, uint8_t *buf, int size)
 {
     TLSContext *c = h->priv_data;
-    int ret = SSL_read(c->ssl, buf, size);
+    int ret;
+    // Set or clear the AVIO_FLAG_NONBLOCK on c->tls_shared.tcp
+    c->tls_shared.tcp->flags &= ~AVIO_FLAG_NONBLOCK;
+    c->tls_shared.tcp->flags |= h->flags & AVIO_FLAG_NONBLOCK;
+    ret = SSL_read(c->ssl, buf, size);
     if (ret > 0)
         return ret;
     if (ret == 0)
@@ -303,7 +317,11 @@ static int tls_read(URLContext *h, uint8_t *buf, int size)
 static int tls_write(URLContext *h, const uint8_t *buf, int size)
 {
     TLSContext *c = h->priv_data;
-    int ret = SSL_write(c->ssl, buf, size);
+    int ret;
+    // Set or clear the AVIO_FLAG_NONBLOCK on c->tls_shared.tcp
+    c->tls_shared.tcp->flags &= ~AVIO_FLAG_NONBLOCK;
+    c->tls_shared.tcp->flags |= h->flags & AVIO_FLAG_NONBLOCK;
+    ret = SSL_write(c->ssl, buf, size);
     if (ret > 0)
         return ret;
     if (ret == 0)