avformat/whip: Add WHIP muxer support for subsecond latency streaming

0. WHIP Version 3.
1. The WHIP muxer has been renamed and refined,
    with improved logging context and error messages for SSL, DTLS, and RTC.
2. Magic numbers have been replaced with macros and extracted to functions,
    and log levels have been altered for better clarity.
3. DTLS curve list has been updated,
    and SRTP profile names have been refined for FFmpeg and OpenSSL.
4. ICE STUN magic number has been refined,
    and RTP payload types have been updated based on Chrome's definition.
5. Fixed frame size has been refined to rtc->audio_par->frame_size,
    and h264_mp4toannexb is now used to convert MP4/ISOM to annexb.
6. OPUS timestamp issue has been addressed,
    and marker setting has been corrected after utilizing BSF.
7. DTLS handshake and ICE handling have been optimized for improved performance,
    with a single handshake timeout and server role to prevent ARQ.
8. Consolidated ICE request/response handling and DTLS handshake into a single function,
    and fixed OpenSSL build errors to work with Pion.
9. Merge TLS & DTLS implementation, shared BIO callbacks, read, write,
    print_ssl_error, openssl_init_ca_key_cert,
    init_bio_method function and shared same data structure
10. Modify configure that whip is enabled only dtls is
    enabled(just support openssl for now) to fix build error

Co-authored-by: winlin <[email protected]>
Co-authored-by: yangrtc <[email protected]>
Co-authored-by: cloudwebrtc <[email protected]>
Co-authored-by: Haibo Chen <[email protected]>
Co-authored-by: Steven Liu <[email protected]>
Co-authored-by: Jun Zhao <[email protected]>
Signed-off-by: Jack Lau <[email protected]>
Signed-off-by: Steven Liu <[email protected]>

Author: 7 people

configure

@@ -3749,6 +3749,7 @@ wav_demuxer_select="riffdec"
 wav_muxer_select="riffenc"
 webm_chunk_muxer_select="webm_muxer"
 webm_dash_manifest_demuxer_select="matroska_demuxer"
+whip_muxer_deps_any="dtls_protocol"
 wtv_demuxer_select="mpegts_demuxer riffdec"
 wtv_muxer_select="mpegts_muxer riffenc"
 xmv_demuxer_select="riffdec"
@@ -3847,6 +3848,9 @@ srtp_protocol_select="rtp_protocol srtp"
 tcp_protocol_select="network"
 tls_protocol_deps_any="gnutls openssl schannel securetransport libtls mbedtls"
 tls_protocol_select="tcp_protocol"
+# TODO: Support libtls, mbedtls, and gnutls.
+dtls_protocol_deps_any="openssl"
+dtls_protocol_select="udp_protocol"
 udp_protocol_select="network"
 udplite_protocol_select="network"
 unix_protocol_deps="sys_un_h"
@@ -7198,6 +7202,14 @@ enabled rkmpp             && { require_pkg_config rkmpp rockchip_mpp  rockchip/r
                              }
 enabled vapoursynth       && require_headers "vapoursynth/VSScript4.h vapoursynth/VapourSynth4.h"
 
+enabled openssl            && {
+    enabled whip_muxer && {
+        $pkg_config --exists --print-errors "openssl >= 1.0.1k" ||
+        require_pkg_config openssl "openssl >= 1.0.1k" openssl/ssl.h SSL_library_init ||
+        require_pkg_config openssl "openssl >= 1.0.1k" openssl/ssl.h OPENSSL_init_ssl
+    }
+}
+
 
 if enabled gcrypt; then
     GCRYPT_CONFIG="${cross_prefix}libgcrypt-config"

doc/muxers.texi

@@ -3879,4 +3879,51 @@ ffmpeg -f webm_dash_manifest -i video1.webm \
        manifest.xml
 @end example
 
+@anchor{whip}
+@section whip
+
+WebRTC (Real-Time Communication) muxer that supports sub-second latency streaming according to
+the WHIP (WebRTC-HTTP ingestion protocol) specification.
+
+It uses HTTP as a signaling protocol to exchange SDP capabilities and ICE lite candidates. Then,
+it uses STUN binding requests and responses to establish a session over UDP. Subsequently, it
+initiates a DTLS handshake to exchange the SRTP encryption keys. Lastly, it splits video and
+audio frames into RTP packets and encrypts them using SRTP.
+
+Ensure that you use H.264 without B frames and Opus for the audio codec. For example, to convert
+an input file with @command{ffmpeg} to WebRTC:
+@example
+ffmpeg -re -i input.mp4 -acodec libopus -ar 48000 -ac 2 \
+  -vcodec libx264 -profile:v baseline -tune zerolatency -threads 1 -bf 0 \
+  -f whip "http://localhost:1985/rtc/v1/whip/?app=live&stream=livestream"
+@end example
+
+For this example, we have employed low latency options, resulting in an end-to-end latency of
+approximately 150ms.
+
+@subsection Options
+
+This muxer supports the following options:
+
+@table @option
+
+@item handshake_timeout @var{integer}
+Set the timeout in milliseconds for ICE and DTLS handshake.
+Default value is 5000.
+
+@item pkt_size @var{integer}
+Set the maximum size, in bytes, of RTP packets that send out.
+Default value is 1500.
+
+@item authorization @var{string}
+The optional Bearer token for WHIP Authorization.
+
+@item cert_file @var{string}
+The optional certificate file path for DTLS.
+
+@item key_file @var{string}
+The optional private key file path for DTLS.
+
+@end table
+
 @c man end MUXERS

libavformat/Makefile

@@ -638,6 +638,7 @@ OBJS-$(CONFIG_WEBM_CHUNK_MUXER)          += webm_chunk.o
 OBJS-$(CONFIG_WEBP_MUXER)                += webpenc.o
 OBJS-$(CONFIG_WEBVTT_DEMUXER)            += webvttdec.o subtitles.o
 OBJS-$(CONFIG_WEBVTT_MUXER)              += webvttenc.o
+OBJS-$(CONFIG_WHIP_MUXER)                += whip.o avc.o http.o srtp.o tls_openssl.o
 OBJS-$(CONFIG_WSAUD_DEMUXER)             += westwood_aud.o
 OBJS-$(CONFIG_WSAUD_MUXER)               += westwood_audenc.o
 OBJS-$(CONFIG_WSD_DEMUXER)               += wsddec.o rawdec.o

libavformat/allformats.c

@@ -515,6 +515,7 @@ extern const FFOutputFormat ff_webp_muxer;
 extern const FFInputFormat  ff_webvtt_demuxer;
 extern const FFOutputFormat ff_webvtt_muxer;
 extern const FFInputFormat  ff_wsaud_demuxer;
+extern const FFOutputFormat ff_whip_muxer;
 extern const FFOutputFormat ff_wsaud_muxer;
 extern const FFInputFormat  ff_wsd_demuxer;
 extern const FFInputFormat  ff_wsvqa_demuxer;

libavformat/avio.c

@@ -339,8 +339,9 @@ static const struct URLProtocol *url_find_protocol(const char *filename)
         }
     }
     av_freep(&protocols);
-    if (av_strstart(filename, "https:", NULL) || av_strstart(filename, "tls:", NULL))
-        av_log(NULL, AV_LOG_WARNING, "https protocol not found, recompile FFmpeg with "
+    if (av_strstart(filename, "https:", NULL) || av_strstart(filename, "tls:", NULL) ||
+        av_strstart(filename, "dtls:", NULL))
+        av_log(NULL, AV_LOG_WARNING, "https or dtls protocol not found, recompile FFmpeg with "
                                      "openssl, gnutls or securetransport enabled.\n");
 
     return NULL;

libavformat/http.c

@@ -562,6 +562,12 @@ int ff_http_averror(int status_code, int default_averror)
         return default_averror;
 }
 
+const char* ff_http_get_new_location(URLContext *h)
+{
+    HTTPContext *s = h->priv_data;
+    return s->new_location;
+}
+
 static int http_write_reply(URLContext* h, int status_code)
 {
     int ret, body = 0, reply_code, message_len;

libavformat/http.h

@@ -62,4 +62,6 @@ int ff_http_do_new_request2(URLContext *h, const char *uri, AVDictionary **optio
 
 int ff_http_averror(int status_code, int default_averror);
 
+const char* ff_http_get_new_location(URLContext *h);
+
 #endif /* AVFORMAT_HTTP_H */

libavformat/protocols.c

@@ -62,6 +62,7 @@ extern const URLProtocol ff_subfile_protocol;
 extern const URLProtocol ff_tee_protocol;
 extern const URLProtocol ff_tcp_protocol;
 extern const URLProtocol ff_tls_protocol;
+extern const URLProtocol ff_dtls_protocol;
 extern const URLProtocol ff_udp_protocol;
 extern const URLProtocol ff_udplite_protocol;
 extern const URLProtocol ff_unix_protocol;

libavformat/srtp.h

@@ -27,7 +27,7 @@
 struct AVAES;
 struct AVHMAC;
 
-struct SRTPContext {
+typedef struct SRTPContext {
     struct AVAES *aes;
     struct AVHMAC *hmac;
     int rtp_hmac_size, rtcp_hmac_size;
@@ -40,7 +40,7 @@ struct SRTPContext {
     uint32_t roc;
 
     uint32_t rtcp_index;
-};
+} SRTPContext;
 
 int ff_srtp_set_crypto(struct SRTPContext *s, const char *suite,
                        const char *params);

libavformat/tls.c

@@ -1,6 +1,7 @@
 /*
- * TLS/SSL Protocol
+ * TLS/DTLS/SSL Protocol
  * Copyright (c) 2011 Martin Storsjo
+ * Copyright (c) 2025 Jack Lau
  *
  * This file is part of FFmpeg.
  *
@@ -20,6 +21,7 @@
  */
 
 #include "avformat.h"
+#include "internal.h"
 #include "network.h"
 #include "os_support.h"
 #include "url.h"
@@ -93,7 +95,7 @@ int ff_tls_open_underlying(TLSShared *c, URLContext *parent, const char *uri, AV
             c->listen = 1;
     }
 
-    ff_url_join(buf, sizeof(buf), "tcp", NULL, c->underlying_host, port, "%s", p);
+    ff_url_join(buf, sizeof(buf), c->is_dtls ? "udp" : "tcp", NULL, c->underlying_host, port, "%s", p);
 
     hints.ai_flags = AI_NUMERICHOST;
     if (!getaddrinfo(c->underlying_host, NULL, &hints, &ai)) {
@@ -124,7 +126,65 @@ int ff_tls_open_underlying(TLSShared *c, URLContext *parent, const char *uri, AV
     }
 
     freeenv_utf8(env_http_proxy);
-    return ffurl_open_whitelist(&c->tcp, buf, AVIO_FLAG_READ_WRITE,
-                                &parent->interrupt_callback, options,
-                                parent->protocol_whitelist, parent->protocol_blacklist, parent);
+    if (c->is_dtls) {
+        av_dict_set_int(options, "connect", 1, 0);
+        av_dict_set_int(options, "fifo_size", 0, 0);
+        /* Set the max packet size to the buffer size. */
+        av_dict_set_int(options, "pkt_size", c->mtu, 0);
+    }
+    ret = ffurl_open_whitelist(c->is_dtls ? &c->udp : &c->tcp, buf, AVIO_FLAG_READ_WRITE,
+                               &parent->interrupt_callback, options,
+                               parent->protocol_whitelist, parent->protocol_blacklist, parent);
+    if (c->is_dtls) {
+        if (ret < 0) {
+            av_log(c, AV_LOG_ERROR, "WHIP: Failed to connect udp://%s:%d\n", c->underlying_host, port);
+            return ret;
+        }
+        /* Make the socket non-blocking, set to READ and WRITE mode after connected */
+        ff_socket_nonblock(ffurl_get_file_handle(c->udp), 1);
+        c->udp->flags |= AVIO_FLAG_READ | AVIO_FLAG_NONBLOCK;
+    }
+    return ret;
+}
+
+/**
+ * Read all data from the given URL url and store it in the given buffer bp.
+ */
+int ff_url_read_all(const char *url, AVBPrint *bp)
+{
+    int ret = 0;
+    AVDictionary *opts = NULL;
+    URLContext *uc = NULL;
+    char buf[MAX_URL_SIZE];
+
+    ret = ffurl_open_whitelist(&uc, url, AVIO_FLAG_READ, NULL, &opts, NULL, NULL, NULL);
+    if (ret < 0) {
+        av_log(NULL, AV_LOG_ERROR, "TLS: Failed to open url %s\n", url);
+        goto end;
+    }
+
+    while (1) {
+        ret = ffurl_read(uc, buf, sizeof(buf));
+        if (ret == AVERROR_EOF) {
+            /* Reset the error because we read all response as answer util EOF. */
+            ret = 0;
+            break;
+        }
+        if (ret <= 0) {
+            av_log(NULL, AV_LOG_ERROR, "TLS: Failed to read from url=%s, key is %s\n", url, bp->str);
+            goto end;
+        }
+
+        av_bprintf(bp, "%.*s", ret, buf);
+        if (!av_bprint_is_complete(bp)) {
+            av_log(NULL, AV_LOG_ERROR, "TLS: Exceed max size %.*s, %s\n", ret, buf, bp->str);
+            ret = AVERROR(EIO);
+            goto end;
+        }
+    }
+
+end:
+    ffurl_closep(&uc);
+    av_dict_free(&opts);
+    return ret;
 }

libavformat/tls.h

@@ -1,6 +1,7 @@
 /*
- * TLS/SSL Protocol
+ * TLS/DTLS/SSL Protocol
  * Copyright (c) 2011 Martin Storsjo
+ * Copyright (c) 2025 Jack Lau
  *
  * This file is part of FFmpeg.
  *
@@ -22,10 +23,27 @@
 #ifndef AVFORMAT_TLS_H
 #define AVFORMAT_TLS_H
 
+#include "libavutil/bprint.h"
 #include "libavutil/opt.h"
 
 #include "url.h"
 
+/**
+ * Maximum size limit of a certificate and private key size.
+ */
+#define MAX_CERTIFICATE_SIZE 8192
+
+enum DTLSState {
+    DTLS_STATE_NONE,
+
+    /* Whether DTLS handshake is finished. */
+    DTLS_STATE_FINISHED,
+    /* Whether DTLS session is closed. */
+    DTLS_STATE_CLOSED,
+    /* Whether DTLS handshake is failed. */
+    DTLS_STATE_FAILED,
+};
+
 typedef struct TLSShared {
     char *ca_file;
     int verify;
@@ -40,6 +58,25 @@ typedef struct TLSShared {
     int numerichost;
 
     URLContext *tcp;
+
+    int is_dtls;
+
+    enum DTLSState state;
+
+    int use_external_udp;
+    URLContext *udp;
+
+    /* The fingerprint of certificate, used in SDP offer. */
+    char *fingerprint;
+
+    /* The certificate and private key content used for DTLS handshake */
+    char* cert_buf;
+    char* key_buf;
+    /**
+     * The size of RTP packet, should generally be set to MTU.
+     * Note that pion requires a smaller value, for example, 1200.
+     */
+    int mtu;
 } TLSShared;
 
 #define TLS_OPTFL (AV_OPT_FLAG_DECODING_PARAM | AV_OPT_FLAG_ENCODING_PARAM)
@@ -51,10 +88,27 @@ typedef struct TLSShared {
     {"key_file",   "Private key file",                    offsetof(pstruct, options_field . key_file),  AV_OPT_TYPE_STRING, .flags = TLS_OPTFL }, \
     {"listen",     "Listen for incoming connections",     offsetof(pstruct, options_field . listen),    AV_OPT_TYPE_INT, { .i64 = 0 }, 0, 1, .flags = TLS_OPTFL }, \
     {"verifyhost", "Verify against a specific hostname",  offsetof(pstruct, options_field . host),      AV_OPT_TYPE_STRING, .flags = TLS_OPTFL }, \
-    {"http_proxy", "Set proxy to tunnel through",         offsetof(pstruct, options_field . http_proxy), AV_OPT_TYPE_STRING, .flags = TLS_OPTFL }
+    {"http_proxy", "Set proxy to tunnel through",         offsetof(pstruct, options_field . http_proxy), AV_OPT_TYPE_STRING, .flags = TLS_OPTFL }, \
+    {"use_external_udp", "Use external UDP from muxer or demuxer", offsetof(pstruct, options_field . use_external_udp), AV_OPT_TYPE_INT, { .i64 = 0}, 0, 1, .flags = TLS_OPTFL }, \
+    {"mtu", "Maximum Transmission Unit", offsetof(pstruct, options_field . mtu), AV_OPT_TYPE_INT,  { .i64 = 0}, INT64_MIN, INT64_MAX, .flags = TLS_OPTFL}, \
+    {"fingerprint", "The optional fingerprint for DTLS", offsetof(pstruct, options_field . fingerprint), AV_OPT_TYPE_STRING, .flags = TLS_OPTFL}, \
+    {"cert_buf", "The optional certificate buffer for DTLS", offsetof(pstruct, options_field . cert_buf), AV_OPT_TYPE_STRING, .flags = TLS_OPTFL}, \
+    {"key_buf", "The optional private key buffer for DTLS", offsetof(pstruct, options_field . key_buf), AV_OPT_TYPE_STRING, .flags = TLS_OPTFL}
 
 int ff_tls_open_underlying(TLSShared *c, URLContext *parent, const char *uri, AVDictionary **options);
 
+int ff_url_read_all(const char *url, AVBPrint *bp);
+
+int ff_dtls_set_udp(URLContext *h, URLContext *udp);
+
+int ff_dtls_export_materials(URLContext *h, char *dtls_srtp_materials, size_t materials_sz);
+
+int ff_dtls_state(URLContext *h);
+
+int ff_ssl_read_key_cert(char *key_url, char *cert_url, char *key_buf, size_t key_sz, char *cert_buf, size_t cert_sz, char **fingerprint);
+
+int ff_ssl_gen_key_cert(char *key_buf, size_t key_sz, char *cert_buf, size_t cert_sz, char **fingerprint);
+
 void ff_gnutls_init(void);
 void ff_gnutls_deinit(void);