From 0c3e3cc431e1a27556c91b0d545af280b558e698 Mon Sep 17 00:00:00 2001 From: sean tearney Date: Thu, 31 Oct 2024 12:51:02 +0800 Subject: [PATCH 01/17] fix stream retrieval bug RTC updates --- src/net/RTP/RTPSession.cs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/net/RTP/RTPSession.cs b/src/net/RTP/RTPSession.cs index a9da214e1..4e28d7247 100644 --- a/src/net/RTP/RTPSession.cs +++ b/src/net/RTP/RTPSession.cs @@ -1,4 +1,4 @@ -//----------------------------------------------------------------------------- +//----------------------------------------------------------------------------- // Filename: RTPSession.cs // // Description: Represents an RTP session constituted of a single media stream. The session @@ -2318,13 +2318,12 @@ private MediaStream GetMediaStream(uint ssrc) return AudioStream; } } - else + + if (HasVideo) { - if (HasVideo) - { - return VideoStream; - } + return VideoStream; } + return null; } From 1a8b920c42d19a6abe7e8d9703dbe04b2d1bb791 Mon Sep 17 00:00:00 2001 From: sean t Date: Sun, 10 Nov 2024 15:50:42 +0800 Subject: [PATCH 02/17] add translations for tray app and installer --- src/SIPSorcery.csproj | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/SIPSorcery.csproj b/src/SIPSorcery.csproj index b378622b3..fcdfdd6b4 100755 --- a/src/SIPSorcery.csproj +++ b/src/SIPSorcery.csproj @@ -90,4 +90,8 @@ + + + + From e484d9c75188d39ddc152bf1265db7e14633281a Mon Sep 17 00:00:00 2001 From: sean t Date: Fri, 15 Nov 2024 13:47:31 +0800 Subject: [PATCH 03/17] fix for ptz serialization use wyze camera name from url --- src/app/Media/Codecs/G722Codec.cs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/app/Media/Codecs/G722Codec.cs b/src/app/Media/Codecs/G722Codec.cs index e087a2050..029570ed7 100644 --- a/src/app/Media/Codecs/G722Codec.cs +++ b/src/app/Media/Codecs/G722Codec.cs @@ -414,7 +414,15 @@ public int Encode(G722CodecState state, byte[] outputBuffer, short[] inputBuffer state.QmfSignalHistory[i] = state.QmfSignalHistory[i + 2]; } state.QmfSignalHistory[22] = inputBuffer[j++]; - state.QmfSignalHistory[23] = inputBuffer[j++]; + if (j < inputBufferCount) + { + state.QmfSignalHistory[23] = inputBuffer[j++]; + } + else + { + //Duplicate the last sample + state.QmfSignalHistory[23] = state.QmfSignalHistory[22]; + } // Discard every other QMF output sumeven = 0; @@ -692,4 +700,4 @@ public enum G722Flags /// Packed = 0x0002 } -} \ No newline at end of file +} From ed8cd70ad92fa6c1fbedc4041ecd46e9a524f26d Mon Sep 17 00:00:00 2001 From: sean t Date: Thu, 21 Nov 2024 09:19:01 +0800 Subject: [PATCH 04/17] fix bug in g722 codec --- src/app/Media/Codecs/G722Codec.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app/Media/Codecs/G722Codec.cs b/src/app/Media/Codecs/G722Codec.cs index 029570ed7..959fe6bb9 100644 --- a/src/app/Media/Codecs/G722Codec.cs +++ b/src/app/Media/Codecs/G722Codec.cs @@ -420,7 +420,7 @@ public int Encode(G722CodecState state, byte[] outputBuffer, short[] inputBuffer } else { - //Duplicate the last sample + //Duplicate the last sample - fix odd shorts issue state.QmfSignalHistory[23] = state.QmfSignalHistory[22]; } From de318f689b23b805219a58a789842ebc61aeb072 Mon Sep 17 00:00:00 2001 From: sean t Date: Sun, 1 Dec 2024 14:57:10 +0800 Subject: [PATCH 05/17] Only get stream format once Add timeout to ice gathering Add prefer H264 flag for compatible formats --- src/net/RTP/AudioStream.cs | 11 +++++++++-- src/net/RTP/VideoStream.cs | 17 +++++++++++++---- src/net/SDP/SDPAudioVideoMediaFormat.cs | 10 +++++++--- src/net/WebRTC/RTCPeerConnection.cs | 13 ++++++++++++- 4 files changed, 41 insertions(+), 10 deletions(-) diff --git a/src/net/RTP/AudioStream.cs b/src/net/RTP/AudioStream.cs index dce756328..8c9152b5a 100755 --- a/src/net/RTP/AudioStream.cs +++ b/src/net/RTP/AudioStream.cs @@ -67,10 +67,17 @@ public bool HasAudio /// The duration in RTP timestamp units of the audio sample. This /// value is added to the previous RTP timestamp when building the RTP header. /// The audio sample to set as the RTP packet payload. + /// + private SDPAudioVideoMediaFormat sendingFormat; + private bool sendingFormatFound = false; public void SendAudio(uint durationRtpUnits, byte[] sample) { - var audioFormat = GetSendingFormat(); - SendAudioFrame(durationRtpUnits, audioFormat.ID, sample); + if (!sendingFormatFound) + { + sendingFormat = GetSendingFormat(); + sendingFormatFound = true; + } + SendAudioFrame(durationRtpUnits, sendingFormat.ID, sample); } /// diff --git a/src/net/RTP/VideoStream.cs b/src/net/RTP/VideoStream.cs index 54dd24161..aef495f89 100644 --- a/src/net/RTP/VideoStream.cs +++ b/src/net/RTP/VideoStream.cs @@ -29,6 +29,7 @@ namespace SIPSorcery.net.RTP public class VideoStream : MediaStream { protected static ILogger logger = Log.Logger; + protected RtpVideoFramer RtpVideoFramer; @@ -244,12 +245,20 @@ public void SendVp8Frame(uint duration, int payloadTypeID, byte[] buffer) /// The duration in RTP timestamp units of the video sample. This /// value is added to the previous RTP timestamp when building the RTP header. /// The video sample to set as the RTP packet payload. + /// + private SDPAudioVideoMediaFormat sendingFormat; + private bool sendingFormatFound = false; public void SendVideo(uint durationRtpUnits, byte[] sample) { - var videoSendingFormat = GetSendingFormat(); - int payloadID = Convert.ToInt32(videoSendingFormat.ID); + if (!sendingFormatFound) + { + sendingFormat = GetSendingFormat(); + sendingFormatFound = true; + } + + int payloadID = Convert.ToInt32(sendingFormat.ID); - switch (videoSendingFormat.Name()) + switch (sendingFormat.Name()) { case "VP8": SendVp8Frame(durationRtpUnits, payloadID, sample); @@ -258,7 +267,7 @@ public void SendVideo(uint durationRtpUnits, byte[] sample) SendH264Frame(durationRtpUnits, payloadID, sample); break; default: - throw new ApplicationException($"Unsupported video format selected {videoSendingFormat.Name()}."); + throw new ApplicationException($"Unsupported video format selected {sendingFormat.Name()}."); } } diff --git a/src/net/SDP/SDPAudioVideoMediaFormat.cs b/src/net/SDP/SDPAudioVideoMediaFormat.cs index 67cafa6c0..fa38d0c9f 100755 --- a/src/net/SDP/SDPAudioVideoMediaFormat.cs +++ b/src/net/SDP/SDPAudioVideoMediaFormat.cs @@ -350,7 +350,7 @@ public static bool AreMatch(SDPAudioVideoMediaFormat format1, SDPAudioVideoMedia /// The first list to match the media formats for. /// The second list to match the media formats for. /// A list of media formats that are compatible for BOTH lists. - public static List GetCompatibleFormats(List a, List b) + public static List GetCompatibleFormats(List a, List b, bool preferH264 = true) { List compatible = new List(); @@ -368,13 +368,17 @@ public static List GetCompatibleFormats(List SDPAudioVideoMediaFormat.AreMatch(format, x))) + if (b.Any(x => AreMatch(format, x))) { + if (preferH264 && (format.Rtpmap ?? "").StartsWith("H264")) + { + return new List { format }; + } compatible.Add(format); } } } - + return compatible; } diff --git a/src/net/WebRTC/RTCPeerConnection.cs b/src/net/WebRTC/RTCPeerConnection.cs index dbad165c2..38978d984 100755 --- a/src/net/WebRTC/RTCPeerConnection.cs +++ b/src/net/WebRTC/RTCPeerConnection.cs @@ -1204,7 +1204,18 @@ private SDP createBaseSdp(List mediaStreamList, bool excludeIceCand // In theory it would be better to an async/await but that would result in a breaking // change to the API and for a one off (once per class instance not once per method call) // delay of a few hundred milliseconds it was decided not to break the API. - _iceGatheringTask.Wait(); + using (var ct = new CancellationTokenSource(TimeSpan.FromMilliseconds(2000))) + { + try + { + _iceGatheringTask.Wait(ct.Token); + } + catch (OperationCanceledException) + { + // Handle the timeout scenario here + + } + } SDP offerSdp = new SDP(IPAddress.Loopback); offerSdp.SessionId = LocalSdpSessionID; From c49c0c9bfbee6e77e4a2947d3e80fa39393793dc Mon Sep 17 00:00:00 2001 From: sean t Date: Mon, 2 Dec 2024 07:53:53 +0800 Subject: [PATCH 06/17] revert --- src/net/SDP/SDPAudioVideoMediaFormat.cs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/net/SDP/SDPAudioVideoMediaFormat.cs b/src/net/SDP/SDPAudioVideoMediaFormat.cs index fa38d0c9f..d58a7d118 100755 --- a/src/net/SDP/SDPAudioVideoMediaFormat.cs +++ b/src/net/SDP/SDPAudioVideoMediaFormat.cs @@ -350,7 +350,7 @@ public static bool AreMatch(SDPAudioVideoMediaFormat format1, SDPAudioVideoMedia /// The first list to match the media formats for. /// The second list to match the media formats for. /// A list of media formats that are compatible for BOTH lists. - public static List GetCompatibleFormats(List a, List b, bool preferH264 = true) + public static List GetCompatibleFormats(List a, List b) { List compatible = new List(); @@ -370,10 +370,6 @@ public static List GetCompatibleFormats(List AreMatch(format, x))) { - if (preferH264 && (format.Rtpmap ?? "").StartsWith("H264")) - { - return new List { format }; - } compatible.Add(format); } } From 8e7b251ca1d57bafc8e6b7110ad7944b5d93b3e1 Mon Sep 17 00:00:00 2001 From: sean t Date: Mon, 2 Dec 2024 08:00:07 +0800 Subject: [PATCH 07/17] Optimise SendVideo and SendAudio (only get SendingFormat once) Add sanity check for ICE Gathering timeout --- src/net/RTP/AudioStream.cs | 6 +++--- src/net/RTP/VideoStream.cs | 7 +++---- src/net/WebRTC/RTCPeerConnection.cs | 7 ++++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/net/RTP/AudioStream.cs b/src/net/RTP/AudioStream.cs index 8c9152b5a..3d1f9935b 100755 --- a/src/net/RTP/AudioStream.cs +++ b/src/net/RTP/AudioStream.cs @@ -31,9 +31,11 @@ namespace SIPSorcery.net.RTP public class AudioStream : MediaStream { protected static ILogger logger = Log.Logger; - protected Boolean rtpEventInProgress = false; + private SDPAudioVideoMediaFormat sendingFormat; + private bool sendingFormatFound = false; + #region EVENTS /// @@ -68,8 +70,6 @@ public bool HasAudio /// value is added to the previous RTP timestamp when building the RTP header. /// The audio sample to set as the RTP packet payload. /// - private SDPAudioVideoMediaFormat sendingFormat; - private bool sendingFormatFound = false; public void SendAudio(uint durationRtpUnits, byte[] sample) { if (!sendingFormatFound) diff --git a/src/net/RTP/VideoStream.cs b/src/net/RTP/VideoStream.cs index aef495f89..71f4e6cb6 100644 --- a/src/net/RTP/VideoStream.cs +++ b/src/net/RTP/VideoStream.cs @@ -29,10 +29,11 @@ namespace SIPSorcery.net.RTP public class VideoStream : MediaStream { protected static ILogger logger = Log.Logger; - - protected RtpVideoFramer RtpVideoFramer; + private SDPAudioVideoMediaFormat sendingFormat; + private bool sendingFormatFound = false; + #region EVENTS /// @@ -246,8 +247,6 @@ public void SendVp8Frame(uint duration, int payloadTypeID, byte[] buffer) /// value is added to the previous RTP timestamp when building the RTP header. /// The video sample to set as the RTP packet payload. /// - private SDPAudioVideoMediaFormat sendingFormat; - private bool sendingFormatFound = false; public void SendVideo(uint durationRtpUnits, byte[] sample) { if (!sendingFormatFound) diff --git a/src/net/WebRTC/RTCPeerConnection.cs b/src/net/WebRTC/RTCPeerConnection.cs index 38978d984..439266269 100755 --- a/src/net/WebRTC/RTCPeerConnection.cs +++ b/src/net/WebRTC/RTCPeerConnection.cs @@ -160,6 +160,7 @@ public class RTCPeerConnection : RTPSession, IRTCPeerConnection private const string NORMAL_CLOSE_REASON = "normal"; private const ushort SCTP_DEFAULT_PORT = 5000; private const string UNKNOWN_DATACHANNEL_ERROR = "unknown"; + private const int GATHER_TIMEOUT_MS = 2000; /// /// The period to wait for the SCTP association to complete before giving up. @@ -1204,7 +1205,7 @@ private SDP createBaseSdp(List mediaStreamList, bool excludeIceCand // In theory it would be better to an async/await but that would result in a breaking // change to the API and for a one off (once per class instance not once per method call) // delay of a few hundred milliseconds it was decided not to break the API. - using (var ct = new CancellationTokenSource(TimeSpan.FromMilliseconds(2000))) + using (var ct = new CancellationTokenSource(TimeSpan.FromMilliseconds(GATHER_TIMEOUT_MS))) { try { @@ -1212,8 +1213,8 @@ private SDP createBaseSdp(List mediaStreamList, bool excludeIceCand } catch (OperationCanceledException) { - // Handle the timeout scenario here - + logger.LogWarning($"ICE gathering timed out after {GATHER_TIMEOUT_MS}Ms"); + } } From 44c733c6c87bcaef1aa835a9c8f8440b3b8f4a76 Mon Sep 17 00:00:00 2001 From: sean t Date: Thu, 21 Nov 2024 09:19:01 +0800 Subject: [PATCH 08/17] Optimise SendVideo and SendAudio (only get SendingFormat once) Add sanity check for ICE Gathering timeout Fix bug in g722 codec --- src/app/Media/Codecs/G722Codec.cs | 2 +- src/net/RTP/AudioStream.cs | 13 ++++++++++--- src/net/RTP/VideoStream.cs | 18 +++++++++++++----- src/net/SDP/SDPAudioVideoMediaFormat.cs | 4 ++-- src/net/WebRTC/RTCPeerConnection.cs | 14 +++++++++++++- 5 files changed, 39 insertions(+), 12 deletions(-) diff --git a/src/app/Media/Codecs/G722Codec.cs b/src/app/Media/Codecs/G722Codec.cs index 029570ed7..959fe6bb9 100644 --- a/src/app/Media/Codecs/G722Codec.cs +++ b/src/app/Media/Codecs/G722Codec.cs @@ -420,7 +420,7 @@ public int Encode(G722CodecState state, byte[] outputBuffer, short[] inputBuffer } else { - //Duplicate the last sample + //Duplicate the last sample - fix odd shorts issue state.QmfSignalHistory[23] = state.QmfSignalHistory[22]; } diff --git a/src/net/RTP/AudioStream.cs b/src/net/RTP/AudioStream.cs index dce756328..3d1f9935b 100755 --- a/src/net/RTP/AudioStream.cs +++ b/src/net/RTP/AudioStream.cs @@ -31,9 +31,11 @@ namespace SIPSorcery.net.RTP public class AudioStream : MediaStream { protected static ILogger logger = Log.Logger; - protected Boolean rtpEventInProgress = false; + private SDPAudioVideoMediaFormat sendingFormat; + private bool sendingFormatFound = false; + #region EVENTS /// @@ -67,10 +69,15 @@ public bool HasAudio /// The duration in RTP timestamp units of the audio sample. This /// value is added to the previous RTP timestamp when building the RTP header. /// The audio sample to set as the RTP packet payload. + /// public void SendAudio(uint durationRtpUnits, byte[] sample) { - var audioFormat = GetSendingFormat(); - SendAudioFrame(durationRtpUnits, audioFormat.ID, sample); + if (!sendingFormatFound) + { + sendingFormat = GetSendingFormat(); + sendingFormatFound = true; + } + SendAudioFrame(durationRtpUnits, sendingFormat.ID, sample); } /// diff --git a/src/net/RTP/VideoStream.cs b/src/net/RTP/VideoStream.cs index 54dd24161..71f4e6cb6 100644 --- a/src/net/RTP/VideoStream.cs +++ b/src/net/RTP/VideoStream.cs @@ -29,9 +29,11 @@ namespace SIPSorcery.net.RTP public class VideoStream : MediaStream { protected static ILogger logger = Log.Logger; - protected RtpVideoFramer RtpVideoFramer; + private SDPAudioVideoMediaFormat sendingFormat; + private bool sendingFormatFound = false; + #region EVENTS /// @@ -244,12 +246,18 @@ public void SendVp8Frame(uint duration, int payloadTypeID, byte[] buffer) /// The duration in RTP timestamp units of the video sample. This /// value is added to the previous RTP timestamp when building the RTP header. /// The video sample to set as the RTP packet payload. + /// public void SendVideo(uint durationRtpUnits, byte[] sample) { - var videoSendingFormat = GetSendingFormat(); - int payloadID = Convert.ToInt32(videoSendingFormat.ID); + if (!sendingFormatFound) + { + sendingFormat = GetSendingFormat(); + sendingFormatFound = true; + } + + int payloadID = Convert.ToInt32(sendingFormat.ID); - switch (videoSendingFormat.Name()) + switch (sendingFormat.Name()) { case "VP8": SendVp8Frame(durationRtpUnits, payloadID, sample); @@ -258,7 +266,7 @@ public void SendVideo(uint durationRtpUnits, byte[] sample) SendH264Frame(durationRtpUnits, payloadID, sample); break; default: - throw new ApplicationException($"Unsupported video format selected {videoSendingFormat.Name()}."); + throw new ApplicationException($"Unsupported video format selected {sendingFormat.Name()}."); } } diff --git a/src/net/SDP/SDPAudioVideoMediaFormat.cs b/src/net/SDP/SDPAudioVideoMediaFormat.cs index 67cafa6c0..d58a7d118 100755 --- a/src/net/SDP/SDPAudioVideoMediaFormat.cs +++ b/src/net/SDP/SDPAudioVideoMediaFormat.cs @@ -368,13 +368,13 @@ public static List GetCompatibleFormats(List SDPAudioVideoMediaFormat.AreMatch(format, x))) + if (b.Any(x => AreMatch(format, x))) { compatible.Add(format); } } } - + return compatible; } diff --git a/src/net/WebRTC/RTCPeerConnection.cs b/src/net/WebRTC/RTCPeerConnection.cs index dbad165c2..439266269 100755 --- a/src/net/WebRTC/RTCPeerConnection.cs +++ b/src/net/WebRTC/RTCPeerConnection.cs @@ -160,6 +160,7 @@ public class RTCPeerConnection : RTPSession, IRTCPeerConnection private const string NORMAL_CLOSE_REASON = "normal"; private const ushort SCTP_DEFAULT_PORT = 5000; private const string UNKNOWN_DATACHANNEL_ERROR = "unknown"; + private const int GATHER_TIMEOUT_MS = 2000; /// /// The period to wait for the SCTP association to complete before giving up. @@ -1204,7 +1205,18 @@ private SDP createBaseSdp(List mediaStreamList, bool excludeIceCand // In theory it would be better to an async/await but that would result in a breaking // change to the API and for a one off (once per class instance not once per method call) // delay of a few hundred milliseconds it was decided not to break the API. - _iceGatheringTask.Wait(); + using (var ct = new CancellationTokenSource(TimeSpan.FromMilliseconds(GATHER_TIMEOUT_MS))) + { + try + { + _iceGatheringTask.Wait(ct.Token); + } + catch (OperationCanceledException) + { + logger.LogWarning($"ICE gathering timed out after {GATHER_TIMEOUT_MS}Ms"); + + } + } SDP offerSdp = new SDP(IPAddress.Loopback); offerSdp.SessionId = LocalSdpSessionID; From 66c7568aba34910524ad89332365400df76df12d Mon Sep 17 00:00:00 2001 From: sean t Date: Mon, 2 Dec 2024 08:03:35 +0800 Subject: [PATCH 09/17] remove turn folder --- src/SIPSorcery.csproj | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/SIPSorcery.csproj b/src/SIPSorcery.csproj index ddfc90d07..18760a6b3 100755 --- a/src/SIPSorcery.csproj +++ b/src/SIPSorcery.csproj @@ -1,5 +1,11 @@  + + + + + + @@ -79,8 +85,4 @@ - - - - From dc1aa2690ce1b47319634962e45d03a640d0a94b Mon Sep 17 00:00:00 2001 From: sean t Date: Mon, 2 Dec 2024 08:06:13 +0800 Subject: [PATCH 10/17] Remove turn folder --- src/SIPSorcery.csproj | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/SIPSorcery.csproj b/src/SIPSorcery.csproj index 18760a6b3..57fd7ea39 100755 --- a/src/SIPSorcery.csproj +++ b/src/SIPSorcery.csproj @@ -1,11 +1,5 @@  - - - - - - From 52076bc5a343012c28bce74f7b2d167b99016be0 Mon Sep 17 00:00:00 2001 From: sean t Date: Mon, 2 Dec 2024 10:44:27 +0800 Subject: [PATCH 11/17] Move gather timeout to config --- src/net/WebRTC/IRTCPeerConnection.cs | 5 +++++ src/net/WebRTC/RTCPeerConnection.cs | 7 +++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/net/WebRTC/IRTCPeerConnection.cs b/src/net/WebRTC/IRTCPeerConnection.cs index f5779e209..ed0ea8e4a 100755 --- a/src/net/WebRTC/IRTCPeerConnection.cs +++ b/src/net/WebRTC/IRTCPeerConnection.cs @@ -345,6 +345,11 @@ public class RTCConfiguration /// is to use ECDSA. Chrome has defaulted to ECDSA since 2016 (see https://developer.chrome.com/blog/webrtc-ecdsa). /// public bool X_UseRsaForDtlsCertificate; + + /// + /// Timeout for gathering local IP addresses + /// + public int X_GatherTimeoutMs = 30000; } /// diff --git a/src/net/WebRTC/RTCPeerConnection.cs b/src/net/WebRTC/RTCPeerConnection.cs index 439266269..7c658a6f3 100755 --- a/src/net/WebRTC/RTCPeerConnection.cs +++ b/src/net/WebRTC/RTCPeerConnection.cs @@ -160,8 +160,7 @@ public class RTCPeerConnection : RTPSession, IRTCPeerConnection private const string NORMAL_CLOSE_REASON = "normal"; private const ushort SCTP_DEFAULT_PORT = 5000; private const string UNKNOWN_DATACHANNEL_ERROR = "unknown"; - private const int GATHER_TIMEOUT_MS = 2000; - + /// /// The period to wait for the SCTP association to complete before giving up. /// In theory this should be very quick as the DTLS connection should already have been established @@ -1205,7 +1204,7 @@ private SDP createBaseSdp(List mediaStreamList, bool excludeIceCand // In theory it would be better to an async/await but that would result in a breaking // change to the API and for a one off (once per class instance not once per method call) // delay of a few hundred milliseconds it was decided not to break the API. - using (var ct = new CancellationTokenSource(TimeSpan.FromMilliseconds(GATHER_TIMEOUT_MS))) + using (var ct = new CancellationTokenSource(TimeSpan.FromMilliseconds(_configuration.X_GatherTimeoutMs))) { try { @@ -1213,7 +1212,7 @@ private SDP createBaseSdp(List mediaStreamList, bool excludeIceCand } catch (OperationCanceledException) { - logger.LogWarning($"ICE gathering timed out after {GATHER_TIMEOUT_MS}Ms"); + logger.LogWarning($"ICE gathering timed out after {_configuration.X_GatherTimeoutMs}Ms"); } } From ebc9a43d7ccb610fb9333eb28a914c22cdffdad9 Mon Sep 17 00:00:00 2001 From: sean t Date: Fri, 6 Dec 2024 15:45:46 +0800 Subject: [PATCH 12/17] Fix rounding bug in SendAudioFrame Fix bug where duplicate durations were being added to local track timestamp in SendAudioFrame Ignore H264 formats that use unsupported packetization modes Clean up logic in AreMatch --- src/net/RTP/AudioStream.cs | 33 ++++++++---- src/net/SDP/SDPAudioVideoMediaFormat.cs | 70 +++++++++++++++++++++---- 2 files changed, 82 insertions(+), 21 deletions(-) diff --git a/src/net/RTP/AudioStream.cs b/src/net/RTP/AudioStream.cs index 3d1f9935b..bd9c3c416 100755 --- a/src/net/RTP/AudioStream.cs +++ b/src/net/RTP/AudioStream.cs @@ -17,7 +17,6 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Net; using System.Net.Sockets; using System.Threading; using System.Threading.Tasks; @@ -107,15 +106,21 @@ public void SendAudioFrame(uint duration, int payloadTypeID, byte[] buffer) // paylaod. // See https://github.com/sipsorcery/sipsorcery/issues/394. - uint payloadDuration = 0; + int maxPayload = RTPSession.RTP_MAX_PAYLOAD; + int totalPackets = (buffer.Length + maxPayload - 1) / maxPayload; - for (int index = 0; index * RTPSession.RTP_MAX_PAYLOAD < buffer.Length; index++) + uint totalIncrement = 0; + uint startTimestamp = LocalTrack.Timestamp; // Keep track of where we started. + + for (int index = 0; index < totalPackets; index++) { - int offset = (index == 0) ? 0 : (index * RTPSession.RTP_MAX_PAYLOAD); - int payloadLength = (offset + RTPSession.RTP_MAX_PAYLOAD < buffer.Length) ? RTPSession.RTP_MAX_PAYLOAD : buffer.Length - offset; - LocalTrack.Timestamp += payloadDuration; - byte[] payload = new byte[payloadLength]; + int offset = index * maxPayload; + int payloadLength = Math.Min(maxPayload, buffer.Length - offset); + + double fraction = (double)payloadLength / buffer.Length; + uint packetDuration = (uint)Math.Round(fraction * duration); + byte[] payload = new byte[payloadLength]; Buffer.BlockCopy(buffer, offset, payload, 0, payloadLength); // RFC3551 specifies that for audio the marker bit should always be 0 except for when returning @@ -123,13 +128,21 @@ public void SendAudioFrame(uint duration, int payloadTypeID, byte[] buffer) // in a frame. int markerBit = 0; + // Send this packet at the current LocalTrack.Timestamp SendRtpRaw(payload, LocalTrack.Timestamp, markerBit, payloadTypeID, true); - //logger.LogDebug($"send audio { audioRtpChannel.RTPLocalEndPoint}->{AudioDestinationEndPoint}."); - payloadDuration = (uint)(((decimal)payloadLength / buffer.Length) * duration); // Get the percentage duration of this payload. + // After sending, increment the timestamp by this packet's portion. + // This ensures the timestamp increments for the next packet, including the first one. + LocalTrack.Timestamp += packetDuration; + totalIncrement += packetDuration; } - LocalTrack.Timestamp += duration; + // After all packets are sent, correct if we haven't incremented exactly by `duration`. + if (totalIncrement != duration) + { + // Add or subtract the difference so total increment equals duration. + LocalTrack.Timestamp += (duration - totalIncrement); + } } catch (SocketException sockExcp) { diff --git a/src/net/SDP/SDPAudioVideoMediaFormat.cs b/src/net/SDP/SDPAudioVideoMediaFormat.cs index d58a7d118..d041d469a 100755 --- a/src/net/SDP/SDPAudioVideoMediaFormat.cs +++ b/src/net/SDP/SDPAudioVideoMediaFormat.cs @@ -18,6 +18,7 @@ using System.Collections.Generic; using System.Linq; using SIPSorceryMedia.Abstractions; +using static Org.BouncyCastle.Bcpg.Attr.ImageAttrib; namespace SIPSorcery.Net { @@ -142,6 +143,51 @@ public SDPAudioVideoMediaFormat(SDPWellKnownMediaFormatsEnum knownFormat) } } + public bool IsH264 + { + get + { + return (Rtpmap ?? "").ToUpperInvariant().Trim().StartsWith("H264"); + } + } + + public bool CheckCompatible() + { + if (IsH264) + { + var parameters = ParseWebRtcParameters(Fmtp); + if (parameters.TryGetValue("packetization-mode", out string packetizationMode)) + { + if (packetizationMode != "1") + { + return false; + } + } + } + return true; + } + + private static Dictionary ParseWebRtcParameters(string input) + { + var parameters = new Dictionary(); + if (string.IsNullOrEmpty(input)) + { + return parameters; + } + + foreach (var pair in input.Split(';')) + { + var keyValue = pair.Split('='); + if (keyValue.Length == 2) + { + parameters[keyValue[0].Trim().ToLowerInvariant()] = keyValue[1].Trim(); + } + } + + return parameters; + } + + /// /// Creates a new SDP media format for a dynamic media type. Dynamic media types are those that use /// ID's between 96 and 127 inclusive and require an rtpmap attribute and optionally an fmtp attribute. @@ -324,23 +370,22 @@ public VideoFormat ToVideoFormat() public static bool AreMatch(SDPAudioVideoMediaFormat format1, SDPAudioVideoMediaFormat format2) { // rtpmap takes priority as well known format ID's can be overruled. - if (format1.Rtpmap != null - && format2.Rtpmap != null && - string.Equals(format1.Rtpmap.Trim(), format2.Rtpmap.Trim(), StringComparison.OrdinalIgnoreCase)) + if (format1.Rtpmap != null && format2.Rtpmap != null) { - return true; + if (string.Equals(format1.Rtpmap.Trim(), format2.Rtpmap.Trim(), StringComparison.OrdinalIgnoreCase)) + { + return true; + } } - else if (format1.ID < DYNAMIC_ID_MIN + if (format1.ID < DYNAMIC_ID_MIN && format1.ID == format2.ID && string.Equals(format1.Name(), format2.Name(), StringComparison.OrdinalIgnoreCase)) { // Well known format type. return true; } - else - { - return false; - } + return false; + } /// @@ -370,11 +415,14 @@ public static List GetCompatibleFormats(List AreMatch(format, x))) { - compatible.Add(format); + if (format.CheckCompatible()) + { + compatible.Add(format); + } } } } - + return compatible; } From 1c65a2d3731e3330441b38eb2d80a9442753d7b5 Mon Sep 17 00:00:00 2001 From: sean t Date: Sun, 8 Dec 2024 08:41:10 +0800 Subject: [PATCH 13/17] Remove comment marker --- src/net/RTP/AudioStream.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/net/RTP/AudioStream.cs b/src/net/RTP/AudioStream.cs index bd9c3c416..062a24880 100755 --- a/src/net/RTP/AudioStream.cs +++ b/src/net/RTP/AudioStream.cs @@ -68,7 +68,6 @@ public bool HasAudio /// The duration in RTP timestamp units of the audio sample. This /// value is added to the previous RTP timestamp when building the RTP header. /// The audio sample to set as the RTP packet payload. - /// public void SendAudio(uint durationRtpUnits, byte[] sample) { if (!sendingFormatFound) From 8eb58373404757e3412f6902278a0899bae7237a Mon Sep 17 00:00:00 2001 From: sean t Date: Sun, 8 Dec 2024 09:02:20 +0800 Subject: [PATCH 14/17] remove comment marker --- src/net/RTP/AudioStream.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/net/RTP/AudioStream.cs b/src/net/RTP/AudioStream.cs index bd9c3c416..062a24880 100755 --- a/src/net/RTP/AudioStream.cs +++ b/src/net/RTP/AudioStream.cs @@ -68,7 +68,6 @@ public bool HasAudio /// The duration in RTP timestamp units of the audio sample. This /// value is added to the previous RTP timestamp when building the RTP header. /// The audio sample to set as the RTP packet payload. - /// public void SendAudio(uint durationRtpUnits, byte[] sample) { if (!sendingFormatFound) From dca74475499c7cc336ab7ad14e0f8e8030035305 Mon Sep 17 00:00:00 2001 From: sean t Date: Tue, 17 Dec 2024 18:39:54 +0800 Subject: [PATCH 15/17] Check for null --- src/net/RTP/RTPSession.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/net/RTP/RTPSession.cs b/src/net/RTP/RTPSession.cs index 0bc2f0b94..d3312896b 100755 --- a/src/net/RTP/RTPSession.cs +++ b/src/net/RTP/RTPSession.cs @@ -705,7 +705,7 @@ private void CloseRtcpSession(MediaStream mediaStream, string reason) } } - mediaStream.RtcpSession.Close(reason); + mediaStream.RtcpSession?.Close(reason); mediaStream.RtcpSession = null; } } From ae9c22cc730816d705fdf3c73162045f6d138d86 Mon Sep 17 00:00:00 2001 From: sean t Date: Mon, 30 Dec 2024 09:28:55 +0800 Subject: [PATCH 16/17] Fix bug with HasVideo and HasAudio --- src/net/RTP/AudioStream.cs | 4 ++-- src/net/RTP/VideoStream.cs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/net/RTP/AudioStream.cs b/src/net/RTP/AudioStream.cs index 062a24880..5644bade7 100755 --- a/src/net/RTP/AudioStream.cs +++ b/src/net/RTP/AudioStream.cs @@ -53,8 +53,8 @@ public bool HasAudio { get { - return LocalTrack != null && LocalTrack.StreamStatus != MediaStreamStatusEnum.Inactive - && RemoteTrack != null && RemoteTrack.StreamStatus != MediaStreamStatusEnum.Inactive; + return (LocalTrack != null && LocalTrack.StreamStatus != MediaStreamStatusEnum.Inactive) + || (RemoteTrack != null && RemoteTrack.StreamStatus != MediaStreamStatusEnum.Inactive); } } diff --git a/src/net/RTP/VideoStream.cs b/src/net/RTP/VideoStream.cs index 71f4e6cb6..b326f9d7b 100644 --- a/src/net/RTP/VideoStream.cs +++ b/src/net/RTP/VideoStream.cs @@ -64,8 +64,8 @@ public bool HasVideo { get { - return LocalTrack != null && LocalTrack.StreamStatus != MediaStreamStatusEnum.Inactive - && RemoteTrack != null && RemoteTrack.StreamStatus != MediaStreamStatusEnum.Inactive; + return (LocalTrack != null && LocalTrack.StreamStatus != MediaStreamStatusEnum.Inactive) + || (RemoteTrack != null && RemoteTrack.StreamStatus != MediaStreamStatusEnum.Inactive); } } From a8050a624d30fb85ee962a857aecc0e51a59a5c6 Mon Sep 17 00:00:00 2001 From: sean t Date: Sun, 23 Feb 2025 09:10:07 +0800 Subject: [PATCH 17/17] Fix for allocate requests into the STUN server resulting in null reference exceptions --- src/net/STUN/STUNServer.cs | 75 +++++++++++++++++++------------------- 1 file changed, 38 insertions(+), 37 deletions(-) diff --git a/src/net/STUN/STUNServer.cs b/src/net/STUN/STUNServer.cs index 8bcf01f72..b8c9a59f2 100644 --- a/src/net/STUN/STUNServer.cs +++ b/src/net/STUN/STUNServer.cs @@ -206,49 +206,50 @@ public void STUNSecondaryReceived(IPEndPoint localEndPoint, IPEndPoint receivedE private STUNMessage GetResponse(IPEndPoint receivedEndPoint, STUNMessage stunRequest, bool primary) { - if (stunRequest.Header.MessageType == STUNMessageTypesEnum.BindingRequest) + switch(stunRequest.Header.MessageType) { - STUNMessage stunResponse = new STUNMessage(); - stunResponse.Header.MessageType = STUNMessageTypesEnum.BindingSuccessResponse; - stunResponse.Header.TransactionId = stunRequest.Header.TransactionId; - - // Add MappedAddress attribute to indicate the socket the request was received from. - STUNAddressAttribute mappedAddressAtt = new STUNAddressAttribute(STUNAttributeTypesEnum.MappedAddress, receivedEndPoint.Port, receivedEndPoint.Address); - stunResponse.Attributes.Add(mappedAddressAtt); - stunResponse.AddXORMappedAddressAttribute(receivedEndPoint.Address, receivedEndPoint.Port);//Compatible with the client code - - // Add SourceAddress attribute to indicate the socket used to send the response. - if (primary) - { - STUNAddressAttribute sourceAddressAtt = new STUNAddressAttribute(STUNAttributeTypesEnum.SourceAddress, m_primaryEndPoint.Port, m_primaryEndPoint.Address); - stunResponse.Attributes.Add(sourceAddressAtt); - } - else - { - STUNAddressAttribute sourceAddressAtt = new STUNAddressAttribute(STUNAttributeTypesEnum.SourceAddress, m_secondaryEndPoint.Port, m_secondaryEndPoint.Address); - stunResponse.Attributes.Add(sourceAddressAtt); - } + case STUNMessageTypesEnum.BindingRequest: + STUNMessage stunResponse = new STUNMessage(); + stunResponse.Header.MessageType = STUNMessageTypesEnum.BindingSuccessResponse; + stunResponse.Header.TransactionId = stunRequest.Header.TransactionId; + + // Add MappedAddress attribute to indicate the socket the request was received from. + STUNAddressAttribute mappedAddressAtt = new STUNAddressAttribute(STUNAttributeTypesEnum.MappedAddress, receivedEndPoint.Port, receivedEndPoint.Address); + stunResponse.Attributes.Add(mappedAddressAtt); + stunResponse.AddXORMappedAddressAttribute(receivedEndPoint.Address, receivedEndPoint.Port);//Compatible with the client code + + // Add SourceAddress attribute to indicate the socket used to send the response. + if (primary) + { + STUNAddressAttribute sourceAddressAtt = new STUNAddressAttribute(STUNAttributeTypesEnum.SourceAddress, m_primaryEndPoint.Port, m_primaryEndPoint.Address); + stunResponse.Attributes.Add(sourceAddressAtt); + } + else + { + STUNAddressAttribute sourceAddressAtt = new STUNAddressAttribute(STUNAttributeTypesEnum.SourceAddress, m_secondaryEndPoint.Port, m_secondaryEndPoint.Address); + stunResponse.Attributes.Add(sourceAddressAtt); + } - // Add ChangedAddress attribute to indicate the servers alternative socket. - if (primary) - { - STUNAddressAttribute changedAddressAtt = new STUNAddressAttribute(STUNAttributeTypesEnum.ChangedAddress, m_secondaryEndPoint.Port, m_secondaryEndPoint.Address); - stunResponse.Attributes.Add(changedAddressAtt); - } - else - { - STUNAddressAttribute changedAddressAtt = new STUNAddressAttribute(STUNAttributeTypesEnum.ChangedAddress, m_primaryEndPoint.Port, m_primaryEndPoint.Address); - stunResponse.Attributes.Add(changedAddressAtt); - } + // Add ChangedAddress attribute to indicate the servers alternative socket. + if (primary) + { + STUNAddressAttribute changedAddressAtt = new STUNAddressAttribute(STUNAttributeTypesEnum.ChangedAddress, m_secondaryEndPoint.Port, m_secondaryEndPoint.Address); + stunResponse.Attributes.Add(changedAddressAtt); + } + else + { + STUNAddressAttribute changedAddressAtt = new STUNAddressAttribute(STUNAttributeTypesEnum.ChangedAddress, m_primaryEndPoint.Port, m_primaryEndPoint.Address); + stunResponse.Attributes.Add(changedAddressAtt); + } - //Console.WriteLine(stunResponse.ToString()); + //Console.WriteLine(stunResponse.ToString()); - //byte[] stunResponseBuffer = stunResponse.ToByteBuffer(); + //byte[] stunResponseBuffer = stunResponse.ToByteBuffer(); - return stunResponse; + return stunResponse; + default://Allocate not supported - this is a STUN server only not a full TURN server + throw new ApplicationException("Unsupported MessageType " + stunRequest.Header.MessageType); } - - return null; } public void Stop()