From b593aa05efe667db934c818543ac01a6f0b3e7a4 Mon Sep 17 00:00:00 2001 From: Marvin W Date: Tue, 9 Nov 2021 22:06:47 +0100 Subject: RTP: Encode with device --- plugins/rtp/src/device.vala | 335 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 260 insertions(+), 75 deletions(-) (limited to 'plugins/rtp/src/device.vala') diff --git a/plugins/rtp/src/device.vala b/plugins/rtp/src/device.vala index e25271b1..3546ab94 100644 --- a/plugins/rtp/src/device.vala +++ b/plugins/rtp/src/device.vala @@ -1,5 +1,9 @@ +using Xmpp.Xep.JingleRtp; +using Gee; + public class Dino.Plugins.Rtp.Device : MediaDevice, Object { public Plugin plugin { get; private set; } + public CodecUtil codec_util { get { return plugin.codec_util; } } public Gst.Device device { get; private set; } private string device_name; @@ -17,28 +21,45 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { return plugin.pipe; }} public string? media { get { - if (device.device_class.has_prefix("Audio/")) { + if (device.has_classes("Audio")) { return "audio"; - } else if (device.device_class.has_prefix("Video/")) { + } else if (device.has_classes("Video")) { return "video"; } else { return null; } }} public bool is_source { get { - return device.device_class.has_suffix("/Source"); + return device.has_classes("Source"); }} public bool is_sink { get { - return device.device_class.has_suffix("/Sink"); + return device.has_classes("Sink"); }} private Gst.Element element; private Gst.Element tee; private Gst.Element dsp; - private Gst.Element mixer; + private Gst.Base.Aggregator mixer; private Gst.Element filter; - private Gst.Element rate; - private int links = 0; + private int links; + + // Codecs + private Gee.Map codecs = new HashMap(PayloadType.hash_func, PayloadType.equals_func); + private Gee.Map codec_tees = new HashMap(PayloadType.hash_func, PayloadType.equals_func); + private Gee.Map> payloaders = new HashMap>(PayloadType.hash_func, PayloadType.equals_func); + private Gee.Map> payloader_tees = new HashMap>(PayloadType.hash_func, PayloadType.equals_func); + private Gee.Map> payloader_links = new HashMap>(PayloadType.hash_func, PayloadType.equals_func); + private Gee.Map> codec_bitrates = new HashMap>(PayloadType.hash_func, PayloadType.equals_func); + + private class CodecBitrate { + public uint bitrate; + public int64 timestamp; + + public CodecBitrate(uint bitrate) { + this.bitrate = bitrate; + this.timestamp = get_monotonic_time(); + } + } public Device(Plugin plugin, Gst.Device device) { this.plugin = plugin; @@ -57,25 +78,154 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { } public Gst.Element? link_sink() { + if (!is_sink) return null; if (element == null) create(); links++; - if (mixer != null) return mixer; - if (is_sink && media == "audio") return filter; + if (mixer != null) { + Gst.Element rate = Gst.ElementFactory.make("audiorate", @"$(id)_rate_$(Random.next_int())"); + pipe.add(rate); + rate.link(mixer); + return rate; + } + if (media == "audio") return filter; return element; } - public Gst.Element? link_source() { + public Gst.Element? link_source(PayloadType? payload_type = null, uint ssrc = Random.next_int(), int seqnum_offset = -1) { + if (!is_source) return null; if (element == null) create(); links++; + if (payload_type != null && tee != null) { + bool new_codec = false; + string? codec = CodecUtil.get_codec_from_payload(media, payload_type); + if (!codecs.has_key(payload_type)) { + codecs[payload_type] = codec_util.get_encode_bin_without_payloader(media, payload_type, @"$(id)_$(codec)_encoder"); + pipe.add(codecs[payload_type]); + new_codec = true; + } + if (!codec_tees.has_key(payload_type)) { + codec_tees[payload_type] = Gst.ElementFactory.make("tee", @"$(id)_$(codec)_tee"); + codec_tees[payload_type].@set("allow-not-linked", true); + pipe.add(codec_tees[payload_type]); + codecs[payload_type].link(codec_tees[payload_type]); + } + if (!payloaders.has_key(payload_type)) { + payloaders[payload_type] = new HashMap(); + } + if (!payloaders[payload_type].has_key(ssrc)) { + payloaders[payload_type][ssrc] = codec_util.get_payloader_bin(media, payload_type, @"$(id)_$(codec)_$(ssrc)"); + var payload = (Gst.RTP.BasePayload) ((Gst.Bin) payloaders[payload_type][ssrc]).get_by_name(@"$(id)_$(codec)_$(ssrc)_rtp_pay"); + payload.ssrc = ssrc; + payload.seqnum_offset = seqnum_offset; + pipe.add(payloaders[payload_type][ssrc]); + codec_tees[payload_type].link(payloaders[payload_type][ssrc]); + } + if (!payloader_tees.has_key(payload_type)) { + payloader_tees[payload_type] = new HashMap(); + } + if (!payloader_tees[payload_type].has_key(ssrc)) { + payloader_tees[payload_type][ssrc] = Gst.ElementFactory.make("tee", @"$(id)_$(codec)_$(ssrc)_tee"); + payloader_tees[payload_type][ssrc].@set("allow-not-linked", true); + pipe.add(payloader_tees[payload_type][ssrc]); + payloaders[payload_type][ssrc].link(payloader_tees[payload_type][ssrc]); + } + if (!payloader_links.has_key(payload_type)) { + payloader_links[payload_type] = new HashMap(); + } + if (!payloader_links[payload_type].has_key(ssrc)) { + payloader_links[payload_type][ssrc] = 1; + } else { + payloader_links[payload_type][ssrc] = payloader_links[payload_type][ssrc] + 1; + } + if (new_codec) { + tee.link(codecs[payload_type]); + } + return payloader_tees[payload_type][ssrc]; + } if (tee != null) return tee; return element; } - public void unlink() { + public void update_bitrate(PayloadType payload_type, uint bitrate) { + if (codecs.has_key(payload_type)) { + lock(codec_bitrates); + if (!codec_bitrates.has_key(payload_type)) { + codec_bitrates[payload_type] = new ArrayList(); + } + codec_bitrates[payload_type].add(new CodecBitrate(bitrate)); + var remove = new ArrayList(); + foreach (CodecBitrate rate in codec_bitrates[payload_type]) { + if (rate.timestamp < get_monotonic_time() - 5000000L) { + remove.add(rate); + continue; + } + if (rate.bitrate < bitrate) { + bitrate = rate.bitrate; + } + } + codec_bitrates[payload_type].remove_all(remove); + codec_util.update_bitrate(media, payload_type, codecs[payload_type], bitrate); + unlock(codec_bitrates); + } + } + + public void unlink(Gst.Element? link = null) { if (links <= 0) { critical("Link count below zero."); return; } + if (link != null && is_source && tee != null) { + PayloadType payload_type = payloader_tees.first_match((entry) => entry.value.any_match((entry) => entry.value == link)).key; + uint ssrc = payloader_tees[payload_type].first_match((entry) => entry.value == link).key; + payloader_links[payload_type][ssrc] = payloader_links[payload_type][ssrc] - 1; + if (payloader_links[payload_type][ssrc] == 0) { + plugin.pause(); + + codec_tees[payload_type].unlink(payloaders[payload_type][ssrc]); + payloaders[payload_type][ssrc].set_locked_state(true); + payloaders[payload_type][ssrc].set_state(Gst.State.NULL); + payloaders[payload_type][ssrc].unlink(payloader_tees[payload_type][ssrc]); + pipe.remove(payloaders[payload_type][ssrc]); + payloaders[payload_type].unset(ssrc); + payloader_tees[payload_type][ssrc].set_locked_state(true); + payloader_tees[payload_type][ssrc].set_state(Gst.State.NULL); + pipe.remove(payloader_tees[payload_type][ssrc]); + payloader_tees[payload_type].unset(ssrc); + + payloader_links[payload_type].unset(ssrc); + plugin.unpause(); + } + if (payloader_links[payload_type].size == 0) { + plugin.pause(); + + tee.unlink(codecs[payload_type]); + codecs[payload_type].set_locked_state(true); + codecs[payload_type].set_state(Gst.State.NULL); + codecs[payload_type].unlink(codec_tees[payload_type]); + pipe.remove(codecs[payload_type]); + codecs.unset(payload_type); + codec_tees[payload_type].set_locked_state(true); + codec_tees[payload_type].set_state(Gst.State.NULL); + pipe.remove(codec_tees[payload_type]); + codec_tees.unset(payload_type); + + payloaders.unset(payload_type); + payloader_tees.unset(payload_type); + payloader_links.unset(payload_type); + plugin.unpause(); + } + } + if (link != null && is_sink && mixer != null) { + plugin.pause(); + link.set_locked_state(true); + Gst.Base.AggregatorPad mixer_sink_pad = (Gst.Base.AggregatorPad) link.get_static_pad("src").get_peer(); + link.get_static_pad("src").unlink(mixer_sink_pad); + mixer_sink_pad.set_active(false); + link.set_state(Gst.State.NULL); + pipe.remove(link); + mixer.release_request_pad(mixer_sink_pad); + plugin.unpause(); + } links--; if (links == 0) { destroy(); @@ -150,15 +300,59 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { return target; } + private static Gst.PadProbeReturn log_probe(Gst.Pad pad, Gst.PadProbeInfo info) { + if ((info.type & Gst.PadProbeType.EVENT_DOWNSTREAM) > 0) { + debug("%s.%s probed downstream event %s", pad.get_parent_element().name, pad.name, info.get_event().type.get_name()); + } + if ((info.type & Gst.PadProbeType.EVENT_UPSTREAM) > 0) { + var event = info.get_event(); + if (event.type == Gst.EventType.RECONFIGURE) return Gst.PadProbeReturn.DROP; + if (event.type == Gst.EventType.QOS) { + Gst.QOSType qos_type; + double proportion; + Gst.ClockTimeDiff diff; + Gst.ClockTime timestamp; + event.parse_qos(out qos_type, out proportion, out diff, out timestamp); + debug("%s.%s probed qos event: type: %s, proportion: %f, diff: %lli, timestamp: %llu", pad.get_parent_element().name, pad.name, @"$qos_type", proportion, diff, timestamp); + } else { + debug("%s.%s probed upstream event %s", pad.get_parent_element().name, pad.name, event.type.get_name()); + } + } + if ((info.type & Gst.PadProbeType.QUERY_DOWNSTREAM) > 0) { + debug("%s.%s probed downstream query %s", pad.get_parent_element().name, pad.name, info.get_query().type.get_name()); + } + if ((info.type & Gst.PadProbeType.QUERY_UPSTREAM) > 0) { + debug("%s.%s probed upstream query %s", pad.get_parent_element().name, pad.name, info.get_query().type.get_name()); + } + if ((info.type & Gst.PadProbeType.BUFFER) > 0) { + uint id = pad.get_data("no_buffer_probe_timeout"); + if (id != 0) { + Source.remove(id); + } + string name = @"$(pad.get_parent_element().name).$(pad.name)"; + id = Timeout.add_seconds(1, () => { + debug("%s probed no buffer for 1 second", name); + return Source.REMOVE; + }); + pad.set_data("no_buffer_probe_timeout", id); + } + return Gst.PadProbeReturn.PASS; + } + private void create() { debug("Creating device %s", id); plugin.pause(); element = device.create_element(id); + if (is_sink) { + element.@set("async", false); + element.@set("sync", false); + } pipe.add(element); if (is_source) { element.@set("do-timestamp", true); filter = Gst.ElementFactory.make("capsfilter", @"caps_filter_$id"); filter.@set("caps", get_best_caps()); + filter.get_static_pad("src").add_probe(Gst.PadProbeType.BLOCK, log_probe); pipe.add(filter); element.link(filter); #if WITH_VOICE_PROCESSOR @@ -174,22 +368,18 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { pipe.add(tee); (dsp ?? filter).link(tee); } - if (is_sink) { - element.@set("async", false); - element.@set("sync", false); - } if (is_sink && media == "audio") { - filter = Gst.ElementFactory.make("capsfilter", @"caps_filter_$id"); - filter.@set("caps", get_best_caps()); - pipe.add(filter); - if (plugin.echoprobe != null) { - rate = Gst.ElementFactory.make("audiorate", @"rate_$id"); - rate.@set("tolerance", 100000000); - pipe.add(rate); - filter.link(rate); - rate.link(plugin.echoprobe); + mixer = (Gst.Base.Aggregator) Gst.ElementFactory.make("audiomixer", @"mixer_$id"); + pipe.add(mixer); + mixer.link(pipe); + if (plugin.echoprobe != null && !plugin.echoprobe.get_static_pad("src").is_linked()) { + mixer.link(plugin.echoprobe); plugin.echoprobe.link(element); } else { + filter = Gst.ElementFactory.make("capsfilter", @"caps_filter_$id"); + filter.@set("caps", get_best_caps()); + pipe.add(filter); + mixer.link(filter); filter.link(element); } } @@ -197,38 +387,25 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { } private void destroy() { - if (mixer != null) { - if (is_sink && media == "audio" && plugin.echoprobe != null) { - plugin.echoprobe.unlink(mixer); - } - int linked_sink_pads = 0; - mixer.foreach_sink_pad((_, pad) => { - if (pad.is_linked()) linked_sink_pads++; - return true; - }); - if (linked_sink_pads > 0) { - warning("%s-mixer still has %i sink pads while being destroyed", id, linked_sink_pads); + if (is_sink) { + if (mixer != null) { + int linked_sink_pads = 0; + mixer.foreach_sink_pad((_, pad) => { + if (pad.is_linked()) linked_sink_pads++; + return true; + }); + if (linked_sink_pads > 0) { + warning("%s-mixer still has %i sink pads while being destroyed", id, linked_sink_pads); + } + mixer.unlink(plugin.echoprobe ?? element); } - mixer.set_locked_state(true); - mixer.set_state(Gst.State.NULL); - mixer.unlink(element); - pipe.remove(mixer); - mixer = null; - } else if (is_sink && media == "audio") { if (filter != null) { filter.set_locked_state(true); filter.set_state(Gst.State.NULL); - filter.unlink(rate ?? ((Gst.Element)plugin.echoprobe) ?? element); + filter.unlink(element); pipe.remove(filter); filter = null; } - if (rate != null) { - rate.set_locked_state(true); - rate.set_state(Gst.State.NULL); - rate.unlink(plugin.echoprobe); - pipe.remove(rate); - rate = null; - } if (plugin.echoprobe != null) { plugin.echoprobe.unlink(element); } @@ -239,34 +416,42 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { else if (is_source) element.unlink(tee); pipe.remove(element); element = null; - if (filter != null) { - filter.set_locked_state(true); - filter.set_state(Gst.State.NULL); - filter.unlink(dsp ?? tee); - pipe.remove(filter); - filter = null; - } - if (dsp != null) { - dsp.set_locked_state(true); - dsp.set_state(Gst.State.NULL); - dsp.unlink(tee); - pipe.remove(dsp); - dsp = null; + if (mixer != null) { + mixer.set_locked_state(true); + mixer.set_state(Gst.State.NULL); + pipe.remove(mixer); + mixer = null; } - if (tee != null) { - int linked_src_pads = 0; - tee.foreach_src_pad((_, pad) => { - if (pad.is_linked()) linked_src_pads++; - return true; - }); - if (linked_src_pads != 0) { - warning("%s-tee still has %d src pads while being destroyed", id, linked_src_pads); + if (is_source) { + if (filter != null) { + filter.set_locked_state(true); + filter.set_state(Gst.State.NULL); + filter.unlink(dsp ?? tee); + pipe.remove(filter); + filter = null; + } + if (dsp != null) { + dsp.set_locked_state(true); + dsp.set_state(Gst.State.NULL); + dsp.unlink(tee); + pipe.remove(dsp); + dsp = null; + } + if (tee != null) { + int linked_src_pads = 0; + tee.foreach_src_pad((_, pad) => { + if (pad.is_linked()) linked_src_pads++; + return true; + }); + if (linked_src_pads != 0) { + warning("%s-tee still has %d src pads while being destroyed", id, linked_src_pads); + } + tee.set_locked_state(true); + tee.set_state(Gst.State.NULL); + pipe.remove(tee); + tee = null; } - tee.set_locked_state(true); - tee.set_state(Gst.State.NULL); - pipe.remove(tee); - tee = null; } debug("Destroyed device %s", id); } -} \ No newline at end of file +} -- cgit v1.2.3 From 0b828a0ae55b3c0eaf1206f0c54cfcee4f60e6af Mon Sep 17 00:00:00 2001 From: Marvin W Date: Mon, 15 Nov 2021 22:49:44 +0100 Subject: Add maximum bitrate and adjust video resolution based on bitrate --- plugins/rtp/src/codec_util.vala | 35 +++++++++++++---- plugins/rtp/src/device.vala | 85 ++++++++++++++++++++++++++++++++++++++++- plugins/rtp/src/plugin.vala | 1 - plugins/rtp/src/stream.vala | 2 +- 4 files changed, 111 insertions(+), 12 deletions(-) (limited to 'plugins/rtp/src/device.vala') diff --git a/plugins/rtp/src/codec_util.vala b/plugins/rtp/src/codec_util.vala index 4c68985f..ec922f97 100644 --- a/plugins/rtp/src/codec_util.vala +++ b/plugins/rtp/src/codec_util.vala @@ -140,13 +140,16 @@ public class Dino.Plugins.Rtp.CodecUtil { public static string? get_encode_args(string media, string codec, string encode, JingleRtp.PayloadType? payload_type) { // H264 if (encode == "msdkh264enc") return @" rate-control=vbr"; - if (encode == "vaapih264enc") return @" tune=low-power"; - if (encode == "x264enc") return @" byte-stream=1 profile=baseline speed-preset=ultrafast tune=zerolatency"; + if (encode == "vaapih264enc") return @" rate-control=vbr tune=low-power"; + if (encode == "x264enc") return @" byte-stream=1 speed-preset=ultrafast tune=zerolatency"; // VP8 - if (encode == "msdkvp8enc") return " rate-control=vbr"; - if (encode == "vaapivp8enc") return " rate-control=vbr"; - if (encode == "vp8enc") return " deadline=1 error-resilient=3 lag-in-frames=0 resize-allowed=true threads=8 dropframe-threshold=30"; + if (encode == "vaapivp8enc" || encode == "msdkvp8enc") return " rate-control=vbr target-percentage=90"; + if (encode == "vp8enc") return " deadline=1 error-resilient=3 lag-in-frames=0 resize-allowed=true threads=8 dropframe-threshold=30 end-usage=vbr"; + + // VP9 + if (encode == "msdkvp9enc" || encode == "vaapivp9enc") return " rate-control=vbr target-percentage=90"; + if (encode == "vp9enc") return " deadline=1 error-resilient=3 lag-in-frames=0 resize-allowed=true threads=8 dropframe-threshold=30 end-usage=vbr"; // OPUS if (encode == "opusenc") { @@ -186,13 +189,29 @@ public class Dino.Plugins.Rtp.CodecUtil { case "vp9enc": case "vp8enc": bitrate = uint.min(2147483, bitrate); - encode.set("target-bitrate", bitrate * 1000); + encode.set("target-bitrate", bitrate * 1024); return bitrate; } return 0; } + public void update_rescale_caps(Gst.Element encode_element, Gst.Caps caps) { + Gst.Bin? encode_bin = encode_element as Gst.Bin; + if (encode_bin == null) return; + Gst.Element rescale_caps = encode_bin.get_by_name(@"$(encode_bin.name)_rescale_caps"); + rescale_caps.set("caps", caps); + } + + public Gst.Caps? get_rescale_caps(Gst.Element encode_element) { + Gst.Bin? encode_bin = encode_element as Gst.Bin; + if (encode_bin == null) return null; + Gst.Element rescale_caps = encode_bin.get_by_name(@"$(encode_bin.name)_rescale_caps"); + Gst.Caps caps; + rescale_caps.get("caps", out caps); + return caps; + } + public static string? get_decode_prefix(string media, string codec, string decode, JingleRtp.PayloadType? payload_type) { return null; } @@ -309,8 +328,8 @@ public class Dino.Plugins.Rtp.CodecUtil { string encode_prefix = get_encode_prefix(media, codec, encode, payload_type) ?? ""; string encode_args = get_encode_args(media, codec, encode, payload_type) ?? ""; string encode_suffix = get_encode_suffix(media, codec, encode, payload_type) ?? ""; - string resample = media == "audio" ? @" ! audioresample name=$(base_name)_resample" : ""; - return @"$(media)convert name=$(base_name)_convert$resample ! queue ! $encode_prefix$encode$encode_args name=$(base_name)_encode$encode_suffix"; + string rescale = media == "audio" ? @" ! audioresample name=$(base_name)_resample" : @" ! videoscale name=$(base_name)_rescale ! capsfilter name=$(base_name)_rescale_caps"; + return @"$(media)convert name=$(base_name)_convert$rescale ! queue ! $encode_prefix$encode$encode_args name=$(base_name)_encode$encode_suffix"; } public Gst.Element? get_encode_bin(string media, JingleRtp.PayloadType payload_type, string? name = null) { diff --git a/plugins/rtp/src/device.vala b/plugins/rtp/src/device.vala index 3546ab94..89d499ed 100644 --- a/plugins/rtp/src/device.vala +++ b/plugins/rtp/src/device.vala @@ -36,6 +36,7 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { return device.has_classes("Sink"); }} + private Gst.Caps device_caps; private Gst.Element element; private Gst.Element tee; private Gst.Element dsp; @@ -46,9 +47,13 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { // Codecs private Gee.Map codecs = new HashMap(PayloadType.hash_func, PayloadType.equals_func); private Gee.Map codec_tees = new HashMap(PayloadType.hash_func, PayloadType.equals_func); + + // Payloaders private Gee.Map> payloaders = new HashMap>(PayloadType.hash_func, PayloadType.equals_func); private Gee.Map> payloader_tees = new HashMap>(PayloadType.hash_func, PayloadType.equals_func); private Gee.Map> payloader_links = new HashMap>(PayloadType.hash_func, PayloadType.equals_func); + + // Bitrate private Gee.Map> codec_bitrates = new HashMap>(PayloadType.hash_func, PayloadType.equals_func); private class CodecBitrate { @@ -146,6 +151,51 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { return element; } + private static double get_target_bitrate(Gst.Caps caps) { + if (caps == null || caps.get_size() == 0) return uint.MAX; + unowned Gst.Structure? that = caps.get_structure(0); + int num = 0, den = 0, width = 0, height = 0; + if (!that.has_field("width") || !that.get_int("width", out width)) return uint.MAX; + if (!that.has_field("height") || !that.get_int("height", out height)) return uint.MAX; + if (!that.has_field("framerate")) return uint.MAX; + Value framerate = that.get_value("framerate"); + if (framerate.type() != typeof(Gst.Fraction)) return uint.MAX; + num = Gst.Value.get_fraction_numerator(framerate); + den = Gst.Value.get_fraction_denominator(framerate); + double pxs = ((double)num/(double)den) * (double)width * (double)height; + double br = Math.sqrt(Math.sqrt(pxs)) * 100.0 - 3700.0; + if (br < 128.0) return 128.0; + return br; + } + + private const int[] common_widths = {320, 480, 640, 960, 1280, 1920, 2560, 3840}; + private Gst.Caps get_active_caps(PayloadType payload_type) { + return codec_util.get_rescale_caps(codecs[payload_type]) ?? device_caps; + } + private void apply_caps(PayloadType payload_type, Gst.Caps caps) { + plugin.pause(); + debug("Set scaled caps to %s", caps.to_string()); + codec_util.update_rescale_caps(codecs[payload_type], caps); + plugin.unpause(); + } + private void apply_width(PayloadType payload_type, int new_width, uint bitrate) { + int device_caps_width, device_caps_height, active_caps_width, device_caps_framerate_num, device_caps_framerate_den; + device_caps.get_structure(0).get_int("width", out device_caps_width); + device_caps.get_structure(0).get_int("height", out device_caps_height); + device_caps.get_structure(0).get_fraction("framerate", out device_caps_framerate_num, out device_caps_framerate_den); + Gst.Caps active_caps = get_active_caps(payload_type); + if (active_caps != null && active_caps.get_size() > 0) { + active_caps.get_structure(0).get_int("width", out active_caps_width); + } else { + active_caps_width = device_caps_width; + } + if (new_width == active_caps_width) return; + int new_height = device_caps_height * new_width / device_caps_width; + Gst.Caps new_caps = new Gst.Caps.simple("video/x-raw", "width", typeof(int), new_width, "height", typeof(int), new_height, "framerate", typeof(Gst.Fraction), device_caps_framerate_num, device_caps_framerate_den, null); + double required_bitrate = get_target_bitrate(new_caps); + if (bitrate < required_bitrate) return; + apply_caps(payload_type, new_caps); + } public void update_bitrate(PayloadType payload_type, uint bitrate) { if (codecs.has_key(payload_type)) { lock(codec_bitrates); @@ -164,6 +214,36 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { } } codec_bitrates[payload_type].remove_all(remove); + if (media == "video") { + if (bitrate < 128) bitrate = 128; + Gst.Caps active_caps = get_active_caps(payload_type); + double max_bitrate = get_target_bitrate(device_caps) * 2; + double current_target_bitrate = get_target_bitrate(active_caps); + int device_caps_width, active_caps_width; + device_caps.get_structure(0).get_int("width", out device_caps_width); + if (active_caps != null && active_caps.get_size() > 0) { + active_caps.get_structure(0).get_int("width", out active_caps_width); + } else { + active_caps_width = device_caps_width; + } + if (bitrate < 0.75 * current_target_bitrate && active_caps_width > common_widths[0]) { + // Lower video resolution + int i = 1; + for(; i < common_widths.length && common_widths[i] < active_caps_width; i++); + apply_width(payload_type, common_widths[i-1], bitrate); + } else if (bitrate > 2 * current_target_bitrate && active_caps_width < device_caps_width) { + // Higher video resolution + int i = 0; + for(; i < common_widths.length && common_widths[i] <= active_caps_width; i++); + if (common_widths[i] > device_caps_width) { + // We never scale up, so just stick with what the device gives + apply_width(payload_type, device_caps_width, bitrate); + } else if (common_widths[i] != active_caps_width) { + apply_width(payload_type, common_widths[i], bitrate); + } + } + if (bitrate > max_bitrate) bitrate = (uint) max_bitrate; + } codec_util.update_bitrate(media, payload_type, codecs[payload_type], bitrate); unlock(codec_bitrates); } @@ -348,10 +428,11 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { element.@set("sync", false); } pipe.add(element); + device_caps = get_best_caps(); if (is_source) { element.@set("do-timestamp", true); filter = Gst.ElementFactory.make("capsfilter", @"caps_filter_$id"); - filter.@set("caps", get_best_caps()); + filter.@set("caps", device_caps); filter.get_static_pad("src").add_probe(Gst.PadProbeType.BLOCK, log_probe); pipe.add(filter); element.link(filter); @@ -377,7 +458,7 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { plugin.echoprobe.link(element); } else { filter = Gst.ElementFactory.make("capsfilter", @"caps_filter_$id"); - filter.@set("caps", get_best_caps()); + filter.@set("caps", device_caps); pipe.add(filter); mixer.link(filter); filter.link(element); diff --git a/plugins/rtp/src/plugin.vala b/plugins/rtp/src/plugin.vala index 3b19318e..d10303a6 100644 --- a/plugins/rtp/src/plugin.vala +++ b/plugins/rtp/src/plugin.vala @@ -163,7 +163,6 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { debug("State of %s changed. Old: %s, New: %s, Pending; %s", ((Gst.Element)message.src).name, @"$oldState", @"$newState", @"$pendingState"); } } - break; case Gst.MessageType.STREAM_STATUS: Gst.StreamStatusType status; diff --git a/plugins/rtp/src/stream.vala b/plugins/rtp/src/stream.vala index 24adcb9a..5e5a556b 100644 --- a/plugins/rtp/src/stream.vala +++ b/plugins/rtp/src/stream.vala @@ -266,7 +266,7 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { last_remb_time = time_now; uint actual_bitrate = (uint)(((double)new_octets * 8.0) * (double)time_diff / 1000.0 / 1000000.0); new_target_receive_bitrate = uint.max(new_target_receive_bitrate, (uint)(0.9 * (double)actual_bitrate)); - max_target_receive_bitrate = uint.max(actual_bitrate * 2, max_target_receive_bitrate); + max_target_receive_bitrate = uint.max((uint)(1.5 * (double)actual_bitrate), max_target_receive_bitrate); new_target_receive_bitrate = uint.min(new_target_receive_bitrate, max_target_receive_bitrate); } new_target_receive_bitrate = uint.max(16, new_target_receive_bitrate); // Never go below 16 -- cgit v1.2.3 From 4f80a9f5ccea5eaf6a78a841af417c597eb7db73 Mon Sep 17 00:00:00 2001 From: Marvin W Date: Sat, 18 Dec 2021 21:43:12 +0100 Subject: RTP: Correctly handle timestamp after re-enabling a stream --- plugins/rtp/src/device.vala | 6 +++++- plugins/rtp/src/stream.vala | 11 +++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) (limited to 'plugins/rtp/src/device.vala') diff --git a/plugins/rtp/src/device.vala b/plugins/rtp/src/device.vala index 89d499ed..de46bea4 100644 --- a/plugins/rtp/src/device.vala +++ b/plugins/rtp/src/device.vala @@ -96,7 +96,7 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { return element; } - public Gst.Element? link_source(PayloadType? payload_type = null, uint ssrc = Random.next_int(), int seqnum_offset = -1) { + public Gst.Element? link_source(PayloadType? payload_type = null, uint ssrc = Random.next_int(), int seqnum_offset = -1, uint32 timestamp_offset = 0) { if (!is_source) return null; if (element == null) create(); links++; @@ -122,8 +122,12 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { var payload = (Gst.RTP.BasePayload) ((Gst.Bin) payloaders[payload_type][ssrc]).get_by_name(@"$(id)_$(codec)_$(ssrc)_rtp_pay"); payload.ssrc = ssrc; payload.seqnum_offset = seqnum_offset; + if (timestamp_offset != 0) { + payload.timestamp_offset = timestamp_offset; + } pipe.add(payloaders[payload_type][ssrc]); codec_tees[payload_type].link(payloaders[payload_type][ssrc]); + debug("Payload for %s with %s using ssrc %u, seqnum_offset %u, timestamp_offset %u", media, codec, ssrc, seqnum_offset, timestamp_offset); } if (!payloader_tees.has_key(payload_type)) { payloader_tees[payload_type] = new HashMap(); diff --git a/plugins/rtp/src/stream.vala b/plugins/rtp/src/stream.vala index 85864022..2cc40783 100644 --- a/plugins/rtp/src/stream.vala +++ b/plugins/rtp/src/stream.vala @@ -29,7 +29,7 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { public Device input_device { get { return _input_device; } set { if (!paused) { var input = this.input; - set_input(value != null ? value.link_source(payload_type, our_ssrc, next_seqnum_offset) : null); + set_input(value != null ? value.link_source(payload_type, our_ssrc, next_seqnum_offset, next_timestamp_offset) : null); if (this._input_device != null) this._input_device.unlink(input); } this._input_device = value; @@ -46,6 +46,13 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { private bool push_recv_data = false; private uint our_ssrc = Random.next_int(); private int next_seqnum_offset = -1; + private uint32 next_timestamp_offset_base = 0; + private int64 next_timestamp_offset_stamp = 0; + private uint32 next_timestamp_offset { get { + if (next_timestamp_offset_base == 0) return 0; + int64 monotonic_diff = get_monotonic_time() - next_timestamp_offset_stamp; + return next_timestamp_offset_base + (uint32)((double)monotonic_diff / 1000000.0 * payload_type.clockrate); + } } private uint32 participant_ssrc = 0; private Gst.Pad recv_rtcp_sink_pad; @@ -657,7 +664,7 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { public void unpause() { if (!paused) return; - set_input_and_pause(input_device != null ? input_device.link_source(payload_type, our_ssrc, next_seqnum_offset) : null, false); + set_input_and_pause(input_device != null ? input_device.link_source(payload_type, our_ssrc, next_seqnum_offset, next_timestamp_offset) : null, false); } public uint get_participant_ssrc(Xmpp.Jid participant) { -- cgit v1.2.3 From b07c4187ef40295c17cc09fea6dc5ba4137fd73b Mon Sep 17 00:00:00 2001 From: Marvin W Date: Sat, 18 Dec 2021 21:47:50 +0100 Subject: RTP: Less log spam --- plugins/rtp/src/device.vala | 40 ---------------------------------------- plugins/rtp/src/plugin.vala | 8 -------- plugins/rtp/src/stream.vala | 40 ---------------------------------------- 3 files changed, 88 deletions(-) (limited to 'plugins/rtp/src/device.vala') diff --git a/plugins/rtp/src/device.vala b/plugins/rtp/src/device.vala index de46bea4..368356ae 100644 --- a/plugins/rtp/src/device.vala +++ b/plugins/rtp/src/device.vala @@ -384,45 +384,6 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { return target; } - private static Gst.PadProbeReturn log_probe(Gst.Pad pad, Gst.PadProbeInfo info) { - if ((info.type & Gst.PadProbeType.EVENT_DOWNSTREAM) > 0) { - debug("%s.%s probed downstream event %s", pad.get_parent_element().name, pad.name, info.get_event().type.get_name()); - } - if ((info.type & Gst.PadProbeType.EVENT_UPSTREAM) > 0) { - var event = info.get_event(); - if (event.type == Gst.EventType.RECONFIGURE) return Gst.PadProbeReturn.DROP; - if (event.type == Gst.EventType.QOS) { - Gst.QOSType qos_type; - double proportion; - Gst.ClockTimeDiff diff; - Gst.ClockTime timestamp; - event.parse_qos(out qos_type, out proportion, out diff, out timestamp); - debug("%s.%s probed qos event: type: %s, proportion: %f, diff: %lli, timestamp: %llu", pad.get_parent_element().name, pad.name, @"$qos_type", proportion, diff, timestamp); - } else { - debug("%s.%s probed upstream event %s", pad.get_parent_element().name, pad.name, event.type.get_name()); - } - } - if ((info.type & Gst.PadProbeType.QUERY_DOWNSTREAM) > 0) { - debug("%s.%s probed downstream query %s", pad.get_parent_element().name, pad.name, info.get_query().type.get_name()); - } - if ((info.type & Gst.PadProbeType.QUERY_UPSTREAM) > 0) { - debug("%s.%s probed upstream query %s", pad.get_parent_element().name, pad.name, info.get_query().type.get_name()); - } - if ((info.type & Gst.PadProbeType.BUFFER) > 0) { - uint id = pad.get_data("no_buffer_probe_timeout"); - if (id != 0) { - Source.remove(id); - } - string name = @"$(pad.get_parent_element().name).$(pad.name)"; - id = Timeout.add_seconds(1, () => { - debug("%s probed no buffer for 1 second", name); - return Source.REMOVE; - }); - pad.set_data("no_buffer_probe_timeout", id); - } - return Gst.PadProbeReturn.PASS; - } - private void create() { debug("Creating device %s", id); plugin.pause(); @@ -437,7 +398,6 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { element.@set("do-timestamp", true); filter = Gst.ElementFactory.make("capsfilter", @"caps_filter_$id"); filter.@set("caps", device_caps); - filter.get_static_pad("src").add_probe(Gst.PadProbeType.BLOCK, log_probe); pipe.add(filter); element.link(filter); #if WITH_VOICE_PROCESSOR diff --git a/plugins/rtp/src/plugin.vala b/plugins/rtp/src/plugin.vala index d10303a6..6d6da79a 100644 --- a/plugins/rtp/src/plugin.vala +++ b/plugins/rtp/src/plugin.vala @@ -155,14 +155,6 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { break; case Gst.MessageType.STATE_CHANGED: // Ignore - { - unowned Gst.Structure struc = message.get_structure(); - if (struc != null && message.src is Gst.Element) { - Gst.State oldState, newState, pendingState; - message.parse_state_changed(out oldState, out newState, out pendingState); - debug("State of %s changed. Old: %s, New: %s, Pending; %s", ((Gst.Element)message.src).name, @"$oldState", @"$newState", @"$pendingState"); - } - } break; case Gst.MessageType.STREAM_STATUS: Gst.StreamStatusType status; diff --git a/plugins/rtp/src/stream.vala b/plugins/rtp/src/stream.vala index 2c8b6356..385a590c 100644 --- a/plugins/rtp/src/stream.vala +++ b/plugins/rtp/src/stream.vala @@ -81,45 +81,6 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { } } - private static Gst.PadProbeReturn log_probe(Gst.Pad pad, Gst.PadProbeInfo info) { - if ((info.type & Gst.PadProbeType.EVENT_DOWNSTREAM) > 0) { - debug("%s.%s probed downstream event %s", pad.get_parent_element().name, pad.name, info.get_event().type.get_name()); - } - if ((info.type & Gst.PadProbeType.EVENT_UPSTREAM) > 0) { - var event = info.get_event(); - if (event.type == Gst.EventType.RECONFIGURE) return Gst.PadProbeReturn.DROP; - if (event.type == Gst.EventType.QOS) { - Gst.QOSType qos_type; - double proportion; - Gst.ClockTimeDiff diff; - Gst.ClockTime timestamp; - event.parse_qos(out qos_type, out proportion, out diff, out timestamp); - debug("%s.%s probed qos event: type: %s, proportion: %f, diff: %lli, timestamp: %llu", pad.get_parent_element().name, pad.name, @"$qos_type", proportion, diff, timestamp); - } else { - debug("%s.%s probed upstream event %s", pad.get_parent_element().name, pad.name, event.type.get_name()); - } - } - if ((info.type & Gst.PadProbeType.QUERY_DOWNSTREAM) > 0) { - debug("%s.%s probed downstream query %s", pad.get_parent_element().name, pad.name, info.get_query().type.get_name()); - } - if ((info.type & Gst.PadProbeType.QUERY_UPSTREAM) > 0) { - debug("%s.%s probed upstream query %s", pad.get_parent_element().name, pad.name, info.get_query().type.get_name()); - } - if ((info.type & Gst.PadProbeType.BUFFER) > 0) { - uint id = pad.steal_data("no_buffer_probe_timeout"); - if (id != 0) { - Source.remove(id); - } - string name = @"$(pad.get_parent_element().name).$(pad.name)"; - id = Timeout.add_seconds(1, () => { - debug("%s probed no buffer for 1 second", name); - return Source.REMOVE; - }); - pad.set_data("no_buffer_probe_timeout", id); - } - return Gst.PadProbeReturn.PASS; - } - public override void create() { plugin.pause(); @@ -160,7 +121,6 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { recv_rtp.do_timestamp = true; recv_rtp.format = Gst.Format.TIME; recv_rtp.is_live = true; - recv_rtp.get_static_pad("src").add_probe(Gst.PadProbeType.BLOCK, log_probe); pipe.add(recv_rtp); recv_rtcp = Gst.ElementFactory.make("appsrc", @"rtcp_src_$rtpid") as Gst.App.Src; -- cgit v1.2.3 From cd6d501c23c8550e7b2d880f8e60e3df8e887c2a Mon Sep 17 00:00:00 2001 From: Marvin W Date: Sun, 19 Dec 2021 22:38:00 +0100 Subject: RTP: Improve screen resolution update logic --- plugins/rtp/src/device.vala | 46 +++++++++++++++++++++++---------------------- plugins/rtp/src/stream.vala | 9 ++++++--- 2 files changed, 30 insertions(+), 25 deletions(-) (limited to 'plugins/rtp/src/device.vala') diff --git a/plugins/rtp/src/device.vala b/plugins/rtp/src/device.vala index 368356ae..97258d0c 100644 --- a/plugins/rtp/src/device.vala +++ b/plugins/rtp/src/device.vala @@ -2,24 +2,19 @@ using Xmpp.Xep.JingleRtp; using Gee; public class Dino.Plugins.Rtp.Device : MediaDevice, Object { + private const int[] common_widths = {320, 360, 400, 480, 640, 960, 1280, 1920, 2560, 3840}; + public Plugin plugin { get; private set; } public CodecUtil codec_util { get { return plugin.codec_util; } } public Gst.Device device { get; private set; } - private string device_name; - public string id { get { - return device_name; - }} - private string device_display_name; - public string display_name { get { - return device_display_name; - }} + public string id { get { return device_name; }} + public string display_name { get { return device_display_name; }} public string detail_name { get { return device.properties.get_string("alsa.card_name") ?? device.properties.get_string("alsa.id") ?? id; }} - public Gst.Pipeline pipe { get { - return plugin.pipe; - }} + + public Gst.Pipeline pipe { get { return plugin.pipe; }} public string? media { get { if (device.has_classes("Audio")) { return "audio"; @@ -29,12 +24,11 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { return null; } }} - public bool is_source { get { - return device.has_classes("Source"); - }} - public bool is_sink { get { - return device.has_classes("Sink"); - }} + public bool is_source { get { return device.has_classes("Source"); }} + public bool is_sink { get { return device.has_classes("Sink"); }} + + private string device_name; + private string device_display_name; private Gst.Caps device_caps; private Gst.Element element; @@ -96,11 +90,11 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { return element; } - public Gst.Element? link_source(PayloadType? payload_type = null, uint ssrc = Random.next_int(), int seqnum_offset = -1, uint32 timestamp_offset = 0) { + public Gst.Element? link_source(PayloadType? payload_type = null, uint ssrc = 0, int seqnum_offset = -1, uint32 timestamp_offset = 0) { if (!is_source) return null; if (element == null) create(); links++; - if (payload_type != null && tee != null) { + if (payload_type != null && ssrc != 0 && tee != null) { bool new_codec = false; string? codec = CodecUtil.get_codec_from_payload(media, payload_type); if (!codecs.has_key(payload_type)) { @@ -172,16 +166,17 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { return br; } - private const int[] common_widths = {320, 480, 640, 960, 1280, 1920, 2560, 3840}; private Gst.Caps get_active_caps(PayloadType payload_type) { return codec_util.get_rescale_caps(codecs[payload_type]) ?? device_caps; } + private void apply_caps(PayloadType payload_type, Gst.Caps caps) { plugin.pause(); debug("Set scaled caps to %s", caps.to_string()); codec_util.update_rescale_caps(codecs[payload_type], caps); plugin.unpause(); } + private void apply_width(PayloadType payload_type, int new_width, uint bitrate) { int device_caps_width, device_caps_height, active_caps_width, device_caps_framerate_num, device_caps_framerate_den; device_caps.get_structure(0).get_int("width", out device_caps_width); @@ -197,9 +192,11 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { int new_height = device_caps_height * new_width / device_caps_width; Gst.Caps new_caps = new Gst.Caps.simple("video/x-raw", "width", typeof(int), new_width, "height", typeof(int), new_height, "framerate", typeof(Gst.Fraction), device_caps_framerate_num, device_caps_framerate_den, null); double required_bitrate = get_target_bitrate(new_caps); - if (bitrate < required_bitrate) return; + debug("Changing resolution width from %d to %d (requires bitrate %f, current target is %u)", active_caps_width, new_width, required_bitrate, bitrate); + if (bitrate < required_bitrate && new_width > active_caps_width) return; apply_caps(payload_type, new_caps); } + public void update_bitrate(PayloadType payload_type, uint bitrate) { if (codecs.has_key(payload_type)) { lock(codec_bitrates); @@ -233,12 +230,17 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { if (bitrate < 0.75 * current_target_bitrate && active_caps_width > common_widths[0]) { // Lower video resolution int i = 1; - for(; i < common_widths.length && common_widths[i] < active_caps_width; i++); + for(; i < common_widths.length && common_widths[i] < active_caps_width; i++);if (common_widths[i] != active_caps_width) { + debug("Decrease resolution to ensure target bitrate (%u) is in reach (current resolution target bitrate is %f)", bitrate, current_target_bitrate); + } apply_width(payload_type, common_widths[i-1], bitrate); } else if (bitrate > 2 * current_target_bitrate && active_caps_width < device_caps_width) { // Higher video resolution int i = 0; for(; i < common_widths.length && common_widths[i] <= active_caps_width; i++); + if (common_widths[i] != active_caps_width) { + debug("Increase resolution to make use of available bandwidth of target bitrate (%u) (current resolution target bitrate is %f)", bitrate, current_target_bitrate); + } if (common_widths[i] > device_caps_width) { // We never scale up, so just stick with what the device gives apply_width(payload_type, device_caps_width, bitrate); diff --git a/plugins/rtp/src/stream.vala b/plugins/rtp/src/stream.vala index 385a590c..dc712b61 100644 --- a/plugins/rtp/src/stream.vala +++ b/plugins/rtp/src/stream.vala @@ -169,7 +169,7 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { Timeout.add(1000, () => remb_adjust()); } if (input_device != null && media == "video") { - input_device.update_bitrate(payload_type, 256); + input_device.update_bitrate(payload_type, target_send_bitrate); } } @@ -214,16 +214,18 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { if (packets_received < last_packets_received) new_received = 0; uint64 new_octets = octets_received - last_octets_received; if (octets_received < last_octets_received) octets_received = 0; + if (new_received == 0) continue; last_packets_lost = packets_lost; last_packets_received = packets_received; last_octets_received = octets_received; - if (new_received == 0) continue; double loss_rate = (double)new_lost / (double)(new_lost + new_received); - uint new_target_receive_bitrate = 256; + uint new_target_receive_bitrate; if (new_lost <= 0 || loss_rate < 0.02) { new_target_receive_bitrate = (uint)(1.08 * (double)target_receive_bitrate); } else if (loss_rate > 0.1) { new_target_receive_bitrate = (uint)((1.0 - 0.5 * loss_rate) * (double)target_receive_bitrate); + } else { + new_target_receive_bitrate = target_receive_bitrate; } if (last_remb_time == 0) { last_remb_time = get_monotonic_time(); @@ -670,6 +672,7 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { public void unpause() { if (!paused) return; set_input_and_pause(input_device != null ? input_device.link_source(payload_type, our_ssrc, next_seqnum_offset, next_timestamp_offset) : null, false); + input_device.update_bitrate(payload_type, target_send_bitrate); } public uint get_participant_ssrc(Xmpp.Jid participant) { -- cgit v1.2.3