From 148cf48d2b68354881066e2587e2673c91d2619a Mon Sep 17 00:00:00 2001 From: hrxi Date: Sat, 28 Dec 2019 03:11:51 +0100 Subject: Add libnice and listen for direct connections in Jingle SOCKS5 (#608) Add libnice as a plugin. If it is present, use libnice to enumerate local IP addresses and listen on them to support direct connections for Jingle SOCKS5. Tested with Conversations and Gajim. Created the nice.vapi file using ``` vapigen --library nice --pkg gio-2.0 --metadatadir metadata /usr/share/gir-1.0/Nice-0.1.gir ``` --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'CMakeLists.txt') diff --git a/CMakeLists.txt b/CMakeLists.txt index b738b585..2af0719b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,7 +11,7 @@ else () endif () # Prepare Plugins -set(DEFAULT_PLUGINS omemo;openpgp;http-files) +set(DEFAULT_PLUGINS omemo;openpgp;http-files;ice) foreach (plugin ${DEFAULT_PLUGINS}) if ("$CACHE{DINO_PLUGIN_ENABLED_${plugin}}" STREQUAL "") if (NOT DEFINED DINO_PLUGIN_ENABLED_${plugin}}) -- cgit v1.2.3 From 0f46facecd558786631c2ad4cf66d27331f16a86 Mon Sep 17 00:00:00 2001 From: fiaxh Date: Fri, 19 Mar 2021 23:09:56 +0100 Subject: Add UI for audio/video calls --- CMakeLists.txt | 4 +- main/CMakeLists.txt | 21 ++ main/data/call_widget.ui | 111 +++++++++ main/data/icons/dino-microphone-off-symbolic.svg | 1 + main/data/icons/dino-microphone-symbolic.svg | 1 + main/data/icons/dino-phone-hangup-symbolic.svg | 1 + main/data/icons/dino-phone-in-talk-symbolic.svg | 1 + main/data/icons/dino-phone-missed-symbolic.svg | 1 + main/data/icons/dino-phone-ring-symbolic.svg | 1 + main/data/icons/dino-phone-symbolic.svg | 1 + main/data/icons/dino-video-off-symbolic.svg | 1 + main/data/icons/dino-video-symbolic.svg | 1 + main/data/theme.css | 97 +++++++- main/src/main.vala | 2 +- main/src/ui/application.vala | 18 ++ .../src/ui/call_window/audio_settings_popover.vala | 127 ++++++++++ main/src/ui/call_window/call_bottom_bar.vala | 165 +++++++++++++ main/src/ui/call_window/call_window.vala | 260 +++++++++++++++++++++ .../src/ui/call_window/call_window_controller.vala | 208 +++++++++++++++++ .../src/ui/call_window/video_settings_popover.vala | 73 ++++++ .../ui/conversation_content_view/call_widget.vala | 215 +++++++++++++++++ .../content_populator.vala | 3 + .../ui/conversation_content_view/file_widget.vala | 3 - .../conversation_selector_row.vala | 8 + main/src/ui/conversation_titlebar/call_entry.vala | 130 +++++++++++ main/src/ui/conversation_view_controller.vala | 1 + main/src/ui/notifier_freedesktop.vala | 38 +++ main/src/ui/notifier_gnotifications.vala | 19 ++ main/src/ui/util/helper.vala | 9 + 29 files changed, 1514 insertions(+), 7 deletions(-) create mode 100644 main/data/call_widget.ui create mode 100644 main/data/icons/dino-microphone-off-symbolic.svg create mode 100644 main/data/icons/dino-microphone-symbolic.svg create mode 100644 main/data/icons/dino-phone-hangup-symbolic.svg create mode 100644 main/data/icons/dino-phone-in-talk-symbolic.svg create mode 100644 main/data/icons/dino-phone-missed-symbolic.svg create mode 100644 main/data/icons/dino-phone-ring-symbolic.svg create mode 100644 main/data/icons/dino-phone-symbolic.svg create mode 100644 main/data/icons/dino-video-off-symbolic.svg create mode 100644 main/data/icons/dino-video-symbolic.svg create mode 100644 main/src/ui/call_window/audio_settings_popover.vala create mode 100644 main/src/ui/call_window/call_bottom_bar.vala create mode 100644 main/src/ui/call_window/call_window.vala create mode 100644 main/src/ui/call_window/call_window_controller.vala create mode 100644 main/src/ui/call_window/video_settings_popover.vala create mode 100644 main/src/ui/conversation_content_view/call_widget.vala create mode 100644 main/src/ui/conversation_titlebar/call_entry.vala (limited to 'CMakeLists.txt') diff --git a/CMakeLists.txt b/CMakeLists.txt index 2af0719b..f480b0b2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,7 +11,7 @@ else () endif () # Prepare Plugins -set(DEFAULT_PLUGINS omemo;openpgp;http-files;ice) +set(DEFAULT_PLUGINS omemo;openpgp;http-files;ice;rtp) foreach (plugin ${DEFAULT_PLUGINS}) if ("$CACHE{DINO_PLUGIN_ENABLED_${plugin}}" STREQUAL "") if (NOT DEFINED DINO_PLUGIN_ENABLED_${plugin}}) @@ -96,6 +96,7 @@ macro(AddCFlagIfSupported list flag) endif () endmacro() + if ("Ninja" STREQUAL ${CMAKE_GENERATOR}) AddCFlagIfSupported(CMAKE_C_FLAGS -fdiagnostics-color) endif () @@ -105,6 +106,7 @@ AddCFlagIfSupported(CMAKE_C_FLAGS -Wall) AddCFlagIfSupported(CMAKE_C_FLAGS -Wextra) AddCFlagIfSupported(CMAKE_C_FLAGS -Werror=format-security) AddCFlagIfSupported(CMAKE_C_FLAGS -Wno-duplicate-decl-specifier) +AddCFlagIfSupported(CMAKE_C_FLAGS -fno-omit-frame-pointer) if (NOT VALA_WARN) set(VALA_WARN "conversion") diff --git a/main/CMakeLists.txt b/main/CMakeLists.txt index 5169e8ae..69992f06 100644 --- a/main/CMakeLists.txt +++ b/main/CMakeLists.txt @@ -5,6 +5,8 @@ gettext_compile(${GETTEXT_PACKAGE} SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/po TAR find_packages(MAIN_PACKAGES REQUIRED Gee + Gst + GstVideo GLib GModule GObject @@ -21,7 +23,14 @@ set(RESOURCE_LIST icons/dino-emoticon-symbolic.svg icons/dino-qr-code-symbolic.svg icons/dino-security-high-symbolic.svg + icons/dino-microphone-off-symbolic.svg + icons/dino-microphone-symbolic.svg icons/dino-party-popper-symbolic.svg + icons/dino-phone-hangup-symbolic.svg + icons/dino-phone-in-talk-symbolic.svg + icons/dino-phone-missed-symbolic.svg + icons/dino-phone-ring-symbolic.svg + icons/dino-phone-symbolic.svg icons/dino-status-away.svg icons/dino-status-chat.svg icons/dino-status-dnd.svg @@ -29,6 +38,8 @@ set(RESOURCE_LIST icons/im.dino.Dino.svg icons/im.dino.Dino-symbolic.svg icons/dino-tick-symbolic.svg + icons/dino-video-off-symbolic.svg + icons/dino-video-symbolic.svg icons/dino-device-desktop-symbolic.svg icons/dino-device-phone-symbolic.svg @@ -46,6 +57,8 @@ set(RESOURCE_LIST add_conversation/conference_details_fragment.ui add_conversation/list_row.ui add_conversation/select_jid_fragment.ui + + call_widget.ui chat_input.ui contact_details_dialog.ui conversation_list_titlebar.ui @@ -124,6 +137,12 @@ SOURCES src/ui/add_conversation/select_contact_dialog.vala src/ui/add_conversation/select_jid_fragment.vala + src/ui/call_window/audio_settings_popover.vala + src/ui/call_window/call_bottom_bar.vala + src/ui/call_window/call_window.vala + src/ui/call_window/call_window_controller.vala + src/ui/call_window/video_settings_popover.vala + src/ui/chat_input/chat_input_controller.vala src/ui/chat_input/chat_text_view.vala src/ui/chat_input/edit_history.vala @@ -142,6 +161,7 @@ SOURCES src/ui/conversation_selector/conversation_selector_row.vala src/ui/conversation_selector/conversation_selector.vala + src/ui/conversation_content_view/call_widget.vala src/ui/conversation_content_view/chat_state_populator.vala src/ui/conversation_content_view/content_populator.vala src/ui/conversation_content_view/conversation_item_skeleton.vala @@ -153,6 +173,7 @@ SOURCES src/ui/conversation_content_view/message_widget.vala src/ui/conversation_content_view/subscription_notification.vala + src/ui/conversation_titlebar/call_entry.vala src/ui/conversation_titlebar/menu_entry.vala src/ui/conversation_titlebar/occupants_entry.vala src/ui/conversation_titlebar/search_entry.vala diff --git a/main/data/call_widget.ui b/main/data/call_widget.ui new file mode 100644 index 00000000..47fb0046 --- /dev/null +++ b/main/data/call_widget.ui @@ -0,0 +1,111 @@ + + + + diff --git a/main/data/icons/dino-microphone-off-symbolic.svg b/main/data/icons/dino-microphone-off-symbolic.svg new file mode 100644 index 00000000..7e5b853d --- /dev/null +++ b/main/data/icons/dino-microphone-off-symbolic.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/main/data/icons/dino-microphone-symbolic.svg b/main/data/icons/dino-microphone-symbolic.svg new file mode 100644 index 00000000..fbf0784a --- /dev/null +++ b/main/data/icons/dino-microphone-symbolic.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/main/data/icons/dino-phone-hangup-symbolic.svg b/main/data/icons/dino-phone-hangup-symbolic.svg new file mode 100644 index 00000000..ecd230ac --- /dev/null +++ b/main/data/icons/dino-phone-hangup-symbolic.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/main/data/icons/dino-phone-in-talk-symbolic.svg b/main/data/icons/dino-phone-in-talk-symbolic.svg new file mode 100644 index 00000000..351035da --- /dev/null +++ b/main/data/icons/dino-phone-in-talk-symbolic.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/main/data/icons/dino-phone-missed-symbolic.svg b/main/data/icons/dino-phone-missed-symbolic.svg new file mode 100644 index 00000000..228f073e --- /dev/null +++ b/main/data/icons/dino-phone-missed-symbolic.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/main/data/icons/dino-phone-ring-symbolic.svg b/main/data/icons/dino-phone-ring-symbolic.svg new file mode 100644 index 00000000..06b8dcbf --- /dev/null +++ b/main/data/icons/dino-phone-ring-symbolic.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/main/data/icons/dino-phone-symbolic.svg b/main/data/icons/dino-phone-symbolic.svg new file mode 100644 index 00000000..0020dddc --- /dev/null +++ b/main/data/icons/dino-phone-symbolic.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/main/data/icons/dino-video-off-symbolic.svg b/main/data/icons/dino-video-off-symbolic.svg new file mode 100644 index 00000000..d438e065 --- /dev/null +++ b/main/data/icons/dino-video-off-symbolic.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/main/data/icons/dino-video-symbolic.svg b/main/data/icons/dino-video-symbolic.svg new file mode 100644 index 00000000..60a1c742 --- /dev/null +++ b/main/data/icons/dino-video-symbolic.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/main/data/theme.css b/main/data/theme.css index 6bacee30..423cbf68 100644 --- a/main/data/theme.css +++ b/main/data/theme.css @@ -86,16 +86,26 @@ window.dino-main .dino-conversation .message-box.edit-mode:hover { background: alpha(@theme_selected_bg_color, 0.12); } -window.dino-main .file-box-outer { +window.dino-main .file-box-outer, +window.dino-main .call-box-outer { background: @theme_base_color; border-radius: 3px; border: 1px solid alpha(@theme_fg_color, 0.1); } -window.dino-main .file-box { +window.dino-main .file-box, +window.dino-main .call-box { margin: 12px 16px 12px 12px; } +window.dino-main .call-box-outer.incoming { + border-color: alpha(@theme_selected_bg_color, 0.3); +} + +window.dino-main .incoming-call-box { + background: alpha(@theme_selected_bg_color, 0.1); +} + window.dino-main .dino-sidebar > frame.collapsed { border-bottom: 1px solid @borders; } @@ -204,3 +214,86 @@ box.dino-input-error label.input-status-highlight-once { animation-iteration-count: 1; animation-name: input-error-highlight; } + +/* Call window */ + +.dino-call-window .titlebar { + min-height: 0; +} + +.dino-call-window headerbar, .call-button { + box-shadow: none; +} + +.dino-call-window .titlebutton.close:hover { + background: rgba(255,255,255,0.15); + border-color: rgba(255,255,255,0); + box-shadow: none; +} + +.dino-call-window button.call-button { + outline: 0; + border-radius: 1000px; +} +.dino-call-window button.white-button { + color: #1d1c1d; + background: rgba(255,255,255,0.9); + border: lightgrey; +} + +.dino-call-window button.transparent-white-button { + color: white; + background: rgba(255,255,255,0.15); + border: none; +} + +.dino-call-window button.call-mediadevice-settings-button { + border-radius: 1000px; + min-height: 0; + min-width: 0; + padding: 3px; + margin: 2px; + transition-duration: 0; +} + +.dino-call-window button.call-mediadevice-settings-button:hover, +.dino-call-window button.call-mediadevice-settings-button:checked { /* Effect that makes the button slightly larger on hover :) */ + border-radius: 1000px; + min-height: 0; + min-width: 0; + padding: 5px; + margin: 0; +} + +.dino-call-window .unencrypted-box { + color: @error_color; + padding: 10px; + border-radius: 5px; + background: rgba(0,0,0,0.5); +} + +.dino-call-window .call-header-bar { + background: linear-gradient(rgba(0,0,0,0.4), rgba(0,0,0,0)); + border: 0; + border-radius: 0; +} + +.dino-call-window .call-header-bar, +.dino-call-window .call-header-bar image { + color: #ededec; +} + +.dino-call-window .call-bottom-bar { + background: linear-gradient(rgba(0,0,0,0), rgba(0,0,0,0.3)); + border: 0; +} + +.dino-call-window .video-placeholder-box { + background-color: #212121; +} + +.dino-call-window .text-no-controls { + background: white; + border-radius: 5px; + padding: 5px 10px; +} \ No newline at end of file diff --git a/main/src/main.vala b/main/src/main.vala index 6274dcdd..afa1f52b 100644 --- a/main/src/main.vala +++ b/main/src/main.vala @@ -17,7 +17,7 @@ void main(string[] args) { Gtk.init(ref args); Dino.Ui.Application app = new Dino.Ui.Application() { search_path_generator=search_path_generator }; Plugins.Loader loader = new Plugins.Loader(app); - loader.loadAll(); + loader.load_all(); app.run(args); loader.shutdown(); diff --git a/main/src/ui/application.vala b/main/src/ui/application.vala index 358097e3..780c37fd 100644 --- a/main/src/ui/application.vala +++ b/main/src/ui/application.vala @@ -199,6 +199,24 @@ public class Dino.Ui.Application : Gtk.Application, Dino.Application { dialog.present(); }); add_action(open_shortcuts_action); + + SimpleAction accept_call_action = new SimpleAction("accept-call", VariantType.INT32); + accept_call_action.activate.connect((variant) => { + Call? call = stream_interactor.get_module(CallStore.IDENTITY).get_call_by_id(variant.get_int32()); + stream_interactor.get_module(Calls.IDENTITY).accept_call(call); + + var call_window = new CallWindow(); + call_window.controller = new CallWindowController(call_window, call, stream_interactor); + call_window.present(); + }); + add_action(accept_call_action); + + SimpleAction deny_call_action = new SimpleAction("deny-call", VariantType.INT32); + deny_call_action.activate.connect((variant) => { + Call? call = stream_interactor.get_module(CallStore.IDENTITY).get_call_by_id(variant.get_int32()); + stream_interactor.get_module(Calls.IDENTITY).reject_call(call); + }); + add_action(deny_call_action); } public bool use_csd() { diff --git a/main/src/ui/call_window/audio_settings_popover.vala b/main/src/ui/call_window/audio_settings_popover.vala new file mode 100644 index 00000000..7d1f39b0 --- /dev/null +++ b/main/src/ui/call_window/audio_settings_popover.vala @@ -0,0 +1,127 @@ +using Gee; +using Gtk; +using Dino.Entities; + +public class Dino.Ui.AudioSettingsPopover : Gtk.Popover { + + public signal void microphone_selected(Plugins.MediaDevice device); + public signal void speaker_selected(Plugins.MediaDevice device); + + public Plugins.MediaDevice? current_microphone_device { get; set; } + public Plugins.MediaDevice? current_speaker_device { get; set; } + + private HashMap row_microphone_device = new HashMap(); + private HashMap row_speaker_device = new HashMap(); + + public AudioSettingsPopover() { + Box box = new Box(Orientation.VERTICAL, 15) { margin=18, visible=true }; + box.add(create_microphone_box()); + box.add(create_speaker_box()); + + this.add(box); + } + + private Widget create_microphone_box() { + Plugins.VideoCallPlugin call_plugin = Dino.Application.get_default().plugin_registry.video_call_plugin; + Gee.List devices = call_plugin.get_devices("audio", false); + + Box micro_box = new Box(Orientation.VERTICAL, 10) { visible=true }; + micro_box.add(new Label("" + _("Microphones") + "") { use_markup=true, xalign=0, visible=true, can_focus=true /* grab initial focus*/ }); + + if (devices.size == 0) { + micro_box.add(new Label("No microphones found.")); + } else { + ListBox micro_list_box = new ListBox() { activate_on_single_click=true, selection_mode=SelectionMode.SINGLE, visible=true }; + micro_list_box.set_header_func(listbox_header_func); + Frame micro_frame = new Frame(null) { visible=true }; + micro_frame.add(micro_list_box); + foreach (Plugins.MediaDevice device in devices) { + Label label = new Label(device.display_name) { xalign=0, visible=true }; + Image image = new Image.from_icon_name("object-select-symbolic", IconSize.BUTTON) { visible=true }; + if (current_microphone_device == null || current_microphone_device.id != device.id) { + image.opacity = 0; + } + this.notify["current-microphone-device"].connect(() => { + if (current_microphone_device == null || current_microphone_device.id != device.id) { + image.opacity = 0; + } else { + image.opacity = 1; + } + }); + Box device_box = new Box(Orientation.HORIZONTAL, 0) { spacing=7, margin=7, visible=true }; + device_box.add(image); + device_box.add(label); + ListBoxRow list_box_row = new ListBoxRow() { visible=true }; + list_box_row.add(device_box); + micro_list_box.add(list_box_row); + + row_microphone_device[list_box_row] = device; + } + micro_list_box.row_activated.connect((row) => { + if (!row_microphone_device.has_key(row)) return; + microphone_selected(row_microphone_device[row]); + micro_list_box.unselect_row(row); + }); + micro_box.add(micro_frame); + } + + return micro_box; + } + + private Widget create_speaker_box() { + Plugins.VideoCallPlugin call_plugin = Dino.Application.get_default().plugin_registry.video_call_plugin; + Gee.List devices = call_plugin.get_devices("audio", true); + + Box speaker_box = new Box(Orientation.VERTICAL, 10) { visible=true }; + speaker_box.add(new Label("" + _("Speakers") +"") { use_markup=true, xalign=0, visible=true }); + + if (devices.size == 0) { + speaker_box.add(new Label("No speakers found.")); + } else { + ListBox speaker_list_box = new ListBox() { activate_on_single_click=true, selection_mode=SelectionMode.SINGLE, visible=true }; + speaker_list_box.set_header_func(listbox_header_func); + speaker_list_box.row_selected.connect((row) => { + + }); + Frame speaker_frame = new Frame(null) { visible=true }; + speaker_frame.add(speaker_list_box); + foreach (Plugins.MediaDevice device in devices) { + Label label = new Label(device.display_name) { xalign=0, visible=true }; + Image image = new Image.from_icon_name("object-select-symbolic", IconSize.BUTTON) { visible=true }; + if (current_speaker_device == null || current_speaker_device.id != device.id) { + image.opacity = 0; + } + this.notify["current-speaker-device"].connect(() => { + if (current_speaker_device == null || current_speaker_device.id != device.id) { + image.opacity = 0; + } else { + image.opacity = 1; + } + }); + Box device_box = new Box(Orientation.HORIZONTAL, 0) { spacing=7, margin=7, visible=true }; + device_box.add(image); + device_box.add(label); + ListBoxRow list_box_row = new ListBoxRow() { visible=true }; + list_box_row.add(device_box); + speaker_list_box.add(list_box_row); + + row_speaker_device[list_box_row] = device; + } + speaker_list_box.row_activated.connect((row) => { + if (!row_speaker_device.has_key(row)) return; + speaker_selected(row_speaker_device[row]); + speaker_list_box.unselect_row(row); + }); + speaker_box.add(speaker_frame); + } + + return speaker_box; + } + + private void listbox_header_func(ListBoxRow row, ListBoxRow? before_row) { + if (row.get_header() == null && before_row != null) { + row.set_header(new Separator(Orientation.HORIZONTAL)); + } + } + +} \ No newline at end of file diff --git a/main/src/ui/call_window/call_bottom_bar.vala b/main/src/ui/call_window/call_bottom_bar.vala new file mode 100644 index 00000000..bc800485 --- /dev/null +++ b/main/src/ui/call_window/call_bottom_bar.vala @@ -0,0 +1,165 @@ +using Dino.Entities; +using Gtk; + +public class Dino.Ui.CallBottomBar : Gtk.Box { + + public signal void hang_up(); + + public bool audio_enabled { get; set; } + public bool video_enabled { get; set; } + + public static IconSize ICON_SIZE_MEDIADEVICE_BUTTON = Gtk.icon_size_register("im.dino.Dino.CALL_MEDIADEVICE_BUTTON", 10, 10); + + public string counterpart_display_name { get; set; } + + private Button audio_button = new Button() { height_request=45, width_request=45, halign=Align.START, valign=Align.START, visible=true }; + private Overlay audio_button_overlay = new Overlay() { visible=true }; + private Image audio_image = new Image() { visible=true }; + private MenuButton audio_settings_button = new MenuButton() { halign=Align.END, valign=Align.END }; + public AudioSettingsPopover? audio_settings_popover; + + private Button video_button = new Button() { height_request=45, width_request=45, halign=Align.START, valign=Align.START, visible=true }; + private Overlay video_button_overlay = new Overlay() { visible=true }; + private Image video_image = new Image() { visible=true }; + private MenuButton video_settings_button = new MenuButton() { halign=Align.END, valign=Align.END }; + public VideoSettingsPopover? video_settings_popover; + + private Label label = new Label("") { margin=20, halign=Align.CENTER, valign=Align.CENTER, wrap=true, wrap_mode=Pango.WrapMode.WORD_CHAR, hexpand=true, visible=true }; + private Stack stack = new Stack() { visible=true }; + + public CallBottomBar() { + Object(orientation:Orientation.HORIZONTAL, spacing:0); + + Overlay default_control = new Overlay() { visible=true }; + Image encryption_image = new Image.from_icon_name("changes-allow-symbolic", IconSize.BUTTON) { margin_start=20, margin_bottom=25, halign=Align.START, valign=Align.END, visible=true }; + encryption_image.tooltip_text = _("Unencrypted"); + encryption_image.get_style_context().add_class("unencrypted-box"); + + default_control.add_overlay(encryption_image); + + Box main_buttons = new Box(Orientation.HORIZONTAL, 20) { margin_start=40, margin_end=40, margin=20, halign=Align.CENTER, hexpand=true, visible=true }; + + audio_button.add(audio_image); + audio_button.get_style_context().add_class("call-button"); + audio_button.clicked.connect(() => { audio_enabled = !audio_enabled; }); + audio_button.margin_end = audio_button.margin_bottom = 5; // space for the small settings button + audio_button_overlay.add(audio_button); + audio_button_overlay.add_overlay(audio_settings_button); + audio_settings_button.set_image(new Image.from_icon_name("go-up-symbolic", ICON_SIZE_MEDIADEVICE_BUTTON) { visible=true }); + audio_settings_button.get_style_context().add_class("call-mediadevice-settings-button"); + audio_settings_button.use_popover = true; + main_buttons.add(audio_button_overlay); + + video_button.add(video_image); + video_button.get_style_context().add_class("call-button"); + video_button.clicked.connect(() => { video_enabled = !video_enabled; }); + video_button.margin_end = video_button.margin_bottom = 5; + video_button_overlay.add(video_button); + video_button_overlay.add_overlay(video_settings_button); + video_settings_button.set_image(new Image.from_icon_name("go-up-symbolic", ICON_SIZE_MEDIADEVICE_BUTTON) { visible=true }); + video_settings_button.get_style_context().add_class("call-mediadevice-settings-button"); + video_settings_button.use_popover = true; + main_buttons.add(video_button_overlay); + + Button button_hang = new Button.from_icon_name("dino-phone-hangup-symbolic", IconSize.LARGE_TOOLBAR) { height_request=45, width_request=45, halign=Align.START, valign=Align.START, visible=true }; + button_hang.get_style_context().add_class("call-button"); + button_hang.get_style_context().add_class("destructive-action"); + button_hang.clicked.connect(() => hang_up()); + main_buttons.add(button_hang); + + default_control.add(main_buttons); + + label.get_style_context().add_class("text-no-controls"); + + stack.add_named(default_control, "control-buttons"); + stack.add_named(label, "label"); + this.add(stack); + + this.notify["audio-enabled"].connect(on_audio_enabled_changed); + this.notify["video-enabled"].connect(on_video_enabled_changed); + + audio_enabled = true; + video_enabled = false; + + on_audio_enabled_changed(); + on_video_enabled_changed(); + + this.get_style_context().add_class("call-bottom-bar"); + } + + public AudioSettingsPopover? show_audio_device_choices(bool show) { + audio_settings_button.visible = show; + if (audio_settings_popover != null) audio_settings_popover.visible = false; + if (!show) return null; + + audio_settings_popover = new AudioSettingsPopover(); + + audio_settings_button.popover = audio_settings_popover; + + audio_settings_popover.set_relative_to(audio_settings_button); + audio_settings_popover.microphone_selected.connect(() => { audio_settings_button.active = false; }); + audio_settings_popover.speaker_selected.connect(() => { audio_settings_button.active = false; }); + + return audio_settings_popover; + } + + public void show_audio_device_error() { + audio_settings_button.set_image(new Image.from_icon_name("dialog-warning-symbolic", IconSize.BUTTON) { visible=true }); + Util.force_error_color(audio_settings_button); + } + + public VideoSettingsPopover? show_video_device_choices(bool show) { + video_settings_button.visible = show; + if (video_settings_popover != null) video_settings_popover.visible = false; + if (!show) return null; + + video_settings_popover = new VideoSettingsPopover(); + + + video_settings_button.popover = video_settings_popover; + + video_settings_popover.set_relative_to(video_settings_button); + video_settings_popover.camera_selected.connect(() => { video_settings_button.active = false; }); + + return video_settings_popover; + } + + public void show_video_device_error() { + video_settings_button.set_image(new Image.from_icon_name("dialog-warning-symbolic", IconSize.BUTTON) { visible=true }); + Util.force_error_color(video_settings_button); + } + + public void on_audio_enabled_changed() { + if (audio_enabled) { + audio_image.set_from_icon_name("dino-microphone-symbolic", IconSize.LARGE_TOOLBAR); + audio_button.get_style_context().add_class("white-button"); + audio_button.get_style_context().remove_class("transparent-white-button"); + } else { + audio_image.set_from_icon_name("dino-microphone-off-symbolic", IconSize.LARGE_TOOLBAR); + audio_button.get_style_context().remove_class("white-button"); + audio_button.get_style_context().add_class("transparent-white-button"); + } + } + + public void on_video_enabled_changed() { + if (video_enabled) { + video_image.set_from_icon_name("dino-video-symbolic", IconSize.LARGE_TOOLBAR); + video_button.get_style_context().add_class("white-button"); + video_button.get_style_context().remove_class("transparent-white-button"); + + } else { + video_image.set_from_icon_name("dino-video-off-symbolic", IconSize.LARGE_TOOLBAR); + video_button.get_style_context().remove_class("white-button"); + video_button.get_style_context().add_class("transparent-white-button"); + } + } + + public void show_counterpart_ended(string text) { + stack.set_visible_child_name("label"); + label.label = text; + } + + public bool is_menu_active() { + return video_settings_button.active || audio_settings_button.active; + } +} \ No newline at end of file diff --git a/main/src/ui/call_window/call_window.vala b/main/src/ui/call_window/call_window.vala new file mode 100644 index 00000000..572f73b6 --- /dev/null +++ b/main/src/ui/call_window/call_window.vala @@ -0,0 +1,260 @@ +using Dino.Entities; +using Gtk; + +namespace Dino.Ui { + + public class CallWindow : Gtk.Window { + public string counterpart_display_name { get; set; } + + // TODO should find another place for this + public CallWindowController controller; + + public Overlay overlay = new Overlay() { visible=true }; + public EventBox event_box = new EventBox() { visible=true }; + public CallBottomBar bottom_bar = new CallBottomBar() { visible=true }; + public Revealer bottom_bar_revealer = new Revealer() { valign=Align.END, transition_type=RevealerTransitionType.CROSSFADE, transition_duration=200, visible=true }; + public HeaderBar header_bar = new HeaderBar() { show_close_button=true, visible=true }; + public Revealer header_bar_revealer = new Revealer() { valign=Align.START, transition_type=RevealerTransitionType.CROSSFADE, transition_duration=200, visible=true }; + public Stack stack = new Stack() { visible=true }; + public Box own_video_box = new Box(Orientation.HORIZONTAL, 0) { expand=true, visible=true }; + private Widget? own_video = null; + private Box? own_video_border = new Box(Orientation.HORIZONTAL, 0) { expand=true }; // hack to draw a border around our own video, since we apparently can't draw a border around the Gst widget + + private int own_video_width = 150; + private int own_video_height = 100; + + private bool hide_controll_elements = false; + private uint hide_controll_handler = 0; + private Widget? main_widget = null; + + construct { + header_bar.get_style_context().add_class("call-header-bar"); + header_bar_revealer.add(header_bar); + + this.get_style_context().add_class("dino-call-window"); + + bottom_bar_revealer.add(bottom_bar); + + overlay.add_overlay(own_video_box); + overlay.add_overlay(own_video_border); + overlay.add_overlay(bottom_bar_revealer); + overlay.add_overlay(header_bar_revealer); + + event_box.add(overlay); + add(event_box); + + Util.force_css(own_video_border, "* { border: 1px solid #616161; background-color: transparent; }"); + } + + public CallWindow() { + event_box.events |= Gdk.EventMask.POINTER_MOTION_MASK; + event_box.events |= Gdk.EventMask.ENTER_NOTIFY_MASK; + event_box.events |= Gdk.EventMask.LEAVE_NOTIFY_MASK; + + this.bind_property("counterpart-display-name", header_bar, "title", BindingFlags.SYNC_CREATE); + this.bind_property("counterpart-display-name", bottom_bar, "counterpart-display-name", BindingFlags.SYNC_CREATE); + + event_box.motion_notify_event.connect(reveal_control_elements); + event_box.enter_notify_event.connect(reveal_control_elements); + event_box.leave_notify_event.connect(reveal_control_elements); + this.configure_event.connect(reveal_control_elements); // upon resizing + this.configure_event.connect(update_own_video_position); + + this.set_titlebar(new OutsideHeaderBar(this.header_bar) { visible=true }); + + reveal_control_elements(); + } + + public void set_video_fallback(StreamInteractor stream_interactor, Conversation conversation) { + hide_controll_elements = false; + + Box box = new Box(Orientation.HORIZONTAL, 0) { visible=true }; + box.get_style_context().add_class("video-placeholder-box"); + AvatarImage avatar = new AvatarImage() { hexpand=true, vexpand=true, halign=Align.CENTER, valign=Align.CENTER, height=100, width=100, visible=true }; + avatar.set_conversation(stream_interactor, conversation); + box.add(avatar); + + set_new_main_widget(box); + } + + public void set_video(Widget widget) { + hide_controll_elements = true; + + widget.visible = true; + set_new_main_widget(widget); + } + + public void set_own_video(Widget? widget_) { + own_video_box.foreach((widget) => { own_video_box.remove(widget); }); + + own_video = widget_; + if (own_video == null) { + own_video = new Box(Orientation.HORIZONTAL, 0) { expand=true }; + } + own_video.visible = true; + own_video.width_request = 150; + own_video.height_request = 100; + own_video_box.add(own_video); + + own_video_border.visible = true; + + update_own_video_position(); + } + + public void set_own_video_ratio(int width, int height) { + if (width / height > 150 / 100) { + this.own_video_width = 150; + this.own_video_height = height * 150 / width; + } else { + this.own_video_width = width * 100 / height; + this.own_video_height = 100; + } + + own_video.width_request = own_video_width; + own_video.height_request = own_video_height; + + update_own_video_position(); + } + + public void unset_own_video() { + own_video_box.foreach((widget) => { own_video_box.remove(widget); }); + + own_video_border.visible = false; + } + + public void set_test_video() { + hide_controll_elements = true; + + var pipeline = new Gst.Pipeline(null); + var src = Gst.ElementFactory.make("videotestsrc", null); + pipeline.add(src); + Gst.Video.Sink sink = (Gst.Video.Sink) Gst.ElementFactory.make("gtksink", null); + Gtk.Widget widget; + sink.get("widget", out widget); + widget.unparent(); + pipeline.add(sink); + src.link(sink); + widget.visible = true; + + pipeline.set_state(Gst.State.PLAYING); + + sink.get_static_pad("sink").notify["caps"].connect(() => { + int width, height; + sink.get_static_pad("sink").caps.get_structure(0).get_int("width", out width); + sink.get_static_pad("sink").caps.get_structure(0).get_int("height", out height); + widget.width_request = width; + widget.height_request = height; + }); + + set_new_main_widget(widget); + } + + private void set_new_main_widget(Widget widget) { + if (main_widget != null) overlay.remove(main_widget); + overlay.add(widget); + main_widget = widget; + } + + public void set_status(string state) { + switch (state) { + case "requested": + header_bar.subtitle = _("Sending a call request…"); + break; + case "ringing": + header_bar.subtitle = _("Ringing…"); + break; + case "establishing": + header_bar.subtitle = _("Establishing a (peer-to-peer) connection…"); + break; + default: + header_bar.subtitle = null; + break; + } + } + + public void show_counterpart_ended(string? reason_name, string? reason_text) { + hide_controll_elements = false; + reveal_control_elements(); + + string text = ""; + if (reason_name == Xmpp.Xep.Jingle.ReasonElement.SUCCESS) { + text = _("%s ended the call").printf(counterpart_display_name); + } else if (reason_name == Xmpp.Xep.Jingle.ReasonElement.DECLINE || reason_name == Xmpp.Xep.Jingle.ReasonElement.BUSY) { + text = _("%s declined the call").printf(counterpart_display_name); + } else { + text = "The call has been terminated: " + (reason_name ?? "") + " " + (reason_text ?? ""); + } + + bottom_bar.show_counterpart_ended(text); + } + + public bool reveal_control_elements() { + if (!bottom_bar_revealer.child_revealed) { + bottom_bar_revealer.set_reveal_child(true); + header_bar_revealer.set_reveal_child(true); + } + + if (hide_controll_handler != 0) { + Source.remove(hide_controll_handler); + hide_controll_handler = 0; + } + + if (!hide_controll_elements) { + return false; + } + + hide_controll_handler = Timeout.add_seconds(3, () => { + if (!hide_controll_elements) { + return false; + } + + if (bottom_bar.is_menu_active()) { + return true; + } + + header_bar_revealer.set_reveal_child(false); + bottom_bar_revealer.set_reveal_child(false); + hide_controll_handler = 0; + return false; + }); + return false; + } + + private bool update_own_video_position() { + if (own_video == null) return false; + + int width, height; + this.get_size(out width,out height); + + own_video.margin_end = own_video.margin_bottom = own_video_border.margin_end = own_video_border.margin_bottom = 20; + own_video.margin_start = own_video_border.margin_start = width - own_video_width - 20; + own_video.margin_top = own_video_border.margin_top = height - own_video_height - 20; + + return false; + } + } + + /* Hack to make the CallHeaderBar feel like a HeaderBar (right click menu, double click, ..) although it isn't set as headerbar. + * OutsideHeaderBar is set as a headerbar and it doesn't take any space, but claims to take space (which is actually taken by CallHeaderBar). + */ + public class OutsideHeaderBar : Gtk.Box { + HeaderBar header_bar; + + public OutsideHeaderBar(HeaderBar header_bar) { + this.header_bar = header_bar; + + size_allocate.connect_after(on_header_bar_size_allocate); + header_bar.size_allocate.connect(on_header_bar_size_allocate); + } + + public void on_header_bar_size_allocate() { + Allocation header_bar_alloc; + header_bar.get_allocation(out header_bar_alloc); + + Allocation alloc; + get_allocation(out alloc); + alloc.height = header_bar_alloc.height; + set_allocation(alloc); + } + } +} \ No newline at end of file diff --git a/main/src/ui/call_window/call_window_controller.vala b/main/src/ui/call_window/call_window_controller.vala new file mode 100644 index 00000000..09c8f88c --- /dev/null +++ b/main/src/ui/call_window/call_window_controller.vala @@ -0,0 +1,208 @@ +using Dino.Entities; +using Gtk; + +public class Dino.Ui.CallWindowController : Object { + + public signal void terminated(); + + private CallWindow call_window; + private Call call; + private Conversation conversation; + private StreamInteractor stream_interactor; + private Calls calls; + private Plugins.VideoCallPlugin call_plugin = Dino.Application.get_default().plugin_registry.video_call_plugin; + + private Plugins.VideoCallWidget? own_video = null; + private Plugins.VideoCallWidget? counterpart_video = null; + + public CallWindowController(CallWindow call_window, Call call, StreamInteractor stream_interactor) { + this.call_window = call_window; + this.call = call; + this.stream_interactor = stream_interactor; + + this.calls = stream_interactor.get_module(Calls.IDENTITY); + this.conversation = stream_interactor.get_module(ConversationManager.IDENTITY).get_conversation(call.counterpart.bare_jid, call.account, Conversation.Type.CHAT); + this.own_video = call_plugin.create_widget(Plugins.WidgetType.GTK); + this.counterpart_video = call_plugin.create_widget(Plugins.WidgetType.GTK); + + call_window.counterpart_display_name = Util.get_conversation_display_name(stream_interactor, conversation); + call_window.set_default_size(640, 480); + call_window.set_video_fallback(stream_interactor, conversation); + + this.call_window.bottom_bar.video_enabled = calls.should_we_send_video(call); + + if (call.direction == Call.DIRECTION_INCOMING) { + call_window.set_status("establishing"); + } else { + call_window.set_status("requested"); + } + + call_window.bottom_bar.hang_up.connect(end_call); + call_window.destroy.connect(end_call); + + call_window.bottom_bar.notify["audio-enabled"].connect(() => { + calls.mute_own_audio(call, !call_window.bottom_bar.audio_enabled); + }); + call_window.bottom_bar.notify["video-enabled"].connect(() => { + calls.mute_own_video(call, !call_window.bottom_bar.video_enabled); + update_own_video(); + }); + + calls.counterpart_sends_video_updated.connect((call, mute) => { + if (!this.call.equals(call)) return; + + if (mute) { + call_window.set_video_fallback(stream_interactor, conversation); + counterpart_video.detach(); + } else { + if (!(counterpart_video is Widget)) return; + Widget widget = (Widget) counterpart_video; + call_window.set_video(widget); + counterpart_video.display_stream(calls.get_video_stream(call)); + } + }); + calls.info_received.connect((call, session_info) => { + if (!this.call.equals(call)) return; + if (session_info == Xmpp.Xep.JingleRtp.CallSessionInfo.RINGING) { + call_window.set_status("ringing"); + } + }); + + own_video.resolution_changed.connect((width, height) => { + if (width == 0 || height == 0) return; + call_window.set_own_video_ratio((int)width, (int)height); + }); + counterpart_video.resolution_changed.connect((width, height) => { + if (width == 0 || height == 0) return; + if (width / height > 640 / 480) { + call_window.resize(640, (int) (height * 640 / width)); + } else { + call_window.resize((int) (width * 480 / height), 480); + } + }); + + call.notify["state"].connect(on_call_state_changed); + calls.call_terminated.connect(on_call_terminated); + + update_own_video(); + } + + private void end_call() { + call.notify["state"].disconnect(on_call_state_changed); + calls.call_terminated.disconnect(on_call_terminated); + + calls.end_call(conversation, call); + call_window.close(); + call_window.destroy(); + terminated(); + } + + private void on_call_state_changed() { + if (call.state == Call.State.IN_PROGRESS) { + call_window.set_status(""); + call_plugin.devices_changed.connect((media, incoming) => { + if (media == "audio") update_audio_device_choices(); + if (media == "video") update_video_device_choices(); + }); + + update_audio_device_choices(); + update_video_device_choices(); + } + } + + private void on_call_terminated(Call call, string? reason_name, string? reason_text) { + call_window.show_counterpart_ended(reason_name, reason_text); + Timeout.add_seconds(3, () => { + call.notify["state"].disconnect(on_call_state_changed); + calls.call_terminated.disconnect(on_call_terminated); + + + call_window.close(); + call_window.destroy(); + + return false; + }); + } + + private void update_audio_device_choices() { + if (call_plugin.get_devices("audio", true).size == 0 || call_plugin.get_devices("audio", false).size == 0) { + call_window.bottom_bar.show_audio_device_error(); + } /*else if (call_plugin.get_devices("audio", true).size == 1 && call_plugin.get_devices("audio", false).size == 1) { + call_window.bottom_bar.show_audio_device_choices(false); + return; + } + + AudioSettingsPopover? audio_settings_popover = call_window.bottom_bar.show_audio_device_choices(true); + update_current_audio_device(audio_settings_popover); + + audio_settings_popover.microphone_selected.connect((device) => { + call_plugin.set_device(calls.get_audio_stream(call), device); + update_current_audio_device(audio_settings_popover); + }); + audio_settings_popover.speaker_selected.connect((device) => { + call_plugin.set_device(calls.get_audio_stream(call), device); + update_current_audio_device(audio_settings_popover); + }); + calls.stream_created.connect((call, media) => { + if (media == "audio") { + update_current_audio_device(audio_settings_popover); + } + });*/ + } + + private void update_current_audio_device(AudioSettingsPopover audio_settings_popover) { + Xmpp.Xep.JingleRtp.Stream stream = calls.get_audio_stream(call); + if (stream != null) { + audio_settings_popover.current_microphone_device = call_plugin.get_device(stream, false); + audio_settings_popover.current_speaker_device = call_plugin.get_device(stream, true); + } + } + + private void update_video_device_choices() { + int device_count = call_plugin.get_devices("video", false).size; + + if (device_count == 0) { + call_window.bottom_bar.show_video_device_error(); + } /*else if (device_count == 1 || calls.get_video_stream(call) == null) { + call_window.bottom_bar.show_video_device_choices(false); + return; + } + + VideoSettingsPopover? video_settings_popover = call_window.bottom_bar.show_video_device_choices(true); + update_current_video_device(video_settings_popover); + + video_settings_popover.camera_selected.connect((device) => { + call_plugin.set_device(calls.get_video_stream(call), device); + update_current_video_device(video_settings_popover); + own_video.display_device(device); + }); + calls.stream_created.connect((call, media) => { + if (media == "video") { + update_current_video_device(video_settings_popover); + } + });*/ + } + + private void update_current_video_device(VideoSettingsPopover video_settings_popover) { + Xmpp.Xep.JingleRtp.Stream stream = calls.get_video_stream(call); + if (stream != null) { + video_settings_popover.current_device = call_plugin.get_device(stream, false); + } + } + + private void update_own_video() { + if (this.call_window.bottom_bar.video_enabled) { + Gee.List devices = call_plugin.get_devices("video", false); + if (!(own_video is Widget) || devices.is_empty) { + call_window.set_own_video(null); + } else { + Widget widget = (Widget) own_video; + call_window.set_own_video(widget); + own_video.display_device(devices.first()); + } + } else { + own_video.detach(); + call_window.unset_own_video(); + } + } +} \ No newline at end of file diff --git a/main/src/ui/call_window/video_settings_popover.vala b/main/src/ui/call_window/video_settings_popover.vala new file mode 100644 index 00000000..396c697c --- /dev/null +++ b/main/src/ui/call_window/video_settings_popover.vala @@ -0,0 +1,73 @@ +using Gee; +using Gtk; +using Dino.Entities; + +public class Dino.Ui.VideoSettingsPopover : Gtk.Popover { + + public signal void camera_selected(Plugins.MediaDevice device); + + public Plugins.MediaDevice? current_device { get; set; } + + private HashMap row_device = new HashMap(); + + public VideoSettingsPopover() { + Box box = new Box(Orientation.VERTICAL, 15) { margin=18, visible=true }; + box.add(create_camera_box()); + + this.add(box); + } + + private Widget create_camera_box() { + Plugins.VideoCallPlugin call_plugin = Dino.Application.get_default().plugin_registry.video_call_plugin; + Gee.List devices = call_plugin.get_devices("video", false); + + Box camera_box = new Box(Orientation.VERTICAL, 10) { visible=true }; + camera_box.add(new Label("" + _("Cameras") + "") { use_markup=true, xalign=0, visible=true, can_focus=true /* grab initial focus*/ }); + + if (devices.size == 0) { + camera_box.add(new Label("No cameras found.") { visible=true }); + } else { + ListBox list_box = new ListBox() { activate_on_single_click=true, selection_mode=SelectionMode.SINGLE, visible=true }; + list_box.set_header_func(listbox_header_func); + Frame frame = new Frame(null) { visible=true }; + frame.add(list_box); + foreach (Plugins.MediaDevice device in devices) { + Label label = new Label(device.display_name) { xalign=0, visible=true }; + Image image = new Image.from_icon_name("object-select-symbolic", IconSize.BUTTON) { visible=true }; + if (current_device == null || current_device.id != device.id) { + image.opacity = 0; + } + this.notify["current-device"].connect(() => { + if (current_device == null || current_device.id != device.id) { + image.opacity = 0; + } else { + image.opacity = 1; + } + }); + Box device_box = new Box(Orientation.HORIZONTAL, 0) { spacing=7, margin=7, visible=true }; + device_box.add(image); + device_box.add(label); + ListBoxRow list_box_row = new ListBoxRow() { visible=true }; + list_box_row.add(device_box); + list_box.add(list_box_row); + + row_device[list_box_row] = device; + } + list_box.row_activated.connect((row) => { + if (!row_device.has_key(row)) return; + camera_selected(row_device[row]); + list_box.unselect_row(row); + }); + camera_box.add(frame); + } + + return camera_box; + } + + private void listbox_header_func(ListBoxRow row, ListBoxRow? before_row) { + if (row.get_header() == null && before_row != null) { + row.set_header(new Separator(Orientation.HORIZONTAL)); + } + } + +} \ No newline at end of file diff --git a/main/src/ui/conversation_content_view/call_widget.vala b/main/src/ui/conversation_content_view/call_widget.vala new file mode 100644 index 00000000..66788e28 --- /dev/null +++ b/main/src/ui/conversation_content_view/call_widget.vala @@ -0,0 +1,215 @@ +using Gee; +using Gdk; +using Gtk; +using Pango; + +using Dino.Entities; + +namespace Dino.Ui { + + public class CallMetaItem : ConversationSummary.ContentMetaItem { + + private StreamInteractor stream_interactor; + + public CallMetaItem(ContentItem content_item, StreamInteractor stream_interactor) { + base(content_item); + this.stream_interactor = stream_interactor; + } + + public override Object? get_widget(Plugins.WidgetType type) { + CallItem call_item = content_item as CallItem; + return new CallWidget(stream_interactor, call_item.call, call_item.conversation) { visible=true }; + } + + public override Gee.List? get_item_actions(Plugins.WidgetType type) { return null; } + } + + [GtkTemplate (ui = "/im/dino/Dino/call_widget.ui")] + public class CallWidget : SizeRequestBox { + + [GtkChild] public Image image; + [GtkChild] public Label title_label; + [GtkChild] public Label subtitle_label; + [GtkChild] public Revealer incoming_call_revealer; + [GtkChild] public Button accept_call_button; + [GtkChild] public Button reject_call_button; + + private StreamInteractor stream_interactor; + private Call call; + private Conversation conversation; + public Call.State call_state { get; set; } // needs to be public for binding + private uint time_update_handler_id = 0; + + construct { + margin_top = 4; + size_request_mode = SizeRequestMode.HEIGHT_FOR_WIDTH; + } + + public CallWidget(StreamInteractor stream_interactor, Call call, Conversation conversation) { + this.stream_interactor = stream_interactor; + this.call = call; + this.conversation = conversation; + + size_allocate.connect((allocation) => { + if (allocation.height > parent.get_allocated_height()) { + Idle.add(() => { parent.queue_resize(); return false; }); + } + }); + + call.bind_property("state", this, "call-state"); + this.notify["call-state"].connect(update_widget); + + accept_call_button.clicked.connect(() => { + stream_interactor.get_module(Calls.IDENTITY).accept_call(call); + + var call_window = new CallWindow(); + call_window.controller = new CallWindowController(call_window, call, stream_interactor); + call_window.present(); + }); + + reject_call_button.clicked.connect(() => { + stream_interactor.get_module(Calls.IDENTITY).reject_call(call); + }); + + update_widget(); + } + + private void update_widget() { + incoming_call_revealer.reveal_child = false; + incoming_call_revealer.get_style_context().remove_class("incoming"); + + switch (call.state) { + case Call.State.RINGING: + image.set_from_icon_name("dino-phone-ring-symbolic", IconSize.LARGE_TOOLBAR); + if (call.direction == Call.DIRECTION_INCOMING) { + bool video = stream_interactor.get_module(Calls.IDENTITY).should_we_send_video(call); + title_label.label = video ? _("Video call incoming") : _("Call incoming"); + subtitle_label.label = "Ring ring…!"; + incoming_call_revealer.reveal_child = true; + incoming_call_revealer.get_style_context().add_class("incoming"); + } else { + title_label.label = _("Establishing call"); + subtitle_label.label = "Ring ring…?"; + } + break; + case Call.State.ESTABLISHING: + image.set_from_icon_name("dino-phone-ring-symbolic", IconSize.LARGE_TOOLBAR); + if (call.direction == Call.DIRECTION_INCOMING) { + bool video = stream_interactor.get_module(Calls.IDENTITY).should_we_send_video(call); + title_label.label = video ? _("Video call establishing") : _("Call establishing"); + subtitle_label.label = "Connecting…"; + } + break; + case Call.State.IN_PROGRESS: + image.set_from_icon_name("dino-phone-in-talk-symbolic", IconSize.LARGE_TOOLBAR); + title_label.label = _("Call in progress…"); + string duration = get_duration_string((new DateTime.now_utc()).difference(call.local_time)); + subtitle_label.label = _("Started %s ago").printf(duration); + + time_update_handler_id = Timeout.add_seconds(get_next_time_change() + 1, () => { + Source.remove(time_update_handler_id); + time_update_handler_id = 0; + update_widget(); + return true; + }); + + break; + case Call.State.OTHER_DEVICE_ACCEPTED: + image.set_from_icon_name("dino-phone-hangup-symbolic", IconSize.LARGE_TOOLBAR); + title_label.label = call.direction == Call.DIRECTION_INCOMING ? _("Incoming call") : _("Outgoing call"); + subtitle_label.label = _("You handled this call on another device"); + + break; + case Call.State.ENDED: + image.set_from_icon_name("dino-phone-hangup-symbolic", IconSize.LARGE_TOOLBAR); + title_label.label = _("Call ended"); + string formated_end = Util.format_time(call.end_time, _("%H∶%M"), _("%l∶%M %p")); + string duration = get_duration_string(call.end_time.difference(call.local_time)); + subtitle_label.label = _("Ended at %s").printf(formated_end) + + " · " + + _("Lasted for %s").printf(duration); + break; + case Call.State.MISSED: + image.set_from_icon_name("dino-phone-missed-symbolic", IconSize.LARGE_TOOLBAR); + title_label.label = _("Call missed"); + string who = null; + if (call.direction == Call.DIRECTION_INCOMING) { + who = "You"; + } else { + who = Util.get_participant_display_name(stream_interactor, conversation, call.to); + } + subtitle_label.label = "%s missed this call".printf(who); + break; + case Call.State.DECLINED: + image.set_from_icon_name("dino-phone-hangup-symbolic", IconSize.LARGE_TOOLBAR); + title_label.label = _("Call declined"); + string who = null; + if (call.direction == Call.DIRECTION_INCOMING) { + who = "You"; + } else { + who = Util.get_participant_display_name(stream_interactor, conversation, call.to); + } + subtitle_label.label = "%s declined this call".printf(who); + break; + case Call.State.FAILED: + image.set_from_icon_name("dino-phone-hangup-symbolic", IconSize.LARGE_TOOLBAR); + title_label.label = _("Call failed"); + subtitle_label.label = "This call failed to establish"; + break; + } + } + + private string get_duration_string(TimeSpan duration) { + DateTime a = new DateTime.now_utc(); + DateTime b = new DateTime.now_utc(); + a.difference(b); + + TimeSpan remainder_duration = duration; + + int hours = (int) Math.floor(remainder_duration / TimeSpan.HOUR); + remainder_duration -= hours * TimeSpan.HOUR; + + int minutes = (int) Math.floor(remainder_duration / TimeSpan.MINUTE); + remainder_duration -= minutes * TimeSpan.MINUTE; + + string ret = ""; + + if (hours > 0) { + ret += n("%i hour", "%i hours", hours).printf(hours); + } + + if (minutes > 0) { + if (ret.length > 0) { + ret += " "; + } + ret += n("%i minute", "%i minutes", minutes).printf(minutes); + } + + if (ret.length > 0) { + return ret; + } + + return _("seconds"); + } + + private int get_next_time_change() { + DateTime now = new DateTime.now_local(); + DateTime item_time = call.local_time; + + if (now.get_second() < item_time.get_second()) { + return item_time.get_second() - now.get_second(); + } else { + return 60 - (now.get_second() - item_time.get_second()); + } + } + + public override void dispose() { + base.dispose(); + + if (time_update_handler_id != 0) { + Source.remove(time_update_handler_id); + time_update_handler_id = 0; + } + } + } +} diff --git a/main/src/ui/conversation_content_view/content_populator.vala b/main/src/ui/conversation_content_view/content_populator.vala index 97f15bf9..d7ce9ce5 100644 --- a/main/src/ui/conversation_content_view/content_populator.vala +++ b/main/src/ui/conversation_content_view/content_populator.vala @@ -68,7 +68,10 @@ public class ContentProvider : ContentItemCollection, Object { return new MessageMetaItem(content_item, stream_interactor); } else if (content_item.type_ == FileItem.TYPE) { return new FileMetaItem(content_item, stream_interactor); + } else if (content_item.type_ == CallItem.TYPE) { + return new CallMetaItem(content_item, stream_interactor); } + critical("Got unknown content item type %s", content_item.type_); return null; } } diff --git a/main/src/ui/conversation_content_view/file_widget.vala b/main/src/ui/conversation_content_view/file_widget.vala index 9b748876..7d77ba11 100644 --- a/main/src/ui/conversation_content_view/file_widget.vala +++ b/main/src/ui/conversation_content_view/file_widget.vala @@ -32,9 +32,6 @@ public class FileWidget : SizeRequestBox { DEFAULT } - private const int MAX_HEIGHT = 300; - private const int MAX_WIDTH = 600; - private StreamInteractor stream_interactor; private FileTransfer file_transfer; public FileTransfer.State file_transfer_state { get; set; } diff --git a/main/src/ui/conversation_selector/conversation_selector_row.vala b/main/src/ui/conversation_selector/conversation_selector_row.vala index cd513d13..6f181a64 100644 --- a/main/src/ui/conversation_selector/conversation_selector_row.vala +++ b/main/src/ui/conversation_selector/conversation_selector_row.vala @@ -198,6 +198,14 @@ public class ConversationSelectorRow : ListBoxRow { message_label.label = (file_is_image ? _("Image received") : _("File received") ); } break; + case CallItem.TYPE: + CallItem call_item = (CallItem) last_content_item; + Call call = call_item.call; + + nick_label.label = call.direction == Call.DIRECTION_OUTGOING ? _("Me") + ": " : ""; + message_label.attributes.insert(attr_style_new(Pango.Style.ITALIC)); + message_label.label = call.direction == Call.DIRECTION_OUTGOING ? _("Outgoing call") : _("Incoming call"); + break; } nick_label.visible = true; message_label.visible = true; diff --git a/main/src/ui/conversation_titlebar/call_entry.vala b/main/src/ui/conversation_titlebar/call_entry.vala new file mode 100644 index 00000000..1ac4dd83 --- /dev/null +++ b/main/src/ui/conversation_titlebar/call_entry.vala @@ -0,0 +1,130 @@ +using Xmpp; +using Gtk; +using Gee; + +using Dino.Entities; + +namespace Dino.Ui { + + public class CallTitlebarEntry : Plugins.ConversationTitlebarEntry, Object { + public string id { get { return "call"; } } + + public CallButton call_button; + + private StreamInteractor stream_interactor; + + public CallTitlebarEntry(StreamInteractor stream_interactor) { + this.stream_interactor = stream_interactor; + + call_button = new CallButton(stream_interactor) { tooltip_text=_("Start call") }; + call_button.set_image(new Gtk.Image.from_icon_name("dino-phone-symbolic", Gtk.IconSize.MENU) { visible=true }); + } + + public double order { get { return 4; } } + public Plugins.ConversationTitlebarWidget? get_widget(Plugins.WidgetType type) { + if (type == Plugins.WidgetType.GTK) { + return call_button; + } + return null; + } + } + + public class CallButton : Plugins.ConversationTitlebarWidget, Gtk.MenuButton { + + private StreamInteractor stream_interactor; + private Conversation conversation; + + public CallButton(StreamInteractor stream_interactor) { + this.stream_interactor = stream_interactor; + + use_popover = true; + image = new Gtk.Image.from_icon_name("dino-phone-symbolic", Gtk.IconSize.MENU) { visible=true }; + + Gtk.PopoverMenu popover_menu = new Gtk.PopoverMenu(); + Box box = new Box(Orientation.VERTICAL, 0) { margin=10, visible=true }; + ModelButton audio_button = new ModelButton() { text="Audio call", visible=true }; + audio_button.clicked.connect(() => { + stream_interactor.get_module(Calls.IDENTITY).initiate_call.begin(conversation, false, (_, res) => { + Call call = stream_interactor.get_module(Calls.IDENTITY).initiate_call.end(res); + open_call_window(call); + }); + }); + box.add(audio_button); + ModelButton video_button = new ModelButton() { text="Video call", visible=true }; + video_button.clicked.connect(() => { + stream_interactor.get_module(Calls.IDENTITY).initiate_call.begin(conversation, true, (_, res) => { + Call call = stream_interactor.get_module(Calls.IDENTITY).initiate_call.end(res); + open_call_window(call); + }); + }); + box.add(video_button); + popover_menu.add(box); + + popover = popover_menu; + + clicked.connect(() => { + popover_menu.visible = true; + }); + + stream_interactor.get_module(Calls.IDENTITY).call_incoming.connect((call, conversation) => { + update_button_state(); + }); + + stream_interactor.get_module(Calls.IDENTITY).call_terminated.connect((call) => { + update_button_state(); + }); + stream_interactor.get_module(PresenceManager.IDENTITY).show_received.connect((jid, account) => { + if (this.conversation.counterpart.equals_bare(jid) && this.conversation.account.equals(account)) { + update_visibility.begin(); + } + }); + stream_interactor.connection_manager.connection_state_changed.connect((account, state) => { + update_visibility.begin(); + }); + } + + private void open_call_window(Call call) { + var call_window = new CallWindow(); + var call_controller = new CallWindowController(call_window, call, stream_interactor); + call_window.controller = call_controller; + call_window.present(); + + update_button_state(); + call_controller.terminated.connect(() => { + update_button_state(); + }); + } + + public new void set_conversation(Conversation conversation) { + this.conversation = conversation; + + update_visibility.begin(); + update_button_state(); + } + + private void update_button_state() { + Jid? call_counterpart = stream_interactor.get_module(Calls.IDENTITY).is_call_in_progress(); + this.sensitive = call_counterpart == null; + + if (call_counterpart != null && call_counterpart.equals_bare(conversation.counterpart)) { + this.set_image(new Gtk.Image.from_icon_name("dino-phone-in-talk-symbolic", Gtk.IconSize.MENU) { visible=true }); + } else { + this.set_image(new Gtk.Image.from_icon_name("dino-phone-symbolic", Gtk.IconSize.MENU) { visible=true }); + } + } + + private async void update_visibility() { + if (conversation.type_ == Conversation.Type.CHAT) { + Conversation conv_bak = conversation; + Gee.List? resources = yield stream_interactor.get_module(Calls.IDENTITY).get_call_resources(conversation); + if (conv_bak != conversation) return; + visible = resources != null && resources.size > 0; + } else { + visible = false; + } + } + + public new void unset_conversation() { } + } + +} diff --git a/main/src/ui/conversation_view_controller.vala b/main/src/ui/conversation_view_controller.vala index dcd3e1c7..a9a94738 100644 --- a/main/src/ui/conversation_view_controller.vala +++ b/main/src/ui/conversation_view_controller.vala @@ -87,6 +87,7 @@ public class ConversationViewController : Object { app.plugin_registry.register_contact_titlebar_entry(new MenuEntry(stream_interactor)); app.plugin_registry.register_contact_titlebar_entry(search_menu_entry); app.plugin_registry.register_contact_titlebar_entry(new OccupantsEntry(stream_interactor)); + app.plugin_registry.register_contact_titlebar_entry(new CallTitlebarEntry(stream_interactor)); foreach(var entry in app.plugin_registry.conversation_titlebar_entries) { titlebar.insert_entry(entry); } diff --git a/main/src/ui/notifier_freedesktop.vala b/main/src/ui/notifier_freedesktop.vala index 00ba0d06..35b95e3e 100644 --- a/main/src/ui/notifier_freedesktop.vala +++ b/main/src/ui/notifier_freedesktop.vala @@ -14,6 +14,7 @@ public class Dino.Ui.FreeDesktopNotifier : NotificationProvider, Object { private HashMap content_notifications = new HashMap(Conversation.hash_func, Conversation.equals_func); private HashMap> conversation_notifications = new HashMap>(Conversation.hash_func, Conversation.equals_func); private HashMap> action_listeners = new HashMap>(); + private HashMap call_notifications = new HashMap(Call.hash_func, Call.equals_func); private FreeDesktopNotifier(StreamInteractor stream_interactor) { this.stream_interactor = stream_interactor; @@ -109,6 +110,43 @@ public class Dino.Ui.FreeDesktopNotifier : NotificationProvider, Object { } } + public async void notify_call(Call call, Conversation conversation, bool video, string conversation_display_name) { + string summary = Markup.escape_text(conversation_display_name); + string body = video ? _("Incoming video call") : _("Incoming call"); + + HashTable hash_table = new HashTable(null, null); + hash_table["image-path"] = "call-start-symbolic"; + hash_table["sound-name"] = new Variant.string("phone-incoming-call"); + hash_table["urgency"] = new Variant.byte(2); + string[] actions = new string[] {"default", "Open conversation", "reject", _("Reject"), "accept", _("Accept")}; + try { + uint32 notification_id = dbus_notifications.notify("Dino", 0, "", summary, body, actions, hash_table, 0); + call_notifications[call] = notification_id; + + add_action_listener(notification_id, "default", () => { + GLib.Application.get_default().activate_action("open-conversation", new Variant.int32(conversation.id)); + }); + add_action_listener(notification_id, "reject", () => { + GLib.Application.get_default().activate_action("deny-call", new Variant.int32(call.id)); + }); + add_action_listener(notification_id, "accept", () => { + GLib.Application.get_default().activate_action("accept-call", new Variant.int32(call.id)); + }); + } catch (Error e) { + warning("Failed showing subscription request notification: %s", e.message); + } + } + + public async void retract_call_notification(Call call, Conversation conversation) { + if (!call_notifications.has_key(call)) return; + uint32 notification_id = call_notifications[call]; + try { + dbus_notifications.close_notification(notification_id); + action_listeners.unset(notification_id); + call_notifications.unset(call); + } catch (Error e) { } + } + public async void notify_subscription_request(Conversation conversation) { string summary = _("Subscription request"); string body = Markup.escape_text(conversation.counterpart.to_string()); diff --git a/main/src/ui/notifier_gnotifications.vala b/main/src/ui/notifier_gnotifications.vala index 31d1ffa3..5fd3be4b 100644 --- a/main/src/ui/notifier_gnotifications.vala +++ b/main/src/ui/notifier_gnotifications.vala @@ -65,6 +65,25 @@ namespace Dino.Ui { } } + public async void notify_call(Call call, Conversation conversation, bool video, string conversation_display_name) { + Notification notification = new Notification(conversation_display_name); + string body = _("Incoming call"); + notification.set_body(body); + notification.set_urgent(true); + + notification.set_icon(new ThemedIcon.from_names(new string[] {"call-start-symbolic"})); + + notification.set_default_action_and_target_value("app.open-conversation", new Variant.int32(conversation.id)); + notification.add_button_with_target_value(_("Deny"), "app.deny-call", new Variant.int32(call.id)); + notification.add_button_with_target_value(_("Accept"), "app.accept-call", new Variant.int32(call.id)); + + GLib.Application.get_default().send_notification(call.id.to_string(), notification); + } + + private async void retract_call_notification(Call call, Conversation conversation) { + GLib.Application.get_default().withdraw_notification(call.id.to_string()); + } + public async void notify_subscription_request(Conversation conversation) { Notification notification = new Notification(_("Subscription request")); notification.set_body(conversation.counterpart.to_string()); diff --git a/main/src/ui/util/helper.vala b/main/src/ui/util/helper.vala index b6c9cb5a..d3ca063b 100644 --- a/main/src/ui/util/helper.vala +++ b/main/src/ui/util/helper.vala @@ -194,6 +194,15 @@ public static bool is_24h_format() { return is24h == 1; } +public static string format_time(DateTime datetime, string format_24h, string format_12h) { + string format = Util.is_24h_format() ? format_24h : format_12h; + if (!get_charset(null)) { + // No UTF-8 support, use simple colon for time instead + format = format.replace("∶", ":"); + } + return datetime.format(format); +} + public static Regex get_url_regex() { if (URL_REGEX == null) { URL_REGEX = /\b(((http|ftp)s?:\/\/|(ircs?|xmpp|mailto|sms|smsto|mms|tel|geo|openpgp4fpr|im|news|nntp|sip|ssh|bitcoin|sftp|magnet|vnc|urn):)\S+)/; -- cgit v1.2.3 From 23ffd37dded3bf872e42d7a00727ab3c4d105a97 Mon Sep 17 00:00:00 2001 From: Marvin W Date: Sat, 1 May 2021 15:19:05 +0200 Subject: Echo Cancellation --- CMakeLists.txt | 6 +- cmake/FindGstAudio.cmake | 14 +++ cmake/FindWebRTCAudioProcessing.cmake | 12 ++ plugins/rtp/CMakeLists.txt | 20 +++- plugins/rtp/src/device.vala | 30 +++-- plugins/rtp/src/plugin.vala | 5 +- plugins/rtp/src/voice_processor.vala | 176 +++++++++++++++++++++++++++++ plugins/rtp/src/voice_processor_native.cpp | 141 +++++++++++++++++++++++ 8 files changed, 385 insertions(+), 19 deletions(-) create mode 100644 cmake/FindGstAudio.cmake create mode 100644 cmake/FindWebRTCAudioProcessing.cmake create mode 100644 plugins/rtp/src/voice_processor.vala create mode 100644 plugins/rtp/src/voice_processor_native.cpp (limited to 'CMakeLists.txt') diff --git a/CMakeLists.txt b/CMakeLists.txt index f480b0b2..b3bd35cc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,11 +2,11 @@ cmake_minimum_required(VERSION 3.3) list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake) include(ComputeVersion) if (NOT VERSION_FOUND) - project(Dino LANGUAGES C) + project(Dino LANGUAGES C CXX) elseif (VERSION_IS_RELEASE) - project(Dino VERSION ${VERSION_FULL} LANGUAGES C) + project(Dino VERSION ${VERSION_FULL} LANGUAGES C CXX) else () - project(Dino LANGUAGES C) + project(Dino LANGUAGES C CXX) set(PROJECT_VERSION ${VERSION_FULL}) endif () diff --git a/cmake/FindGstAudio.cmake b/cmake/FindGstAudio.cmake new file mode 100644 index 00000000..d5fc5dfb --- /dev/null +++ b/cmake/FindGstAudio.cmake @@ -0,0 +1,14 @@ +include(PkgConfigWithFallback) +find_pkg_config_with_fallback(GstAudio + PKG_CONFIG_NAME gstreamer-audio-1.0 + LIB_NAMES gstaudio + LIB_DIR_HINTS gstreamer-1.0 + INCLUDE_NAMES gst/audio/audio.h + INCLUDE_DIR_SUFFIXES gstreamer-1.0 gstreamer-1.0/include gstreamer-audio-1.0 gstreamer-audio-1.0/include + DEPENDS Gst +) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(GstAudio + REQUIRED_VARS GstAudio_LIBRARY + VERSION_VAR GstAudio_VERSION) diff --git a/cmake/FindWebRTCAudioProcessing.cmake b/cmake/FindWebRTCAudioProcessing.cmake new file mode 100644 index 00000000..5f17805d --- /dev/null +++ b/cmake/FindWebRTCAudioProcessing.cmake @@ -0,0 +1,12 @@ +include(PkgConfigWithFallback) +find_pkg_config_with_fallback(WebRTCAudioProcessing + PKG_CONFIG_NAME webrtc-audio-processing + LIB_NAMES webrtc_audio_processing + INCLUDE_NAMES webrtc/modules/audio_processing/include/audio_processing.h + INCLUDE_DIR_SUFFIXES webrtc-audio-processing webrtc_audio_processing +) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(WebRTCAudioProcessing + REQUIRED_VARS WebRTCAudioProcessing_LIBRARY + VERSION_VAR WebRTCAudioProcessing_VERSION) diff --git a/plugins/rtp/CMakeLists.txt b/plugins/rtp/CMakeLists.txt index 92ec1b97..b19c8a8f 100644 --- a/plugins/rtp/CMakeLists.txt +++ b/plugins/rtp/CMakeLists.txt @@ -1,4 +1,5 @@ find_package(GstRtp REQUIRED) +find_package(WebRTCAudioProcessing 0.2) find_packages(RTP_PACKAGES REQUIRED Gee GLib @@ -8,12 +9,26 @@ find_packages(RTP_PACKAGES REQUIRED GTK3 Gst GstApp + GstAudio ) if(Gst_VERSION VERSION_GREATER "1.16") set(RTP_DEFINITIONS GST_1_16) endif() +if(WebRTCAudioProcessing_VERSION GREATER "0.4") + message(WARNING "Ignoring WebRTCAudioProcessing, only versions < 0.4 supported so far") + unset(WebRTCAudioProcessing_FOUND) +endif() + +if(WebRTCAudioProcessing_FOUND) + set(RTP_DEFINITIONS ${RTP_DEFINITIONS} WITH_VOICE_PROCESSOR) + set(RTP_VOICE_PROCESSOR_VALA src/voice_processor.vala) + set(RTP_VOICE_PROCESSOR_CXX src/voice_processor_native.cpp) +else() + message(WARNING "WebRTCAudioProcessing not found, build without voice pre-processing!") +endif() + vala_precompile(RTP_VALA_C SOURCES src/codec_util.vala @@ -23,6 +38,7 @@ SOURCES src/stream.vala src/video_widget.vala src/register_plugin.vala + ${RTP_VOICE_PROCESSOR_VALA} CUSTOM_VAPIS ${CMAKE_BINARY_DIR}/exports/crypto-vala.vapi ${CMAKE_BINARY_DIR}/exports/xmpp-vala.vapi @@ -36,8 +52,8 @@ DEFINITIONS ) add_definitions(${VALA_CFLAGS} -DG_LOG_DOMAIN="rtp" -I${CMAKE_CURRENT_SOURCE_DIR}/src) -add_library(rtp SHARED ${RTP_VALA_C}) -target_link_libraries(rtp libdino crypto-vala ${RTP_PACKAGES} gstreamer-rtp-1.0) +add_library(rtp SHARED ${RTP_VALA_C} ${RTP_VOICE_PROCESSOR_CXX}) +target_link_libraries(rtp libdino crypto-vala ${RTP_PACKAGES} gstreamer-rtp-1.0 webrtc-audio-processing) set_target_properties(rtp PROPERTIES PREFIX "") set_target_properties(rtp PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/plugins/) diff --git a/plugins/rtp/src/device.vala b/plugins/rtp/src/device.vala index 785f853a..f8894502 100644 --- a/plugins/rtp/src/device.vala +++ b/plugins/rtp/src/device.vala @@ -37,6 +37,7 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { private Gst.Element dsp; private Gst.Element mixer; private Gst.Element filter; + private Gst.Element rate; private int links = 0; public Device(Plugin plugin, Gst.Device device) { @@ -132,12 +133,10 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { pipe.add(filter); element.link(filter); if (media == "audio" && plugin.echoprobe != null) { - dsp = Gst.ElementFactory.make("webrtcdsp", @"dsp_$id"); - if (dsp != null) { - dsp.@set("probe", plugin.echoprobe.name); - pipe.add(dsp); - filter.link(dsp); - } + dsp = new VoiceProcessor(plugin.echoprobe, element as Gst.Audio.StreamVolume); + dsp.name = @"dsp_$id"; + pipe.add(dsp); + filter.link(dsp); } tee = Gst.ElementFactory.make("tee", @"tee_$id"); tee.@set("allow-not-linked", true); @@ -153,7 +152,11 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { filter.@set("caps", get_best_caps()); pipe.add(filter); if (plugin.echoprobe != null) { - filter.link(plugin.echoprobe); + rate = Gst.ElementFactory.make("audiorate", @"rate_$id"); + rate.@set("tolerance", 100000000); + pipe.add(rate); + filter.link(rate); + rate.link(plugin.echoprobe); plugin.echoprobe.link(element); } else { filter.link(element); @@ -184,14 +187,17 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { if (filter != null) { filter.set_locked_state(true); filter.set_state(Gst.State.NULL); - if (plugin.echoprobe != null) { - filter.unlink(plugin.echoprobe); - } else { - filter.unlink(element); - } + filter.unlink(rate ?? ((Gst.Element)plugin.echoprobe) ?? element); pipe.remove(filter); filter = null; } + if (rate != null) { + rate.set_locked_state(true); + rate.set_state(Gst.State.NULL); + rate.unlink(plugin.echoprobe); + pipe.remove(rate); + rate = null; + } if (plugin.echoprobe != null) { plugin.echoprobe.unlink(element); } diff --git a/plugins/rtp/src/plugin.vala b/plugins/rtp/src/plugin.vala index d43588b4..e3d5ee41 100644 --- a/plugins/rtp/src/plugin.vala +++ b/plugins/rtp/src/plugin.vala @@ -8,7 +8,7 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { public Gst.DeviceMonitor device_monitor { get; private set; } public Gst.Pipeline pipe { get; private set; } public Gst.Bin rtpbin { get; private set; } - public Gst.Element echoprobe { get; private set; } + public EchoProbe echoprobe { get; private set; } private Gee.List streams = new ArrayList(); private Gee.List devices = new ArrayList(); @@ -72,7 +72,8 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { pipe.add(rtpbin); // Audio echo probe - echoprobe = Gst.ElementFactory.make("webrtcechoprobe", "echo-probe"); +// echoprobe = Gst.ElementFactory.make("webrtcechoprobe", "echo-probe"); + echoprobe = new EchoProbe(); if (echoprobe != null) pipe.add(echoprobe); // Pipeline diff --git a/plugins/rtp/src/voice_processor.vala b/plugins/rtp/src/voice_processor.vala new file mode 100644 index 00000000..e6dc7e8f --- /dev/null +++ b/plugins/rtp/src/voice_processor.vala @@ -0,0 +1,176 @@ +using Gst; + +namespace Dino.Plugins.Rtp { +public static extern Buffer adjust_to_running_time(Base.Transform transform, Buffer buf); +} + +public class Dino.Plugins.Rtp.EchoProbe : Audio.Filter { + private static StaticPadTemplate sink_template = {"sink", PadDirection.SINK, PadPresence.ALWAYS, {null, "audio/x-raw,rate=48000,channels=1,layout=interleaved,format=S16LE"}}; + private static StaticPadTemplate src_template = {"src", PadDirection.SRC, PadPresence.ALWAYS, {null, "audio/x-raw,rate=48000,channels=1,layout=interleaved,format=S16LE"}}; + public Audio.Info audio_info { get; private set; } + public signal void on_new_buffer(Buffer buffer); + private uint period_samples; + private uint period_size; + private Base.Adapter adapter = new Base.Adapter(); + + static construct { + add_static_pad_template(sink_template); + add_static_pad_template(src_template); + set_static_metadata("Acoustic Echo Canceller probe", "Generic/Audio", "Gathers playback buffers for echo cancellation", "Dino Team "); + } + + construct { + set_passthrough(true); + } + + public override bool setup(Audio.Info info) { + audio_info = info; + period_samples = info.rate / 100; // 10ms buffers + period_size = period_samples * info.bpf; + return true; + } + + + public override FlowReturn transform_ip(Buffer buf) { + lock (adapter) { + adapter.push(adjust_to_running_time(this, buf)); + while (adapter.available() > period_size) { + on_new_buffer(adapter.take_buffer(period_size)); + } + } + return FlowReturn.OK; + } + + public override bool stop() { + adapter.clear(); + return true; + } +} + +public class Dino.Plugins.Rtp.VoiceProcessor : Audio.Filter { + private static StaticPadTemplate sink_template = {"sink", PadDirection.SINK, PadPresence.ALWAYS, {null, "audio/x-raw,rate=48000,channels=1,layout=interleaved,format=S16LE"}}; + private static StaticPadTemplate src_template = {"src", PadDirection.SRC, PadPresence.ALWAYS, {null, "audio/x-raw,rate=48000,channels=1,layout=interleaved,format=S16LE"}}; + public Audio.Info audio_info { get; private set; } + private ulong process_outgoing_buffer_handler_id; + private uint adjust_delay_timeout_id; + private uint period_samples; + private uint period_size; + private Base.Adapter adapter = new Base.Adapter(); + private EchoProbe? echo_probe; + private Audio.StreamVolume? stream_volume; + private ClockTime last_reverse; + private void* native; + + static construct { + add_static_pad_template(sink_template); + add_static_pad_template(src_template); + set_static_metadata("Voice Processor (AGC, AEC, filters, etc.)", "Generic/Audio", "Pre-processes voice with WebRTC Audio Processing Library", "Dino Team "); + } + + construct { + set_passthrough(false); + } + + public VoiceProcessor(EchoProbe? echo_probe = null, Audio.StreamVolume? stream_volume = null) { + this.echo_probe = echo_probe; + this.stream_volume = stream_volume; + } + + private static extern void* init_native(int stream_delay); + private static extern void setup_native(void* native); + private static extern void destroy_native(void* native); + private static extern void analyze_reverse_stream(void* native, Audio.Info info, Buffer buffer); + private static extern void process_stream(void* native, Audio.Info info, Buffer buffer); + private static extern void adjust_stream_delay(void* native); + private static extern void notify_gain_level(void* native, int gain_level); + private static extern int get_suggested_gain_level(void* native); + private static extern bool get_stream_has_voice(void* native); + + public override bool setup(Audio.Info info) { + debug("VoiceProcessor.setup(%s)", info.to_caps().to_string()); + audio_info = info; + period_samples = info.rate / 100; // 10ms buffers + period_size = period_samples * info.bpf; + adapter.clear(); + setup_native(native); + return true; + } + + public override bool start() { + native = init_native(150); + if (process_outgoing_buffer_handler_id == 0 && echo_probe != null) { + process_outgoing_buffer_handler_id = echo_probe.on_new_buffer.connect(process_outgoing_buffer); + } + if (stream_volume == null && sinkpad.get_peer() != null && sinkpad.get_peer().get_parent_element() is Audio.StreamVolume) { + stream_volume = sinkpad.get_peer().get_parent_element() as Audio.StreamVolume; + } + return true; + } + + private bool adjust_delay() { + if (native != null) { + adjust_stream_delay(native); + return Source.CONTINUE; + } else { + adjust_delay_timeout_id = 0; + return Source.REMOVE; + } + } + + private void process_outgoing_buffer(Buffer buffer) { + if (buffer.pts != uint64.MAX) { + last_reverse = buffer.pts; + } + analyze_reverse_stream(native, echo_probe.audio_info, buffer); + if (adjust_delay_timeout_id == 0 && echo_probe != null) { + adjust_delay_timeout_id = Timeout.add(5000, adjust_delay); + } + } + + public override FlowReturn submit_input_buffer(bool is_discont, Buffer input) { + lock (adapter) { + if (is_discont) { + adapter.clear(); + } + adapter.push(adjust_to_running_time(this, input)); + } + return FlowReturn.OK; + } + + public override FlowReturn generate_output(out Buffer output_buffer) { + lock (adapter) { + if (adapter.available() >= period_size) { + output_buffer = (Gst.Buffer) adapter.take_buffer(period_size).make_writable(); + int old_gain_level = 0; + if (stream_volume != null) { + old_gain_level = (int) (stream_volume.get_volume(Audio.StreamVolumeFormat.LINEAR) * 255.0); + notify_gain_level(native, old_gain_level); + } + process_stream(native, audio_info, output_buffer); + if (stream_volume != null) { + int new_gain_level = get_suggested_gain_level(native); + if (old_gain_level != new_gain_level) { + debug("Gain: %i -> %i", old_gain_level, new_gain_level); + stream_volume.set_volume(Audio.StreamVolumeFormat.LINEAR, ((double)new_gain_level) / 255.0); + } + } + } + } + return FlowReturn.OK; + } + + public override bool stop() { + if (process_outgoing_buffer_handler_id != 0) { + echo_probe.disconnect(process_outgoing_buffer_handler_id); + process_outgoing_buffer_handler_id = 0; + } + if (adjust_delay_timeout_id != 0) { + Source.remove(adjust_delay_timeout_id); + adjust_delay_timeout_id = 0; + } + adapter.clear(); + destroy_native(native); + native = null; + return true; + } +} \ No newline at end of file diff --git a/plugins/rtp/src/voice_processor_native.cpp b/plugins/rtp/src/voice_processor_native.cpp new file mode 100644 index 00000000..9b3292b8 --- /dev/null +++ b/plugins/rtp/src/voice_processor_native.cpp @@ -0,0 +1,141 @@ +#include +#include +#include +#include +#include +#include + +#define SAMPLE_RATE 48000 +#define SAMPLE_CHANNELS 1 + +struct _DinoPluginsRtpVoiceProcessorNative { + webrtc::AudioProcessing *apm; + gint stream_delay; +}; + +extern "C" void *dino_plugins_rtp_adjust_to_running_time(GstBaseTransform *transform, GstBuffer *buffer) { + GstBuffer *copy = gst_buffer_copy(buffer); + GST_BUFFER_PTS(copy) = gst_segment_to_running_time(&transform->segment, GST_FORMAT_TIME, GST_BUFFER_PTS(buffer)); + return copy; +} + +extern "C" void *dino_plugins_rtp_voice_processor_init_native(gint stream_delay) { + _DinoPluginsRtpVoiceProcessorNative *native = new _DinoPluginsRtpVoiceProcessorNative(); + webrtc::Config config; + config.Set(new webrtc::ExtendedFilter(true)); + config.Set(new webrtc::ExperimentalAgc(true, 85)); + native->apm = webrtc::AudioProcessing::Create(config); + native->stream_delay = stream_delay; + return native; +} + +extern "C" void dino_plugins_rtp_voice_processor_setup_native(void *native_ptr) { + _DinoPluginsRtpVoiceProcessorNative *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr; + webrtc::AudioProcessing *apm = native->apm; + webrtc::ProcessingConfig pconfig; + pconfig.streams[webrtc::ProcessingConfig::kInputStream] = + webrtc::StreamConfig(SAMPLE_RATE, SAMPLE_CHANNELS, false); + pconfig.streams[webrtc::ProcessingConfig::kOutputStream] = + webrtc::StreamConfig(SAMPLE_RATE, SAMPLE_CHANNELS, false); + pconfig.streams[webrtc::ProcessingConfig::kReverseInputStream] = + webrtc::StreamConfig(SAMPLE_RATE, SAMPLE_CHANNELS, false); + pconfig.streams[webrtc::ProcessingConfig::kReverseOutputStream] = + webrtc::StreamConfig(SAMPLE_RATE, SAMPLE_CHANNELS, false); + apm->Initialize(pconfig); + apm->high_pass_filter()->Enable(true); + apm->echo_cancellation()->enable_drift_compensation(false); + apm->echo_cancellation()->set_suppression_level(webrtc::EchoCancellation::kModerateSuppression); + apm->echo_cancellation()->enable_delay_logging(true); + apm->echo_cancellation()->Enable(true); + apm->noise_suppression()->set_level(webrtc::NoiseSuppression::kModerate); + apm->noise_suppression()->Enable(true); + apm->gain_control()->set_analog_level_limits(0, 255); + apm->gain_control()->set_mode(webrtc::GainControl::kAdaptiveAnalog); + apm->gain_control()->set_target_level_dbfs(3); + apm->gain_control()->set_compression_gain_db(9); + apm->gain_control()->enable_limiter(true); + apm->gain_control()->Enable(true); + apm->voice_detection()->set_likelihood(webrtc::VoiceDetection::Likelihood::kLowLikelihood); + apm->voice_detection()->Enable(true); +} + +extern "C" void +dino_plugins_rtp_voice_processor_analyze_reverse_stream(void *native_ptr, GstAudioInfo *info, GstBuffer *buffer) { + _DinoPluginsRtpVoiceProcessorNative *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr; + webrtc::StreamConfig config(SAMPLE_RATE, SAMPLE_CHANNELS, false); + webrtc::AudioProcessing *apm = native->apm; + + GstAudioBuffer audio_buffer; + gst_audio_buffer_map(&audio_buffer, info, buffer, GST_MAP_READ); + + webrtc::AudioFrame frame; + frame.num_channels_ = info->channels; + frame.sample_rate_hz_ = info->rate; + frame.samples_per_channel_ = gst_buffer_get_size(buffer) / info->bpf; + memcpy(frame.data_, audio_buffer.planes[0], frame.samples_per_channel_ * info->bpf); + + int err = apm->AnalyzeReverseStream(&frame); + if (err < 0) g_warning("ProcessReverseStream %i", err); + + gst_audio_buffer_unmap(&audio_buffer); +} + +extern "C" void dino_plugins_rtp_voice_processor_notify_gain_level(void *native_ptr, gint gain_level) { + _DinoPluginsRtpVoiceProcessorNative *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr; + webrtc::AudioProcessing *apm = native->apm; + apm->gain_control()->set_stream_analog_level(gain_level); +} + +extern "C" gint dino_plugins_rtp_voice_processor_get_suggested_gain_level(void *native_ptr) { + _DinoPluginsRtpVoiceProcessorNative *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr; + webrtc::AudioProcessing *apm = native->apm; + return apm->gain_control()->stream_analog_level(); +} + +extern "C" bool dino_plugins_rtp_voice_processor_get_stream_has_voice(void *native_ptr) { + _DinoPluginsRtpVoiceProcessorNative *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr; + webrtc::AudioProcessing *apm = native->apm; + return apm->voice_detection()->stream_has_voice(); +} + +extern "C" void dino_plugins_rtp_voice_processor_adjust_stream_delay(void *native_ptr) { + _DinoPluginsRtpVoiceProcessorNative *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr; + webrtc::AudioProcessing *apm = native->apm; + int median, std; + float fraction_poor_delays; + apm->echo_cancellation()->GetDelayMetrics(&median, &std, &fraction_poor_delays); + if (fraction_poor_delays < 0) return; + g_debug("voice_processor_native.cpp: Stream delay metrics: %i %i %f", median, std, fraction_poor_delays); + if (fraction_poor_delays > 0.5) { + native->stream_delay = std::max(0, native->stream_delay + std::min(-10, std::max(median, 10))); + g_debug("Adjusted stream delay %i", native->stream_delay); + } +} + +extern "C" void +dino_plugins_rtp_voice_processor_process_stream(void *native_ptr, GstAudioInfo *info, GstBuffer *buffer) { + _DinoPluginsRtpVoiceProcessorNative *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr; + webrtc::StreamConfig config(SAMPLE_RATE, SAMPLE_CHANNELS, false); + webrtc::AudioProcessing *apm = native->apm; + + GstAudioBuffer audio_buffer; + gst_audio_buffer_map(&audio_buffer, info, buffer, GST_MAP_READWRITE); + + webrtc::AudioFrame frame; + frame.num_channels_ = info->channels; + frame.sample_rate_hz_ = info->rate; + frame.samples_per_channel_ = info->rate / 100; + memcpy(frame.data_, audio_buffer.planes[0], frame.samples_per_channel_ * info->bpf); + + apm->set_stream_delay_ms(native->stream_delay); + int err = apm->ProcessStream(&frame); + if (err >= 0) memcpy(audio_buffer.planes[0], frame.data_, frame.samples_per_channel_ * info->bpf); + if (err < 0) g_warning("ProcessStream %i", err); + + gst_audio_buffer_unmap(&audio_buffer); +} + +extern "C" void dino_plugins_rtp_voice_processor_destroy_native(void *native_ptr) { + _DinoPluginsRtpVoiceProcessorNative *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr; + delete native; +} \ No newline at end of file -- cgit v1.2.3