Merge pull request #484 from trilene/screenshare-x11

Support screen sharing on X11
This commit is contained in:
DeepBlueV7.X 2021-02-25 19:16:18 +00:00 committed by GitHub
commit 0bf3f4634d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 834 additions and 281 deletions

View File

@ -447,11 +447,17 @@ else()
endif() endif()
include(FindPkgConfig) include(FindPkgConfig)
pkg_check_modules(GSTREAMER IMPORTED_TARGET gstreamer-sdp-1.0>=1.16 gstreamer-webrtc-1.0>=1.16) pkg_check_modules(GSTREAMER IMPORTED_TARGET gstreamer-sdp-1.0>=1.18 gstreamer-webrtc-1.0>=1.18)
if (TARGET PkgConfig::GSTREAMER) if (TARGET PkgConfig::GSTREAMER)
add_feature_info(voip ON "GStreamer found. Call support is enabled automatically.") add_feature_info(voip ON "GStreamer found. Call support is enabled automatically.")
pkg_check_modules(XCB IMPORTED_TARGET xcb xcb-ewmh)
if (TARGET PkgConfig::XCB)
add_feature_info("Window selection when screen sharing (X11)" ON "XCB-EWMH found. Window selection is enabled when screen sharing (X11).")
else()
add_feature_info("Window selection when screen sharing (X11)" OFF "XCB-EWMH could not be found on your system. Screen sharing (X11) is limited to the entire screen only. To enable window selection, make sure xcb and xcb-ewmh can be found via pkgconfig.")
endif()
else() else()
add_feature_info(voip OFF "GStreamer could not be found on your system. As a consequence call support has been disabled. If you don't want that, make sure gstreamer-sdp-1.0>=1.16 gstreamer-webrtc-1.0>=1.16 can be found via pkgconfig.") add_feature_info(voip OFF "GStreamer could not be found on your system. As a consequence call support has been disabled. If you don't want that, make sure gstreamer-sdp-1.0>=1.18 gstreamer-webrtc-1.0>=1.18 can be found via pkgconfig.")
endif() endif()
# single instance functionality # single instance functionality
@ -639,6 +645,10 @@ endif()
if (TARGET PkgConfig::GSTREAMER) if (TARGET PkgConfig::GSTREAMER)
target_link_libraries(nheko PRIVATE PkgConfig::GSTREAMER) target_link_libraries(nheko PRIVATE PkgConfig::GSTREAMER)
target_compile_definitions(nheko PRIVATE GSTREAMER_AVAILABLE) target_compile_definitions(nheko PRIVATE GSTREAMER_AVAILABLE)
if (TARGET PkgConfig::XCB)
target_link_libraries(nheko PRIVATE PkgConfig::XCB)
target_compile_definitions(nheko PRIVATE XCB_AVAILABLE)
endif()
endif() endif()
if(MSVC) if(MSVC)

Binary file not shown.

After

Width:  |  Height:  |  Size: 773 B

View File

@ -44,7 +44,6 @@ Rectangle {
} else if (CallManager.isOnCall) { } else if (CallManager.isOnCall) {
CallManager.hangUp(); CallManager.hangUp();
} else { } else {
CallManager.refreshDevices();
var dialog = placeCallDialog.createObject(timelineRoot); var dialog = placeCallDialog.createObject(timelineRoot);
dialog.open(); dialog.open();
} }

View File

@ -267,7 +267,7 @@ Page {
} }
Loader { Loader {
source: CallManager.isOnCall && CallManager.isVideo ? "voip/VideoCall.qml" : "" source: CallManager.isOnCall && CallManager.callType != CallType.VOICE ? "voip/VideoCall.qml" : ""
onLoaded: TimelineManager.setVideoCallItem() onLoaded: TimelineManager.setVideoCallItem()
} }

View File

@ -12,7 +12,7 @@ Rectangle {
MouseArea { MouseArea {
anchors.fill: parent anchors.fill: parent
onClicked: { onClicked: {
if (CallManager.isVideo) if (CallManager.callType != CallType.VOICE)
stackLayout.currentIndex = stackLayout.currentIndex ? 0 : 1; stackLayout.currentIndex = stackLayout.currentIndex ? 0 : 1;
} }
@ -42,10 +42,46 @@ Rectangle {
} }
Image { Image {
id: callTypeIcon
Layout.leftMargin: 4 Layout.leftMargin: 4
Layout.preferredWidth: 24 Layout.preferredWidth: 24
Layout.preferredHeight: 24 Layout.preferredHeight: 24
source: CallManager.isVideo ? "qrc:/icons/icons/ui/video-call.png" : "qrc:/icons/icons/ui/place-call.png" }
Item {
states: [
State {
name: "VOICE"
when: CallManager.callType == CallType.VOICE
PropertyChanges {
target: callTypeIcon
source: "qrc:/icons/icons/ui/place-call.png"
}
},
State {
name: "VIDEO"
when: CallManager.callType == CallType.VIDEO
PropertyChanges {
target: callTypeIcon
source: "qrc:/icons/icons/ui/video-call.png"
}
},
State {
name: "SCREEN"
when: CallManager.callType == CallType.SCREEN
PropertyChanges {
target: callTypeIcon
source: "qrc:/icons/icons/ui/screen-share.png"
}
}
]
} }
Label { Label {
@ -103,7 +139,7 @@ Rectangle {
PropertyChanges { PropertyChanges {
target: stackLayout target: stackLayout
currentIndex: CallManager.isVideo ? 1 : 0 currentIndex: CallManager.callType != CallType.VOICE ? 1 : 0
} }
}, },
@ -147,20 +183,28 @@ Rectangle {
} }
} }
Label {
Layout.leftMargin: 16
visible: CallManager.callType == CallType.SCREEN && CallManager.callState == WebRTCState.CONNECTED
text: qsTr("You are screen sharing")
font.pointSize: fontMetrics.font.pointSize * 1.1
color: "#000000"
}
Item { Item {
Layout.fillWidth: true Layout.fillWidth: true
} }
ImageButton { ImageButton {
visible: CallManager.haveLocalVideo visible: CallManager.haveLocalPiP
width: 24 width: 24
height: 24 height: 24
buttonTextColor: "#000000" buttonTextColor: "#000000"
image: ":/icons/icons/ui/toggle-camera-view.png" image: ":/icons/icons/ui/toggle-camera-view.png"
hoverEnabled: true hoverEnabled: true
ToolTip.visible: hovered ToolTip.visible: hovered
ToolTip.text: qsTr("Toggle camera view") ToolTip.text: qsTr("Hide/Show Picture-in-Picture")
onClicked: CallManager.toggleCameraView() onClicked: CallManager.toggleLocalPiP()
} }
ImageButton { ImageButton {

View File

@ -40,7 +40,7 @@ Popup {
} }
RowLayout { RowLayout {
visible: CallManager.isVideo && CallManager.cameras.length > 0 visible: CallManager.callType == CallType.VIDEO && CallManager.cameras.length > 0
Image { Image {
Layout.preferredWidth: 22 Layout.preferredWidth: 22

View File

@ -53,7 +53,7 @@ Popup {
Layout.bottomMargin: msgView.height / 25 Layout.bottomMargin: msgView.height / 25
Image { Image {
property string image: CallManager.isVideo ? ":/icons/icons/ui/video-call.png" : ":/icons/icons/ui/place-call.png" property string image: CallManager.callType == CallType.VIDEO ? ":/icons/icons/ui/video-call.png" : ":/icons/icons/ui/place-call.png"
Layout.alignment: Qt.AlignCenter Layout.alignment: Qt.AlignCenter
Layout.preferredWidth: msgView.height / 10 Layout.preferredWidth: msgView.height / 10
@ -63,7 +63,7 @@ Popup {
Label { Label {
Layout.alignment: Qt.AlignCenter Layout.alignment: Qt.AlignCenter
text: CallManager.isVideo ? qsTr("Video Call") : qsTr("Voice Call") text: CallManager.callType == CallType.VIDEO ? qsTr("Video Call") : qsTr("Voice Call")
font.pointSize: fontMetrics.font.pointSize * 2 font.pointSize: fontMetrics.font.pointSize * 2
color: colors.windowText color: colors.windowText
} }
@ -97,7 +97,7 @@ Popup {
} }
RowLayout { RowLayout {
visible: CallManager.isVideo && CallManager.cameras.length > 0 visible: CallManager.callType == CallType.VIDEO && CallManager.cameras.length > 0
Layout.alignment: Qt.AlignCenter Layout.alignment: Qt.AlignCenter
Image { Image {
@ -159,7 +159,7 @@ Popup {
RoundButton { RoundButton {
id: acceptButton id: acceptButton
property string image: CallManager.isVideo ? ":/icons/icons/ui/video-call.png" : ":/icons/icons/ui/place-call.png" property string image: CallManager.callType == CallType.VIDEO ? ":/icons/icons/ui/video-call.png" : ":/icons/icons/ui/place-call.png"
implicitWidth: buttonLayout.buttonSize implicitWidth: buttonLayout.buttonSize
implicitHeight: buttonLayout.buttonSize implicitHeight: buttonLayout.buttonSize

View File

@ -52,12 +52,12 @@ Rectangle {
Layout.leftMargin: 4 Layout.leftMargin: 4
Layout.preferredWidth: 24 Layout.preferredWidth: 24
Layout.preferredHeight: 24 Layout.preferredHeight: 24
source: CallManager.isVideo ? "qrc:/icons/icons/ui/video-call.png" : "qrc:/icons/icons/ui/place-call.png" source: CallManager.callType == CallType.VIDEO ? "qrc:/icons/icons/ui/video-call.png" : "qrc:/icons/icons/ui/place-call.png"
} }
Label { Label {
font.pointSize: fontMetrics.font.pointSize * 1.1 font.pointSize: fontMetrics.font.pointSize * 1.1
text: CallManager.isVideo ? qsTr("Video Call") : qsTr("Voice Call") text: CallManager.callType == CallType.VIDEO ? qsTr("Video Call") : qsTr("Voice Call")
color: "#000000" color: "#000000"
} }
@ -75,7 +75,6 @@ Rectangle {
ToolTip.visible: hovered ToolTip.visible: hovered
ToolTip.text: qsTr("Devices") ToolTip.text: qsTr("Devices")
onClicked: { onClicked: {
CallManager.refreshDevices();
var dialog = devicesDialog.createObject(timelineRoot); var dialog = devicesDialog.createObject(timelineRoot);
dialog.open(); dialog.open();
} }
@ -83,7 +82,7 @@ Rectangle {
Button { Button {
Layout.rightMargin: 4 Layout.rightMargin: 4
icon.source: CallManager.isVideo ? "qrc:/icons/icons/ui/video-call.png" : "qrc:/icons/icons/ui/place-call.png" icon.source: CallManager.callType == CallType.VIDEO ? "qrc:/icons/icons/ui/video-call.png" : "qrc:/icons/icons/ui/place-call.png"
text: qsTr("Accept") text: qsTr("Accept")
palette: colors palette: colors
onClicked: { onClicked: {
@ -102,7 +101,7 @@ Rectangle {
dialog.open(); dialog.open();
return ; return ;
} }
if (CallManager.isVideo && CallManager.cameras.length > 0 && !CallManager.cameras.includes(Settings.camera)) { if (CallManager.callType == CallType.VIDEO && CallManager.cameras.length > 0 && !CallManager.cameras.includes(Settings.camera)) {
var dialog = deviceError.createObject(timelineRoot, { var dialog = deviceError.createObject(timelineRoot, {
"errorString": qsTr("Unknown camera: %1").arg(Settings.camera), "errorString": qsTr("Unknown camera: %1").arg(Settings.camera),
"image": ":/icons/icons/ui/video-call.png" "image": ":/icons/icons/ui/video-call.png"

View File

@ -23,6 +23,14 @@ Popup {
} }
Component {
id: screenShareDialog
ScreenShare {
}
}
ColumnLayout { ColumnLayout {
id: columnLayout id: columnLayout
@ -76,7 +84,7 @@ Popup {
onClicked: { onClicked: {
if (buttonLayout.validateMic()) { if (buttonLayout.validateMic()) {
Settings.microphone = micCombo.currentText; Settings.microphone = micCombo.currentText;
CallManager.sendInvite(TimelineManager.timeline.roomId(), false); CallManager.sendInvite(TimelineManager.timeline.roomId(), CallType.VOICE);
close(); close();
} }
} }
@ -90,12 +98,23 @@ Popup {
if (buttonLayout.validateMic()) { if (buttonLayout.validateMic()) {
Settings.microphone = micCombo.currentText; Settings.microphone = micCombo.currentText;
Settings.camera = cameraCombo.currentText; Settings.camera = cameraCombo.currentText;
CallManager.sendInvite(TimelineManager.timeline.roomId(), true); CallManager.sendInvite(TimelineManager.timeline.roomId(), CallType.VIDEO);
close(); close();
} }
} }
} }
Button {
visible: CallManager.screenShareSupported
text: qsTr("Screen")
icon.source: "qrc:/icons/icons/ui/screen-share.png"
onClicked: {
var dialog = screenShareDialog.createObject(timelineRoot);
dialog.open();
close();
}
}
Button { Button {
text: qsTr("Cancel") text: qsTr("Cancel")
onClicked: { onClicked: {

View File

@ -0,0 +1,153 @@
import "../"
import QtQuick 2.9
import QtQuick.Controls 2.3
import QtQuick.Layouts 1.2
import im.nheko 1.0
Popup {
modal: true
// only set the anchors on Qt 5.12 or higher
// see https://doc.qt.io/qt-5/qml-qtquick-controls2-popup.html#anchors.centerIn-prop
Component.onCompleted: {
if (anchors)
anchors.centerIn = parent;
frameRateCombo.currentIndex = frameRateCombo.find(Settings.screenShareFrameRate);
}
palette: colors
ColumnLayout {
Label {
Layout.topMargin: 16
Layout.bottomMargin: 16
Layout.leftMargin: 8
Layout.rightMargin: 8
Layout.alignment: Qt.AlignLeft
text: qsTr("Share desktop with %1?").arg(TimelineManager.timeline.roomName)
color: colors.windowText
}
RowLayout {
Layout.leftMargin: 8
Layout.rightMargin: 8
Layout.bottomMargin: 8
Label {
Layout.alignment: Qt.AlignLeft
text: qsTr("Window:")
color: colors.windowText
}
ComboBox {
id: windowCombo
Layout.fillWidth: true
model: CallManager.windowList()
}
}
RowLayout {
Layout.leftMargin: 8
Layout.rightMargin: 8
Layout.bottomMargin: 8
Label {
Layout.alignment: Qt.AlignLeft
text: qsTr("Frame rate:")
color: colors.windowText
}
ComboBox {
id: frameRateCombo
Layout.fillWidth: true
model: ["25", "20", "15", "10", "5", "2", "1"]
}
}
CheckBox {
id: pipCheckBox
enabled: CallManager.cameras.length > 0
checked: Settings.screenSharePiP
Layout.alignment: Qt.AlignLeft
Layout.leftMargin: 8
Layout.rightMargin: 8
text: qsTr("Include your camera picture-in-picture")
}
CheckBox {
id: remoteVideoCheckBox
Layout.alignment: Qt.AlignLeft
Layout.leftMargin: 8
Layout.rightMargin: 8
text: qsTr("Request remote camera")
checked: Settings.screenShareRemoteVideo
ToolTip.text: qsTr("View your callee's camera like a regular video call")
ToolTip.visible: hovered
}
CheckBox {
id: hideCursorCheckBox
Layout.alignment: Qt.AlignLeft
Layout.leftMargin: 8
Layout.rightMargin: 8
Layout.bottomMargin: 8
text: qsTr("Hide mouse cursor")
checked: Settings.screenShareHideCursor
}
RowLayout {
Layout.margins: 8
Item {
Layout.fillWidth: true
}
Button {
text: qsTr("Share")
icon.source: "qrc:/icons/icons/ui/screen-share.png"
onClicked: {
if (buttonLayout.validateMic()) {
Settings.microphone = micCombo.currentText;
if (pipCheckBox.checked)
Settings.camera = cameraCombo.currentText;
Settings.screenShareFrameRate = frameRateCombo.currentText;
Settings.screenSharePiP = pipCheckBox.checked;
Settings.screenShareRemoteVideo = remoteVideoCheckBox.checked;
Settings.screenShareHideCursor = hideCursorCheckBox.checked;
CallManager.sendInvite(TimelineManager.timeline.roomId(), CallType.SCREEN, windowCombo.currentIndex);
close();
}
}
}
Button {
text: qsTr("Preview")
onClicked: {
CallManager.previewWindow(windowCombo.currentIndex);
}
}
Button {
text: qsTr("Cancel")
onClicked: {
close();
}
}
}
}
background: Rectangle {
color: colors.window
border.color: colors.windowText
}
}

View File

@ -74,6 +74,7 @@
<file>icons/ui/end-call.png</file> <file>icons/ui/end-call.png</file>
<file>icons/ui/microphone-mute.png</file> <file>icons/ui/microphone-mute.png</file>
<file>icons/ui/microphone-unmute.png</file> <file>icons/ui/microphone-unmute.png</file>
<file>icons/ui/screen-share.png</file>
<file>icons/ui/toggle-camera-view.png</file> <file>icons/ui/toggle-camera-view.png</file>
<file>icons/ui/video-call.png</file> <file>icons/ui/video-call.png</file>
@ -167,6 +168,7 @@
<file>qml/voip/CallInviteBar.qml</file> <file>qml/voip/CallInviteBar.qml</file>
<file>qml/voip/DeviceError.qml</file> <file>qml/voip/DeviceError.qml</file>
<file>qml/voip/PlaceCall.qml</file> <file>qml/voip/PlaceCall.qml</file>
<file>qml/voip/ScreenShare.qml</file>
<file>qml/voip/VideoCall.qml</file> <file>qml/voip/VideoCall.qml</file>
</qresource> </qresource>
<qresource prefix="/media"> <qresource prefix="/media">

View File

@ -152,7 +152,6 @@ addDevice(GstDevice *device)
setDefaultDevice(true); setDefaultDevice(true);
} }
#if GST_CHECK_VERSION(1, 18, 0)
template<typename T> template<typename T>
bool bool
removeDevice(T &sources, GstDevice *device, bool changed) removeDevice(T &sources, GstDevice *device, bool changed)
@ -212,7 +211,6 @@ newBusMessage(GstBus *bus G_GNUC_UNUSED, GstMessage *msg, gpointer user_data G_G
} }
return TRUE; return TRUE;
} }
#endif
template<typename T> template<typename T>
std::vector<std::string> std::vector<std::string>
@ -257,7 +255,6 @@ tokenise(std::string_view str, char delim)
void void
CallDevices::init() CallDevices::init()
{ {
#if GST_CHECK_VERSION(1, 18, 0)
static GstDeviceMonitor *monitor = nullptr; static GstDeviceMonitor *monitor = nullptr;
if (!monitor) { if (!monitor) {
monitor = gst_device_monitor_new(); monitor = gst_device_monitor_new();
@ -278,43 +275,6 @@ CallDevices::init()
return; return;
} }
} }
#endif
}
void
CallDevices::refresh()
{
#if !GST_CHECK_VERSION(1, 18, 0)
static GstDeviceMonitor *monitor = nullptr;
if (!monitor) {
monitor = gst_device_monitor_new();
GstCaps *caps = gst_caps_new_empty_simple("audio/x-raw");
gst_device_monitor_add_filter(monitor, "Audio/Source", caps);
gst_device_monitor_add_filter(monitor, "Audio/Duplex", caps);
gst_caps_unref(caps);
caps = gst_caps_new_empty_simple("video/x-raw");
gst_device_monitor_add_filter(monitor, "Video/Source", caps);
gst_device_monitor_add_filter(monitor, "Video/Duplex", caps);
gst_caps_unref(caps);
}
auto clearDevices = [](auto &sources) {
std::for_each(
sources.begin(), sources.end(), [](auto &s) { gst_object_unref(s.device); });
sources.clear();
};
clearDevices(audioSources_);
clearDevices(videoSources_);
GList *devices = gst_device_monitor_get_devices(monitor);
if (devices) {
for (GList *l = devices; l != nullptr; l = l->next)
addDevice(GST_DEVICE_CAST(l->data));
g_list_free(devices);
}
emit devicesChanged();
#endif
} }
bool bool
@ -400,10 +360,6 @@ CallDevices::videoDevice(std::pair<int, int> &resolution, std::pair<int, int> &f
#else #else
void
CallDevices::refresh()
{}
bool bool
CallDevices::haveMic() const CallDevices::haveMic() const
{ {

View File

@ -19,7 +19,6 @@ public:
return instance; return instance;
} }
void refresh();
bool haveMic() const; bool haveMic() const;
bool haveCamera() const; bool haveCamera() const;
std::vector<std::string> names(bool isVideo, const std::string &defaultDevice) const; std::vector<std::string> names(bool isVideo, const std::string &defaultDevice) const;

View File

@ -2,6 +2,8 @@
#include <cctype> #include <cctype>
#include <chrono> #include <chrono>
#include <cstdint> #include <cstdint>
#include <cstdlib>
#include <memory>
#include <QMediaPlaylist> #include <QMediaPlaylist>
#include <QUrl> #include <QUrl>
@ -17,6 +19,18 @@
#include "mtx/responses/turn_server.hpp" #include "mtx/responses/turn_server.hpp"
#ifdef XCB_AVAILABLE
#include <xcb/xcb.h>
#include <xcb/xcb_ewmh.h>
#endif
#ifdef GSTREAMER_AVAILABLE
extern "C"
{
#include "gst/gst.h"
}
#endif
Q_DECLARE_METATYPE(std::vector<mtx::events::msg::CallCandidates::Candidate>) Q_DECLARE_METATYPE(std::vector<mtx::events::msg::CallCandidates::Candidate>)
Q_DECLARE_METATYPE(mtx::events::msg::CallCandidates::Candidate) Q_DECLARE_METATYPE(mtx::events::msg::CallCandidates::Candidate)
Q_DECLARE_METATYPE(mtx::responses::TurnServer) Q_DECLARE_METATYPE(mtx::responses::TurnServer)
@ -24,6 +38,8 @@ Q_DECLARE_METATYPE(mtx::responses::TurnServer)
using namespace mtx::events; using namespace mtx::events;
using namespace mtx::events::msg; using namespace mtx::events::msg;
using webrtc::CallType;
namespace { namespace {
std::vector<std::string> std::vector<std::string>
getTurnURIs(const mtx::responses::TurnServer &turnServer); getTurnURIs(const mtx::responses::TurnServer &turnServer);
@ -148,10 +164,18 @@ CallManager::CallManager(QObject *parent)
} }
void void
CallManager::sendInvite(const QString &roomid, bool isVideo) CallManager::sendInvite(const QString &roomid, CallType callType, unsigned int windowIndex)
{ {
if (isOnCall()) if (isOnCall())
return; return;
if (callType == CallType::SCREEN) {
if (!screenShareSupported())
return;
if (windows_.empty() || windowIndex >= windows_.size()) {
nhlog::ui()->error("WebRTC: window index out of range");
return;
}
}
auto roomInfo = cache::singleRoomInfo(roomid.toStdString()); auto roomInfo = cache::singleRoomInfo(roomid.toStdString());
if (roomInfo.member_count != 2) { if (roomInfo.member_count != 2) {
@ -161,17 +185,20 @@ CallManager::sendInvite(const QString &roomid, bool isVideo)
std::string errorMessage; std::string errorMessage;
if (!session_.havePlugins(false, &errorMessage) || if (!session_.havePlugins(false, &errorMessage) ||
(isVideo && !session_.havePlugins(true, &errorMessage))) { ((callType == CallType::VIDEO || callType == CallType::SCREEN) &&
!session_.havePlugins(true, &errorMessage))) {
emit ChatPage::instance()->showNotification(QString::fromStdString(errorMessage)); emit ChatPage::instance()->showNotification(QString::fromStdString(errorMessage));
return; return;
} }
isVideo_ = isVideo; callType_ = callType;
roomid_ = roomid; roomid_ = roomid;
session_.setTurnServers(turnURIs_); session_.setTurnServers(turnURIs_);
generateCallID(); generateCallID();
nhlog::ui()->debug( std::string strCallType = callType_ == CallType::VOICE
"WebRTC: call id: {} - creating {} invite", callid_, isVideo ? "video" : "voice"); ? "voice"
: (callType_ == CallType::VIDEO ? "video" : "screen");
nhlog::ui()->debug("WebRTC: call id: {} - creating {} invite", callid_, strCallType);
std::vector<RoomMember> members(cache::getMembers(roomid.toStdString())); std::vector<RoomMember> members(cache::getMembers(roomid.toStdString()));
const RoomMember &callee = const RoomMember &callee =
members.front().user_id == utils::localUser() ? members.back() : members.front(); members.front().user_id == utils::localUser() ? members.back() : members.front();
@ -179,7 +206,8 @@ CallManager::sendInvite(const QString &roomid, bool isVideo)
callPartyAvatarUrl_ = QString::fromStdString(roomInfo.avatar_url); callPartyAvatarUrl_ = QString::fromStdString(roomInfo.avatar_url);
emit newInviteState(); emit newInviteState();
playRingtone(QUrl("qrc:/media/media/ringback.ogg"), true); playRingtone(QUrl("qrc:/media/media/ringback.ogg"), true);
if (!session_.createOffer(isVideo)) { if (!session_.createOffer(
callType, callType == CallType::SCREEN ? windows_[windowIndex].second : 0)) {
emit ChatPage::instance()->showNotification("Problem setting up call."); emit ChatPage::instance()->showNotification("Problem setting up call.");
endCall(); endCall();
} }
@ -215,8 +243,8 @@ void
CallManager::syncEvent(const mtx::events::collections::TimelineEvents &event) CallManager::syncEvent(const mtx::events::collections::TimelineEvents &event)
{ {
#ifdef GSTREAMER_AVAILABLE #ifdef GSTREAMER_AVAILABLE
if (handleEvent_<CallInvite>(event) || handleEvent_<CallCandidates>(event) || if (handleEvent<CallInvite>(event) || handleEvent<CallCandidates>(event) ||
handleEvent_<CallAnswer>(event) || handleEvent_<CallHangUp>(event)) handleEvent<CallAnswer>(event) || handleEvent<CallHangUp>(event))
return; return;
#else #else
(void)event; (void)event;
@ -225,7 +253,7 @@ CallManager::syncEvent(const mtx::events::collections::TimelineEvents &event)
template<typename T> template<typename T>
bool bool
CallManager::handleEvent_(const mtx::events::collections::TimelineEvents &event) CallManager::handleEvent(const mtx::events::collections::TimelineEvents &event)
{ {
if (std::holds_alternative<RoomEvent<T>>(event)) { if (std::holds_alternative<RoomEvent<T>>(event)) {
handleEvent(std::get<RoomEvent<T>>(event)); handleEvent(std::get<RoomEvent<T>>(event));
@ -280,9 +308,8 @@ CallManager::handleEvent(const RoomEvent<CallInvite> &callInviteEvent)
callPartyAvatarUrl_ = QString::fromStdString(roomInfo.avatar_url); callPartyAvatarUrl_ = QString::fromStdString(roomInfo.avatar_url);
haveCallInvite_ = true; haveCallInvite_ = true;
isVideo_ = isVideo; callType_ = isVideo ? CallType::VIDEO : CallType::VOICE;
inviteSDP_ = callInviteEvent.content.sdp; inviteSDP_ = callInviteEvent.content.sdp;
CallDevices::instance().refresh();
emit newInviteState(); emit newInviteState();
} }
@ -295,7 +322,7 @@ CallManager::acceptInvite()
stopRingtone(); stopRingtone();
std::string errorMessage; std::string errorMessage;
if (!session_.havePlugins(false, &errorMessage) || if (!session_.havePlugins(false, &errorMessage) ||
(isVideo_ && !session_.havePlugins(true, &errorMessage))) { (callType_ == CallType::VIDEO && !session_.havePlugins(true, &errorMessage))) {
emit ChatPage::instance()->showNotification(QString::fromStdString(errorMessage)); emit ChatPage::instance()->showNotification(QString::fromStdString(errorMessage));
hangUp(); hangUp();
return; return;
@ -383,7 +410,7 @@ CallManager::toggleMicMute()
} }
bool bool
CallManager::callsSupported() const CallManager::callsSupported()
{ {
#ifdef GSTREAMER_AVAILABLE #ifdef GSTREAMER_AVAILABLE
return true; return true;
@ -392,6 +419,12 @@ CallManager::callsSupported() const
#endif #endif
} }
bool
CallManager::screenShareSupported()
{
return std::getenv("DISPLAY") && !std::getenv("WAYLAND_DISPLAY");
}
QStringList QStringList
CallManager::devices(bool isVideo) const CallManager::devices(bool isVideo) const
{ {
@ -424,7 +457,7 @@ CallManager::clear()
callParty_.clear(); callParty_.clear();
callPartyAvatarUrl_.clear(); callPartyAvatarUrl_.clear();
callid_.clear(); callid_.clear();
isVideo_ = false; callType_ = CallType::VOICE;
haveCallInvite_ = false; haveCallInvite_ = false;
emit newInviteState(); emit newInviteState();
inviteSDP_.clear(); inviteSDP_.clear();
@ -477,6 +510,150 @@ CallManager::stopRingtone()
player_.setPlaylist(nullptr); player_.setPlaylist(nullptr);
} }
QStringList
CallManager::windowList()
{
windows_.clear();
windows_.push_back({tr("Entire screen"), 0});
#ifdef XCB_AVAILABLE
std::unique_ptr<xcb_connection_t, std::function<void(xcb_connection_t *)>> connection(
xcb_connect(nullptr, nullptr), [](xcb_connection_t *c) { xcb_disconnect(c); });
if (xcb_connection_has_error(connection.get())) {
nhlog::ui()->error("Failed to connect to X server");
return {};
}
xcb_ewmh_connection_t ewmh;
if (!xcb_ewmh_init_atoms_replies(
&ewmh, xcb_ewmh_init_atoms(connection.get(), &ewmh), nullptr)) {
nhlog::ui()->error("Failed to connect to EWMH server");
return {};
}
std::unique_ptr<xcb_ewmh_connection_t, std::function<void(xcb_ewmh_connection_t *)>>
ewmhconnection(&ewmh, [](xcb_ewmh_connection_t *c) { xcb_ewmh_connection_wipe(c); });
for (int i = 0; i < ewmh.nb_screens; i++) {
xcb_ewmh_get_windows_reply_t clients;
if (!xcb_ewmh_get_client_list_reply(
&ewmh, xcb_ewmh_get_client_list(&ewmh, i), &clients, nullptr)) {
nhlog::ui()->error("Failed to request window list");
return {};
}
for (uint32_t w = 0; w < clients.windows_len; w++) {
xcb_window_t window = clients.windows[w];
std::string name;
xcb_ewmh_get_utf8_strings_reply_t data;
auto getName = [](xcb_ewmh_get_utf8_strings_reply_t *r) {
std::string name(r->strings, r->strings_len);
xcb_ewmh_get_utf8_strings_reply_wipe(r);
return name;
};
xcb_get_property_cookie_t cookie = xcb_ewmh_get_wm_name(&ewmh, window);
if (xcb_ewmh_get_wm_name_reply(&ewmh, cookie, &data, nullptr))
name = getName(&data);
cookie = xcb_ewmh_get_wm_visible_name(&ewmh, window);
if (xcb_ewmh_get_wm_visible_name_reply(&ewmh, cookie, &data, nullptr))
name = getName(&data);
windows_.push_back({QString::fromStdString(name), window});
}
xcb_ewmh_get_windows_reply_wipe(&clients);
}
#endif
QStringList ret;
ret.reserve(windows_.size());
for (const auto &w : windows_)
ret.append(w.first);
return ret;
}
#ifdef GSTREAMER_AVAILABLE
namespace {
GstElement *pipe_ = nullptr;
unsigned int busWatchId_ = 0;
gboolean
newBusMessage(GstBus *bus G_GNUC_UNUSED, GstMessage *msg, gpointer G_GNUC_UNUSED)
{
switch (GST_MESSAGE_TYPE(msg)) {
case GST_MESSAGE_EOS:
if (pipe_) {
gst_element_set_state(GST_ELEMENT(pipe_), GST_STATE_NULL);
gst_object_unref(pipe_);
pipe_ = nullptr;
}
if (busWatchId_) {
g_source_remove(busWatchId_);
busWatchId_ = 0;
}
break;
default:
break;
}
return TRUE;
}
}
#endif
void
CallManager::previewWindow(unsigned int index) const
{
#ifdef GSTREAMER_AVAILABLE
if (windows_.empty() || index >= windows_.size() || !gst_is_initialized())
return;
GstElement *ximagesrc = gst_element_factory_make("ximagesrc", nullptr);
if (!ximagesrc) {
nhlog::ui()->error("Failed to create ximagesrc");
return;
}
GstElement *videoconvert = gst_element_factory_make("videoconvert", nullptr);
GstElement *videoscale = gst_element_factory_make("videoscale", nullptr);
GstElement *capsfilter = gst_element_factory_make("capsfilter", nullptr);
GstElement *ximagesink = gst_element_factory_make("ximagesink", nullptr);
g_object_set(ximagesrc, "use-damage", FALSE, nullptr);
g_object_set(ximagesrc, "show-pointer", FALSE, nullptr);
g_object_set(ximagesrc, "xid", windows_[index].second, nullptr);
GstCaps *caps = gst_caps_new_simple(
"video/x-raw", "width", G_TYPE_INT, 480, "height", G_TYPE_INT, 360, nullptr);
g_object_set(capsfilter, "caps", caps, nullptr);
gst_caps_unref(caps);
pipe_ = gst_pipeline_new(nullptr);
gst_bin_add_many(
GST_BIN(pipe_), ximagesrc, videoconvert, videoscale, capsfilter, ximagesink, nullptr);
if (!gst_element_link_many(
ximagesrc, videoconvert, videoscale, capsfilter, ximagesink, nullptr)) {
nhlog::ui()->error("Failed to link preview window elements");
gst_object_unref(pipe_);
pipe_ = nullptr;
return;
}
if (gst_element_set_state(pipe_, GST_STATE_PLAYING) == GST_STATE_CHANGE_FAILURE) {
nhlog::ui()->error("Unable to start preview pipeline");
gst_object_unref(pipe_);
pipe_ = nullptr;
return;
}
GstBus *bus = gst_pipeline_get_bus(GST_PIPELINE(pipe_));
busWatchId_ = gst_bus_add_watch(bus, newBusMessage, nullptr);
gst_object_unref(bus);
#else
(void)index;
#endif
}
namespace { namespace {
std::vector<std::string> std::vector<std::string>
getTurnURIs(const mtx::responses::TurnServer &turnServer) getTurnURIs(const mtx::responses::TurnServer &turnServer)

View File

@ -25,41 +25,45 @@ class CallManager : public QObject
Q_OBJECT Q_OBJECT
Q_PROPERTY(bool haveCallInvite READ haveCallInvite NOTIFY newInviteState) Q_PROPERTY(bool haveCallInvite READ haveCallInvite NOTIFY newInviteState)
Q_PROPERTY(bool isOnCall READ isOnCall NOTIFY newCallState) Q_PROPERTY(bool isOnCall READ isOnCall NOTIFY newCallState)
Q_PROPERTY(bool isVideo READ isVideo NOTIFY newInviteState) Q_PROPERTY(webrtc::CallType callType READ callType NOTIFY newInviteState)
Q_PROPERTY(bool haveLocalVideo READ haveLocalVideo NOTIFY newCallState)
Q_PROPERTY(webrtc::State callState READ callState NOTIFY newCallState) Q_PROPERTY(webrtc::State callState READ callState NOTIFY newCallState)
Q_PROPERTY(QString callParty READ callParty NOTIFY newInviteState) Q_PROPERTY(QString callParty READ callParty NOTIFY newInviteState)
Q_PROPERTY(QString callPartyAvatarUrl READ callPartyAvatarUrl NOTIFY newInviteState) Q_PROPERTY(QString callPartyAvatarUrl READ callPartyAvatarUrl NOTIFY newInviteState)
Q_PROPERTY(bool isMicMuted READ isMicMuted NOTIFY micMuteChanged) Q_PROPERTY(bool isMicMuted READ isMicMuted NOTIFY micMuteChanged)
Q_PROPERTY(bool callsSupported READ callsSupported CONSTANT) Q_PROPERTY(bool haveLocalPiP READ haveLocalPiP NOTIFY newCallState)
Q_PROPERTY(QStringList mics READ mics NOTIFY devicesChanged) Q_PROPERTY(QStringList mics READ mics NOTIFY devicesChanged)
Q_PROPERTY(QStringList cameras READ cameras NOTIFY devicesChanged) Q_PROPERTY(QStringList cameras READ cameras NOTIFY devicesChanged)
Q_PROPERTY(bool callsSupported READ callsSupported CONSTANT)
Q_PROPERTY(bool screenShareSupported READ screenShareSupported CONSTANT)
public: public:
CallManager(QObject *); CallManager(QObject *);
bool haveCallInvite() const { return haveCallInvite_; } bool haveCallInvite() const { return haveCallInvite_; }
bool isOnCall() const { return session_.state() != webrtc::State::DISCONNECTED; } bool isOnCall() const { return session_.state() != webrtc::State::DISCONNECTED; }
bool isVideo() const { return isVideo_; } webrtc::CallType callType() const { return callType_; }
bool haveLocalVideo() const { return session_.haveLocalVideo(); }
webrtc::State callState() const { return session_.state(); } webrtc::State callState() const { return session_.state(); }
QString callParty() const { return callParty_; } QString callParty() const { return callParty_; }
QString callPartyAvatarUrl() const { return callPartyAvatarUrl_; } QString callPartyAvatarUrl() const { return callPartyAvatarUrl_; }
bool isMicMuted() const { return session_.isMicMuted(); } bool isMicMuted() const { return session_.isMicMuted(); }
bool callsSupported() const; bool haveLocalPiP() const { return session_.haveLocalPiP(); }
QStringList mics() const { return devices(false); } QStringList mics() const { return devices(false); }
QStringList cameras() const { return devices(true); } QStringList cameras() const { return devices(true); }
void refreshTurnServer(); void refreshTurnServer();
static bool callsSupported();
static bool screenShareSupported();
public slots: public slots:
void sendInvite(const QString &roomid, bool isVideo); void sendInvite(const QString &roomid, webrtc::CallType, unsigned int windowIndex = 0);
void syncEvent(const mtx::events::collections::TimelineEvents &event); void syncEvent(const mtx::events::collections::TimelineEvents &event);
void refreshDevices() { CallDevices::instance().refresh(); }
void toggleMicMute(); void toggleMicMute();
void toggleCameraView() { session_.toggleCameraView(); } void toggleLocalPiP() { session_.toggleLocalPiP(); }
void acceptInvite(); void acceptInvite();
void hangUp( void hangUp(
mtx::events::msg::CallHangUp::Reason = mtx::events::msg::CallHangUp::Reason::User); mtx::events::msg::CallHangUp::Reason = mtx::events::msg::CallHangUp::Reason::User);
QStringList windowList();
void previewWindow(unsigned int windowIndex) const;
signals: signals:
void newMessage(const QString &roomid, const mtx::events::msg::CallInvite &); void newMessage(const QString &roomid, const mtx::events::msg::CallInvite &);
@ -81,17 +85,18 @@ private:
QString callParty_; QString callParty_;
QString callPartyAvatarUrl_; QString callPartyAvatarUrl_;
std::string callid_; std::string callid_;
const uint32_t timeoutms_ = 120000; const uint32_t timeoutms_ = 120000;
bool isVideo_ = false; webrtc::CallType callType_ = webrtc::CallType::VOICE;
bool haveCallInvite_ = false; bool haveCallInvite_ = false;
std::string inviteSDP_; std::string inviteSDP_;
std::vector<mtx::events::msg::CallCandidates::Candidate> remoteICECandidates_; std::vector<mtx::events::msg::CallCandidates::Candidate> remoteICECandidates_;
std::vector<std::string> turnURIs_; std::vector<std::string> turnURIs_;
QTimer turnServerTimer_; QTimer turnServerTimer_;
QMediaPlayer player_; QMediaPlayer player_;
std::vector<std::pair<QString, uint32_t>> windows_;
template<typename T> template<typename T>
bool handleEvent_(const mtx::events::collections::TimelineEvents &event); bool handleEvent(const mtx::events::collections::TimelineEvents &event);
void handleEvent(const mtx::events::RoomEvent<mtx::events::msg::CallInvite> &); void handleEvent(const mtx::events::RoomEvent<mtx::events::msg::CallInvite> &);
void handleEvent(const mtx::events::RoomEvent<mtx::events::msg::CallCandidates> &); void handleEvent(const mtx::events::RoomEvent<mtx::events::msg::CallCandidates> &);
void handleEvent(const mtx::events::RoomEvent<mtx::events::msg::CallAnswer> &); void handleEvent(const mtx::events::RoomEvent<mtx::events::msg::CallAnswer> &);

View File

@ -107,13 +107,17 @@ UserSettings::load(std::optional<QString> profile)
auto presenceValue = QMetaEnum::fromType<Presence>().keyToValue(tempPresence.c_str()); auto presenceValue = QMetaEnum::fromType<Presence>().keyToValue(tempPresence.c_str());
if (presenceValue < 0) if (presenceValue < 0)
presenceValue = 0; presenceValue = 0;
presence_ = static_cast<Presence>(presenceValue); presence_ = static_cast<Presence>(presenceValue);
ringtone_ = settings.value("user/ringtone", "Default").toString(); ringtone_ = settings.value("user/ringtone", "Default").toString();
microphone_ = settings.value("user/microphone", QString()).toString(); microphone_ = settings.value("user/microphone", QString()).toString();
camera_ = settings.value("user/camera", QString()).toString(); camera_ = settings.value("user/camera", QString()).toString();
cameraResolution_ = settings.value("user/camera_resolution", QString()).toString(); cameraResolution_ = settings.value("user/camera_resolution", QString()).toString();
cameraFrameRate_ = settings.value("user/camera_frame_rate", QString()).toString(); cameraFrameRate_ = settings.value("user/camera_frame_rate", QString()).toString();
useStunServer_ = settings.value("user/use_stun_server", false).toBool(); screenShareFrameRate_ = settings.value("user/screen_share_frame_rate", 5).toInt();
screenSharePiP_ = settings.value("user/screen_share_pip", true).toBool();
screenShareRemoteVideo_ = settings.value("user/screen_share_remote_video", false).toBool();
screenShareHideCursor_ = settings.value("user/screen_share_hide_cursor", false).toBool();
useStunServer_ = settings.value("user/use_stun_server", false).toBool();
if (profile) // set to "" if it's the default to maintain compatibility if (profile) // set to "" if it's the default to maintain compatibility
profile_ = (*profile == "default") ? "" : *profile; profile_ = (*profile == "default") ? "" : *profile;
@ -444,6 +448,46 @@ UserSettings::setCameraFrameRate(QString frameRate)
save(); save();
} }
void
UserSettings::setScreenShareFrameRate(int frameRate)
{
if (frameRate == screenShareFrameRate_)
return;
screenShareFrameRate_ = frameRate;
emit screenShareFrameRateChanged(frameRate);
save();
}
void
UserSettings::setScreenSharePiP(bool state)
{
if (state == screenSharePiP_)
return;
screenSharePiP_ = state;
emit screenSharePiPChanged(state);
save();
}
void
UserSettings::setScreenShareRemoteVideo(bool state)
{
if (state == screenShareRemoteVideo_)
return;
screenShareRemoteVideo_ = state;
emit screenShareRemoteVideoChanged(state);
save();
}
void
UserSettings::setScreenShareHideCursor(bool state)
{
if (state == screenShareHideCursor_)
return;
screenShareHideCursor_ = state;
emit screenShareHideCursorChanged(state);
save();
}
void void
UserSettings::setProfile(QString profile) UserSettings::setProfile(QString profile)
{ {
@ -593,6 +637,10 @@ UserSettings::save()
settings.setValue("camera", camera_); settings.setValue("camera", camera_);
settings.setValue("camera_resolution", cameraResolution_); settings.setValue("camera_resolution", cameraResolution_);
settings.setValue("camera_frame_rate", cameraFrameRate_); settings.setValue("camera_frame_rate", cameraFrameRate_);
settings.setValue("screen_share_frame_rate", screenShareFrameRate_);
settings.setValue("screen_share_pip", screenSharePiP_);
settings.setValue("screen_share_remote_video", screenShareRemoteVideo_);
settings.setValue("screen_share_hide_cursor", screenShareHideCursor_);
settings.setValue("use_stun_server", useStunServer_); settings.setValue("use_stun_server", useStunServer_);
settings.setValue("currentProfile", profile_); settings.setValue("currentProfile", profile_);
@ -1240,7 +1288,6 @@ UserSettingsPage::showEvent(QShowEvent *)
timelineMaxWidthSpin_->setValue(settings_->timelineMaxWidth()); timelineMaxWidthSpin_->setValue(settings_->timelineMaxWidth());
privacyScreenTimeout_->setValue(settings_->privacyScreenTimeout()); privacyScreenTimeout_->setValue(settings_->privacyScreenTimeout());
CallDevices::instance().refresh();
auto mics = CallDevices::instance().names(false, settings_->microphone().toStdString()); auto mics = CallDevices::instance().names(false, settings_->microphone().toStdString());
microphoneCombo_->clear(); microphoneCombo_->clear();
for (const auto &m : mics) for (const auto &m : mics)

View File

@ -86,6 +86,14 @@ class UserSettings : public QObject
cameraResolutionChanged) cameraResolutionChanged)
Q_PROPERTY(QString cameraFrameRate READ cameraFrameRate WRITE setCameraFrameRate NOTIFY Q_PROPERTY(QString cameraFrameRate READ cameraFrameRate WRITE setCameraFrameRate NOTIFY
cameraFrameRateChanged) cameraFrameRateChanged)
Q_PROPERTY(int screenShareFrameRate READ screenShareFrameRate WRITE setScreenShareFrameRate
NOTIFY screenShareFrameRateChanged)
Q_PROPERTY(bool screenSharePiP READ screenSharePiP WRITE setScreenSharePiP NOTIFY
screenSharePiPChanged)
Q_PROPERTY(bool screenShareRemoteVideo READ screenShareRemoteVideo WRITE
setScreenShareRemoteVideo NOTIFY screenShareRemoteVideoChanged)
Q_PROPERTY(bool screenShareHideCursor READ screenShareHideCursor WRITE
setScreenShareHideCursor NOTIFY screenShareHideCursorChanged)
Q_PROPERTY( Q_PROPERTY(
bool useStunServer READ useStunServer WRITE setUseStunServer NOTIFY useStunServerChanged) bool useStunServer READ useStunServer WRITE setUseStunServer NOTIFY useStunServerChanged)
Q_PROPERTY(bool shareKeysWithTrustedUsers READ shareKeysWithTrustedUsers WRITE Q_PROPERTY(bool shareKeysWithTrustedUsers READ shareKeysWithTrustedUsers WRITE
@ -143,6 +151,10 @@ public:
void setCamera(QString camera); void setCamera(QString camera);
void setCameraResolution(QString resolution); void setCameraResolution(QString resolution);
void setCameraFrameRate(QString frameRate); void setCameraFrameRate(QString frameRate);
void setScreenShareFrameRate(int frameRate);
void setScreenSharePiP(bool state);
void setScreenShareRemoteVideo(bool state);
void setScreenShareHideCursor(bool state);
void setUseStunServer(bool state); void setUseStunServer(bool state);
void setShareKeysWithTrustedUsers(bool state); void setShareKeysWithTrustedUsers(bool state);
void setProfile(QString profile); void setProfile(QString profile);
@ -191,6 +203,10 @@ public:
QString camera() const { return camera_; } QString camera() const { return camera_; }
QString cameraResolution() const { return cameraResolution_; } QString cameraResolution() const { return cameraResolution_; }
QString cameraFrameRate() const { return cameraFrameRate_; } QString cameraFrameRate() const { return cameraFrameRate_; }
int screenShareFrameRate() const { return screenShareFrameRate_; }
bool screenSharePiP() const { return screenSharePiP_; }
bool screenShareRemoteVideo() const { return screenShareRemoteVideo_; }
bool screenShareHideCursor() const { return screenShareHideCursor_; }
bool useStunServer() const { return useStunServer_; } bool useStunServer() const { return useStunServer_; }
bool shareKeysWithTrustedUsers() const { return shareKeysWithTrustedUsers_; } bool shareKeysWithTrustedUsers() const { return shareKeysWithTrustedUsers_; }
QString profile() const { return profile_; } QString profile() const { return profile_; }
@ -229,6 +245,10 @@ signals:
void cameraChanged(QString camera); void cameraChanged(QString camera);
void cameraResolutionChanged(QString resolution); void cameraResolutionChanged(QString resolution);
void cameraFrameRateChanged(QString frameRate); void cameraFrameRateChanged(QString frameRate);
void screenShareFrameRateChanged(int frameRate);
void screenSharePiPChanged(bool state);
void screenShareRemoteVideoChanged(bool state);
void screenShareHideCursorChanged(bool state);
void useStunServerChanged(bool state); void useStunServerChanged(bool state);
void shareKeysWithTrustedUsersChanged(bool state); void shareKeysWithTrustedUsersChanged(bool state);
void profileChanged(QString profile); void profileChanged(QString profile);
@ -272,6 +292,10 @@ private:
QString camera_; QString camera_;
QString cameraResolution_; QString cameraResolution_;
QString cameraFrameRate_; QString cameraFrameRate_;
int screenShareFrameRate_;
bool screenSharePiP_;
bool screenShareRemoteVideo_;
bool screenShareHideCursor_;
bool useStunServer_; bool useStunServer_;
QString profile_; QString profile_;
QString userId_; QString userId_;

View File

@ -10,6 +10,7 @@
#include <thread> #include <thread>
#include <utility> #include <utility>
#include "CallDevices.h"
#include "ChatPage.h" #include "ChatPage.h"
#include "Logging.h" #include "Logging.h"
#include "UserSettingsPage.h" #include "UserSettingsPage.h"
@ -29,14 +30,20 @@ extern "C"
// https://github.com/vector-im/riot-web/issues/10173 // https://github.com/vector-im/riot-web/issues/10173
#define STUN_SERVER "stun://turn.matrix.org:3478" #define STUN_SERVER "stun://turn.matrix.org:3478"
Q_DECLARE_METATYPE(webrtc::CallType)
Q_DECLARE_METATYPE(webrtc::State) Q_DECLARE_METATYPE(webrtc::State)
using webrtc::CallType;
using webrtc::State; using webrtc::State;
WebRTCSession::WebRTCSession() WebRTCSession::WebRTCSession()
: QObject() : QObject()
, devices_(CallDevices::instance()) , devices_(CallDevices::instance())
{ {
qRegisterMetaType<webrtc::CallType>();
qmlRegisterUncreatableMetaObject(
webrtc::staticMetaObject, "im.nheko", 1, 0, "CallType", "Can't instantiate enum");
qRegisterMetaType<webrtc::State>(); qRegisterMetaType<webrtc::State>();
qmlRegisterUncreatableMetaObject( qmlRegisterUncreatableMetaObject(
webrtc::staticMetaObject, "im.nheko", 1, 0, "WebRTCState", "Can't instantiate enum"); webrtc::staticMetaObject, "im.nheko", 1, 0, "WebRTCState", "Can't instantiate enum");
@ -82,9 +89,10 @@ namespace {
std::string localsdp_; std::string localsdp_;
std::vector<mtx::events::msg::CallCandidates::Candidate> localcandidates_; std::vector<mtx::events::msg::CallCandidates::Candidate> localcandidates_;
bool haveAudioStream_; bool haveAudioStream_ = false;
bool haveVideoStream_; bool haveVideoStream_ = false;
GstPad *insetSinkPad_ = nullptr; GstPad *localPiPSinkPad_ = nullptr;
GstPad *remotePiPSinkPad_ = nullptr;
gboolean gboolean
newBusMessage(GstBus *bus G_GNUC_UNUSED, GstMessage *msg, gpointer user_data) newBusMessage(GstBus *bus G_GNUC_UNUSED, GstMessage *msg, gpointer user_data)
@ -166,7 +174,6 @@ createAnswer(GstPromise *promise, gpointer webrtc)
g_signal_emit_by_name(webrtc, "create-answer", nullptr, promise); g_signal_emit_by_name(webrtc, "create-answer", nullptr, promise);
} }
#if GST_CHECK_VERSION(1, 18, 0)
void void
iceGatheringStateChanged(GstElement *webrtc, iceGatheringStateChanged(GstElement *webrtc,
GParamSpec *pspec G_GNUC_UNUSED, GParamSpec *pspec G_GNUC_UNUSED,
@ -186,23 +193,6 @@ iceGatheringStateChanged(GstElement *webrtc,
} }
} }
#else
gboolean
onICEGatheringCompletion(gpointer timerid)
{
*(guint *)(timerid) = 0;
if (WebRTCSession::instance().isOffering()) {
emit WebRTCSession::instance().offerCreated(localsdp_, localcandidates_);
emit WebRTCSession::instance().stateChanged(State::OFFERSENT);
} else {
emit WebRTCSession::instance().answerCreated(localsdp_, localcandidates_);
emit WebRTCSession::instance().stateChanged(State::ANSWERSENT);
}
return FALSE;
}
#endif
void void
addLocalICECandidate(GstElement *webrtc G_GNUC_UNUSED, addLocalICECandidate(GstElement *webrtc G_GNUC_UNUSED,
guint mlineIndex, guint mlineIndex,
@ -210,28 +200,7 @@ addLocalICECandidate(GstElement *webrtc G_GNUC_UNUSED,
gpointer G_GNUC_UNUSED) gpointer G_GNUC_UNUSED)
{ {
nhlog::ui()->debug("WebRTC: local candidate: (m-line:{}):{}", mlineIndex, candidate); nhlog::ui()->debug("WebRTC: local candidate: (m-line:{}):{}", mlineIndex, candidate);
#if GST_CHECK_VERSION(1, 18, 0)
localcandidates_.push_back({std::string() /*max-bundle*/, (uint16_t)mlineIndex, candidate}); localcandidates_.push_back({std::string() /*max-bundle*/, (uint16_t)mlineIndex, candidate});
return;
#else
if (WebRTCSession::instance().state() >= State::OFFERSENT) {
emit WebRTCSession::instance().newICECandidate(
{std::string() /*max-bundle*/, (uint16_t)mlineIndex, candidate});
return;
}
localcandidates_.push_back({std::string() /*max-bundle*/, (uint16_t)mlineIndex, candidate});
// GStreamer v1.16: webrtcbin's notify::ice-gathering-state triggers
// GST_WEBRTC_ICE_GATHERING_STATE_COMPLETE too early. Fixed in v1.18.
// Use a 1s timeout in the meantime
static guint timerid = 0;
if (timerid)
g_source_remove(timerid);
timerid = g_timeout_add(1000, onICEGatheringCompletion, &timerid);
#endif
} }
void void
@ -320,7 +289,6 @@ testPacketLoss(gpointer G_GNUC_UNUSED)
return FALSE; return FALSE;
} }
#if GST_CHECK_VERSION(1, 18, 0)
void void
setWaitForKeyFrame(GstBin *decodebin G_GNUC_UNUSED, GstElement *element, gpointer G_GNUC_UNUSED) setWaitForKeyFrame(GstBin *decodebin G_GNUC_UNUSED, GstElement *element, gpointer G_GNUC_UNUSED)
{ {
@ -329,7 +297,6 @@ setWaitForKeyFrame(GstBin *decodebin G_GNUC_UNUSED, GstElement *element, gpointe
"rtpvp8depay")) "rtpvp8depay"))
g_object_set(element, "wait-for-keyframe", TRUE, nullptr); g_object_set(element, "wait-for-keyframe", TRUE, nullptr);
} }
#endif
GstElement * GstElement *
newAudioSinkChain(GstElement *pipe) newAudioSinkChain(GstElement *pipe)
@ -357,6 +324,7 @@ newVideoSinkChain(GstElement *pipe)
GstElement *glcolorconvert = gst_element_factory_make("glcolorconvert", nullptr); GstElement *glcolorconvert = gst_element_factory_make("glcolorconvert", nullptr);
GstElement *qmlglsink = gst_element_factory_make("qmlglsink", nullptr); GstElement *qmlglsink = gst_element_factory_make("qmlglsink", nullptr);
GstElement *glsinkbin = gst_element_factory_make("glsinkbin", nullptr); GstElement *glsinkbin = gst_element_factory_make("glsinkbin", nullptr);
g_object_set(compositor, "background", 1, nullptr);
g_object_set(qmlglsink, "widget", WebRTCSession::instance().getVideoItem(), nullptr); g_object_set(qmlglsink, "widget", WebRTCSession::instance().getVideoItem(), nullptr);
g_object_set(glsinkbin, "sink", qmlglsink, nullptr); g_object_set(glsinkbin, "sink", qmlglsink, nullptr);
gst_bin_add_many( gst_bin_add_many(
@ -382,44 +350,98 @@ getResolution(GstPad *pad)
return ret; return ret;
} }
void std::pair<int, int>
addCameraView(GstElement *pipe, const std::pair<int, int> &videoCallSize) getResolution(GstElement *pipe, const gchar *elementName, const gchar *padName)
{ {
GstElement *element = gst_bin_get_by_name(GST_BIN(pipe), elementName);
GstPad *pad = gst_element_get_static_pad(element, padName);
auto ret = getResolution(pad);
gst_object_unref(pad);
gst_object_unref(element);
return ret;
}
std::pair<int, int>
getPiPDimensions(const std::pair<int, int> &resolution, int fullWidth, double scaleFactor)
{
int pipWidth = fullWidth * scaleFactor;
int pipHeight = static_cast<double>(resolution.second) / resolution.first * pipWidth;
return {pipWidth, pipHeight};
}
void
addLocalPiP(GstElement *pipe, const std::pair<int, int> &videoCallSize)
{
// embed localUser's camera into received video (CallType::VIDEO)
// OR embed screen share into received video (CallType::SCREEN)
GstElement *tee = gst_bin_get_by_name(GST_BIN(pipe), "videosrctee"); GstElement *tee = gst_bin_get_by_name(GST_BIN(pipe), "videosrctee");
if (!tee) if (!tee)
return; return;
GstElement *queue = gst_element_factory_make("queue", nullptr); GstElement *queue = gst_element_factory_make("queue", nullptr);
GstElement *videorate = gst_element_factory_make("videorate", nullptr); gst_bin_add(GST_BIN(pipe), queue);
gst_bin_add_many(GST_BIN(pipe), queue, videorate, nullptr); gst_element_link(tee, queue);
gst_element_link_many(tee, queue, videorate, nullptr);
gst_element_sync_state_with_parent(queue); gst_element_sync_state_with_parent(queue);
gst_element_sync_state_with_parent(videorate);
gst_object_unref(tee); gst_object_unref(tee);
GstElement *camerafilter = gst_bin_get_by_name(GST_BIN(pipe), "camerafilter");
GstPad *filtersinkpad = gst_element_get_static_pad(camerafilter, "sink");
auto cameraResolution = getResolution(filtersinkpad);
int insetWidth = videoCallSize.first / 4;
int insetHeight =
static_cast<double>(cameraResolution.second) / cameraResolution.first * insetWidth;
nhlog::ui()->debug("WebRTC: picture-in-picture size: {}x{}", insetWidth, insetHeight);
gst_object_unref(filtersinkpad);
gst_object_unref(camerafilter);
GstPad *camerapad = gst_element_get_static_pad(videorate, "src");
GstElement *compositor = gst_bin_get_by_name(GST_BIN(pipe), "compositor"); GstElement *compositor = gst_bin_get_by_name(GST_BIN(pipe), "compositor");
insetSinkPad_ = gst_element_get_request_pad(compositor, "sink_%u"); localPiPSinkPad_ = gst_element_get_request_pad(compositor, "sink_%u");
g_object_set(insetSinkPad_, "zorder", 2, nullptr); g_object_set(localPiPSinkPad_, "zorder", 2, nullptr);
g_object_set(insetSinkPad_, "width", insetWidth, "height", insetHeight, nullptr);
bool isVideo = WebRTCSession::instance().callType() == CallType::VIDEO;
const gchar *element = isVideo ? "camerafilter" : "screenshare";
const gchar *pad = isVideo ? "sink" : "src";
auto resolution = getResolution(pipe, element, pad);
auto pipSize = getPiPDimensions(resolution, videoCallSize.first, 0.25);
nhlog::ui()->debug(
"WebRTC: local picture-in-picture: {}x{}", pipSize.first, pipSize.second);
g_object_set(localPiPSinkPad_, "width", pipSize.first, "height", pipSize.second, nullptr);
gint offset = videoCallSize.first / 80; gint offset = videoCallSize.first / 80;
g_object_set(insetSinkPad_, "xpos", offset, "ypos", offset, nullptr); g_object_set(localPiPSinkPad_, "xpos", offset, "ypos", offset, nullptr);
if (GST_PAD_LINK_FAILED(gst_pad_link(camerapad, insetSinkPad_)))
nhlog::ui()->error("WebRTC: failed to link camera view chain"); GstPad *srcpad = gst_element_get_static_pad(queue, "src");
gst_object_unref(camerapad); if (GST_PAD_LINK_FAILED(gst_pad_link(srcpad, localPiPSinkPad_)))
nhlog::ui()->error("WebRTC: failed to link local PiP elements");
gst_object_unref(srcpad);
gst_object_unref(compositor); gst_object_unref(compositor);
} }
void
addRemotePiP(GstElement *pipe)
{
// embed localUser's camera into screen image being shared
if (remotePiPSinkPad_) {
auto camRes = getResolution(pipe, "camerafilter", "sink");
auto shareRes = getResolution(pipe, "screenshare", "src");
auto pipSize = getPiPDimensions(camRes, shareRes.first, 0.2);
nhlog::ui()->debug(
"WebRTC: screen share picture-in-picture: {}x{}", pipSize.first, pipSize.second);
gint offset = shareRes.first / 100;
g_object_set(remotePiPSinkPad_, "zorder", 2, nullptr);
g_object_set(
remotePiPSinkPad_, "width", pipSize.first, "height", pipSize.second, nullptr);
g_object_set(remotePiPSinkPad_,
"xpos",
shareRes.first - pipSize.first - offset,
"ypos",
shareRes.second - pipSize.second - offset,
nullptr);
}
}
void
addLocalVideo(GstElement *pipe)
{
GstElement *queue = newVideoSinkChain(pipe);
GstElement *tee = gst_bin_get_by_name(GST_BIN(pipe), "videosrctee");
GstPad *srcpad = gst_element_get_request_pad(tee, "src_%u");
GstPad *sinkpad = gst_element_get_static_pad(queue, "sink");
if (GST_PAD_LINK_FAILED(gst_pad_link(srcpad, sinkpad)))
nhlog::ui()->error("WebRTC: failed to link videosrctee -> video sink chain");
gst_object_unref(srcpad);
}
void void
linkNewPad(GstElement *decodebin, GstPad *newpad, GstElement *pipe) linkNewPad(GstElement *decodebin, GstPad *newpad, GstElement *pipe)
{ {
@ -455,7 +477,7 @@ linkNewPad(GstElement *decodebin, GstPad *newpad, GstElement *pipe)
nhlog::ui()->info("WebRTC: incoming video resolution: {}x{}", nhlog::ui()->info("WebRTC: incoming video resolution: {}x{}",
videoCallSize.first, videoCallSize.first,
videoCallSize.second); videoCallSize.second);
addCameraView(pipe, videoCallSize); addLocalPiP(pipe, videoCallSize);
} else { } else {
g_free(mediaType); g_free(mediaType);
nhlog::ui()->error("WebRTC: unknown pad type: {}", GST_PAD_NAME(newpad)); nhlog::ui()->error("WebRTC: unknown pad type: {}", GST_PAD_NAME(newpad));
@ -467,7 +489,7 @@ linkNewPad(GstElement *decodebin, GstPad *newpad, GstElement *pipe)
if (GST_PAD_LINK_FAILED(gst_pad_link(newpad, queuepad))) if (GST_PAD_LINK_FAILED(gst_pad_link(newpad, queuepad)))
nhlog::ui()->error("WebRTC: unable to link new pad"); nhlog::ui()->error("WebRTC: unable to link new pad");
else { else {
if (!session->isVideo() || if (session->callType() == CallType::VOICE ||
(haveAudioStream_ && (haveAudioStream_ &&
(haveVideoStream_ || session->isRemoteVideoRecvOnly()))) { (haveVideoStream_ || session->isRemoteVideoRecvOnly()))) {
emit session->stateChanged(State::CONNECTED); emit session->stateChanged(State::CONNECTED);
@ -477,6 +499,9 @@ linkNewPad(GstElement *decodebin, GstPad *newpad, GstElement *pipe)
keyFrameRequestData_.timerid = keyFrameRequestData_.timerid =
g_timeout_add_seconds(3, testPacketLoss, nullptr); g_timeout_add_seconds(3, testPacketLoss, nullptr);
} }
addRemotePiP(pipe);
if (session->isRemoteVideoRecvOnly())
addLocalVideo(pipe);
} }
} }
gst_object_unref(queuepad); gst_object_unref(queuepad);
@ -495,9 +520,7 @@ addDecodeBin(GstElement *webrtc G_GNUC_UNUSED, GstPad *newpad, GstElement *pipe)
// hardware decoding needs investigation; eg rendering fails if vaapi plugin installed // hardware decoding needs investigation; eg rendering fails if vaapi plugin installed
g_object_set(decodebin, "force-sw-decoders", TRUE, nullptr); g_object_set(decodebin, "force-sw-decoders", TRUE, nullptr);
g_signal_connect(decodebin, "pad-added", G_CALLBACK(linkNewPad), pipe); g_signal_connect(decodebin, "pad-added", G_CALLBACK(linkNewPad), pipe);
#if GST_CHECK_VERSION(1, 18, 0)
g_signal_connect(decodebin, "element-added", G_CALLBACK(setWaitForKeyFrame), nullptr); g_signal_connect(decodebin, "element-added", G_CALLBACK(setWaitForKeyFrame), nullptr);
#endif
gst_bin_add(GST_BIN(pipe), decodebin); gst_bin_add(GST_BIN(pipe), decodebin);
gst_element_sync_state_with_parent(decodebin); gst_element_sync_state_with_parent(decodebin);
GstPad *sinkpad = gst_element_get_static_pad(decodebin, "sink"); GstPad *sinkpad = gst_element_get_static_pad(decodebin, "sink");
@ -523,14 +546,17 @@ getMediaAttributes(const GstSDPMessage *sdp,
const char *mediaType, const char *mediaType,
const char *encoding, const char *encoding,
int &payloadType, int &payloadType,
bool &recvOnly) bool &recvOnly,
bool &sendOnly)
{ {
payloadType = -1; payloadType = -1;
recvOnly = false; recvOnly = false;
sendOnly = false;
for (guint mlineIndex = 0; mlineIndex < gst_sdp_message_medias_len(sdp); ++mlineIndex) { for (guint mlineIndex = 0; mlineIndex < gst_sdp_message_medias_len(sdp); ++mlineIndex) {
const GstSDPMedia *media = gst_sdp_message_get_media(sdp, mlineIndex); const GstSDPMedia *media = gst_sdp_message_get_media(sdp, mlineIndex);
if (!std::strcmp(gst_sdp_media_get_media(media), mediaType)) { if (!std::strcmp(gst_sdp_media_get_media(media), mediaType)) {
recvOnly = gst_sdp_media_get_attribute_val(media, "recvonly") != nullptr; recvOnly = gst_sdp_media_get_attribute_val(media, "recvonly") != nullptr;
sendOnly = gst_sdp_media_get_attribute_val(media, "sendonly") != nullptr;
const gchar *rtpval = nullptr; const gchar *rtpval = nullptr;
for (guint n = 0; n == 0 || rtpval; ++n) { for (guint n = 0; n == 0 || rtpval; ++n) {
rtpval = gst_sdp_media_get_attribute_val_n(media, "rtpmap", n); rtpval = gst_sdp_media_get_attribute_val_n(media, "rtpmap", n);
@ -603,17 +629,12 @@ WebRTCSession::havePlugins(bool isVideo, std::string *errorMessage)
} }
bool bool
WebRTCSession::createOffer(bool isVideo) WebRTCSession::createOffer(CallType callType, uint32_t shareWindowId)
{ {
isOffering_ = true; clear();
isVideo_ = isVideo; isOffering_ = true;
isRemoteVideoRecvOnly_ = false; callType_ = callType;
videoItem_ = nullptr; shareWindowId_ = shareWindowId;
haveAudioStream_ = false;
haveVideoStream_ = false;
insetSinkPad_ = nullptr;
localsdp_.clear();
localcandidates_.clear();
// opus and vp8 rtp payload types must be defined dynamically // opus and vp8 rtp payload types must be defined dynamically
// therefore from the range [96-127] // therefore from the range [96-127]
@ -630,22 +651,15 @@ WebRTCSession::acceptOffer(const std::string &sdp)
if (state_ != State::DISCONNECTED) if (state_ != State::DISCONNECTED)
return false; return false;
isOffering_ = false; clear();
isRemoteVideoRecvOnly_ = false;
videoItem_ = nullptr;
haveAudioStream_ = false;
haveVideoStream_ = false;
insetSinkPad_ = nullptr;
localsdp_.clear();
localcandidates_.clear();
GstWebRTCSessionDescription *offer = parseSDP(sdp, GST_WEBRTC_SDP_TYPE_OFFER); GstWebRTCSessionDescription *offer = parseSDP(sdp, GST_WEBRTC_SDP_TYPE_OFFER);
if (!offer) if (!offer)
return false; return false;
int opusPayloadType; int opusPayloadType;
bool recvOnly; bool recvOnly;
if (getMediaAttributes(offer->sdp, "audio", "opus", opusPayloadType, recvOnly)) { bool sendOnly;
if (getMediaAttributes(offer->sdp, "audio", "opus", opusPayloadType, recvOnly, sendOnly)) {
if (opusPayloadType == -1) { if (opusPayloadType == -1) {
nhlog::ui()->error("WebRTC: remote audio offer - no opus encoding"); nhlog::ui()->error("WebRTC: remote audio offer - no opus encoding");
gst_webrtc_session_description_free(offer); gst_webrtc_session_description_free(offer);
@ -658,13 +672,18 @@ WebRTCSession::acceptOffer(const std::string &sdp)
} }
int vp8PayloadType; int vp8PayloadType;
isVideo_ = bool isVideo = getMediaAttributes(offer->sdp,
getMediaAttributes(offer->sdp, "video", "vp8", vp8PayloadType, isRemoteVideoRecvOnly_); "video",
if (isVideo_ && vp8PayloadType == -1) { "vp8",
vp8PayloadType,
isRemoteVideoRecvOnly_,
isRemoteVideoSendOnly_);
if (isVideo && vp8PayloadType == -1) {
nhlog::ui()->error("WebRTC: remote video offer - no vp8 encoding"); nhlog::ui()->error("WebRTC: remote video offer - no vp8 encoding");
gst_webrtc_session_description_free(offer); gst_webrtc_session_description_free(offer);
return false; return false;
} }
callType_ = isVideo ? CallType::VIDEO : CallType::VOICE;
if (!startPipeline(opusPayloadType, vp8PayloadType)) { if (!startPipeline(opusPayloadType, vp8PayloadType)) {
gst_webrtc_session_description_free(offer); gst_webrtc_session_description_free(offer);
@ -695,10 +714,14 @@ WebRTCSession::acceptAnswer(const std::string &sdp)
return false; return false;
} }
if (isVideo_) { if (callType_ != CallType::VOICE) {
int unused; int unused;
if (!getMediaAttributes( if (!getMediaAttributes(answer->sdp,
answer->sdp, "video", "vp8", unused, isRemoteVideoRecvOnly_)) "video",
"vp8",
unused,
isRemoteVideoRecvOnly_,
isRemoteVideoSendOnly_))
isRemoteVideoRecvOnly_ = true; isRemoteVideoRecvOnly_ = true;
} }
@ -769,11 +792,10 @@ WebRTCSession::startPipeline(int opusPayloadType, int vp8PayloadType)
gst_element_set_state(pipe_, GST_STATE_READY); gst_element_set_state(pipe_, GST_STATE_READY);
g_signal_connect(webrtc_, "pad-added", G_CALLBACK(addDecodeBin), pipe_); g_signal_connect(webrtc_, "pad-added", G_CALLBACK(addDecodeBin), pipe_);
#if GST_CHECK_VERSION(1, 18, 0)
// capture ICE gathering completion // capture ICE gathering completion
g_signal_connect( g_signal_connect(
webrtc_, "notify::ice-gathering-state", G_CALLBACK(iceGatheringStateChanged), nullptr); webrtc_, "notify::ice-gathering-state", G_CALLBACK(iceGatheringStateChanged), nullptr);
#endif
// webrtcbin lifetime is the same as that of the pipeline // webrtcbin lifetime is the same as that of the pipeline
gst_object_unref(webrtc_); gst_object_unref(webrtc_);
@ -855,40 +877,115 @@ WebRTCSession::createPipeline(int opusPayloadType, int vp8PayloadType)
return false; return false;
} }
return isVideo_ ? addVideoPipeline(vp8PayloadType) : true; return callType_ == CallType::VOICE || isRemoteVideoSendOnly_
? true
: addVideoPipeline(vp8PayloadType);
} }
bool bool
WebRTCSession::addVideoPipeline(int vp8PayloadType) WebRTCSession::addVideoPipeline(int vp8PayloadType)
{ {
// allow incoming video calls despite localUser having no webcam // allow incoming video calls despite localUser having no webcam
if (!devices_.haveCamera()) if (callType_ == CallType::VIDEO && !devices_.haveCamera())
return !isOffering_; return !isOffering_;
std::pair<int, int> resolution; auto settings = ChatPage::instance()->userSettings();
std::pair<int, int> frameRate; GstElement *camerafilter = nullptr;
GstDevice *device = devices_.videoDevice(resolution, frameRate);
if (!device)
return false;
GstElement *source = gst_device_create_element(device, nullptr);
GstElement *videoconvert = gst_element_factory_make("videoconvert", nullptr); GstElement *videoconvert = gst_element_factory_make("videoconvert", nullptr);
GstElement *capsfilter = gst_element_factory_make("capsfilter", "camerafilter"); GstElement *tee = gst_element_factory_make("tee", "videosrctee");
GstCaps *caps = gst_caps_new_simple("video/x-raw", gst_bin_add_many(GST_BIN(pipe_), videoconvert, tee, nullptr);
"width", if (callType_ == CallType::VIDEO || (settings->screenSharePiP() && devices_.haveCamera())) {
G_TYPE_INT, std::pair<int, int> resolution;
resolution.first, std::pair<int, int> frameRate;
"height", GstDevice *device = devices_.videoDevice(resolution, frameRate);
G_TYPE_INT, if (!device)
resolution.second, return false;
"framerate",
GST_TYPE_FRACTION, GstElement *camera = gst_device_create_element(device, nullptr);
frameRate.first, GstCaps *caps = gst_caps_new_simple("video/x-raw",
frameRate.second, "width",
nullptr); G_TYPE_INT,
g_object_set(capsfilter, "caps", caps, nullptr); resolution.first,
gst_caps_unref(caps); "height",
GstElement *tee = gst_element_factory_make("tee", "videosrctee"); G_TYPE_INT,
resolution.second,
"framerate",
GST_TYPE_FRACTION,
frameRate.first,
frameRate.second,
nullptr);
camerafilter = gst_element_factory_make("capsfilter", "camerafilter");
g_object_set(camerafilter, "caps", caps, nullptr);
gst_caps_unref(caps);
gst_bin_add_many(GST_BIN(pipe_), camera, camerafilter, nullptr);
if (!gst_element_link_many(camera, videoconvert, camerafilter, nullptr)) {
nhlog::ui()->error("WebRTC: failed to link camera elements");
return false;
}
if (callType_ == CallType::VIDEO && !gst_element_link(camerafilter, tee)) {
nhlog::ui()->error("WebRTC: failed to link camerafilter -> tee");
return false;
}
}
if (callType_ == CallType::SCREEN) {
nhlog::ui()->debug("WebRTC: screen share frame rate: {} fps",
settings->screenShareFrameRate());
nhlog::ui()->debug("WebRTC: screen share picture-in-picture: {}",
settings->screenSharePiP());
nhlog::ui()->debug("WebRTC: screen share request remote camera: {}",
settings->screenShareRemoteVideo());
nhlog::ui()->debug("WebRTC: screen share hide mouse cursor: {}",
settings->screenShareHideCursor());
GstElement *ximagesrc = gst_element_factory_make("ximagesrc", "screenshare");
if (!ximagesrc) {
nhlog::ui()->error("WebRTC: failed to create ximagesrc");
return false;
}
g_object_set(ximagesrc, "use-damage", FALSE, nullptr);
g_object_set(ximagesrc, "xid", shareWindowId_, nullptr);
g_object_set(
ximagesrc, "show-pointer", !settings->screenShareHideCursor(), nullptr);
GstCaps *caps = gst_caps_new_simple("video/x-raw",
"framerate",
GST_TYPE_FRACTION,
settings->screenShareFrameRate(),
1,
nullptr);
GstElement *capsfilter = gst_element_factory_make("capsfilter", nullptr);
g_object_set(capsfilter, "caps", caps, nullptr);
gst_caps_unref(caps);
gst_bin_add_many(GST_BIN(pipe_), ximagesrc, capsfilter, nullptr);
if (settings->screenSharePiP() && devices_.haveCamera()) {
GstElement *compositor = gst_element_factory_make("compositor", nullptr);
g_object_set(compositor, "background", 1, nullptr);
gst_bin_add(GST_BIN(pipe_), compositor);
if (!gst_element_link_many(
ximagesrc, compositor, capsfilter, tee, nullptr)) {
nhlog::ui()->error("WebRTC: failed to link screen share elements");
return false;
}
GstPad *srcpad = gst_element_get_static_pad(camerafilter, "src");
remotePiPSinkPad_ = gst_element_get_request_pad(compositor, "sink_%u");
if (GST_PAD_LINK_FAILED(gst_pad_link(srcpad, remotePiPSinkPad_))) {
nhlog::ui()->error(
"WebRTC: failed to link camerafilter -> compositor");
gst_object_unref(srcpad);
return false;
}
gst_object_unref(srcpad);
} else if (!gst_element_link_many(
ximagesrc, videoconvert, capsfilter, tee, nullptr)) {
nhlog::ui()->error("WebRTC: failed to link screen share elements");
return false;
}
}
GstElement *queue = gst_element_factory_make("queue", nullptr); GstElement *queue = gst_element_factory_make("queue", nullptr);
GstElement *vp8enc = gst_element_factory_make("vp8enc", nullptr); GstElement *vp8enc = gst_element_factory_make("vp8enc", nullptr);
g_object_set(vp8enc, "deadline", 1, nullptr); g_object_set(vp8enc, "deadline", 1, nullptr);
@ -910,46 +1007,45 @@ WebRTCSession::addVideoPipeline(int vp8PayloadType)
g_object_set(rtpcapsfilter, "caps", rtpcaps, nullptr); g_object_set(rtpcapsfilter, "caps", rtpcaps, nullptr);
gst_caps_unref(rtpcaps); gst_caps_unref(rtpcaps);
gst_bin_add_many(GST_BIN(pipe_), gst_bin_add_many(
source, GST_BIN(pipe_), queue, vp8enc, rtpvp8pay, rtpqueue, rtpcapsfilter, nullptr);
videoconvert,
capsfilter,
tee,
queue,
vp8enc,
rtpvp8pay,
rtpqueue,
rtpcapsfilter,
nullptr);
GstElement *webrtcbin = gst_bin_get_by_name(GST_BIN(pipe_), "webrtcbin"); GstElement *webrtcbin = gst_bin_get_by_name(GST_BIN(pipe_), "webrtcbin");
if (!gst_element_link_many(source, if (!gst_element_link_many(
videoconvert, tee, queue, vp8enc, rtpvp8pay, rtpqueue, rtpcapsfilter, webrtcbin, nullptr)) {
capsfilter, nhlog::ui()->error("WebRTC: failed to link rtp video elements");
tee,
queue,
vp8enc,
rtpvp8pay,
rtpqueue,
rtpcapsfilter,
webrtcbin,
nullptr)) {
nhlog::ui()->error("WebRTC: failed to link video pipeline elements");
gst_object_unref(webrtcbin); gst_object_unref(webrtcbin);
return false; return false;
} }
if (callType_ == CallType::SCREEN &&
!ChatPage::instance()->userSettings()->screenShareRemoteVideo()) {
GArray *transceivers;
g_signal_emit_by_name(webrtcbin, "get-transceivers", &transceivers);
GstWebRTCRTPTransceiver *transceiver =
g_array_index(transceivers, GstWebRTCRTPTransceiver *, 1);
transceiver->direction = GST_WEBRTC_RTP_TRANSCEIVER_DIRECTION_SENDONLY;
g_array_unref(transceivers);
}
gst_object_unref(webrtcbin); gst_object_unref(webrtcbin);
return true; return true;
} }
bool bool
WebRTCSession::haveLocalVideo() const WebRTCSession::haveLocalPiP() const
{ {
if (isVideo_ && state_ >= State::INITIATED) { if (state_ >= State::INITIATED) {
GstElement *tee = gst_bin_get_by_name(GST_BIN(pipe_), "videosrctee"); if (callType_ == CallType::VOICE || isRemoteVideoRecvOnly_)
if (tee) { return false;
gst_object_unref(tee); else if (callType_ == CallType::SCREEN)
return true; return true;
else {
GstElement *tee = gst_bin_get_by_name(GST_BIN(pipe_), "videosrctee");
if (tee) {
gst_object_unref(tee);
return true;
}
} }
} }
return false; return false;
@ -983,15 +1079,35 @@ WebRTCSession::toggleMicMute()
} }
void void
WebRTCSession::toggleCameraView() WebRTCSession::toggleLocalPiP()
{ {
if (insetSinkPad_) { if (localPiPSinkPad_) {
guint zorder; guint zorder;
g_object_get(insetSinkPad_, "zorder", &zorder, nullptr); g_object_get(localPiPSinkPad_, "zorder", &zorder, nullptr);
g_object_set(insetSinkPad_, "zorder", zorder ? 0 : 2, nullptr); g_object_set(localPiPSinkPad_, "zorder", zorder ? 0 : 2, nullptr);
} }
} }
void
WebRTCSession::clear()
{
callType_ = webrtc::CallType::VOICE;
isOffering_ = false;
isRemoteVideoRecvOnly_ = false;
isRemoteVideoSendOnly_ = false;
videoItem_ = nullptr;
pipe_ = nullptr;
webrtc_ = nullptr;
busWatchId_ = 0;
shareWindowId_ = 0;
haveAudioStream_ = false;
haveVideoStream_ = false;
localPiPSinkPad_ = nullptr;
remotePiPSinkPad_ = nullptr;
localsdp_.clear();
localcandidates_.clear();
}
void void
WebRTCSession::end() WebRTCSession::end()
{ {
@ -1007,12 +1123,7 @@ WebRTCSession::end()
} }
} }
webrtc_ = nullptr; clear();
isVideo_ = false;
isOffering_ = false;
isRemoteVideoRecvOnly_ = false;
videoItem_ = nullptr;
insetSinkPad_ = nullptr;
if (state_ != State::DISCONNECTED) if (state_ != State::DISCONNECTED)
emit stateChanged(State::DISCONNECTED); emit stateChanged(State::DISCONNECTED);
} }
@ -1026,16 +1137,12 @@ WebRTCSession::havePlugins(bool, std::string *)
} }
bool bool
WebRTCSession::haveLocalVideo() const WebRTCSession::haveLocalPiP() const
{ {
return false; return false;
} }
bool bool WebRTCSession::createOffer(webrtc::CallType, uint32_t) { return false; }
WebRTCSession::createOffer(bool)
{
return false;
}
bool bool
WebRTCSession::acceptOffer(const std::string &) WebRTCSession::acceptOffer(const std::string &)
@ -1066,7 +1173,7 @@ WebRTCSession::toggleMicMute()
} }
void void
WebRTCSession::toggleCameraView() WebRTCSession::toggleLocalPiP()
{} {}
void void

View File

@ -5,15 +5,23 @@
#include <QObject> #include <QObject>
#include "CallDevices.h"
#include "mtx/events/voip.hpp" #include "mtx/events/voip.hpp"
typedef struct _GstElement GstElement; typedef struct _GstElement GstElement;
class CallDevices;
class QQuickItem; class QQuickItem;
namespace webrtc { namespace webrtc {
Q_NAMESPACE Q_NAMESPACE
enum class CallType
{
VOICE,
VIDEO,
SCREEN // localUser is sharing screen
};
Q_ENUM_NS(CallType)
enum class State enum class State
{ {
DISCONNECTED, DISCONNECTED,
@ -42,20 +50,21 @@ public:
} }
bool havePlugins(bool isVideo, std::string *errorMessage = nullptr); bool havePlugins(bool isVideo, std::string *errorMessage = nullptr);
webrtc::CallType callType() const { return callType_; }
webrtc::State state() const { return state_; } webrtc::State state() const { return state_; }
bool isVideo() const { return isVideo_; } bool haveLocalPiP() const;
bool haveLocalVideo() const;
bool isOffering() const { return isOffering_; } bool isOffering() const { return isOffering_; }
bool isRemoteVideoRecvOnly() const { return isRemoteVideoRecvOnly_; } bool isRemoteVideoRecvOnly() const { return isRemoteVideoRecvOnly_; }
bool isRemoteVideoSendOnly() const { return isRemoteVideoSendOnly_; }
bool createOffer(bool isVideo); bool createOffer(webrtc::CallType, uint32_t shareWindowId);
bool acceptOffer(const std::string &sdp); bool acceptOffer(const std::string &sdp);
bool acceptAnswer(const std::string &sdp); bool acceptAnswer(const std::string &sdp);
void acceptICECandidates(const std::vector<mtx::events::msg::CallCandidates::Candidate> &); void acceptICECandidates(const std::vector<mtx::events::msg::CallCandidates::Candidate> &);
bool isMicMuted() const; bool isMicMuted() const;
bool toggleMicMute(); bool toggleMicMute();
void toggleCameraView(); void toggleLocalPiP();
void end(); void end();
void setTurnServers(const std::vector<std::string> &uris) { turnServers_ = uris; } void setTurnServers(const std::vector<std::string> &uris) { turnServers_ = uris; }
@ -81,20 +90,23 @@ private:
bool initialised_ = false; bool initialised_ = false;
bool haveVoicePlugins_ = false; bool haveVoicePlugins_ = false;
bool haveVideoPlugins_ = false; bool haveVideoPlugins_ = false;
webrtc::CallType callType_ = webrtc::CallType::VOICE;
webrtc::State state_ = webrtc::State::DISCONNECTED; webrtc::State state_ = webrtc::State::DISCONNECTED;
bool isVideo_ = false;
bool isOffering_ = false; bool isOffering_ = false;
bool isRemoteVideoRecvOnly_ = false; bool isRemoteVideoRecvOnly_ = false;
bool isRemoteVideoSendOnly_ = false;
QQuickItem *videoItem_ = nullptr; QQuickItem *videoItem_ = nullptr;
GstElement *pipe_ = nullptr; GstElement *pipe_ = nullptr;
GstElement *webrtc_ = nullptr; GstElement *webrtc_ = nullptr;
unsigned int busWatchId_ = 0; unsigned int busWatchId_ = 0;
std::vector<std::string> turnServers_; std::vector<std::string> turnServers_;
uint32_t shareWindowId_ = 0;
bool init(std::string *errorMessage = nullptr); bool init(std::string *errorMessage = nullptr);
bool startPipeline(int opusPayloadType, int vp8PayloadType); bool startPipeline(int opusPayloadType, int vp8PayloadType);
bool createPipeline(int opusPayloadType, int vp8PayloadType); bool createPipeline(int opusPayloadType, int vp8PayloadType);
bool addVideoPipeline(int vp8PayloadType); bool addVideoPipeline(int vp8PayloadType);
void clear();
public: public:
WebRTCSession(WebRTCSession const &) = delete; WebRTCSession(WebRTCSession const &) = delete;