rnhmjoj
/
nheko
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Support desktop screen sharing on X11

master
trilene 2021-02-18 15:55:29 -05:00
parent 0d971f543b
commit 8df10eeeca
15 changed files with 376 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
resources/icons/ui/screen-share.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 773 B

2
resources/qml/TimelineView.qml
View File

@ -249,7 +249,7 @@ Page {
}
Loader {
source: CallManager.isOnCall && CallManager.isVideo ? "voip/VideoCall.qml" : ""
source: CallManager.isOnCall && CallManager.haveVideo ? "voip/VideoCall.qml" : ""
onLoaded: TimelineManager.setVideoCallItem()
}

52
resources/qml/voip/ActiveCallBar.qml
View File

@ -12,7 +12,7 @@ Rectangle {
MouseArea {
anchors.fill: parent
onClicked: {
if (CallManager.isVideo)
if (CallManager.haveVideo)
stackLayout.currentIndex = stackLayout.currentIndex ? 0 : 1;
}
@ -42,10 +42,46 @@ Rectangle {
}
Image {
id: callTypeIcon
Layout.leftMargin: 4
Layout.preferredWidth: 24
Layout.preferredHeight: 24
source: CallManager.isVideo ? "qrc:/icons/icons/ui/video-call.png" : "qrc:/icons/icons/ui/place-call.png"
}
Item {
states: [
State {
name: "VOICE"
when: CallManager.callType == CallType.VOICE
PropertyChanges {
target: callTypeIcon
source: "qrc:/icons/icons/ui/place-call.png"
}
},
State {
name: "VIDEO"
when: CallManager.callType == CallType.VIDEO
PropertyChanges {
target: callTypeIcon
source: "qrc:/icons/icons/ui/video-call.png"
}
},
State {
name: "SCREEN"
when: CallManager.callType == CallType.SCREEN
PropertyChanges {
target: callTypeIcon
source: "qrc:/icons/icons/ui/screen-share.png"
}
}
]
}
Label {
@ -103,7 +139,7 @@ Rectangle {
PropertyChanges {
target: stackLayout
currentIndex: CallManager.isVideo ? 1 : 0
currentIndex: CallManager.haveVideo ? 1 : 0
}
},
@ -147,12 +183,20 @@ Rectangle {
}
}
Label {
Layout.leftMargin: 16
visible: CallManager.callType == CallType.SCREEN && CallManager.callState == WebRTCState.CONNECTED
text: qsTr("You are screen sharing")
font.pointSize: fontMetrics.font.pointSize * 1.1
color: "#000000"
}
Item {
Layout.fillWidth: true
}
ImageButton {
visible: CallManager.haveLocalVideo
visible: CallManager.haveLocalCamera
width: 24
height: 24
buttonTextColor: "#000000"

2
resources/qml/voip/CallDevices.qml
View File

@ -40,7 +40,7 @@ Popup {
}
RowLayout {
visible: CallManager.isVideo && CallManager.cameras.length > 0
visible: CallManager.callType == CallType.VIDEO && CallManager.cameras.length > 0
Image {
Layout.preferredWidth: 22

8
resources/qml/voip/CallInvite.qml
View File

@ -53,7 +53,7 @@ Popup {
Layout.bottomMargin: msgView.height / 25
Image {
property string image: CallManager.isVideo ? ":/icons/icons/ui/video-call.png" : ":/icons/icons/ui/place-call.png"
property string image: CallManager.callType == CallType.VIDEO ? ":/icons/icons/ui/video-call.png" : ":/icons/icons/ui/place-call.png"
Layout.alignment: Qt.AlignCenter
Layout.preferredWidth: msgView.height / 10
@ -63,7 +63,7 @@ Popup {
Label {
Layout.alignment: Qt.AlignCenter
text: CallManager.isVideo ? qsTr("Video Call") : qsTr("Voice Call")
text: CallManager.callType == CallType.VIDEO ? qsTr("Video Call") : qsTr("Voice Call")
font.pointSize: fontMetrics.font.pointSize * 2
color: colors.windowText
}
@ -97,7 +97,7 @@ Popup {
}
RowLayout {
visible: CallManager.isVideo && CallManager.cameras.length > 0
visible: CallManager.callType == CallType.VIDEO && CallManager.cameras.length > 0
Layout.alignment: Qt.AlignCenter
Image {
@ -159,7 +159,7 @@ Popup {
RoundButton {
id: acceptButton
property string image: CallManager.isVideo ? ":/icons/icons/ui/video-call.png" : ":/icons/icons/ui/place-call.png"
property string image: CallManager.callType == CallType.VIDEO ? ":/icons/icons/ui/video-call.png" : ":/icons/icons/ui/place-call.png"
implicitWidth: buttonLayout.buttonSize
implicitHeight: buttonLayout.buttonSize

8
resources/qml/voip/CallInviteBar.qml
View File

@ -52,12 +52,12 @@ Rectangle {
Layout.leftMargin: 4
Layout.preferredWidth: 24
Layout.preferredHeight: 24
source: CallManager.isVideo ? "qrc:/icons/icons/ui/video-call.png" : "qrc:/icons/icons/ui/place-call.png"
source: CallManager.callType == CallType.VIDEO ? "qrc:/icons/icons/ui/video-call.png" : "qrc:/icons/icons/ui/place-call.png"
}
Label {
font.pointSize: fontMetrics.font.pointSize * 1.1
text: CallManager.isVideo ? qsTr("Video Call") : qsTr("Voice Call")
text: CallManager.callType == CallType.VIDEO ? qsTr("Video Call") : qsTr("Voice Call")
color: "#000000"
}
@ -83,7 +83,7 @@ Rectangle {
Button {
Layout.rightMargin: 4
icon.source: CallManager.isVideo ? "qrc:/icons/icons/ui/video-call.png" : "qrc:/icons/icons/ui/place-call.png"
icon.source: CallManager.callType == CallType.VIDEO ? "qrc:/icons/icons/ui/video-call.png" : "qrc:/icons/icons/ui/place-call.png"
text: qsTr("Accept")
palette: colors
onClicked: {
@ -102,7 +102,7 @@ Rectangle {
dialog.open();
return ;
}
if (CallManager.isVideo && CallManager.cameras.length > 0 && !CallManager.cameras.includes(Settings.camera)) {
if (CallManager.callType == CallType.VIDEO && CallManager.cameras.length > 0 && !CallManager.cameras.includes(Settings.camera)) {
var dialog = deviceError.createObject(timelineRoot, {
"errorString": qsTr("Unknown camera: %1").arg(Settings.camera),
"image": ":/icons/icons/ui/video-call.png"

23
resources/qml/voip/PlaceCall.qml
View File

@ -23,6 +23,14 @@ Popup {
}
Component {
id: screenShareDialog
ScreenShare {
}
}
ColumnLayout {
id: columnLayout
@ -76,7 +84,7 @@ Popup {
onClicked: {
if (buttonLayout.validateMic()) {
Settings.microphone = micCombo.currentText;
CallManager.sendInvite(TimelineManager.timeline.roomId(), false);
CallManager.sendInvite(TimelineManager.timeline.roomId(), CallType.VOICE);
close();
}
}
@ -90,12 +98,23 @@ Popup {
if (buttonLayout.validateMic()) {
Settings.microphone = micCombo.currentText;
Settings.camera = cameraCombo.currentText;
CallManager.sendInvite(TimelineManager.timeline.roomId(), true);
CallManager.sendInvite(TimelineManager.timeline.roomId(), CallType.VIDEO);
close();
}
}
}
Button {
visible: CallManager.screenShareSupported
text: qsTr("Screen")
icon.source: "qrc:/icons/icons/ui/screen-share.png"
onClicked: {
var dialog = screenShareDialog.createObject(timelineRoot);
dialog.open();
close();
}
}
Button {
text: qsTr("Cancel")
onClicked: {

95
resources/qml/voip/ScreenShare.qml Normal file
View File

@ -0,0 +1,95 @@
import "../"
import QtQuick 2.9
import QtQuick.Controls 2.3
import QtQuick.Layouts 1.2
import im.nheko 1.0
Popup {
modal: true
// only set the anchors on Qt 5.12 or higher
// see https://doc.qt.io/qt-5/qml-qtquick-controls2-popup.html#anchors.centerIn-prop
Component.onCompleted: {
if (anchors)
anchors.centerIn = parent;
frameRateCombo.currentIndex = frameRateCombo.find(Settings.screenShareFrameRate);
remoteVideoCheckBox.checked = Settings.screenShareRemoteVideo;
}
palette: colors
ColumnLayout {
Label {
Layout.margins: 8
Layout.alignment: Qt.AlignLeft
text: qsTr("Share desktop with %1?").arg(TimelineManager.timeline.roomName)
color: colors.windowText
}
RowLayout {
Layout.leftMargin: 8
Layout.rightMargin: 8
Label {
Layout.alignment: Qt.AlignLeft
text: qsTr("Frame rate:")
color: colors.windowText
}
ComboBox {
id: frameRateCombo
Layout.alignment: Qt.AlignRight
model: ["25", "20", "15", "10", "5", "2", "1"]
}
}
CheckBox {
id: remoteVideoCheckBox
Layout.alignment: Qt.AlignLeft
Layout.leftMargin: 8
Layout.rightMargin: 8
text: qsTr("Request remote camera")
ToolTip.text: qsTr("View your callee's camera like a regular video call")
ToolTip.visible: hovered
}
RowLayout {
Layout.margins: 8
Item {
Layout.fillWidth: true
}
Button {
text: qsTr("Share")
icon.source: "qrc:/icons/icons/ui/screen-share.png"
onClicked: {
if (buttonLayout.validateMic()) {
Settings.microphone = micCombo.currentText;
Settings.screenShareFrameRate = frameRateCombo.currentText;
Settings.screenShareRemoteVideo = remoteVideoCheckBox.checked;
CallManager.sendInvite(TimelineManager.timeline.roomId(), CallType.SCREEN);
close();
}
}
}
Button {
text: qsTr("Cancel")
onClicked: {
close();
}
}
}
}
background: Rectangle {
color: colors.window
border.color: colors.windowText
}
}

2
resources/res.qrc
View File

@ -74,6 +74,7 @@
<file>icons/ui/end-call.png</file>
<file>icons/ui/microphone-mute.png</file>
<file>icons/ui/microphone-unmute.png</file>
<file>icons/ui/screen-share.png</file>
<file>icons/ui/toggle-camera-view.png</file>
<file>icons/ui/video-call.png</file>
@ -165,6 +166,7 @@
<file>qml/voip/CallInviteBar.qml</file>
<file>qml/voip/DeviceError.qml</file>
<file>qml/voip/PlaceCall.qml</file>
<file>qml/voip/ScreenShare.qml</file>
<file>qml/voip/VideoCall.qml</file>
</qresource>
<qresource prefix="/media">

45
src/CallManager.cpp
View File

@ -2,6 +2,7 @@
#include <cctype>
#include <chrono>
#include <cstdint>
#include <cstdlib>
#include <QMediaPlaylist>
#include <QUrl>
@ -24,6 +25,8 @@ Q_DECLARE_METATYPE(mtx::responses::TurnServer)
using namespace mtx::events;
using namespace mtx::events::msg;
using webrtc::CallType;
namespace {
std::vector<std::string>
getTurnURIs(const mtx::responses::TurnServer &turnServer);
@ -148,10 +151,12 @@ CallManager::CallManager(QObject *parent)
}
void
CallManager::sendInvite(const QString &roomid, bool isVideo)
CallManager::sendInvite(const QString &roomid, CallType callType)
{
if (isOnCall())
return;
if (callType == CallType::SCREEN && !screenShareSupported())
return;
auto roomInfo = cache::singleRoomInfo(roomid.toStdString());
if (roomInfo.member_count != 2) {
@ -161,17 +166,20 @@ CallManager::sendInvite(const QString &roomid, bool isVideo)
std::string errorMessage;
if (!session_.havePlugins(false, &errorMessage) ||
(isVideo && !session_.havePlugins(true, &errorMessage))) {
((callType == CallType::VIDEO || callType == CallType::SCREEN) &&
!session_.havePlugins(true, &errorMessage))) {
emit ChatPage::instance()->showNotification(QString::fromStdString(errorMessage));
return;
}
isVideo_ = isVideo;
roomid_ = roomid;
callType_ = callType;
roomid_ = roomid;
session_.setTurnServers(turnURIs_);
generateCallID();
nhlog::ui()->debug(
"WebRTC: call id: {} - creating {} invite", callid_, isVideo ? "video" : "voice");
std::string strCallType = callType_ == CallType::VOICE
? "voice"
: (callType_ == CallType::VIDEO ? "video" : "screen");
nhlog::ui()->debug("WebRTC: call id: {} - creating {} invite", callid_, strCallType);
std::vector<RoomMember> members(cache::getMembers(roomid.toStdString()));
const RoomMember &callee =
members.front().user_id == utils::localUser() ? members.back() : members.front();
@ -179,7 +187,7 @@ CallManager::sendInvite(const QString &roomid, bool isVideo)
callPartyAvatarUrl_ = QString::fromStdString(roomInfo.avatar_url);
emit newInviteState();
playRingtone(QUrl("qrc:/media/media/ringback.ogg"), true);
if (!session_.createOffer(isVideo)) {
if (!session_.createOffer(callType)) {
emit ChatPage::instance()->showNotification("Problem setting up call.");
endCall();
}
@ -280,7 +288,7 @@ CallManager::handleEvent(const RoomEvent<CallInvite> &callInviteEvent)
callPartyAvatarUrl_ = QString::fromStdString(roomInfo.avatar_url);
haveCallInvite_ = true;
isVideo_ = isVideo;
callType_ = isVideo ? CallType::VIDEO : CallType::VOICE;
inviteSDP_ = callInviteEvent.content.sdp;
CallDevices::instance().refresh();
emit newInviteState();
@ -295,7 +303,7 @@ CallManager::acceptInvite()
stopRingtone();
std::string errorMessage;
if (!session_.havePlugins(false, &errorMessage) ||
(isVideo_ && !session_.havePlugins(true, &errorMessage))) {
(callType_ == CallType::VIDEO && !session_.havePlugins(true, &errorMessage))) {
emit ChatPage::instance()->showNotification(QString::fromStdString(errorMessage));
hangUp();
return;
@ -383,7 +391,7 @@ CallManager::toggleMicMute()
}
bool
CallManager::callsSupported() const
CallManager::callsSupported()
{
#ifdef GSTREAMER_AVAILABLE
return true;
@ -392,6 +400,21 @@ CallManager::callsSupported() const
#endif
}
bool
CallManager::screenShareSupported()
{
return std::getenv("DISPLAY") != nullptr;
}
bool
CallManager::haveVideo() const
{
return callType() == CallType::VIDEO ||
(callType() == CallType::SCREEN &&
(ChatPage::instance()->userSettings()->screenShareRemoteVideo() &&
!session_.isRemoteVideoRecvOnly()));
}
QStringList
CallManager::devices(bool isVideo) const
{
@ -424,7 +447,7 @@ CallManager::clear()
callParty_.clear();
callPartyAvatarUrl_.clear();
callid_.clear();
isVideo_ = false;
callType_ = CallType::VOICE;
haveCallInvite_ = false;
emit newInviteState();
inviteSDP_.clear();

25
src/CallManager.h
View File

@ -25,34 +25,39 @@ class CallManager : public QObject
Q_OBJECT
Q_PROPERTY(bool haveCallInvite READ haveCallInvite NOTIFY newInviteState)
Q_PROPERTY(bool isOnCall READ isOnCall NOTIFY newCallState)
Q_PROPERTY(bool isVideo READ isVideo NOTIFY newInviteState)
Q_PROPERTY(bool haveLocalVideo READ haveLocalVideo NOTIFY newCallState)
Q_PROPERTY(webrtc::CallType callType READ callType NOTIFY newInviteState)
Q_PROPERTY(webrtc::State callState READ callState NOTIFY newCallState)
Q_PROPERTY(QString callParty READ callParty NOTIFY newInviteState)
Q_PROPERTY(QString callPartyAvatarUrl READ callPartyAvatarUrl NOTIFY newInviteState)
Q_PROPERTY(bool isMicMuted READ isMicMuted NOTIFY micMuteChanged)
Q_PROPERTY(bool callsSupported READ callsSupported CONSTANT)
Q_PROPERTY(bool haveLocalCamera READ haveLocalCamera NOTIFY newCallState)
Q_PROPERTY(bool haveVideo READ haveVideo NOTIFY newInviteState)
Q_PROPERTY(QStringList mics READ mics NOTIFY devicesChanged)
Q_PROPERTY(QStringList cameras READ cameras NOTIFY devicesChanged)
Q_PROPERTY(bool callsSupported READ callsSupported CONSTANT)
Q_PROPERTY(bool screenShareSupported READ screenShareSupported CONSTANT)
public:
CallManager(QObject *);
bool haveCallInvite() const { return haveCallInvite_; }
bool isOnCall() const { return session_.state() != webrtc::State::DISCONNECTED; }
bool isVideo() const { return isVideo_; }
bool haveLocalVideo() const { return session_.haveLocalVideo(); }
webrtc::CallType callType() const { return callType_; }
webrtc::State callState() const { return session_.state(); }
QString callParty() const { return callParty_; }
QString callPartyAvatarUrl() const { return callPartyAvatarUrl_; }
bool isMicMuted() const { return session_.isMicMuted(); }
bool callsSupported() const;
bool haveLocalCamera() const { return session_.haveLocalCamera(); }
bool haveVideo() const;
QStringList mics() const { return devices(false); }
QStringList cameras() const { return devices(true); }
void refreshTurnServer();
static bool callsSupported();
static bool screenShareSupported();
public slots:
void sendInvite(const QString &roomid, bool isVideo);
void sendInvite(const QString &roomid, webrtc::CallType);
void syncEvent(const mtx::events::collections::TimelineEvents &event);
void refreshDevices() { CallDevices::instance().refresh(); }
void toggleMicMute();
@ -81,9 +86,9 @@ private:
QString callParty_;
QString callPartyAvatarUrl_;
std::string callid_;
const uint32_t timeoutms_ = 120000;
bool isVideo_ = false;
bool haveCallInvite_ = false;
const uint32_t timeoutms_ = 120000;
webrtc::CallType callType_ = webrtc::CallType::VOICE;
bool haveCallInvite_ = false;
std::string inviteSDP_;
std::vector<mtx::events::msg::CallCandidates::Candidate> remoteICECandidates_;
std::vector<std::string> turnURIs_;

38
src/UserSettingsPage.cpp
View File

@ -107,13 +107,15 @@ UserSettings::load(std::optional<QString> profile)
auto presenceValue = QMetaEnum::fromType<Presence>().keyToValue(tempPresence.c_str());
if (presenceValue < 0)
presenceValue = 0;
presence_ = static_cast<Presence>(presenceValue);
ringtone_ = settings.value("user/ringtone", "Default").toString();
microphone_ = settings.value("user/microphone", QString()).toString();
camera_ = settings.value("user/camera", QString()).toString();
cameraResolution_ = settings.value("user/camera_resolution", QString()).toString();
cameraFrameRate_ = settings.value("user/camera_frame_rate", QString()).toString();
useStunServer_ = settings.value("user/use_stun_server", false).toBool();
presence_ = static_cast<Presence>(presenceValue);
ringtone_ = settings.value("user/ringtone", "Default").toString();
microphone_ = settings.value("user/microphone", QString()).toString();
camera_ = settings.value("user/camera", QString()).toString();
cameraResolution_ = settings.value("user/camera_resolution", QString()).toString();
cameraFrameRate_ = settings.value("user/camera_frame_rate", QString()).toString();
screenShareFrameRate_ = settings.value("user/screen_share_frame_rate", 5).toInt();
screenShareRemoteVideo_ = settings.value("user/screen_share_remote_video", false).toBool();
useStunServer_ = settings.value("user/use_stun_server", false).toBool();
if (profile) // set to "" if it's the default to maintain compatibility
profile_ = (*profile == "default") ? "" : *profile;
@ -444,6 +446,26 @@ UserSettings::setCameraFrameRate(QString frameRate)
save();
}
void
UserSettings::setScreenShareFrameRate(int frameRate)
{
if (frameRate == screenShareFrameRate_)
return;
screenShareFrameRate_ = frameRate;
emit screenShareFrameRateChanged(frameRate);
save();
}
void
UserSettings::setScreenShareRemoteVideo(bool state)
{
if (state == screenShareRemoteVideo_)
return;
screenShareRemoteVideo_ = state;
emit screenShareRemoteVideoChanged(state);
save();
}
void
UserSettings::setProfile(QString profile)
{
@ -593,6 +615,8 @@ UserSettings::save()
settings.setValue("camera", camera_);
settings.setValue("camera_resolution", cameraResolution_);
settings.setValue("camera_frame_rate", cameraFrameRate_);
settings.setValue("screen_share_frame_rate", screenShareFrameRate_);
settings.setValue("screen_share_remote_video", screenShareRemoteVideo_);
settings.setValue("use_stun_server", useStunServer_);
settings.setValue("currentProfile", profile_);

12
src/UserSettingsPage.h
View File

@ -86,6 +86,10 @@ class UserSettings : public QObject
cameraResolutionChanged)
Q_PROPERTY(QString cameraFrameRate READ cameraFrameRate WRITE setCameraFrameRate NOTIFY
cameraFrameRateChanged)
Q_PROPERTY(int screenShareFrameRate READ screenShareFrameRate WRITE setScreenShareFrameRate
NOTIFY screenShareFrameRateChanged)
Q_PROPERTY(bool screenShareRemoteVideo READ screenShareRemoteVideo WRITE
setScreenShareRemoteVideo NOTIFY screenShareRemoteVideoChanged)
Q_PROPERTY(
bool useStunServer READ useStunServer WRITE setUseStunServer NOTIFY useStunServerChanged)
Q_PROPERTY(bool shareKeysWithTrustedUsers READ shareKeysWithTrustedUsers WRITE
@ -143,6 +147,8 @@ public:
void setCamera(QString camera);
void setCameraResolution(QString resolution);
void setCameraFrameRate(QString frameRate);
void setScreenShareFrameRate(int frameRate);
void setScreenShareRemoteVideo(bool state);
void setUseStunServer(bool state);
void setShareKeysWithTrustedUsers(bool state);
void setProfile(QString profile);
@ -191,6 +197,8 @@ public:
QString camera() const { return camera_; }
QString cameraResolution() const { return cameraResolution_; }
QString cameraFrameRate() const { return cameraFrameRate_; }
int screenShareFrameRate() const { return screenShareFrameRate_; }
bool screenShareRemoteVideo() const { return screenShareRemoteVideo_; }
bool useStunServer() const { return useStunServer_; }
bool shareKeysWithTrustedUsers() const { return shareKeysWithTrustedUsers_; }
QString profile() const { return profile_; }
@ -229,6 +237,8 @@ signals:
void cameraChanged(QString camera);
void cameraResolutionChanged(QString resolution);
void cameraFrameRateChanged(QString frameRate);
void screenShareFrameRateChanged(int frameRate);
void screenShareRemoteVideoChanged(bool state);
void useStunServerChanged(bool state);
void shareKeysWithTrustedUsersChanged(bool state);
void profileChanged(QString profile);
@ -272,6 +282,8 @@ private:
QString camera_;
QString cameraResolution_;
QString cameraFrameRate_;
int screenShareFrameRate_;
bool screenShareRemoteVideo_;
bool useStunServer_;
QString profile_;
QString userId_;

134
src/WebRTCSession.cpp
View File

@ -10,6 +10,7 @@
#include <thread>
#include <utility>
#include "CallDevices.h"
#include "ChatPage.h"
#include "Logging.h"
#include "UserSettingsPage.h"
@ -29,14 +30,20 @@ extern "C"
// https://github.com/vector-im/riot-web/issues/10173
#define STUN_SERVER "stun://turn.matrix.org:3478"
Q_DECLARE_METATYPE(webrtc::CallType)
Q_DECLARE_METATYPE(webrtc::State)
using webrtc::CallType;
using webrtc::State;
WebRTCSession::WebRTCSession()
: QObject()
, devices_(CallDevices::instance())
{
qRegisterMetaType<webrtc::CallType>();
qmlRegisterUncreatableMetaObject(
webrtc::staticMetaObject, "im.nheko", 1, 0, "CallType", "Can't instantiate enum");
qRegisterMetaType<webrtc::State>();
qmlRegisterUncreatableMetaObject(
webrtc::staticMetaObject, "im.nheko", 1, 0, "WebRTCState", "Can't instantiate enum");
@ -455,7 +462,8 @@ linkNewPad(GstElement *decodebin, GstPad *newpad, GstElement *pipe)
nhlog::ui()->info("WebRTC: incoming video resolution: {}x{}",
videoCallSize.first,
videoCallSize.second);
addCameraView(pipe, videoCallSize);
if (session->callType() == CallType::VIDEO)
addCameraView(pipe, videoCallSize);
} else {
g_free(mediaType);
nhlog::ui()->error("WebRTC: unknown pad type: {}", GST_PAD_NAME(newpad));
@ -467,7 +475,7 @@ linkNewPad(GstElement *decodebin, GstPad *newpad, GstElement *pipe)
if (GST_PAD_LINK_FAILED(gst_pad_link(newpad, queuepad)))
nhlog::ui()->error("WebRTC: unable to link new pad");
else {
if (!session->isVideo() ||
if (session->callType() == CallType::VOICE ||
(haveAudioStream_ &&
(haveVideoStream_ || session->isRemoteVideoRecvOnly()))) {
emit session->stateChanged(State::CONNECTED);
@ -523,14 +531,17 @@ getMediaAttributes(const GstSDPMessage *sdp,
const char *mediaType,
const char *encoding,
int &payloadType,
bool &recvOnly)
bool &recvOnly,
bool &sendOnly)
{
payloadType = -1;
recvOnly = false;
sendOnly = false;
for (guint mlineIndex = 0; mlineIndex < gst_sdp_message_medias_len(sdp); ++mlineIndex) {
const GstSDPMedia *media = gst_sdp_message_get_media(sdp, mlineIndex);
if (!std::strcmp(gst_sdp_media_get_media(media), mediaType)) {
recvOnly = gst_sdp_media_get_attribute_val(media, "recvonly") != nullptr;
sendOnly = gst_sdp_media_get_attribute_val(media, "sendonly") != nullptr;
const gchar *rtpval = nullptr;
for (guint n = 0; n == 0 || rtpval; ++n) {
rtpval = gst_sdp_media_get_attribute_val_n(media, "rtpmap", n);
@ -603,11 +614,12 @@ WebRTCSession::havePlugins(bool isVideo, std::string *errorMessage)
}
bool
WebRTCSession::createOffer(bool isVideo)
WebRTCSession::createOffer(CallType callType)
{
isOffering_ = true;
isVideo_ = isVideo;
callType_ = callType;
isRemoteVideoRecvOnly_ = false;
isRemoteVideoSendOnly_ = false;
videoItem_ = nullptr;
haveAudioStream_ = false;
haveVideoStream_ = false;
@ -630,8 +642,10 @@ WebRTCSession::acceptOffer(const std::string &sdp)
if (state_ != State::DISCONNECTED)
return false;
callType_ = webrtc::CallType::VOICE;
isOffering_ = false;
isRemoteVideoRecvOnly_ = false;
isRemoteVideoSendOnly_ = false;
videoItem_ = nullptr;
haveAudioStream_ = false;
haveVideoStream_ = false;
@ -645,7 +659,8 @@ WebRTCSession::acceptOffer(const std::string &sdp)
int opusPayloadType;
bool recvOnly;
if (getMediaAttributes(offer->sdp, "audio", "opus", opusPayloadType, recvOnly)) {
bool sendOnly;
if (getMediaAttributes(offer->sdp, "audio", "opus", opusPayloadType, recvOnly, sendOnly)) {
if (opusPayloadType == -1) {
nhlog::ui()->error("WebRTC: remote audio offer - no opus encoding");
gst_webrtc_session_description_free(offer);
@ -658,13 +673,18 @@ WebRTCSession::acceptOffer(const std::string &sdp)
}
int vp8PayloadType;
isVideo_ =
getMediaAttributes(offer->sdp, "video", "vp8", vp8PayloadType, isRemoteVideoRecvOnly_);
if (isVideo_ && vp8PayloadType == -1) {
bool isVideo = getMediaAttributes(offer->sdp,
"video",
"vp8",
vp8PayloadType,
isRemoteVideoRecvOnly_,
isRemoteVideoSendOnly_);
if (isVideo && vp8PayloadType == -1) {
nhlog::ui()->error("WebRTC: remote video offer - no vp8 encoding");
gst_webrtc_session_description_free(offer);
return false;
}
callType_ = isVideo ? CallType::VIDEO : CallType::VOICE;
if (!startPipeline(opusPayloadType, vp8PayloadType)) {
gst_webrtc_session_description_free(offer);
@ -695,10 +715,14 @@ WebRTCSession::acceptAnswer(const std::string &sdp)
return false;
}
if (isVideo_) {
if (callType_ != CallType::VOICE) {
int unused;
if (!getMediaAttributes(
answer->sdp, "video", "vp8", unused, isRemoteVideoRecvOnly_))
if (!getMediaAttributes(answer->sdp,
"video",
"vp8",
unused,
isRemoteVideoRecvOnly_,
isRemoteVideoSendOnly_))
isRemoteVideoRecvOnly_ = true;
}
@ -855,39 +879,59 @@ WebRTCSession::createPipeline(int opusPayloadType, int vp8PayloadType)
return false;
}
return isVideo_ ? addVideoPipeline(vp8PayloadType) : true;
return callType_ == CallType::VOICE || isRemoteVideoSendOnly_
? true
: addVideoPipeline(vp8PayloadType);
}
bool
WebRTCSession::addVideoPipeline(int vp8PayloadType)
{
// allow incoming video calls despite localUser having no webcam
if (!devices_.haveCamera())
if (callType_ == CallType::VIDEO && !devices_.haveCamera())
return !isOffering_;
std::pair<int, int> resolution;
std::pair<int, int> frameRate;
GstDevice *device = devices_.videoDevice(resolution, frameRate);
if (!device)
return false;
GstElement *source = nullptr;
GstCaps *caps = nullptr;
if (callType_ == CallType::VIDEO) {
std::pair<int, int> resolution;
std::pair<int, int> frameRate;
GstDevice *device = devices_.videoDevice(resolution, frameRate);
if (!device)
return false;
source = gst_device_create_element(device, nullptr);
caps = gst_caps_new_simple("video/x-raw",
"width",
G_TYPE_INT,
resolution.first,
"height",
G_TYPE_INT,
resolution.second,
"framerate",
GST_TYPE_FRACTION,
frameRate.first,
frameRate.second,
nullptr);
} else {
source = gst_element_factory_make("ximagesrc", nullptr);
if (!source) {
nhlog::ui()->error("WebRTC: failed to create ximagesrc");
return false;
}
g_object_set(source, "use-damage", 0, nullptr);
g_object_set(source, "xid", 0, nullptr);
int frameRate = ChatPage::instance()->userSettings()->screenShareFrameRate();
caps = gst_caps_new_simple(
"video/x-raw", "framerate", GST_TYPE_FRACTION, frameRate, 1, nullptr);
nhlog::ui()->debug("WebRTC: screen share frame rate: {} fps", frameRate);
}
GstElement *source = gst_device_create_element(device, nullptr);
GstElement *videoconvert = gst_element_factory_make("videoconvert", nullptr);
GstElement *capsfilter = gst_element_factory_make("capsfilter", "camerafilter");
GstCaps *caps = gst_caps_new_simple("video/x-raw",
"width",
G_TYPE_INT,
resolution.first,
"height",
G_TYPE_INT,
resolution.second,
"framerate",
GST_TYPE_FRACTION,
frameRate.first,
frameRate.second,
nullptr);
g_object_set(capsfilter, "caps", caps, nullptr);
gst_caps_unref(caps);
GstElement *tee = gst_element_factory_make("tee", "videosrctee");
GstElement *queue = gst_element_factory_make("queue", nullptr);
GstElement *vp8enc = gst_element_factory_make("vp8enc", nullptr);
@ -938,14 +982,25 @@ WebRTCSession::addVideoPipeline(int vp8PayloadType)
gst_object_unref(webrtcbin);
return false;
}
if (callType_ == CallType::SCREEN &&
!ChatPage::instance()->userSettings()->screenShareRemoteVideo()) {
GArray *transceivers;
g_signal_emit_by_name(webrtcbin, "get-transceivers", &transceivers);
GstWebRTCRTPTransceiver *transceiver =
g_array_index(transceivers, GstWebRTCRTPTransceiver *, 1);
transceiver->direction = GST_WEBRTC_RTP_TRANSCEIVER_DIRECTION_SENDONLY;
g_array_unref(transceivers);
}
gst_object_unref(webrtcbin);
return true;
}
bool
WebRTCSession::haveLocalVideo() const
WebRTCSession::haveLocalCamera() const
{
if (isVideo_ && state_ >= State::INITIATED) {
if (callType_ == CallType::VIDEO && state_ >= State::INITIATED) {
GstElement *tee = gst_bin_get_by_name(GST_BIN(pipe_), "videosrctee");
if (tee) {
gst_object_unref(tee);
@ -1008,9 +1063,10 @@ WebRTCSession::end()
}
webrtc_ = nullptr;
isVideo_ = false;
callType_ = CallType::VOICE;
isOffering_ = false;
isRemoteVideoRecvOnly_ = false;
isRemoteVideoSendOnly_ = false;
videoItem_ = nullptr;
insetSinkPad_ = nullptr;
if (state_ != State::DISCONNECTED)
@ -1026,16 +1082,12 @@ WebRTCSession::havePlugins(bool, std::string *)
}
bool
WebRTCSession::haveLocalVideo() const
WebRTCSession::haveLocalCamera() const
{
return false;
}
bool
WebRTCSession::createOffer(bool)
{
return false;
}
bool WebRTCSession::createOffer(webrtc::CallType) { return false; }
bool
WebRTCSession::acceptOffer(const std::string &)

20
src/WebRTCSession.h
View File

@ -5,15 +5,23 @@
#include <QObject>
#include "CallDevices.h"
#include "mtx/events/voip.hpp"
typedef struct _GstElement GstElement;
class CallDevices;
class QQuickItem;
namespace webrtc {
Q_NAMESPACE
enum class CallType
{
VOICE,
VIDEO,
SCREEN // localUser is sharing screen
};
Q_ENUM_NS(CallType)
enum class State
{
DISCONNECTED,
@ -42,13 +50,14 @@ public:
}
bool havePlugins(bool isVideo, std::string *errorMessage = nullptr);
webrtc::CallType callType() const { return callType_; }
webrtc::State state() const { return state_; }
bool isVideo() const { return isVideo_; }
bool haveLocalVideo() const;
bool haveLocalCamera() const;
bool isOffering() const { return isOffering_; }
bool isRemoteVideoRecvOnly() const { return isRemoteVideoRecvOnly_; }
bool isRemoteVideoSendOnly() const { return isRemoteVideoSendOnly_; }
bool createOffer(bool isVideo);
bool createOffer(webrtc::CallType);
bool acceptOffer(const std::string &sdp);
bool acceptAnswer(const std::string &sdp);
void acceptICECandidates(const std::vector<mtx::events::msg::CallCandidates::Candidate> &);
@ -81,10 +90,11 @@ private:
bool initialised_ = false;
bool haveVoicePlugins_ = false;
bool haveVideoPlugins_ = false;
webrtc::CallType callType_ = webrtc::CallType::VOICE;
webrtc::State state_ = webrtc::State::DISCONNECTED;
bool isVideo_ = false;
bool isOffering_ = false;
bool isRemoteVideoRecvOnly_ = false;
bool isRemoteVideoSendOnly_ = false;
QQuickItem *videoItem_ = nullptr;
GstElement *pipe_ = nullptr;
GstElement *webrtc_ = nullptr;