From 09c041c8ac40d2d3608c7224614fde69e1e4f08b Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Sat, 28 Aug 2021 00:38:33 +0200 Subject: [PATCH] Use in memory media player instead of storing unencrypted files on disk --- CMakeLists.txt | 4 +- resources/qml/ForwardCompleter.qml | 5 +- resources/qml/MessageInput.qml | 9 +- resources/qml/QuickSwitcher.qml | 5 +- .../qml/delegates/PlayableMediaMessage.qml | 52 +++---- src/timeline/TimelineModel.h | 9 ++ src/timeline/TimelineViewManager.cpp | 2 + src/ui/MxcMediaProxy.cpp | 142 ++++++++++++++++++ src/ui/MxcMediaProxy.h | 80 ++++++++++ 9 files changed, 267 insertions(+), 41 deletions(-) create mode 100644 src/ui/MxcMediaProxy.cpp create mode 100644 src/ui/MxcMediaProxy.h diff --git a/CMakeLists.txt b/CMakeLists.txt index c2cef7b7..f24cffef 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -311,6 +311,7 @@ set(SRC_FILES src/ui/InfoMessage.cpp src/ui/Label.cpp src/ui/LoadingIndicator.cpp + src/ui/MxcMediaProxy.cpp src/ui/NhekoCursorShape.cpp src/ui/NhekoDropArea.cpp src/ui/NhekoGlobalObject.cpp @@ -350,7 +351,7 @@ set(SRC_FILES src/MemberList.cpp src/MxcImageProvider.cpp src/Olm.cpp - src/ReadReceiptsModel.cpp + src/ReadReceiptsModel.cpp src/RegisterPage.cpp src/SSOHandler.cpp src/CombinedImagePackModel.cpp @@ -521,6 +522,7 @@ qt5_wrap_cpp(MOC_HEADERS src/ui/InfoMessage.h src/ui/Label.h src/ui/LoadingIndicator.h + src/ui/MxcMediaProxy.h src/ui/Menu.h src/ui/NhekoCursorShape.h src/ui/NhekoDropArea.h diff --git a/resources/qml/ForwardCompleter.qml b/resources/qml/ForwardCompleter.qml index fdfcec6f..eccd6ce9 100644 --- a/resources/qml/ForwardCompleter.qml +++ b/resources/qml/ForwardCompleter.qml @@ -85,11 +85,10 @@ Popup { completerPopup.up(); } else if ((event.key == Qt.Key_Down || event.key == Qt.Key_Tab) && completerPopup.opened) { event.accepted = true; - if (event.key == Qt.Key_Tab && (event.modifiers & Qt.ShiftModifier)) { + if (event.key == Qt.Key_Tab && (event.modifiers & Qt.ShiftModifier)) completerPopup.up(); - } else { + else completerPopup.down(); - } } else if (event.matches(StandardKey.InsertParagraphSeparator)) { completerPopup.finishCompletion(); event.accepted = true; diff --git a/resources/qml/MessageInput.qml b/resources/qml/MessageInput.qml index 36d8fbce..e1bf3f06 100644 --- a/resources/qml/MessageInput.qml +++ b/resources/qml/MessageInput.qml @@ -134,9 +134,9 @@ Rectangle { return ; room.input.updateState(selectionStart, selectionEnd, cursorPosition, text); - if (popup.opened && cursorPosition <= completerTriggeredAt) { + if (popup.opened && cursorPosition <= completerTriggeredAt) popup.close(); - } + if (popup.opened) popup.completer.setSearchString(messageInput.getText(completerTriggeredAt, cursorPosition)); @@ -195,11 +195,10 @@ Rectangle { } else if (event.key == Qt.Key_Tab) { event.accepted = true; if (popup.opened) { - if (event.modifiers & Qt.ShiftModifier) { + if (event.modifiers & Qt.ShiftModifier) popup.down(); - } else { + else popup.up(); - } } else { var pos = cursorPosition - 1; while (pos > -1) { diff --git a/resources/qml/QuickSwitcher.qml b/resources/qml/QuickSwitcher.qml index fe1936af..c7141c81 100644 --- a/resources/qml/QuickSwitcher.qml +++ b/resources/qml/QuickSwitcher.qml @@ -44,11 +44,10 @@ Popup { completerPopup.up(); } else if ((event.key == Qt.Key_Down || event.key == Qt.Key_Tab) && completerPopup.opened) { event.accepted = true; - if (event.key == Qt.Key_Tab && (event.modifiers & Qt.ShiftModifier)) { + if (event.key == Qt.Key_Tab && (event.modifiers & Qt.ShiftModifier)) completerPopup.up(); - } else { + else completerPopup.down(); - } } else if (event.matches(StandardKey.InsertParagraphSeparator)) { completerPopup.finishCompletion(); event.accepted = true; diff --git a/resources/qml/delegates/PlayableMediaMessage.qml b/resources/qml/delegates/PlayableMediaMessage.qml index 73c74ec0..94c058ab 100644 --- a/resources/qml/delegates/PlayableMediaMessage.qml +++ b/resources/qml/delegates/PlayableMediaMessage.qml @@ -3,9 +3,9 @@ // SPDX-License-Identifier: GPL-3.0-or-later import "../" -import QtMultimedia 5.6 -import QtQuick 2.12 -import QtQuick.Controls 2.1 +import QtMultimedia 5.15 +import QtQuick 2.15 +import QtQuick.Controls 2.15 import QtQuick.Layouts 1.2 import im.nheko 1.0 @@ -55,7 +55,8 @@ Rectangle { VideoOutput { anchors.fill: parent fillMode: VideoOutput.PreserveAspectFit - source: media + flushMode: VideoOutput.FirstFrame + source: mxcmedia } } @@ -93,15 +94,15 @@ Rectangle { return hh + ":" + mm + ":" + ss; } - positionText.text = formatTime(new Date(media.position)); - durationText.text = formatTime(new Date(media.duration)); + positionText.text = formatTime(new Date(mxcmedia.position)); + durationText.text = formatTime(new Date(mxcmedia.duration)); } Layout.fillWidth: true - value: media.position + value: mxcmedia.position from: 0 - to: media.duration - onMoved: media.seek(value) + to: mxcmedia.duration + onMoved: mxcmedia.position = value onValueChanged: updatePositionTexts() palette: Nheko.colors } @@ -132,15 +133,15 @@ Rectangle { onClicked: { switch (button.state) { case "": - room.cacheMedia(eventId); + mxcmedia.eventId = eventId; break; case "stopped": - media.play(); + mxcmedia.play(); console.log("play"); button.state = "playing"; break; case "playing": - media.pause(); + mxcmedia.pause(); console.log("pause"); button.state = "stopped"; break; @@ -172,29 +173,22 @@ Rectangle { cursorShape: Qt.PointingHandCursor } - MediaPlayer { - id: media + MxcMedia { + id: mxcmedia + roomm: room onError: console.log(errorString) - onStatusChanged: { - if (status == MediaPlayer.Loaded) + onMediaStatusChanged: { + if (status == MxcMedia.LoadedMedia) { progress.updatePositionTexts(); - - } - onStopped: button.state = "stopped" - } - - Connections { - function onMediaCached(mxcUrl, cacheUrl) { - if (mxcUrl == url) { - media.source = cacheUrl; button.state = "stopped"; - console.log("media loaded: " + mxcUrl + " at " + cacheUrl); } - console.log("media cached: " + mxcUrl + " at " + cacheUrl); } - - target: room + onStateChanged: { + if (state == MxcMedia.StoppedState) { + button.state = "stopped"; + } + } } } diff --git a/src/timeline/TimelineModel.h b/src/timeline/TimelineModel.h index e3ca8811..417fbb7f 100644 --- a/src/timeline/TimelineModel.h +++ b/src/timeline/TimelineModel.h @@ -293,6 +293,15 @@ public: crypto::Trust trustlevel() const; int roomMemberCount() const; + std::optional eventById(const QString &id) + { + auto e = events.get(id.toStdString(), ""); + if (e) + return *e; + else + return std::nullopt; + } + public slots: void setCurrentIndex(int index); int currentIndex() const { return idToIndex(currentId); } diff --git a/src/timeline/TimelineViewManager.cpp b/src/timeline/TimelineViewManager.cpp index 97b60b0c..f42ec02f 100644 --- a/src/timeline/TimelineViewManager.cpp +++ b/src/timeline/TimelineViewManager.cpp @@ -35,6 +35,7 @@ #include "dialogs/ImageOverlay.h" #include "emoji/EmojiModel.h" #include "emoji/Provider.h" +#include "ui/MxcMediaProxy.h" #include "ui/NhekoCursorShape.h" #include "ui/NhekoDropArea.h" #include "ui/NhekoGlobalObject.h" @@ -176,6 +177,7 @@ TimelineViewManager::TimelineViewManager(CallManager *callManager, ChatPage *par qmlRegisterType("im.nheko", 1, 0, "DelegateChooser"); qmlRegisterType("im.nheko", 1, 0, "NhekoDropArea"); qmlRegisterType("im.nheko", 1, 0, "CursorShape"); + qmlRegisterType("im.nheko", 1, 0, "MxcMedia"); qmlRegisterUncreatableType( "im.nheko", 1, 0, "DeviceVerificationFlow", "Can't create verification flow from QML!"); qmlRegisterUncreatableType( diff --git a/src/ui/MxcMediaProxy.cpp b/src/ui/MxcMediaProxy.cpp new file mode 100644 index 00000000..c1de2c31 --- /dev/null +++ b/src/ui/MxcMediaProxy.cpp @@ -0,0 +1,142 @@ +// SPDX-FileCopyrightText: 2021 Nheko Contributors +// +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "MxcMediaProxy.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "EventAccessors.h" +#include "Logging.h" +#include "MatrixClient.h" +#include "timeline/TimelineModel.h" + +void +MxcMediaProxy::setVideoSurface(QAbstractVideoSurface *surface) +{ + qDebug() << "Changing surface"; + m_surface = surface; + setVideoOutput(m_surface); +} + +QAbstractVideoSurface * +MxcMediaProxy::getVideoSurface() +{ + return m_surface; +} + +void +MxcMediaProxy::startDownload() +{ + if (!room_) + return; + if (eventId_.isEmpty()) + return; + + auto event = room_->eventById(eventId_); + if (!event) { + nhlog::ui()->error("Failed to load media for event {}, event not found.", + eventId_.toStdString()); + return; + } + + QString mxcUrl = QString::fromStdString(mtx::accessors::url(*event)); + QString originalFilename = QString::fromStdString(mtx::accessors::filename(*event)); + QString mimeType = QString::fromStdString(mtx::accessors::mimetype(*event)); + + auto encryptionInfo = mtx::accessors::file(*event); + + // If the message is a link to a non mxcUrl, don't download it + if (!mxcUrl.startsWith("mxc://")) { + return; + } + + QString suffix = QMimeDatabase().mimeTypeForName(mimeType).preferredSuffix(); + + const auto url = mxcUrl.toStdString(); + const auto name = QString(mxcUrl).remove("mxc://"); + QFileInfo filename(QString("%1/media_cache/media/%2.%3") + .arg(QStandardPaths::writableLocation(QStandardPaths::CacheLocation)) + .arg(name) + .arg(suffix)); + if (QDir::cleanPath(name) != name) { + nhlog::net()->warn("mxcUrl '{}' is not safe, not downloading file", url); + return; + } + + QDir().mkpath(filename.path()); + + QPointer self = this; + + auto processBuffer = [this, encryptionInfo, filename, self](QIODevice &device) { + if (!self) + return; + + if (encryptionInfo) { + QByteArray ba = device.readAll(); + std::string temp(ba.constData(), ba.size()); + temp = mtx::crypto::to_string( + mtx::crypto::decrypt_file(temp, encryptionInfo.value())); + buffer.setData(temp.data(), temp.size()); + } else { + buffer.setData(device.readAll()); + } + buffer.open(QIODevice::ReadOnly); + buffer.reset(); + + QTimer::singleShot(0, this, [this, self, filename] { + nhlog::ui()->info("Playing buffer with size: {}, {}", + buffer.bytesAvailable(), + buffer.isOpen()); + self->setMedia(QMediaContent(filename.fileName()), &buffer); + emit loadedChanged(); + }); + }; + + if (filename.isReadable()) { + QFile f(filename.filePath()); + if (f.open(QIODevice::ReadOnly)) { + processBuffer(f); + return; + } + } + + http::client()->download( + url, + [filename, url, processBuffer](const std::string &data, + const std::string &, + const std::string &, + mtx::http::RequestErr err) { + if (err) { + nhlog::net()->warn("failed to retrieve media {}: {} {}", + url, + err->matrix_error.error, + static_cast(err->status_code)); + return; + } + + try { + QFile file(filename.filePath()); + + if (!file.open(QIODevice::WriteOnly)) + return; + + QByteArray ba(data.data(), (int)data.size()); + file.write(ba); + file.close(); + + QBuffer buf(&ba); + buf.open(QBuffer::ReadOnly); + processBuffer(buf); + } catch (const std::exception &e) { + nhlog::ui()->warn("Error while saving file to: {}", e.what()); + } + }); +} diff --git a/src/ui/MxcMediaProxy.h b/src/ui/MxcMediaProxy.h new file mode 100644 index 00000000..14541815 --- /dev/null +++ b/src/ui/MxcMediaProxy.h @@ -0,0 +1,80 @@ +// SPDX-FileCopyrightText: 2021 Nheko Contributors +// +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "Logging.h" + +class TimelineModel; + +// I failed to get my own buffer into the MediaPlayer in qml, so just make our own. For that we just +// need the videoSurface property, so that part is really easy! +class MxcMediaProxy : public QMediaPlayer +{ + Q_OBJECT + Q_PROPERTY(TimelineModel *roomm READ room WRITE setRoom NOTIFY roomChanged REQUIRED) + Q_PROPERTY(QString eventId READ eventId WRITE setEventId NOTIFY eventIdChanged) + Q_PROPERTY(QAbstractVideoSurface *videoSurface READ getVideoSurface WRITE setVideoSurface) + Q_PROPERTY(bool loaded READ loaded NOTIFY loadedChanged) +public: + MxcMediaProxy(QObject *parent = nullptr) + : QMediaPlayer(parent) + { + connect(this, &MxcMediaProxy::eventIdChanged, &MxcMediaProxy::startDownload); + connect(this, &MxcMediaProxy::roomChanged, &MxcMediaProxy::startDownload); + connect(this, + qOverload(&MxcMediaProxy::error), + [this](QMediaPlayer::Error error) { + nhlog::ui()->info("Media player error {} and errorStr {}", + error, + this->errorString().toStdString()); + }); + connect(this, + &MxcMediaProxy::mediaStatusChanged, + [this](QMediaPlayer::MediaStatus status) { + nhlog::ui()->info( + "Media player status {} and error {}", status, this->error()); + }); + } + + bool loaded() const { return buffer.size() > 0; } + QString eventId() const { return eventId_; } + TimelineModel *room() const { return room_; } + void setEventId(QString newEventId) + { + eventId_ = newEventId; + emit eventIdChanged(); + } + void setRoom(TimelineModel *room) + { + room_ = room; + emit roomChanged(); + } + void setVideoSurface(QAbstractVideoSurface *surface); + QAbstractVideoSurface *getVideoSurface(); + +signals: + void roomChanged(); + void eventIdChanged(); + void loadedChanged(); + void newBuffer(QMediaContent, QIODevice *buf); + +private slots: + void startDownload(); + +private: + TimelineModel *room_ = nullptr; + QString eventId_; + QString filename_; + QBuffer buffer; + QAbstractVideoSurface *m_surface = nullptr; +};