rnhmjoj
/
nheko
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Use in memory media player instead of storing unencrypted files on disk

This commit is contained in:
Nicolas Werner 2021-08-28 00:38:33 +02:00
parent 4ddf067408
commit 09c041c8ac
No known key found for this signature in database
GPG Key ID: C8D75E610773F2D9
9 changed files with 267 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
CMakeLists.txt
View File

@ -311,6 +311,7 @@ set(SRC_FILES
src/ui/InfoMessage.cpp
src/ui/Label.cpp
src/ui/LoadingIndicator.cpp
src/ui/MxcMediaProxy.cpp
src/ui/NhekoCursorShape.cpp
src/ui/NhekoDropArea.cpp
src/ui/NhekoGlobalObject.cpp
@ -350,7 +351,7 @@ set(SRC_FILES
src/MemberList.cpp
src/MxcImageProvider.cpp
src/Olm.cpp
src/ReadReceiptsModel.cpp
src/ReadReceiptsModel.cpp
src/RegisterPage.cpp
src/SSOHandler.cpp
src/CombinedImagePackModel.cpp
@ -521,6 +522,7 @@ qt5_wrap_cpp(MOC_HEADERS
src/ui/InfoMessage.h
src/ui/Label.h
src/ui/LoadingIndicator.h
src/ui/MxcMediaProxy.h
src/ui/Menu.h
src/ui/NhekoCursorShape.h
src/ui/NhekoDropArea.h

5
resources/qml/ForwardCompleter.qml
View File

@ -85,11 +85,10 @@ Popup {
completerPopup.up();
} else if ((event.key == Qt.Key_Down || event.key == Qt.Key_Tab) && completerPopup.opened) {
event.accepted = true;
if (event.key == Qt.Key_Tab && (event.modifiers & Qt.ShiftModifier)) {
if (event.key == Qt.Key_Tab && (event.modifiers & Qt.ShiftModifier))
completerPopup.up();
} else {
else
completerPopup.down();
}
} else if (event.matches(StandardKey.InsertParagraphSeparator)) {
completerPopup.finishCompletion();
event.accepted = true;

9
resources/qml/MessageInput.qml
View File

@ -134,9 +134,9 @@ Rectangle {
return ;
room.input.updateState(selectionStart, selectionEnd, cursorPosition, text);
if (popup.opened && cursorPosition <= completerTriggeredAt) {
if (popup.opened && cursorPosition <= completerTriggeredAt)
popup.close();
}
if (popup.opened)
popup.completer.setSearchString(messageInput.getText(completerTriggeredAt, cursorPosition));
@ -195,11 +195,10 @@ Rectangle {
} else if (event.key == Qt.Key_Tab) {
event.accepted = true;
if (popup.opened) {
if (event.modifiers & Qt.ShiftModifier) {
if (event.modifiers & Qt.ShiftModifier)
popup.down();
} else {
else
popup.up();
}
} else {
var pos = cursorPosition - 1;
while (pos > -1) {

5
resources/qml/QuickSwitcher.qml
View File

@ -44,11 +44,10 @@ Popup {
completerPopup.up();
} else if ((event.key == Qt.Key_Down || event.key == Qt.Key_Tab) && completerPopup.opened) {
event.accepted = true;
if (event.key == Qt.Key_Tab && (event.modifiers & Qt.ShiftModifier)) {
if (event.key == Qt.Key_Tab && (event.modifiers & Qt.ShiftModifier))
completerPopup.up();
} else {
else
completerPopup.down();
}
} else if (event.matches(StandardKey.InsertParagraphSeparator)) {
completerPopup.finishCompletion();
event.accepted = true;

52
resources/qml/delegates/PlayableMediaMessage.qml
View File

@ -3,9 +3,9 @@
// SPDX-License-Identifier: GPL-3.0-or-later
import "../"
import QtMultimedia 5.6
import QtQuick 2.12
import QtQuick.Controls 2.1
import QtMultimedia 5.15
import QtQuick 2.15
import QtQuick.Controls 2.15
import QtQuick.Layouts 1.2
import im.nheko 1.0
@ -55,7 +55,8 @@ Rectangle {
VideoOutput {
anchors.fill: parent
fillMode: VideoOutput.PreserveAspectFit
source: media
flushMode: VideoOutput.FirstFrame
source: mxcmedia
}
}
@ -93,15 +94,15 @@ Rectangle {
return hh + ":" + mm + ":" + ss;
}
positionText.text = formatTime(new Date(media.position));
durationText.text = formatTime(new Date(media.duration));
positionText.text = formatTime(new Date(mxcmedia.position));
durationText.text = formatTime(new Date(mxcmedia.duration));
}
Layout.fillWidth: true
value: media.position
value: mxcmedia.position
from: 0
to: media.duration
onMoved: media.seek(value)
to: mxcmedia.duration
onMoved: mxcmedia.position = value
onValueChanged: updatePositionTexts()
palette: Nheko.colors
}
@ -132,15 +133,15 @@ Rectangle {
onClicked: {
switch (button.state) {
case "":
room.cacheMedia(eventId);
mxcmedia.eventId = eventId;
break;
case "stopped":
media.play();
mxcmedia.play();
console.log("play");
button.state = "playing";
break;
case "playing":
media.pause();
mxcmedia.pause();
console.log("pause");
button.state = "stopped";
break;
@ -172,29 +173,22 @@ Rectangle {
cursorShape: Qt.PointingHandCursor
}
MediaPlayer {
id: media
MxcMedia {
id: mxcmedia
roomm: room
onError: console.log(errorString)
onStatusChanged: {
if (status == MediaPlayer.Loaded)
onMediaStatusChanged: {
if (status == MxcMedia.LoadedMedia) {
progress.updatePositionTexts();
}
onStopped: button.state = "stopped"
}
Connections {
function onMediaCached(mxcUrl, cacheUrl) {
if (mxcUrl == url) {
media.source = cacheUrl;
button.state = "stopped";
console.log("media loaded: " + mxcUrl + " at " + cacheUrl);
}
console.log("media cached: " + mxcUrl + " at " + cacheUrl);
}
target: room
onStateChanged: {
if (state == MxcMedia.StoppedState) {
button.state = "stopped";
}
}
}
}

9
src/timeline/TimelineModel.h
View File

@ -293,6 +293,15 @@ public:
crypto::Trust trustlevel() const;
int roomMemberCount() const;
std::optional<mtx::events::collections::TimelineEvents> eventById(const QString &id)
{
auto e = events.get(id.toStdString(), "");
if (e)
return *e;
else
return std::nullopt;
}
public slots:
void setCurrentIndex(int index);
int currentIndex() const { return idToIndex(currentId); }

2
src/timeline/TimelineViewManager.cpp
View File

@ -35,6 +35,7 @@
#include "dialogs/ImageOverlay.h"
#include "emoji/EmojiModel.h"
#include "emoji/Provider.h"
#include "ui/MxcMediaProxy.h"
#include "ui/NhekoCursorShape.h"
#include "ui/NhekoDropArea.h"
#include "ui/NhekoGlobalObject.h"
@ -176,6 +177,7 @@ TimelineViewManager::TimelineViewManager(CallManager *callManager, ChatPage *par
qmlRegisterType<DelegateChooser>("im.nheko", 1, 0, "DelegateChooser");
qmlRegisterType<NhekoDropArea>("im.nheko", 1, 0, "NhekoDropArea");
qmlRegisterType<NhekoCursorShape>("im.nheko", 1, 0, "CursorShape");
qmlRegisterType<MxcMediaProxy>("im.nheko", 1, 0, "MxcMedia");
qmlRegisterUncreatableType<DeviceVerificationFlow>(
"im.nheko", 1, 0, "DeviceVerificationFlow", "Can't create verification flow from QML!");
qmlRegisterUncreatableType<UserProfile>(

142
src/ui/MxcMediaProxy.cpp Normal file
View File

@ -0,0 +1,142 @@
// SPDX-FileCopyrightText: 2021 Nheko Contributors
//
// SPDX-License-Identifier: GPL-3.0-or-later
#include "MxcMediaProxy.h"
#include <QDir>
#include <QFile>
#include <QFileInfo>
#include <QMediaObject>
#include <QMediaPlayer>
#include <QMimeDatabase>
#include <QStandardPaths>
#include <QUrl>
#include "EventAccessors.h"
#include "Logging.h"
#include "MatrixClient.h"
#include "timeline/TimelineModel.h"
void
MxcMediaProxy::setVideoSurface(QAbstractVideoSurface *surface)
{
qDebug() << "Changing surface";
m_surface = surface;
setVideoOutput(m_surface);
}
QAbstractVideoSurface *
MxcMediaProxy::getVideoSurface()
{
return m_surface;
}
void
MxcMediaProxy::startDownload()
{
if (!room_)
return;
if (eventId_.isEmpty())
return;
auto event = room_->eventById(eventId_);
if (!event) {
nhlog::ui()->error("Failed to load media for event {}, event not found.",
eventId_.toStdString());
return;
}
QString mxcUrl = QString::fromStdString(mtx::accessors::url(*event));
QString originalFilename = QString::fromStdString(mtx::accessors::filename(*event));
QString mimeType = QString::fromStdString(mtx::accessors::mimetype(*event));
auto encryptionInfo = mtx::accessors::file(*event);
// If the message is a link to a non mxcUrl, don't download it
if (!mxcUrl.startsWith("mxc://")) {
return;
}
QString suffix = QMimeDatabase().mimeTypeForName(mimeType).preferredSuffix();
const auto url = mxcUrl.toStdString();
const auto name = QString(mxcUrl).remove("mxc://");
QFileInfo filename(QString("%1/media_cache/media/%2.%3")
.arg(QStandardPaths::writableLocation(QStandardPaths::CacheLocation))
.arg(name)
.arg(suffix));
if (QDir::cleanPath(name) != name) {
nhlog::net()->warn("mxcUrl '{}' is not safe, not downloading file", url);
return;
}
QDir().mkpath(filename.path());
QPointer<MxcMediaProxy> self = this;
auto processBuffer = [this, encryptionInfo, filename, self](QIODevice &device) {
if (!self)
return;
if (encryptionInfo) {
QByteArray ba = device.readAll();
std::string temp(ba.constData(), ba.size());
temp = mtx::crypto::to_string(
mtx::crypto::decrypt_file(temp, encryptionInfo.value()));
buffer.setData(temp.data(), temp.size());
} else {
buffer.setData(device.readAll());
}
buffer.open(QIODevice::ReadOnly);
buffer.reset();
QTimer::singleShot(0, this, [this, self, filename] {
nhlog::ui()->info("Playing buffer with size: {}, {}",
buffer.bytesAvailable(),
buffer.isOpen());
self->setMedia(QMediaContent(filename.fileName()), &buffer);
emit loadedChanged();
});
};
if (filename.isReadable()) {
QFile f(filename.filePath());
if (f.open(QIODevice::ReadOnly)) {
processBuffer(f);
return;
}
}
http::client()->download(
url,
[filename, url, processBuffer](const std::string &data,
const std::string &,
const std::string &,
mtx::http::RequestErr err) {
if (err) {
nhlog::net()->warn("failed to retrieve media {}: {} {}",
url,
err->matrix_error.error,
static_cast<int>(err->status_code));
return;
}
try {
QFile file(filename.filePath());
if (!file.open(QIODevice::WriteOnly))
return;
QByteArray ba(data.data(), (int)data.size());
file.write(ba);
file.close();
QBuffer buf(&ba);
buf.open(QBuffer::ReadOnly);
processBuffer(buf);
} catch (const std::exception &e) {
nhlog::ui()->warn("Error while saving file to: {}", e.what());
}
});
}

80
src/ui/MxcMediaProxy.h Normal file
View File

@ -0,0 +1,80 @@
// SPDX-FileCopyrightText: 2021 Nheko Contributors
//
// SPDX-License-Identifier: GPL-3.0-or-later
#pragma once
#include <QAbstractVideoSurface>
#include <QBuffer>
#include <QMediaContent>
#include <QMediaPlayer>
#include <QObject>
#include <QPointer>
#include <QString>
#include "Logging.h"
class TimelineModel;
// I failed to get my own buffer into the MediaPlayer in qml, so just make our own. For that we just
// need the videoSurface property, so that part is really easy!
class MxcMediaProxy : public QMediaPlayer
{
Q_OBJECT
Q_PROPERTY(TimelineModel *roomm READ room WRITE setRoom NOTIFY roomChanged REQUIRED)
Q_PROPERTY(QString eventId READ eventId WRITE setEventId NOTIFY eventIdChanged)
Q_PROPERTY(QAbstractVideoSurface *videoSurface READ getVideoSurface WRITE setVideoSurface)
Q_PROPERTY(bool loaded READ loaded NOTIFY loadedChanged)
public:
MxcMediaProxy(QObject *parent = nullptr)
: QMediaPlayer(parent)
{
connect(this, &MxcMediaProxy::eventIdChanged, &MxcMediaProxy::startDownload);
connect(this, &MxcMediaProxy::roomChanged, &MxcMediaProxy::startDownload);
connect(this,
qOverload<QMediaPlayer::Error>(&MxcMediaProxy::error),
[this](QMediaPlayer::Error error) {
nhlog::ui()->info("Media player error {} and errorStr {}",
error,
this->errorString().toStdString());
});
connect(this,
&MxcMediaProxy::mediaStatusChanged,
[this](QMediaPlayer::MediaStatus status) {
nhlog::ui()->info(
"Media player status {} and error {}", status, this->error());
});
}
bool loaded() const { return buffer.size() > 0; }
QString eventId() const { return eventId_; }
TimelineModel *room() const { return room_; }
void setEventId(QString newEventId)
{
eventId_ = newEventId;
emit eventIdChanged();
}
void setRoom(TimelineModel *room)
{
room_ = room;
emit roomChanged();
}
void setVideoSurface(QAbstractVideoSurface *surface);
QAbstractVideoSurface *getVideoSurface();
signals:
void roomChanged();
void eventIdChanged();
void loadedChanged();
void newBuffer(QMediaContent, QIODevice *buf);
private slots:
void startDownload();
private:
TimelineModel *room_ = nullptr;
QString eventId_;
QString filename_;
QBuffer buffer;
QAbstractVideoSurface *m_surface = nullptr;
};