From fd83858715dfde7ad61284200821c0357d68ebf8 Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Mon, 21 Mar 2022 00:48:27 +0100 Subject: [PATCH] Add duration and resolution to files --- resources/qml/MessageView.qml | 2 + resources/qml/TimelineRow.qml | 3 + resources/qml/delegates/MessageDelegate.qml | 3 + .../qml/delegates/PlayableMediaMessage.qml | 3 +- resources/qml/delegates/Reply.qml | 2 + resources/qml/ui/media/MediaControls.qml | 2 +- src/EventAccessors.cpp | 19 ++++ src/EventAccessors.h | 2 + src/timeline/InputBar.cpp | 98 ++++++++++++++++- src/timeline/InputBar.h | 100 +++++++++++++++++- src/timeline/TimelineModel.cpp | 4 + src/timeline/TimelineModel.h | 1 + 12 files changed, 230 insertions(+), 9 deletions(-) diff --git a/resources/qml/MessageView.qml b/resources/qml/MessageView.qml index 4fce9a75..bbe61ee9 100644 --- a/resources/qml/MessageView.qml +++ b/resources/qml/MessageView.qml @@ -376,6 +376,7 @@ Item { required property string filesize required property string url required property string thumbnailUrl + required property string duration required property bool isOnlyEmoji required property bool isSender required property bool isEncrypted @@ -492,6 +493,7 @@ Item { filesize: wrapper.filesize url: wrapper.url thumbnailUrl: wrapper.thumbnailUrl + duration: wrapper.duration isOnlyEmoji: wrapper.isOnlyEmoji isSender: wrapper.isSender isEncrypted: wrapper.isEncrypted diff --git a/resources/qml/TimelineRow.qml b/resources/qml/TimelineRow.qml index bb6514d1..032821ba 100644 --- a/resources/qml/TimelineRow.qml +++ b/resources/qml/TimelineRow.qml @@ -41,6 +41,7 @@ Item { required property var reactions required property int trustlevel required property int encryptionError + required property int duration required property var timestamp required property int status required property int relatedEventCacheBuster @@ -128,6 +129,7 @@ Item { userId: r.relatedEventCacheBuster, fromModel(Room.UserId) ?? "" userName: r.relatedEventCacheBuster, fromModel(Room.UserName) ?? "" thumbnailUrl: r.relatedEventCacheBuster, fromModel(Room.ThumbnailUrl) ?? "" + duration: r.relatedEventCacheBuster, fromModel(Room.Duration) ?? "" roomTopic: r.relatedEventCacheBuster, fromModel(Room.RoomTopic) ?? "" roomName: r.relatedEventCacheBuster, fromModel(Room.RoomName) ?? "" callType: r.relatedEventCacheBuster, fromModel(Room.CallType) ?? "" @@ -154,6 +156,7 @@ Item { typeString: r.typeString ?? "" url: r.url thumbnailUrl: r.thumbnailUrl + duration: r.duration originalWidth: r.originalWidth isOnlyEmoji: r.isOnlyEmoji isStateEvent: r.isStateEvent diff --git a/resources/qml/delegates/MessageDelegate.qml b/resources/qml/delegates/MessageDelegate.qml index 08b2098e..0e211ded 100644 --- a/resources/qml/delegates/MessageDelegate.qml +++ b/resources/qml/delegates/MessageDelegate.qml @@ -18,6 +18,7 @@ Item { required property int type required property string typeString required property int originalWidth + required property int duration required property string blurhash required property string body required property string formattedBody @@ -161,6 +162,7 @@ Item { url: d.url body: d.body filesize: d.filesize + duration: d.duration metadataWidth: d.metadataWidth } @@ -178,6 +180,7 @@ Item { url: d.url body: d.body filesize: d.filesize + duration: d.duration metadataWidth: d.metadataWidth } diff --git a/resources/qml/delegates/PlayableMediaMessage.qml b/resources/qml/delegates/PlayableMediaMessage.qml index 5d7beaad..40572704 100644 --- a/resources/qml/delegates/PlayableMediaMessage.qml +++ b/resources/qml/delegates/PlayableMediaMessage.qml @@ -17,6 +17,7 @@ Item { required property double proportionalHeight required property int type required property int originalWidth + required property int duration required property string thumbnailUrl required property string eventId required property string url @@ -85,7 +86,7 @@ Item { anchors.bottom: fileInfoLabel.top playingVideo: type == MtxEvent.VideoMessage positionValue: mxcmedia.position - duration: mxcmedia.duration + duration: mediaLoaded ? mxcmedia.duration : content.duration mediaLoaded: mxcmedia.loaded mediaState: mxcmedia.state onPositionChanged: mxcmedia.position = position diff --git a/resources/qml/delegates/Reply.qml b/resources/qml/delegates/Reply.qml index 513b7c0b..27fb4e07 100644 --- a/resources/qml/delegates/Reply.qml +++ b/resources/qml/delegates/Reply.qml @@ -34,6 +34,7 @@ Item { property string roomTopic property string roomName property string callType + property int duration property int encryptionError property int relatedEventCacheBuster property int maxWidth @@ -112,6 +113,7 @@ Item { typeString: r.typeString ?? "" url: r.url thumbnailUrl: r.thumbnailUrl + duration: r.duration originalWidth: r.originalWidth isOnlyEmoji: r.isOnlyEmoji isStateEvent: r.isStateEvent diff --git a/resources/qml/ui/media/MediaControls.qml b/resources/qml/ui/media/MediaControls.qml index 1844af73..d73957ee 100644 --- a/resources/qml/ui/media/MediaControls.qml +++ b/resources/qml/ui/media/MediaControls.qml @@ -214,7 +214,7 @@ Rectangle { Label { Layout.alignment: Qt.AlignRight - text: (!control.mediaLoaded) ? "-- / --" : (durationToString(control.positionValue) + " / " + durationToString(control.duration)) + text: (!control.mediaLoaded ? "-- " : durationToString(control.positionValue)) + " / " + durationToString(control.duration) color: Nheko.colors.text } diff --git a/src/EventAccessors.cpp b/src/EventAccessors.cpp index 935ff73a..00cea86e 100644 --- a/src/EventAccessors.cpp +++ b/src/EventAccessors.cpp @@ -169,6 +169,20 @@ struct EventThumbnailUrl } }; +struct EventDuration +{ + template + using thumbnail_url_t = decltype(Content::info.duration); + template + uint64_t operator()(const mtx::events::Event &e) + { + if constexpr (is_detected::value) { + return e.content.info.duration; + } + return 0; + } +}; + struct EventBlurhash { template @@ -420,6 +434,11 @@ mtx::accessors::thumbnail_url(const mtx::events::collections::TimelineEvents &ev { return std::visit(EventThumbnailUrl{}, event); } +uint64_t +mtx::accessors::duration(const mtx::events::collections::TimelineEvents &event) +{ + return std::visit(EventDuration{}, event); +} std::string mtx::accessors::blurhash(const mtx::events::collections::TimelineEvents &event) { diff --git a/src/EventAccessors.h b/src/EventAccessors.h index e46d4786..a74c58bc 100644 --- a/src/EventAccessors.h +++ b/src/EventAccessors.h @@ -83,6 +83,8 @@ std::string url(const mtx::events::collections::TimelineEvents &event); std::string thumbnail_url(const mtx::events::collections::TimelineEvents &event); +uint64_t +duration(const mtx::events::collections::TimelineEvents &event); std::string blurhash(const mtx::events::collections::TimelineEvents &event); std::string diff --git a/src/timeline/InputBar.cpp b/src/timeline/InputBar.cpp index 1b7d6efb..eda4507a 100644 --- a/src/timeline/InputBar.cpp +++ b/src/timeline/InputBar.cpp @@ -11,6 +11,8 @@ #include #include #include +#include +#include #include #include #include @@ -452,7 +454,8 @@ InputBar::audio(const QString &filename, const std::optional &file, const QString &url, const QString &mime, - uint64_t dsize) + uint64_t dsize, + uint64_t duration) { mtx::events::msg::Audio audio; audio.info.mimetype = mime.toStdString(); @@ -460,6 +463,9 @@ InputBar::audio(const QString &filename, audio.body = filename.toStdString(); audio.url = url.toStdString(); + if (duration > 0) + audio.info.duration = duration; + if (file) audio.file = file; else @@ -482,13 +488,22 @@ InputBar::video(const QString &filename, const std::optional &file, const QString &url, const QString &mime, - uint64_t dsize) + uint64_t dsize, + uint64_t duration, + const QSize &dimensions) { mtx::events::msg::Video video; video.info.mimetype = mime.toStdString(); video.info.size = dsize; video.body = filename.toStdString(); + if (duration > 0) + video.info.duration = duration; + if (dimensions.isValid()) { + video.info.h = dimensions.height(); + video.info.w = dimensions.width(); + } + if (file) video.file = file; else @@ -645,6 +660,7 @@ MediaUpload::MediaUpload(std::unique_ptr source_, source->open(QIODevice::ReadOnly); data = source->readAll(); + source->reset(); if (!data.size()) { nhlog::ui()->warn("Attempted to upload zero-byte file?! Mimetype {}, filename {}", @@ -657,6 +673,8 @@ MediaUpload::MediaUpload(std::unique_ptr source_, nhlog::ui()->debug("Mime: {}", mimetype_.toStdString()); if (mimeClass_ == u"image") { QImage img = utils::readImage(data); + setThumbnail(img.scaled( + std::min(800, img.width()), std::min(800, img.height()), Qt::KeepAspectRatioByExpanding)); dimensions_ = img.size(); if (img.height() > 200 && img.width() > 360) @@ -672,6 +690,78 @@ MediaUpload::MediaUpload(std::unique_ptr source_, } blurhash_ = QString::fromStdString(blurhash::encode(data_.data(), img.width(), img.height(), 4, 3)); + } else if (mimeClass_ == u"video" || mimeClass_ == u"audio") { + auto mediaPlayer = new QMediaPlayer( + this, + mimeClass_ == u"video" ? QFlags{QMediaPlayer::StreamPlayback, QMediaPlayer::VideoSurface} + : QFlags{QMediaPlayer::StreamPlayback}); + mediaPlayer->setMuted(true); + + if (mimeClass_ == u"video") { + auto newSurface = new InputVideoSurface(this); + connect( + newSurface, &InputVideoSurface::newImage, this, [this, mediaPlayer](QImage img) { + mediaPlayer->stop(); + + nhlog::ui()->debug("Got image {}x{}", img.width(), img.height()); + + this->setThumbnail(img); + + if (!dimensions_.isValid()) + this->dimensions_ = img.size(); + + if (img.height() > 200 && img.width() > 360) + img = img.scaled(360, 200, Qt::KeepAspectRatioByExpanding); + std::vector data_; + for (int y = 0; y < img.height(); y++) { + for (int x = 0; x < img.width(); x++) { + auto p = img.pixel(x, y); + data_.push_back(static_cast(qRed(p))); + data_.push_back(static_cast(qGreen(p))); + data_.push_back(static_cast(qBlue(p))); + } + } + blurhash_ = QString::fromStdString( + blurhash::encode(data_.data(), img.width(), img.height(), 4, 3)); + }); + mediaPlayer->setVideoOutput(newSurface); + } + + connect(mediaPlayer, + qOverload(&QMediaPlayer::error), + this, + [this, mediaPlayer](QMediaPlayer::Error error) { + nhlog::ui()->info("Media player error {} and errorStr {}", + error, + mediaPlayer->errorString().toStdString()); + }); + connect(mediaPlayer, + &QMediaPlayer::mediaStatusChanged, + [this, mediaPlayer](QMediaPlayer::MediaStatus status) { + nhlog::ui()->info( + "Media player status {} and error {}", status, mediaPlayer->error()); + }); + connect(mediaPlayer, + qOverload(&QMediaPlayer::metaDataChanged), + [this, mediaPlayer](QString t, QVariant) { + nhlog::ui()->info("Got metadata {}", t.toStdString()); + + if (mediaPlayer->duration() > 0) + this->duration_ = mediaPlayer->duration(); + + dimensions_ = mediaPlayer->metaData(QMediaMetaData::Resolution).toSize(); + auto orientation = mediaPlayer->metaData(QMediaMetaData::Orientation).toInt(); + if (orientation == 90 || orientation == 270) { + dimensions_.transpose(); + } + }); + connect(mediaPlayer, &QMediaPlayer::durationChanged, [this, mediaPlayer](qint64 duration) { + if (duration > 0) + this->duration_ = mediaPlayer->duration(); + nhlog::ui()->info("Duration changed {}", duration); + }); + mediaPlayer->setMedia(QMediaContent(originalFilename_), source.get()); + mediaPlayer->play(); } } @@ -721,9 +811,9 @@ InputBar::finalizeUpload(MediaUpload *upload, QString url) if (mimeClass == u"image") image(filename, encryptedFile, url, mime, size, upload->dimensions(), upload->blurhash()); else if (mimeClass == u"audio") - audio(filename, encryptedFile, url, mime, size); + audio(filename, encryptedFile, url, mime, size, upload->duration()); else if (mimeClass == u"video") - video(filename, encryptedFile, url, mime, size); + video(filename, encryptedFile, url, mime, size, upload->duration(), upload->dimensions()); else file(filename, encryptedFile, url, mime, size); diff --git a/src/timeline/InputBar.h b/src/timeline/InputBar.h index 607736b6..97d262cc 100644 --- a/src/timeline/InputBar.h +++ b/src/timeline/InputBar.h @@ -5,7 +5,9 @@ #pragma once +#include #include +#include #include #include #include @@ -29,6 +31,90 @@ enum class MarkdownOverride OFF, }; +class InputVideoSurface : public QAbstractVideoSurface +{ + Q_OBJECT + +public: + InputVideoSurface(QObject *parent) + : QAbstractVideoSurface(parent) + {} + + bool present(const QVideoFrame &frame) override + { + QImage::Format format = QImage::Format_Invalid; + + switch (frame.pixelFormat()) { + case QVideoFrame::Format_ARGB32: + format = QImage::Format_ARGB32; + break; + case QVideoFrame::Format_ARGB32_Premultiplied: + format = QImage::Format_ARGB32_Premultiplied; + break; + case QVideoFrame::Format_RGB24: + format = QImage::Format_RGB888; + break; + case QVideoFrame::Format_BGR24: + format = QImage::Format_BGR888; + break; + case QVideoFrame::Format_RGB32: + format = QImage::Format_RGB32; + break; + case QVideoFrame::Format_RGB565: + format = QImage::Format_RGB16; + break; + case QVideoFrame::Format_RGB555: + format = QImage::Format_RGB555; + break; + default: + format = QImage::Format_Invalid; + } + + if (format == QImage::Format_Invalid) { + emit newImage({}); + return false; + } else { + QVideoFrame frametodraw(frame); + + if (!frametodraw.map(QAbstractVideoBuffer::ReadOnly)) { + emit newImage({}); + return false; + } + + // this is a shallow operation. it just refer the frame buffer + QImage image(frametodraw.bits(), + frametodraw.width(), + frametodraw.height(), + frametodraw.bytesPerLine(), + QImage::Format_RGB444); + + emit newImage(std::move(image)); + return true; + } + } + + QList + supportedPixelFormats(QAbstractVideoBuffer::HandleType type) const override + { + if (type == QAbstractVideoBuffer::NoHandle) { + return { + QVideoFrame::Format_ARGB32, + QVideoFrame::Format_ARGB32_Premultiplied, + QVideoFrame::Format_RGB24, + QVideoFrame::Format_BGR24, + QVideoFrame::Format_RGB32, + QVideoFrame::Format_RGB565, + QVideoFrame::Format_RGB555, + }; + } else { + return {}; + } + } + +signals: + void newImage(QImage img); +}; + class MediaUpload : public QObject { Q_OBJECT @@ -67,6 +153,7 @@ public: [[nodiscard]] QString filename() const { return originalFilename_; } [[nodiscard]] QString blurhash() const { return blurhash_; } [[nodiscard]] uint64_t size() const { return size_; } + [[nodiscard]] uint64_t duration() const { return duration_; } [[nodiscard]] std::optional encryptedFile_() { return encryptedFile; @@ -82,6 +169,7 @@ public slots: private slots: void updateThumbnailUrl(QString url) { this->thumbnailUrl_ = std::move(url); } + void setThumbnail(QImage img) { this->thumbnail_ = std::move(img); } public: // void uploadThumbnail(QImage img); @@ -96,8 +184,11 @@ public: QString url_; std::optional encryptedFile; + QImage thumbnail_; + QSize dimensions_; - uint64_t size_ = 0; + uint64_t size_ = 0; + uint64_t duration_ = 0; bool encrypt_; }; @@ -181,12 +272,15 @@ private: const std::optional &file, const QString &url, const QString &mime, - uint64_t dsize); + uint64_t dsize, + uint64_t duration); void video(const QString &filename, const std::optional &file, const QString &url, const QString &mime, - uint64_t dsize); + uint64_t dsize, + uint64_t duration, + const QSize &dimensions); void startUploadFromPath(const QString &path); void startUploadFromMimeData(const QMimeData &source, const QString &format); diff --git a/src/timeline/TimelineModel.cpp b/src/timeline/TimelineModel.cpp index 8e6c7235..4c1ce2dc 100644 --- a/src/timeline/TimelineModel.cpp +++ b/src/timeline/TimelineModel.cpp @@ -474,6 +474,7 @@ TimelineModel::roleNames() const {Timestamp, "timestamp"}, {Url, "url"}, {ThumbnailUrl, "thumbnailUrl"}, + {Duration, "duration"}, {Blurhash, "blurhash"}, {Filename, "filename"}, {Filesize, "filesize"}, @@ -627,6 +628,8 @@ TimelineModel::data(const mtx::events::collections::TimelineEvents &event, int r return QVariant(QString::fromStdString(url(event))); case ThumbnailUrl: return QVariant(QString::fromStdString(thumbnail_url(event))); + case Duration: + return QVariant(static_cast(duration(event))); case Blurhash: return QVariant(QString::fromStdString(blurhash(event))); case Filename: @@ -739,6 +742,7 @@ TimelineModel::data(const mtx::events::collections::TimelineEvents &event, int r m.insert(names[Timestamp], data(event, static_cast(Timestamp))); m.insert(names[Url], data(event, static_cast(Url))); m.insert(names[ThumbnailUrl], data(event, static_cast(ThumbnailUrl))); + m.insert(names[Duration], data(event, static_cast(Duration))); m.insert(names[Blurhash], data(event, static_cast(Blurhash))); m.insert(names[Filename], data(event, static_cast(Filename))); m.insert(names[Filesize], data(event, static_cast(Filesize))); diff --git a/src/timeline/TimelineModel.h b/src/timeline/TimelineModel.h index f47203f0..7e21a394 100644 --- a/src/timeline/TimelineModel.h +++ b/src/timeline/TimelineModel.h @@ -215,6 +215,7 @@ public: Timestamp, Url, ThumbnailUrl, + Duration, Blurhash, Filename, Filesize,