Add duration and resolution to files

resources/qml/MessageView.qml

@@ -376,6 +376,7 @@ Item {
             required property string filesize
             required property string url
             required property string thumbnailUrl
+            required property string duration
             required property bool isOnlyEmoji
             required property bool isSender
             required property bool isEncrypted
@@ -492,6 +493,7 @@ Item {
                 filesize: wrapper.filesize
                 url: wrapper.url
                 thumbnailUrl: wrapper.thumbnailUrl
+                duration: wrapper.duration
                 isOnlyEmoji: wrapper.isOnlyEmoji
                 isSender: wrapper.isSender
                 isEncrypted: wrapper.isEncrypted

resources/qml/TimelineRow.qml

@@ -41,6 +41,7 @@ Item {
     required property var reactions
     required property int trustlevel
     required property int encryptionError
+    required property int duration
     required property var timestamp
     required property int status
     required property int relatedEventCacheBuster
@@ -128,6 +129,7 @@ Item {
                 userId: r.relatedEventCacheBuster, fromModel(Room.UserId) ?? ""
                 userName: r.relatedEventCacheBuster, fromModel(Room.UserName) ?? ""
                 thumbnailUrl: r.relatedEventCacheBuster, fromModel(Room.ThumbnailUrl) ?? ""
+                duration: r.relatedEventCacheBuster, fromModel(Room.Duration) ?? ""
                 roomTopic: r.relatedEventCacheBuster, fromModel(Room.RoomTopic) ?? ""
                 roomName: r.relatedEventCacheBuster, fromModel(Room.RoomName) ?? ""
                 callType: r.relatedEventCacheBuster, fromModel(Room.CallType) ?? ""
@@ -154,6 +156,7 @@ Item {
                 typeString: r.typeString ?? ""
                 url: r.url
                 thumbnailUrl: r.thumbnailUrl
+                duration: r.duration
                 originalWidth: r.originalWidth
                 isOnlyEmoji: r.isOnlyEmoji
                 isStateEvent: r.isStateEvent

resources/qml/delegates/MessageDelegate.qml

@@ -18,6 +18,7 @@ Item {
     required property int type
     required property string typeString
     required property int originalWidth
+    required property int duration
     required property string blurhash
     required property string body
     required property string formattedBody
@@ -161,6 +162,7 @@ Item {
                 url: d.url
                 body: d.body
                 filesize: d.filesize
+                duration: d.duration
                 metadataWidth: d.metadataWidth
             }
 
@@ -178,6 +180,7 @@ Item {
                 url: d.url
                 body: d.body
                 filesize: d.filesize
+                duration: d.duration
                 metadataWidth: d.metadataWidth
             }
 

resources/qml/delegates/PlayableMediaMessage.qml

@@ -17,6 +17,7 @@ Item {
     required property double proportionalHeight
     required property int type
     required property int originalWidth
+    required property int duration
     required property string thumbnailUrl
     required property string eventId
     required property string url
@@ -85,7 +86,7 @@ Item {
         anchors.bottom: fileInfoLabel.top
         playingVideo: type == MtxEvent.VideoMessage
         positionValue: mxcmedia.position
-        duration: mxcmedia.duration
+        duration: mediaLoaded ? mxcmedia.duration : content.duration
         mediaLoaded: mxcmedia.loaded
         mediaState: mxcmedia.state
         onPositionChanged: mxcmedia.position = position

resources/qml/delegates/Reply.qml

@@ -34,6 +34,7 @@ Item {
     property string roomTopic
     property string roomName
     property string callType
+    property int duration
     property int encryptionError
     property int relatedEventCacheBuster
     property int maxWidth
@@ -112,6 +113,7 @@ Item {
             typeString: r.typeString ?? ""
             url: r.url
             thumbnailUrl: r.thumbnailUrl
+            duration: r.duration
             originalWidth: r.originalWidth
             isOnlyEmoji: r.isOnlyEmoji
             isStateEvent: r.isStateEvent

resources/qml/ui/media/MediaControls.qml

@@ -214,7 +214,7 @@ Rectangle {
 
             Label {
                 Layout.alignment: Qt.AlignRight
-                text: (!control.mediaLoaded) ? "-- / --" : (durationToString(control.positionValue) + " / " + durationToString(control.duration))
+                text: (!control.mediaLoaded ? "-- " : durationToString(control.positionValue)) + " / " + durationToString(control.duration)
                 color: Nheko.colors.text
             }
 

src/EventAccessors.cpp

@@ -169,6 +169,20 @@ struct EventThumbnailUrl
     }
 };
 
+struct EventDuration
+{
+    template<class Content>
+    using thumbnail_url_t = decltype(Content::info.duration);
+    template<class T>
+    uint64_t operator()(const mtx::events::Event<T> &e)
+    {
+        if constexpr (is_detected<thumbnail_url_t, T>::value) {
+            return e.content.info.duration;
+        }
+        return 0;
+    }
+};
+
 struct EventBlurhash
 {
     template<class Content>
@@ -420,6 +434,11 @@ mtx::accessors::thumbnail_url(const mtx::events::collections::TimelineEvents &ev
 {
     return std::visit(EventThumbnailUrl{}, event);
 }
+uint64_t
+mtx::accessors::duration(const mtx::events::collections::TimelineEvents &event)
+{
+    return std::visit(EventDuration{}, event);
+}
 std::string
 mtx::accessors::blurhash(const mtx::events::collections::TimelineEvents &event)
 {

src/EventAccessors.h

@@ -83,6 +83,8 @@ std::string
 url(const mtx::events::collections::TimelineEvents &event);
 std::string
 thumbnail_url(const mtx::events::collections::TimelineEvents &event);
+uint64_t
+duration(const mtx::events::collections::TimelineEvents &event);
 std::string
 blurhash(const mtx::events::collections::TimelineEvents &event);
 std::string

src/timeline/InputBar.cpp

@@ -11,6 +11,8 @@
 #include <QFileDialog>
 #include <QGuiApplication>
 #include <QInputMethod>
+#include <QMediaMetaData>
+#include <QMediaPlayer>
 #include <QMimeData>
 #include <QMimeDatabase>
 #include <QStandardPaths>
@@ -452,7 +454,8 @@ InputBar::audio(const QString &filename,
                 const std::optional<mtx::crypto::EncryptedFile> &file,
                 const QString &url,
                 const QString &mime,
-                uint64_t dsize)
+                uint64_t dsize,
+                uint64_t duration)
 {
     mtx::events::msg::Audio audio;
     audio.info.mimetype = mime.toStdString();
@@ -460,6 +463,9 @@ InputBar::audio(const QString &filename,
     audio.body          = filename.toStdString();
     audio.url           = url.toStdString();
 
+    if (duration > 0)
+        audio.info.duration = duration;
+
     if (file)
         audio.file = file;
     else
@@ -482,13 +488,22 @@ InputBar::video(const QString &filename,
                 const std::optional<mtx::crypto::EncryptedFile> &file,
                 const QString &url,
                 const QString &mime,
-                uint64_t dsize)
+                uint64_t dsize,
+                uint64_t duration,
+                const QSize &dimensions)
 {
     mtx::events::msg::Video video;
     video.info.mimetype = mime.toStdString();
     video.info.size     = dsize;
     video.body          = filename.toStdString();
 
+    if (duration > 0)
+        video.info.duration = duration;
+    if (dimensions.isValid()) {
+        video.info.h = dimensions.height();
+        video.info.w = dimensions.width();
+    }
+
     if (file)
         video.file = file;
     else
@@ -645,6 +660,7 @@ MediaUpload::MediaUpload(std::unique_ptr<QIODevice> source_,
         source->open(QIODevice::ReadOnly);
 
     data = source->readAll();
+    source->reset();
 
     if (!data.size()) {
         nhlog::ui()->warn("Attempted to upload zero-byte file?! Mimetype {}, filename {}",
@@ -657,6 +673,8 @@ MediaUpload::MediaUpload(std::unique_ptr<QIODevice> source_,
     nhlog::ui()->debug("Mime: {}", mimetype_.toStdString());
     if (mimeClass_ == u"image") {
         QImage img = utils::readImage(data);
+        setThumbnail(img.scaled(
+          std::min(800, img.width()), std::min(800, img.height()), Qt::KeepAspectRatioByExpanding));
 
         dimensions_ = img.size();
         if (img.height() > 200 && img.width() > 360)
@@ -672,6 +690,78 @@ MediaUpload::MediaUpload(std::unique_ptr<QIODevice> source_,
         }
         blurhash_ =
           QString::fromStdString(blurhash::encode(data_.data(), img.width(), img.height(), 4, 3));
+    } else if (mimeClass_ == u"video" || mimeClass_ == u"audio") {
+        auto mediaPlayer = new QMediaPlayer(
+          this,
+          mimeClass_ == u"video" ? QFlags{QMediaPlayer::StreamPlayback, QMediaPlayer::VideoSurface}
+                                 : QFlags{QMediaPlayer::StreamPlayback});
+        mediaPlayer->setMuted(true);
+
+        if (mimeClass_ == u"video") {
+            auto newSurface = new InputVideoSurface(this);
+            connect(
+              newSurface, &InputVideoSurface::newImage, this, [this, mediaPlayer](QImage img) {
+                  mediaPlayer->stop();
+
+                  nhlog::ui()->debug("Got image {}x{}", img.width(), img.height());
+
+                  this->setThumbnail(img);
+
+                  if (!dimensions_.isValid())
+                      this->dimensions_ = img.size();
+
+                  if (img.height() > 200 && img.width() > 360)
+                      img = img.scaled(360, 200, Qt::KeepAspectRatioByExpanding);
+                  std::vector<unsigned char> data_;
+                  for (int y = 0; y < img.height(); y++) {
+                      for (int x = 0; x < img.width(); x++) {
+                          auto p = img.pixel(x, y);
+                          data_.push_back(static_cast<unsigned char>(qRed(p)));
+                          data_.push_back(static_cast<unsigned char>(qGreen(p)));
+                          data_.push_back(static_cast<unsigned char>(qBlue(p)));
+                      }
+                  }
+                  blurhash_ = QString::fromStdString(
+                    blurhash::encode(data_.data(), img.width(), img.height(), 4, 3));
+              });
+            mediaPlayer->setVideoOutput(newSurface);
+        }
+
+        connect(mediaPlayer,
+                qOverload<QMediaPlayer::Error>(&QMediaPlayer::error),
+                this,
+                [this, mediaPlayer](QMediaPlayer::Error error) {
+                    nhlog::ui()->info("Media player error {} and errorStr {}",
+                                      error,
+                                      mediaPlayer->errorString().toStdString());
+                });
+        connect(mediaPlayer,
+                &QMediaPlayer::mediaStatusChanged,
+                [this, mediaPlayer](QMediaPlayer::MediaStatus status) {
+                    nhlog::ui()->info(
+                      "Media player status {} and error {}", status, mediaPlayer->error());
+                });
+        connect(mediaPlayer,
+                qOverload<const QString &, const QVariant &>(&QMediaPlayer::metaDataChanged),
+                [this, mediaPlayer](QString t, QVariant) {
+                    nhlog::ui()->info("Got metadata {}", t.toStdString());
+
+                    if (mediaPlayer->duration() > 0)
+                        this->duration_ = mediaPlayer->duration();
+
+                    dimensions_      = mediaPlayer->metaData(QMediaMetaData::Resolution).toSize();
+                    auto orientation = mediaPlayer->metaData(QMediaMetaData::Orientation).toInt();
+                    if (orientation == 90 || orientation == 270) {
+                        dimensions_.transpose();
+                    }
+                });
+        connect(mediaPlayer, &QMediaPlayer::durationChanged, [this, mediaPlayer](qint64 duration) {
+            if (duration > 0)
+                this->duration_ = mediaPlayer->duration();
+            nhlog::ui()->info("Duration changed {}", duration);
+        });
+        mediaPlayer->setMedia(QMediaContent(originalFilename_), source.get());
+        mediaPlayer->play();
     }
 }
 
@@ -721,9 +811,9 @@ InputBar::finalizeUpload(MediaUpload *upload, QString url)
     if (mimeClass == u"image")
         image(filename, encryptedFile, url, mime, size, upload->dimensions(), upload->blurhash());
     else if (mimeClass == u"audio")
-        audio(filename, encryptedFile, url, mime, size);
+        audio(filename, encryptedFile, url, mime, size, upload->duration());
     else if (mimeClass == u"video")
-        video(filename, encryptedFile, url, mime, size);
+        video(filename, encryptedFile, url, mime, size, upload->duration(), upload->dimensions());
     else
         file(filename, encryptedFile, url, mime, size);
 

src/timeline/InputBar.h

@@ -5,7 +5,9 @@
 
 #pragma once
 
+#include <QAbstractVideoSurface>
 #include <QIODevice>
+#include <QImage>
 #include <QObject>
 #include <QSize>
 #include <QStringList>
@@ -29,6 +31,90 @@ enum class MarkdownOverride
     OFF,
 };
 
+class InputVideoSurface : public QAbstractVideoSurface
+{
+    Q_OBJECT
+
+public:
+    InputVideoSurface(QObject *parent)
+      : QAbstractVideoSurface(parent)
+    {}
+
+    bool present(const QVideoFrame &frame) override
+    {
+        QImage::Format format = QImage::Format_Invalid;
+
+        switch (frame.pixelFormat()) {
+        case QVideoFrame::Format_ARGB32:
+            format = QImage::Format_ARGB32;
+            break;
+        case QVideoFrame::Format_ARGB32_Premultiplied:
+            format = QImage::Format_ARGB32_Premultiplied;
+            break;
+        case QVideoFrame::Format_RGB24:
+            format = QImage::Format_RGB888;
+            break;
+        case QVideoFrame::Format_BGR24:
+            format = QImage::Format_BGR888;
+            break;
+        case QVideoFrame::Format_RGB32:
+            format = QImage::Format_RGB32;
+            break;
+        case QVideoFrame::Format_RGB565:
+            format = QImage::Format_RGB16;
+            break;
+        case QVideoFrame::Format_RGB555:
+            format = QImage::Format_RGB555;
+            break;
+        default:
+            format = QImage::Format_Invalid;
+        }
+
+        if (format == QImage::Format_Invalid) {
+            emit newImage({});
+            return false;
+        } else {
+            QVideoFrame frametodraw(frame);
+
+            if (!frametodraw.map(QAbstractVideoBuffer::ReadOnly)) {
+                emit newImage({});
+                return false;
+            }
+
+            // this is a shallow operation. it just refer the frame buffer
+            QImage image(frametodraw.bits(),
+                         frametodraw.width(),
+                         frametodraw.height(),
+                         frametodraw.bytesPerLine(),
+                         QImage::Format_RGB444);
+
+            emit newImage(std::move(image));
+            return true;
+        }
+    }
+
+    QList<QVideoFrame::PixelFormat>
+    supportedPixelFormats(QAbstractVideoBuffer::HandleType type) const override
+    {
+        if (type == QAbstractVideoBuffer::NoHandle) {
+            return {
+              QVideoFrame::Format_ARGB32,
+              QVideoFrame::Format_ARGB32_Premultiplied,
+              QVideoFrame::Format_RGB24,
+              QVideoFrame::Format_BGR24,
+              QVideoFrame::Format_RGB32,
+              QVideoFrame::Format_RGB565,
+              QVideoFrame::Format_RGB555,
+            };
+        } else {
+            return {};
+        }
+    }
+
+signals:
+    void newImage(QImage img);
+};
+
 class MediaUpload : public QObject
 {
     Q_OBJECT
@@ -67,6 +153,7 @@ public:
     [[nodiscard]] QString filename() const { return originalFilename_; }
     [[nodiscard]] QString blurhash() const { return blurhash_; }
     [[nodiscard]] uint64_t size() const { return size_; }
+    [[nodiscard]] uint64_t duration() const { return duration_; }
     [[nodiscard]] std::optional<mtx::crypto::EncryptedFile> encryptedFile_()
     {
         return encryptedFile;
@@ -82,6 +169,7 @@ public slots:
 
 private slots:
     void updateThumbnailUrl(QString url) { this->thumbnailUrl_ = std::move(url); }
+    void setThumbnail(QImage img) { this->thumbnail_ = std::move(img); }
 
 public:
     // void uploadThumbnail(QImage img);
@@ -96,8 +184,11 @@ public:
     QString url_;
     std::optional<mtx::crypto::EncryptedFile> encryptedFile;
 
+    QImage thumbnail_;
+
     QSize dimensions_;
     uint64_t size_     = 0;
+    uint64_t duration_ = 0;
     bool encrypt_;
 };
 
@@ -181,12 +272,15 @@ private:
                const std::optional<mtx::crypto::EncryptedFile> &file,
                const QString &url,
                const QString &mime,
-               uint64_t dsize);
+               uint64_t dsize,
+               uint64_t duration);
     void video(const QString &filename,
                const std::optional<mtx::crypto::EncryptedFile> &file,
                const QString &url,
                const QString &mime,
-               uint64_t dsize);
+               uint64_t dsize,
+               uint64_t duration,
+               const QSize &dimensions);
 
     void startUploadFromPath(const QString &path);
     void startUploadFromMimeData(const QMimeData &source, const QString &format);

src/timeline/TimelineModel.cpp

@@ -474,6 +474,7 @@ TimelineModel::roleNames() const
       {Timestamp, "timestamp"},
       {Url, "url"},
       {ThumbnailUrl, "thumbnailUrl"},
+      {Duration, "duration"},
       {Blurhash, "blurhash"},
       {Filename, "filename"},
       {Filesize, "filesize"},
@@ -627,6 +628,8 @@ TimelineModel::data(const mtx::events::collections::TimelineEvents &event, int r
         return QVariant(QString::fromStdString(url(event)));
     case ThumbnailUrl:
         return QVariant(QString::fromStdString(thumbnail_url(event)));
+    case Duration:
+        return QVariant(static_cast<qulonglong>(duration(event)));
     case Blurhash:
         return QVariant(QString::fromStdString(blurhash(event)));
     case Filename:
@@ -739,6 +742,7 @@ TimelineModel::data(const mtx::events::collections::TimelineEvents &event, int r
         m.insert(names[Timestamp], data(event, static_cast<int>(Timestamp)));
         m.insert(names[Url], data(event, static_cast<int>(Url)));
         m.insert(names[ThumbnailUrl], data(event, static_cast<int>(ThumbnailUrl)));
+        m.insert(names[Duration], data(event, static_cast<int>(Duration)));
         m.insert(names[Blurhash], data(event, static_cast<int>(Blurhash)));
         m.insert(names[Filename], data(event, static_cast<int>(Filename)));
         m.insert(names[Filesize], data(event, static_cast<int>(Filesize)));

src/timeline/TimelineModel.h

@@ -215,6 +215,7 @@ public:
         Timestamp,
         Url,
         ThumbnailUrl,
+        Duration,
         Blurhash,
         Filename,
         Filesize,