From 4344b6964f63921aa300112bc3b62fdbaa64866a Mon Sep 17 00:00:00 2001 From: Konstantinos Sideris Date: Thu, 28 Jun 2018 16:16:43 +0300 Subject: [PATCH] Save timeline messages in cache for faster startup times --- deps/CMakeLists.txt | 2 +- include/Cache.h | 47 ++++++++++ include/ChatPage.h | 6 +- include/RoomInfoListItem.h | 11 +-- include/Utils.h | 31 +++++-- include/timeline/TimelineView.h | 1 + include/timeline/TimelineViewManager.h | 2 + src/Cache.cc | 121 +++++++++++++++++++++++++ src/ChatPage.cc | 41 +++------ src/MainWindow.cc | 3 +- src/RoomList.cc | 8 ++ src/Utils.cc | 36 ++++++-- src/timeline/TimelineView.cc | 5 +- src/timeline/TimelineViewManager.cc | 22 +++++ 14 files changed, 272 insertions(+), 64 deletions(-) diff --git a/deps/CMakeLists.txt b/deps/CMakeLists.txt index 4994df38..99abbf35 100644 --- a/deps/CMakeLists.txt +++ b/deps/CMakeLists.txt @@ -37,7 +37,7 @@ set(BOOST_SHA256 5721818253e6a0989583192f96782c4a98eb6204965316df9f5ad75819225ca9) set(MATRIX_STRUCTS_URL https://github.com/mujx/matrix-structs) -set(MATRIX_STRUCTS_TAG c24cb9b38312dfa24b33413847e3238600c678cd) +set(MATRIX_STRUCTS_TAG 3a052a95c555ce3ae12b8a2e0508e8bb73266fa1) set(MTXCLIENT_URL https://github.com/mujx/mtxclient) set(MTXCLIENT_TAG 73491268f94ddeb606284836bb5f512d11b0e249) diff --git a/include/Cache.h b/include/Cache.h index 3d906f02..5d65c80c 100644 --- a/include/Cache.h +++ b/include/Cache.h @@ -19,8 +19,10 @@ #include +#include #include #include +#include #include #include @@ -46,9 +48,24 @@ struct SearchResult QString display_name; }; +inline int +numeric_key_comparison(const MDB_val *a, const MDB_val *b) +{ + auto lhs = std::stoul(std::string((char *)a->mv_data, a->mv_size)); + auto rhs = std::stoul(std::string((char *)b->mv_data, b->mv_size)); + + if (lhs < rhs) + return 1; + else if (lhs == rhs) + return 0; + + return -1; +} + Q_DECLARE_METATYPE(SearchResult) Q_DECLARE_METATYPE(QVector) Q_DECLARE_METATYPE(RoomMember) +Q_DECLARE_METATYPE(mtx::responses::Timeline) //! Used to uniquely identify a list of read receipts. struct ReadReceiptKey @@ -70,6 +87,15 @@ from_json(const json &j, ReadReceiptKey &key) key.room_id = j.at("room_id").get(); } +struct DescInfo +{ + QString username; + QString userid; + QString body; + QString timestamp; + QDateTime datetime; +}; + //! UI info associated with a room. struct RoomInfo { @@ -86,6 +112,8 @@ struct RoomInfo //! Who can access to the room. JoinRule join_rule = JoinRule::Public; bool guest_access = false; + //! Metadata describing the last message in the timeline. + DescInfo msgInfo; }; inline void @@ -289,6 +317,8 @@ public: bool isFormatValid(); void setCurrentFormat(); + std::map roomMessages(); + //! Retrieve all the user ids from a room. std::vector roomMembers(const std::string &room_id); @@ -402,6 +432,13 @@ private: QString getInviteRoomTopic(lmdb::txn &txn, lmdb::dbi &statesdb); QString getInviteRoomAvatarUrl(lmdb::txn &txn, lmdb::dbi &statesdb, lmdb::dbi &membersdb); + DescInfo getLastMessageInfo(lmdb::txn &txn, const std::string &room_id); + void saveTimelineMessages(lmdb::txn &txn, + const std::string &room_id, + const mtx::responses::Timeline &res); + + mtx::responses::Timeline getTimelineMessages(lmdb::txn &txn, const std::string &room_id); + //! Remove a room from the cache. // void removeLeftRoom(lmdb::txn &txn, const std::string &room_id); template @@ -500,6 +537,7 @@ private: mpark::holds_alternative>(e) || mpark::holds_alternative>(e) || mpark::holds_alternative>(e) || + mpark::holds_alternative>(e) || mpark::holds_alternative>(e) || mpark::holds_alternative>(e); } @@ -544,6 +582,15 @@ private: } } + lmdb::dbi getMessagesDb(lmdb::txn &txn, const std::string &room_id) + { + auto db = + lmdb::dbi::open(txn, std::string(room_id + "/messages").c_str(), MDB_CREATE); + lmdb::dbi_set_compare(txn, db, numeric_key_comparison); + + return db; + } + lmdb::dbi getInviteStatesDb(lmdb::txn &txn, const std::string &room_id) { return lmdb::dbi::open( diff --git a/include/ChatPage.h b/include/ChatPage.h index ffea2914..a4c6ccc5 100644 --- a/include/ChatPage.h +++ b/include/ChatPage.h @@ -108,7 +108,6 @@ signals: void showLoginPage(const QString &msg); void showUserSettingsPage(); void showOverlayProgressBar(); - void startConsesusTimer(); void removeTimelineEvent(const QString &room_id, const QString &event_id); @@ -124,7 +123,7 @@ signals: void initializeRoomList(QMap); void initializeViews(const mtx::responses::Rooms &rooms); - void initializeEmptyViews(const std::vector &rooms); + void initializeEmptyViews(const std::map &msgs); void syncUI(const mtx::responses::Rooms &rooms); void syncRoomlist(const std::map &updates); void syncTopBar(const std::map &updates); @@ -206,9 +205,6 @@ private: TextInputWidget *text_input_; TypingDisplay *typingDisplay_; - // Safety net if consensus is not possible or too slow. - QTimer *showContentTimer_; - QTimer *consensusTimer_; QTimer connectivityTimer_; std::atomic_bool isConnected_; diff --git a/include/RoomInfoListItem.h b/include/RoomInfoListItem.h index aebc2216..95db1d75 100644 --- a/include/RoomInfoListItem.h +++ b/include/RoomInfoListItem.h @@ -22,20 +22,11 @@ #include #include +#include "Cache.h" #include class Menu; class RippleOverlay; -struct RoomInfo; - -struct DescInfo -{ - QString username; - QString userid; - QString body; - QString timestamp; - QDateTime datetime; -}; class RoomInfoListItem : public QWidget { diff --git a/include/Utils.h b/include/Utils.h index ad8e2073..7db405b1 100644 --- a/include/Utils.h +++ b/include/Utils.h @@ -41,14 +41,15 @@ template QString messageDescription(const QString &username = "", const QString &body = "") { - using Audio = mtx::events::RoomEvent; - using Emote = mtx::events::RoomEvent; - using File = mtx::events::RoomEvent; - using Image = mtx::events::RoomEvent; - using Notice = mtx::events::RoomEvent; - using Sticker = mtx::events::Sticker; - using Text = mtx::events::RoomEvent; - using Video = mtx::events::RoomEvent; + using Audio = mtx::events::RoomEvent; + using Emote = mtx::events::RoomEvent; + using File = mtx::events::RoomEvent; + using Image = mtx::events::RoomEvent; + using Notice = mtx::events::RoomEvent; + using Sticker = mtx::events::Sticker; + using Text = mtx::events::RoomEvent; + using Video = mtx::events::RoomEvent; + using Encrypted = mtx::events::EncryptedEvent; if (std::is_same::value || std::is_same::value) return QString("sent an audio clip"); @@ -66,6 +67,8 @@ messageDescription(const QString &username = "", const QString &body = "") return QString(": %1").arg(body); else if (std::is_same::value) return QString("* %1 %2").arg(username).arg(body); + else if (std::is_same::value) + return QString("sent an encrypted message"); } template @@ -135,6 +138,18 @@ erase_if(ContainerT &items, const PredicateT &predicate) } } +inline uint64_t +event_timestamp(const mtx::events::collections::TimelineEvents &event) +{ + return mpark::visit([](auto msg) { return msg.origin_server_ts; }, event); +} + +inline nlohmann::json +serialize_event(const mtx::events::collections::TimelineEvents &event) +{ + return mpark::visit([](auto msg) { return json(msg); }, event); +} + inline mtx::events::EventType event_type(const mtx::events::collections::TimelineEvents &event) { diff --git a/include/timeline/TimelineView.h b/include/timeline/TimelineView.h index bbe1dcad..7f1912ea 100644 --- a/include/timeline/TimelineView.h +++ b/include/timeline/TimelineView.h @@ -158,6 +158,7 @@ public: //! Remove an item from the timeline with the given Event ID. void removeEvent(const QString &event_id); + void setPrevBatchToken(const QString &token) { prev_batch_token_ = token; } public slots: void sliderRangeChanged(int min, int max); diff --git a/include/timeline/TimelineViewManager.h b/include/timeline/TimelineViewManager.h index 9e31ecbf..590adb2b 100644 --- a/include/timeline/TimelineViewManager.h +++ b/include/timeline/TimelineViewManager.h @@ -27,6 +27,7 @@ class QFile; class RoomInfoListItem; class TimelineView; struct DescInfo; +struct SavedMessages; class TimelineViewManager : public QStackedWidget { @@ -57,6 +58,7 @@ signals: public slots: void removeTimelineEvent(const QString &room_id, const QString &event_id); + void initWithMessages(const std::map &msgs); void setHistoryView(const QString &room_id); void queueTextMessage(const QString &msg); diff --git a/src/Cache.cc b/src/Cache.cc index ed4194ec..a276f554 100644 --- a/src/Cache.cc +++ b/src/Cache.cc @@ -21,8 +21,10 @@ #include #include #include +#include #include +#include #include #include "Cache.h" @@ -38,6 +40,8 @@ static const lmdb::val NEXT_BATCH_KEY("next_batch"); static const lmdb::val OLM_ACCOUNT_KEY("olm_account"); static const lmdb::val CACHE_FORMAT_VERSION_KEY("cache_format_version"); +constexpr size_t MAX_RESTORED_MESSAGES = 30; + //! Cache databases and their format. //! //! Contains UI information for the joined rooms. (i.e name, topic, avatar url etc). @@ -85,6 +89,7 @@ init(const QString &user_id) qRegisterMetaType(); qRegisterMetaType>(); qRegisterMetaType>(); + qRegisterMetaType>(); instance_ = std::make_unique(user_id); } @@ -744,6 +749,8 @@ Cache::saveState(const mtx::responses::Sync &res) saveStateEvents(txn, statesdb, membersdb, room.first, room.second.state.events); saveStateEvents(txn, statesdb, membersdb, room.first, room.second.timeline.events); + saveTimelineMessages(txn, room.first, room.second.timeline); + RoomInfo updatedInfo; updatedInfo.name = getRoomName(txn, statesdb, membersdb).toStdString(); updatedInfo.topic = getRoomTopic(txn, statesdb).toStdString(); @@ -944,6 +951,57 @@ Cache::getRoomInfo(const std::vector &rooms) return room_info; } +std::map +Cache::roomMessages() +{ + auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + + std::map msgs; + std::string room_id, unused; + + auto roomsCursor = lmdb::cursor::open(txn, roomsDb_); + while (roomsCursor.get(room_id, unused, MDB_NEXT)) + msgs.emplace(QString::fromStdString(room_id), getTimelineMessages(txn, room_id)); + + roomsCursor.close(); + txn.commit(); + + return msgs; +} + +mtx::responses::Timeline +Cache::getTimelineMessages(lmdb::txn &txn, const std::string &room_id) +{ + auto db = getMessagesDb(txn, room_id); + + mtx::responses::Timeline timeline; + std::string timestamp, msg; + + auto cursor = lmdb::cursor::open(txn, db); + + size_t index = 0; + + while (cursor.get(timestamp, msg, MDB_NEXT) && index < MAX_RESTORED_MESSAGES) { + auto obj = json::parse(msg); + + if (obj.count("event") == 0 || obj.count("token") == 0) + continue; + + mtx::events::collections::TimelineEvents event; + mtx::events::collections::from_json(obj.at("event"), event); + + index += 1; + + timeline.events.push_back(event); + timeline.prev_batch = obj.at("token").get(); + } + cursor.close(); + + std::reverse(timeline.events.begin(), timeline.events.end()); + + return timeline; +} + QMap Cache::roomInfo(bool withInvites) { @@ -959,6 +1017,8 @@ Cache::roomInfo(bool withInvites) while (roomsCursor.get(room_id, room_data, MDB_NEXT)) { RoomInfo tmp = json::parse(std::move(room_data)); tmp.member_count = getMembersDb(txn, room_id).size(txn); + tmp.msgInfo = getLastMessageInfo(txn, room_id); + result.insert(QString::fromStdString(std::move(room_id)), std::move(tmp)); } roomsCursor.close(); @@ -979,6 +1039,38 @@ Cache::roomInfo(bool withInvites) return result; } +DescInfo +Cache::getLastMessageInfo(lmdb::txn &txn, const std::string &room_id) +{ + auto db = getMessagesDb(txn, room_id); + + if (db.size(txn) == 0) + return DescInfo{}; + + std::string timestamp, msg; + + QSettings settings; + auto local_user = settings.value("auth/user_id").toString(); + + auto cursor = lmdb::cursor::open(txn, db); + while (cursor.get(timestamp, msg, MDB_NEXT)) { + auto obj = json::parse(msg); + + if (obj.count("event") == 0) + continue; + + mtx::events::collections::TimelineEvents event; + mtx::events::collections::from_json(obj.at("event"), event); + + cursor.close(); + return utils::getMessageDescription( + event, local_user, QString::fromStdString(room_id)); + } + cursor.close(); + + return DescInfo{}; +} + std::map Cache::invites() { @@ -1512,6 +1604,35 @@ Cache::getMembers(const std::string &room_id, std::size_t startIndex, std::size_ return members; } +void +Cache::saveTimelineMessages(lmdb::txn &txn, + const std::string &room_id, + const mtx::responses::Timeline &res) +{ + auto db = getMessagesDb(txn, room_id); + + using namespace mtx::events; + using namespace mtx::events::state; + + for (const auto &e : res.events) { + if (isStateEvent(e)) + continue; + + if (mpark::holds_alternative>(e)) + continue; + + json obj = json::object(); + + obj["event"] = utils::serialize_event(e); + obj["token"] = res.prev_batch; + + lmdb::dbi_put(txn, + db, + lmdb::val(std::to_string(utils::event_timestamp(e))), + lmdb::val(obj.dump())); + } +} + void Cache::markSentNotification(const std::string &event_id) { diff --git a/src/ChatPage.cc b/src/ChatPage.cc index cc9473e6..2b8a6b89 100644 --- a/src/ChatPage.cc +++ b/src/ChatPage.cc @@ -516,23 +516,6 @@ ChatPage::ChatPage(QSharedPointer userSettings, QWidget *parent) connect(this, &ChatPage::leftRoom, this, &ChatPage::removeRoom); connect(this, &ChatPage::notificationsRetrieved, this, &ChatPage::sendDesktopNotifications); - showContentTimer_ = new QTimer(this); - showContentTimer_->setSingleShot(true); - connect(showContentTimer_, &QTimer::timeout, this, [this]() { - consensusTimer_->stop(); - emit contentLoaded(); - }); - - consensusTimer_ = new QTimer(this); - connect(consensusTimer_, &QTimer::timeout, this, [this]() { - if (view_manager_->hasLoaded()) { - // Remove the spinner overlay. - emit contentLoaded(); - showContentTimer_->stop(); - consensusTimer_->stop(); - } - }); - connect(communitiesList_, &CommunitiesList::communityChanged, this, @@ -552,20 +535,15 @@ ChatPage::ChatPage(QSharedPointer userSettings, QWidget *parent) this, &ChatPage::setGroupViewState); - connect(this, &ChatPage::startConsesusTimer, this, [this]() { - consensusTimer_->start(CONSENSUS_TIMEOUT); - showContentTimer_->start(SHOW_CONTENT_TIMEOUT); - }); connect(this, &ChatPage::initializeRoomList, room_list_, &RoomList::initialize); connect(this, &ChatPage::initializeViews, view_manager_, [this](const mtx::responses::Rooms &rooms) { view_manager_->initialize(rooms); }); - connect( - this, - &ChatPage::initializeEmptyViews, - this, - [this](const std::vector &rooms) { view_manager_->initialize(rooms); }); + connect(this, + &ChatPage::initializeEmptyViews, + view_manager_, + &TimelineViewManager::initWithMessages); connect(this, &ChatPage::syncUI, this, [this](const mtx::responses::Rooms &rooms) { try { room_list_->cleanupInvites(cache::client()->invites()); @@ -817,6 +795,8 @@ ChatPage::showUnreadMessageNotification(int count) void ChatPage::loadStateFromCache() { + emit contentLoaded(); + nhlog::db()->info("restoring state from cache"); getProfileInfo(); @@ -829,8 +809,9 @@ ChatPage::loadStateFromCache() cache::client()->populateMembers(); - emit initializeEmptyViews(cache::client()->joinedRooms()); + emit initializeEmptyViews(cache::client()->roomMessages()); emit initializeRoomList(cache::client()->roomInfo()); + } catch (const mtx::crypto::olm_exception &e) { nhlog::crypto()->critical("failed to restore olm account: {}", e.what()); emit dropToLoginPageCb( @@ -841,6 +822,9 @@ ChatPage::loadStateFromCache() emit dropToLoginPageCb( tr("Failed to restore save data. Please login again.")); return; + } catch (const json::exception &e) { + nhlog::db()->critical("failed to parse cache data: {}", e.what()); + return; } nhlog::crypto()->info("ed25519 : {}", olm::client()->identity_keys().ed25519); @@ -848,9 +832,6 @@ ChatPage::loadStateFromCache() // Start receiving events. emit trySyncCb(); - - // Check periodically if the timelines have been loaded. - emit startConsesusTimer(); }); } diff --git a/src/MainWindow.cc b/src/MainWindow.cc index 088bb5c0..c7c3432f 100644 --- a/src/MainWindow.cc +++ b/src/MainWindow.cc @@ -234,7 +234,8 @@ MainWindow::showChatPage() showOverlayProgressBar(); - QTimer::singleShot(100, this, [this]() { pageStack_->setCurrentWidget(chat_page_); }); + welcome_page_->hide(); + pageStack_->setCurrentWidget(chat_page_); login_page_->reset(); chat_page_->bootstrap(userid, homeserver, token); diff --git a/src/RoomList.cc b/src/RoomList.cc index b5bcdad6..5f094a1c 100644 --- a/src/RoomList.cc +++ b/src/RoomList.cc @@ -15,6 +15,7 @@ * along with this program. If not, see . */ +#include #include #include #include @@ -171,6 +172,8 @@ RoomList::initialize(const QMap &info) rooms_.clear(); + setUpdatesEnabled(false); + for (auto it = info.begin(); it != info.end(); it++) { if (it.value().is_invite) addInvitedRoom(it.key(), it.value()); @@ -178,6 +181,11 @@ RoomList::initialize(const QMap &info) addRoom(it.key(), it.value()); } + for (auto it = info.begin(); it != info.end(); it++) + updateRoomDescription(it.key(), it.value().msgInfo); + + setUpdatesEnabled(true); + if (rooms_.empty()) return; diff --git a/src/Utils.cc b/src/Utils.cc index 7b3574db..705a9e21 100644 --- a/src/Utils.cc +++ b/src/Utils.cc @@ -28,13 +28,14 @@ utils::getMessageDescription(const TimelineEvent &event, const QString &localUser, const QString &room_id) { - using Audio = mtx::events::RoomEvent; - using Emote = mtx::events::RoomEvent; - using File = mtx::events::RoomEvent; - using Image = mtx::events::RoomEvent; - using Notice = mtx::events::RoomEvent; - using Text = mtx::events::RoomEvent; - using Video = mtx::events::RoomEvent; + using Audio = mtx::events::RoomEvent; + using Emote = mtx::events::RoomEvent; + using File = mtx::events::RoomEvent; + using Image = mtx::events::RoomEvent; + using Notice = mtx::events::RoomEvent; + using Text = mtx::events::RoomEvent; + using Video = mtx::events::RoomEvent; + using Encrypted = mtx::events::EncryptedEvent; if (mpark::holds_alternative