From 233b3c06ce79ee1c4445863430c0ebbee1f8faf1 Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Wed, 1 Jul 2020 20:15:39 +0200 Subject: [PATCH 01/50] Store events in room specific db --- src/Cache.cpp | 30 ++++++++++++++++++++---------- src/Cache_p.h | 5 +++++ 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/src/Cache.cpp b/src/Cache.cpp index d9d1134e..27f4e694 100644 --- a/src/Cache.cpp +++ b/src/Cache.cpp @@ -1955,24 +1955,34 @@ Cache::saveTimelineMessages(lmdb::txn &txn, const mtx::responses::Timeline &res) { auto db = getMessagesDb(txn, room_id); + auto eventsDb = getEventsDb(txn, room_id); using namespace mtx::events; using namespace mtx::events::state; for (const auto &e : res.events) { - if (std::holds_alternative>(e)) - continue; + auto event = mtx::accessors::serialize_event(e); + if (auto redaction = + std::get_if>(&e)) { + lmdb::dbi_put( + txn, eventsDb, lmdb::val(redaction->redacts), lmdb::val(event.dump())); + } else { + json obj = json::object(); - json obj = json::object(); + obj["event"] = event; + obj["token"] = res.prev_batch; - obj["event"] = mtx::accessors::serialize_event(e); - obj["token"] = res.prev_batch; + lmdb::dbi_put( + txn, + db, + lmdb::val(std::to_string(event["origin_server_ts"].get())), + lmdb::val(obj.dump())); - lmdb::dbi_put( - txn, - db, - lmdb::val(std::to_string(obj["event"]["origin_server_ts"].get())), - lmdb::val(obj.dump())); + lmdb::dbi_put(txn, + eventsDb, + lmdb::val(event["event_id"].get()), + lmdb::val(event.dump())); + } } } diff --git a/src/Cache_p.h b/src/Cache_p.h index 892b66a5..e657447b 100644 --- a/src/Cache_p.h +++ b/src/Cache_p.h @@ -411,6 +411,11 @@ private: return db; } + lmdb::dbi getEventsDb(lmdb::txn &txn, const std::string &room_id) + { + return lmdb::dbi::open(txn, std::string(room_id + "/events").c_str(), MDB_CREATE); + } + lmdb::dbi getInviteStatesDb(lmdb::txn &txn, const std::string &room_id) { return lmdb::dbi::open( From 79a29953dd92b9c025a8b9915aeb56d1d78e5607 Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Sat, 4 Jul 2020 02:09:12 +0200 Subject: [PATCH 02/50] Persist event order --- src/Cache.cpp | 20 ++++++++++++++++++++ src/Cache_p.h | 6 ++++++ 2 files changed, 26 insertions(+) diff --git a/src/Cache.cpp b/src/Cache.cpp index 27f4e694..2824960b 100644 --- a/src/Cache.cpp +++ b/src/Cache.cpp @@ -1957,9 +1957,21 @@ Cache::saveTimelineMessages(lmdb::txn &txn, auto db = getMessagesDb(txn, room_id); auto eventsDb = getEventsDb(txn, room_id); + auto orderDb = getEventOrderDb(txn, room_id); + if (res.limited) + lmdb::dbi_drop(txn, orderDb, false); + using namespace mtx::events; using namespace mtx::events::state; + lmdb::val indexVal, val; + int64_t index = 0; + auto cursor = lmdb::cursor::open(txn, orderDb); + if (cursor.get(indexVal, val, MDB_LAST)) { + index = *indexVal.data(); + } + + bool first = true; for (const auto &e : res.events) { auto event = mtx::accessors::serialize_event(e); if (auto redaction = @@ -1982,6 +1994,14 @@ Cache::saveTimelineMessages(lmdb::txn &txn, eventsDb, lmdb::val(event["event_id"].get()), lmdb::val(event.dump())); + + ++index; + + lmdb::cursor_put(cursor.handle(), + lmdb::val(&index, sizeof(index)), + lmdb::val(first ? res.prev_batch : ""), + MDB_APPEND); + first = false; } } } diff --git a/src/Cache_p.h b/src/Cache_p.h index e657447b..5f01f736 100644 --- a/src/Cache_p.h +++ b/src/Cache_p.h @@ -416,6 +416,12 @@ private: return lmdb::dbi::open(txn, std::string(room_id + "/events").c_str(), MDB_CREATE); } + lmdb::dbi getEventOrderDb(lmdb::txn &txn, const std::string &room_id) + { + return lmdb::dbi::open( + txn, std::string(room_id + "/event_order").c_str(), MDB_CREATE | MDB_INTEGERKEY); + } + lmdb::dbi getInviteStatesDb(lmdb::txn &txn, const std::string &room_id) { return lmdb::dbi::open( From c79205c26a77df9086bd6294ae6285a7346e6656 Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Sun, 5 Jul 2020 05:29:07 +0200 Subject: [PATCH 03/50] Use new timeline cache structure --- src/Cache.cpp | 250 +++++++++++++++++++++++++++++--------------------- src/Cache_p.h | 24 +++-- 2 files changed, 156 insertions(+), 118 deletions(-) diff --git a/src/Cache.cpp b/src/Cache.cpp index 2824960b..26291cfd 100644 --- a/src/Cache.cpp +++ b/src/Cache.cpp @@ -37,7 +37,7 @@ //! Should be changed when a breaking change occurs in the cache format. //! This will reset client's data. -static const std::string CURRENT_CACHE_FORMAT_VERSION("2020.05.01"); +static const std::string CURRENT_CACHE_FORMAT_VERSION("2020.07.05"); static const std::string SECRET("secret"); static lmdb::val NEXT_BATCH_KEY("next_batch"); @@ -46,8 +46,9 @@ static lmdb::val CACHE_FORMAT_VERSION_KEY("cache_format_version"); constexpr size_t MAX_RESTORED_MESSAGES = 30'000; -constexpr auto DB_SIZE = 32ULL * 1024ULL * 1024ULL * 1024ULL; // 32 GB -constexpr auto MAX_DBS = 8092UL; +constexpr auto DB_SIZE = 32ULL * 1024ULL * 1024ULL * 1024ULL; // 32 GB +constexpr auto MAX_DBS = 8092UL; +constexpr auto BATCH_SIZE = 100; //! Cache databases and their format. //! @@ -63,7 +64,6 @@ constexpr auto SYNC_STATE_DB("sync_state"); //! Read receipts per room/event. constexpr auto READ_RECEIPTS_DB("read_receipts"); constexpr auto NOTIFICATIONS_DB("sent_notifications"); -//! TODO: delete pending_receipts database on old cache versions //! Encryption related databases. @@ -93,20 +93,6 @@ namespace { std::unique_ptr instance_ = nullptr; } -int -numeric_key_comparison(const MDB_val *a, const MDB_val *b) -{ - auto lhs = std::stoull(std::string((char *)a->mv_data, a->mv_size)); - auto rhs = std::stoull(std::string((char *)b->mv_data, b->mv_size)); - - if (lhs < rhs) - return 1; - else if (lhs == rhs) - return 0; - - return -1; -} - Cache::Cache(const QString &userId, QObject *parent) : QObject{parent} , env_{nullptr} @@ -697,6 +683,27 @@ Cache::runMigrations() return false; } + nhlog::db()->info("Successfully deleted pending receipts database."); + return true; + }}, + {"2020.07.05", + [this]() { + try { + auto txn = lmdb::txn::begin(env_, nullptr); + auto room_ids = getRoomIds(txn); + + for (const auto &room_id : room_ids) { + auto messagesDb = lmdb::dbi::open( + txn, std::string(room_id + "/messages").c_str(), MDB_CREATE); + lmdb::dbi_drop(txn, messagesDb, true); + } + txn.commit(); + } catch (const lmdb::error &) { + nhlog::db()->critical( + "Failed to delete messages database in migration!"); + return false; + } + nhlog::db()->info("Successfully deleted pending receipts database."); return true; }}, @@ -1232,38 +1239,64 @@ Cache::getTimelineMentions() return notifs; } -mtx::responses::Timeline -Cache::getTimelineMessages(lmdb::txn &txn, const std::string &room_id) +Cache::Messages +Cache::getTimelineMessages(lmdb::txn &txn, const std::string &room_id, int64_t index, bool forward) { // TODO(nico): Limit the messages returned by this maybe? - auto db = getMessagesDb(txn, room_id); + auto orderDb = getEventOrderDb(txn, room_id); + auto eventsDb = getEventsDb(txn, room_id); - mtx::responses::Timeline timeline; - std::string timestamp, msg; + Messages messages{}; - auto cursor = lmdb::cursor::open(txn, db); + lmdb::val indexVal, val; - size_t index = 0; + auto cursor = lmdb::cursor::open(txn, orderDb); + if (index == std::numeric_limits::max()) { + if (cursor.get(indexVal, val, forward ? MDB_FIRST : MDB_LAST)) { + index = *indexVal.data(); + } else { + messages.end_of_cache = true; + return messages; + } + } else { + if (cursor.get(indexVal, val, MDB_SET)) { + index = *indexVal.data(); + } else { + messages.end_of_cache = true; + return messages; + } + } - while (cursor.get(timestamp, msg, MDB_NEXT) && index < MAX_RESTORED_MESSAGES) { - auto obj = json::parse(msg); + int counter = 0; - if (obj.count("event") == 0 || obj.count("token") == 0) + bool ret; + while ((ret = cursor.get(indexVal, val, forward ? MDB_NEXT : MDB_LAST)) && + counter++ < BATCH_SIZE) { + auto obj = json::parse(std::string(val.data(), val.size())); + + if (obj.count("event_id") == 0) + break; + + lmdb::val event; + bool success = lmdb::dbi_get( + txn, eventsDb, lmdb::val(obj["event_id"].get()), event); + if (!success) continue; - mtx::events::collections::TimelineEvent event; - mtx::events::collections::from_json(obj.at("event"), event); + mtx::events::collections::TimelineEvent te; + mtx::events::collections::from_json( + json::parse(std::string(event.data(), event.size())), te); - index += 1; - - timeline.events.push_back(event.data); - timeline.prev_batch = obj.at("token").get(); + messages.timeline.events.push_back(std::move(te.data)); + // timeline.prev_batch = obj.at("token").get(); } cursor.close(); - std::reverse(timeline.events.begin(), timeline.events.end()); + // std::reverse(timeline.events.begin(), timeline.events.end()); + messages.next_index = *indexVal.data(); + messages.end_of_cache = !ret; - return timeline; + return messages; } QMap @@ -1306,55 +1339,59 @@ Cache::roomInfo(bool withInvites) std::string Cache::getLastEventId(lmdb::txn &txn, const std::string &room_id) { - auto db = getMessagesDb(txn, room_id); + auto orderDb = getEventOrderDb(txn, room_id); - if (db.size(txn) == 0) + lmdb::val indexVal, val; + + auto cursor = lmdb::cursor::open(txn, orderDb); + if (!cursor.get(indexVal, val, MDB_LAST)) { return {}; - - std::string timestamp, msg; - - auto cursor = lmdb::cursor::open(txn, db); - while (cursor.get(timestamp, msg, MDB_NEXT)) { - auto obj = json::parse(msg); - - if (obj.count("event") == 0) - continue; - - cursor.close(); - return obj["event"]["event_id"]; } - cursor.close(); - return {}; + auto obj = json::parse(std::string(val.data(), val.size())); + + if (obj.count("event_id") == 0) + return {}; + else + return obj["event_id"]; } DescInfo Cache::getLastMessageInfo(lmdb::txn &txn, const std::string &room_id) { - auto db = getMessagesDb(txn, room_id); - - if (db.size(txn) == 0) + auto orderDb = getEventOrderDb(txn, room_id); + auto eventsDb = getEventsDb(txn, room_id); + if (orderDb.size(txn) == 0) return DescInfo{}; - std::string timestamp, msg; - const auto local_user = utils::localUser(); DescInfo fallbackDesc{}; - auto cursor = lmdb::cursor::open(txn, db); - while (cursor.get(timestamp, msg, MDB_NEXT)) { - auto obj = json::parse(msg); + lmdb::val indexVal, val; - if (obj.count("event") == 0) + auto cursor = lmdb::cursor::open(txn, orderDb); + cursor.get(indexVal, val, MDB_LAST); + while (cursor.get(indexVal, val, MDB_PREV)) { + auto temp = json::parse(std::string(val.data(), val.size())); + + if (temp.count("event_id") == 0) + break; + + lmdb::val event; + bool success = lmdb::dbi_get( + txn, eventsDb, lmdb::val(temp["event_id"].get()), event); + if (!success) continue; - if (fallbackDesc.event_id.isEmpty() && obj["event"]["type"] == "m.room.member" && - obj["event"]["state_key"] == local_user.toStdString() && - obj["event"]["content"]["membership"] == "join") { - uint64_t ts = obj["event"]["origin_server_ts"]; + auto obj = json::parse(std::string(event.data(), event.size())); + + if (fallbackDesc.event_id.isEmpty() && obj["type"] == "m.room.member" && + obj["state_key"] == local_user.toStdString() && + obj["content"]["membership"] == "join") { + uint64_t ts = obj["origin_server_ts"]; auto time = QDateTime::fromMSecsSinceEpoch(ts); - fallbackDesc = DescInfo{QString::fromStdString(obj["event"]["event_id"]), + fallbackDesc = DescInfo{QString::fromStdString(obj["event_id"]), local_user, tr("You joined this room."), utils::descriptiveTime(time), @@ -1362,17 +1399,16 @@ Cache::getLastMessageInfo(lmdb::txn &txn, const std::string &room_id) time}; } - if (!(obj["event"]["type"] == "m.room.message" || - obj["event"]["type"] == "m.sticker" || - obj["event"]["type"] == "m.room.encrypted")) + if (!(obj["type"] == "m.room.message" || obj["type"] == "m.sticker" || + obj["type"] == "m.room.encrypted")) continue; - mtx::events::collections::TimelineEvent event; - mtx::events::collections::from_json(obj.at("event"), event); + mtx::events::collections::TimelineEvent te; + mtx::events::collections::from_json(obj, te); cursor.close(); return utils::getMessageDescription( - event.data, local_user, QString::fromStdString(room_id)); + te.data, local_user, QString::fromStdString(room_id)); } cursor.close(); @@ -1954,7 +1990,6 @@ Cache::saveTimelineMessages(lmdb::txn &txn, const std::string &room_id, const mtx::responses::Timeline &res) { - auto db = getMessagesDb(txn, room_id); auto eventsDb = getEventsDb(txn, room_id); auto orderDb = getEventOrderDb(txn, room_id); @@ -1966,7 +2001,7 @@ Cache::saveTimelineMessages(lmdb::txn &txn, lmdb::val indexVal, val; int64_t index = 0; - auto cursor = lmdb::cursor::open(txn, orderDb); + auto cursor = lmdb::cursor::open(txn, orderDb); if (cursor.get(indexVal, val, MDB_LAST)) { index = *indexVal.data(); } @@ -1979,17 +2014,6 @@ Cache::saveTimelineMessages(lmdb::txn &txn, lmdb::dbi_put( txn, eventsDb, lmdb::val(redaction->redacts), lmdb::val(event.dump())); } else { - json obj = json::object(); - - obj["event"] = event; - obj["token"] = res.prev_batch; - - lmdb::dbi_put( - txn, - db, - lmdb::val(std::to_string(event["origin_server_ts"].get())), - lmdb::val(obj.dump())); - lmdb::dbi_put(txn, eventsDb, lmdb::val(event["event_id"].get()), @@ -1997,9 +2021,16 @@ Cache::saveTimelineMessages(lmdb::txn &txn, ++index; + json orderEntry = json::object(); + orderEntry["event_id"] = event["event_id"]; + if (first) + orderEntry["prev_batch"] = res.prev_batch; + + nhlog::db()->debug("saving '{}'", orderEntry.dump()); + lmdb::cursor_put(cursor.handle(), lmdb::val(&index, sizeof(index)), - lmdb::val(first ? res.prev_batch : ""), + lmdb::val(orderEntry.dump()), MDB_APPEND); first = false; } @@ -2138,34 +2169,43 @@ Cache::getRoomIds(lmdb::txn &txn) void Cache::deleteOldMessages() { + lmdb::val indexVal, val; + auto txn = lmdb::txn::begin(env_); auto room_ids = getRoomIds(txn); - for (const auto &id : room_ids) { - auto msg_db = getMessagesDb(txn, id); + for (const auto &room_id : room_ids) { + auto orderDb = getEventOrderDb(txn, room_id); + auto eventsDb = getEventsDb(txn, room_id); + auto cursor = lmdb::cursor::open(txn, orderDb); - std::string ts, event; - uint64_t idx = 0; - - const auto db_size = msg_db.size(txn); - if (db_size <= 3 * MAX_RESTORED_MESSAGES) + int64_t first, last; + if (cursor.get(indexVal, val, MDB_LAST)) { + last = *indexVal.data(); + } else { + continue; + } + if (cursor.get(indexVal, val, MDB_FIRST)) { + first = *indexVal.data(); + } else { continue; - - nhlog::db()->info("[{}] message count: {}", id, db_size); - - auto cursor = lmdb::cursor::open(txn, msg_db); - while (cursor.get(ts, event, MDB_NEXT)) { - idx += 1; - - if (idx > MAX_RESTORED_MESSAGES) - lmdb::cursor_del(cursor); } + size_t message_count = static_cast(last - first); + if (message_count < MAX_RESTORED_MESSAGES) + continue; + + while (cursor.get(indexVal, val, MDB_NEXT) && + message_count-- < MAX_RESTORED_MESSAGES) { + auto obj = json::parse(std::string(val.data(), val.size())); + + if (obj.count("event_id") != 0) + lmdb::dbi_del( + txn, eventsDb, lmdb::val(obj["event_id"].get())); + lmdb::cursor_del(cursor); + } cursor.close(); - - nhlog::db()->info("[{}] updated message count: {}", id, msg_db.size(txn)); } - txn.commit(); } diff --git a/src/Cache_p.h b/src/Cache_p.h index 5f01f736..37486ca0 100644 --- a/src/Cache_p.h +++ b/src/Cache_p.h @@ -18,6 +18,7 @@ #pragma once +#include #include #include @@ -38,9 +39,6 @@ #include "CacheCryptoStructs.h" #include "CacheStructs.h" -int -numeric_key_comparison(const MDB_val *a, const MDB_val *b); - class Cache : public QObject { Q_OBJECT @@ -250,7 +248,16 @@ private: const std::string &room_id, const mtx::responses::Timeline &res); - mtx::responses::Timeline getTimelineMessages(lmdb::txn &txn, const std::string &room_id); + struct Messages + { + mtx::responses::Timeline timeline; + uint64_t next_index; + bool end_of_cache = false; + }; + Messages getTimelineMessages(lmdb::txn &txn, + const std::string &room_id, + int64_t index = std::numeric_limits::max(), + bool forward = false); //! Remove a room from the cache. // void removeLeftRoom(lmdb::txn &txn, const std::string &room_id); @@ -402,15 +409,6 @@ private: return lmdb::dbi::open(txn, "pending_receipts", MDB_CREATE); } - lmdb::dbi getMessagesDb(lmdb::txn &txn, const std::string &room_id) - { - auto db = - lmdb::dbi::open(txn, std::string(room_id + "/messages").c_str(), MDB_CREATE); - lmdb::dbi_set_compare(txn, db, numeric_key_comparison); - - return db; - } - lmdb::dbi getEventsDb(lmdb::txn &txn, const std::string &room_id) { return lmdb::dbi::open(txn, std::string(room_id + "/events").c_str(), MDB_CREATE); From 82eff09062c51a3136d169e97c70bbed2f439f26 Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Mon, 6 Jul 2020 03:43:14 +0200 Subject: [PATCH 04/50] Fetch event from db and use string_view where possible --- src/Cache.cpp | 83 +++++++++++++++++++++++++++++++++------------------ src/Cache_p.h | 4 +++ 2 files changed, 58 insertions(+), 29 deletions(-) diff --git a/src/Cache.cpp b/src/Cache.cpp index 26291cfd..f07c3855 100644 --- a/src/Cache.cpp +++ b/src/Cache.cpp @@ -778,8 +778,9 @@ Cache::readReceipts(const QString &event_id, const QString &room_id) txn.commit(); if (res) { - auto json_response = json::parse(std::string(value.data(), value.size())); - auto values = json_response.get>(); + auto json_response = + json::parse(std::string_view(value.data(), value.size())); + auto values = json_response.get>(); for (const auto &v : values) // timestamp, user_id @@ -817,8 +818,8 @@ Cache::updateReadReceipt(lmdb::txn &txn, const std::string &room_id, const Recei // If an entry for the event id already exists, we would // merge the existing receipts with the new ones. if (exists) { - auto json_value = - json::parse(std::string(prev_value.data(), prev_value.size())); + auto json_value = json::parse( + std::string_view(prev_value.data(), prev_value.size())); // Retrieve the saved receipts. saved_receipts = json_value.get>(); @@ -937,7 +938,7 @@ Cache::saveState(const mtx::responses::Sync &res) if (lmdb::dbi_get(txn, roomsDb_, lmdb::val(room.first), data)) { try { RoomInfo tmp = - json::parse(std::string(data.data(), data.size())); + json::parse(std::string_view(data.data(), data.size())); updatedInfo.tags = tmp.tags; } catch (const json::exception &e) { nhlog::db()->warn( @@ -1129,7 +1130,7 @@ Cache::singleRoomInfo(const std::string &room_id) // Check if the room is joined. if (lmdb::dbi_get(txn, roomsDb_, lmdb::val(room_id), data)) { try { - RoomInfo tmp = json::parse(std::string(data.data(), data.size())); + RoomInfo tmp = json::parse(std::string_view(data.data(), data.size())); tmp.member_count = getMembersDb(txn, room_id).size(txn); tmp.join_rule = getRoomJoinRule(txn, statesdb); tmp.guest_access = getRoomGuestAccess(txn, statesdb); @@ -1164,7 +1165,8 @@ Cache::getRoomInfo(const std::vector &rooms) // Check if the room is joined. if (lmdb::dbi_get(txn, roomsDb_, lmdb::val(room), data)) { try { - RoomInfo tmp = json::parse(std::string(data.data(), data.size())); + RoomInfo tmp = + json::parse(std::string_view(data.data(), data.size())); tmp.member_count = getMembersDb(txn, room).size(txn); tmp.join_rule = getRoomJoinRule(txn, statesdb); tmp.guest_access = getRoomGuestAccess(txn, statesdb); @@ -1180,7 +1182,7 @@ Cache::getRoomInfo(const std::vector &rooms) if (lmdb::dbi_get(txn, invitesDb_, lmdb::val(room), data)) { try { RoomInfo tmp = - json::parse(std::string(data.data(), data.size())); + json::parse(std::string_view(data.data(), data.size())); tmp.member_count = getInviteMembersDb(txn, room).size(txn); room_info.emplace(QString::fromStdString(room), @@ -1272,7 +1274,7 @@ Cache::getTimelineMessages(lmdb::txn &txn, const std::string &room_id, int64_t i bool ret; while ((ret = cursor.get(indexVal, val, forward ? MDB_NEXT : MDB_LAST)) && counter++ < BATCH_SIZE) { - auto obj = json::parse(std::string(val.data(), val.size())); + auto obj = json::parse(std::string_view(val.data(), val.size())); if (obj.count("event_id") == 0) break; @@ -1285,10 +1287,14 @@ Cache::getTimelineMessages(lmdb::txn &txn, const std::string &room_id, int64_t i mtx::events::collections::TimelineEvent te; mtx::events::collections::from_json( - json::parse(std::string(event.data(), event.size())), te); + json::parse(std::string_view(event.data(), event.size())), te); messages.timeline.events.push_back(std::move(te.data)); - // timeline.prev_batch = obj.at("token").get(); + + if (forward && messages.timeline.prev_batch.empty() && obj.contains("prev_batch")) + messages.timeline.prev_batch = obj["prev_batch"]; + else if (!forward && obj.contains("prev_batch")) + messages.timeline.prev_batch = obj["prev_batch"]; } cursor.close(); @@ -1299,6 +1305,24 @@ Cache::getTimelineMessages(lmdb::txn &txn, const std::string &room_id, int64_t i return messages; } +std::optional +Cache::getEvent(const std::string &room_id, const std::string &event_id) +{ + auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + auto eventsDb = getEventsDb(txn, room_id); + + lmdb::val event{}; + bool success = lmdb::dbi_get(txn, eventsDb, lmdb::val(event_id), event); + if (!success) + return {}; + + mtx::events::collections::TimelineEvent te; + mtx::events::collections::from_json( + json::parse(std::string_view(event.data(), event.size())), te); + + return te; +} + QMap Cache::roomInfo(bool withInvites) { @@ -1348,7 +1372,7 @@ Cache::getLastEventId(lmdb::txn &txn, const std::string &room_id) return {}; } - auto obj = json::parse(std::string(val.data(), val.size())); + auto obj = json::parse(std::string_view(val.data(), val.size())); if (obj.count("event_id") == 0) return {}; @@ -1373,7 +1397,7 @@ Cache::getLastMessageInfo(lmdb::txn &txn, const std::string &room_id) auto cursor = lmdb::cursor::open(txn, orderDb); cursor.get(indexVal, val, MDB_LAST); while (cursor.get(indexVal, val, MDB_PREV)) { - auto temp = json::parse(std::string(val.data(), val.size())); + auto temp = json::parse(std::string_view(val.data(), val.size())); if (temp.count("event_id") == 0) break; @@ -1384,7 +1408,7 @@ Cache::getLastMessageInfo(lmdb::txn &txn, const std::string &room_id) if (!success) continue; - auto obj = json::parse(std::string(event.data(), event.size())); + auto obj = json::parse(std::string_view(event.data(), event.size())); if (fallbackDesc.event_id.isEmpty() && obj["type"] == "m.room.member" && obj["state_key"] == local_user.toStdString() && @@ -1450,7 +1474,7 @@ Cache::getRoomAvatarUrl(lmdb::txn &txn, if (res) { try { StateEvent msg = - json::parse(std::string(event.data(), event.size())); + json::parse(std::string_view(event.data(), event.size())); if (!msg.content.url.empty()) return QString::fromStdString(msg.content.url); @@ -1500,7 +1524,8 @@ Cache::getRoomName(lmdb::txn &txn, lmdb::dbi &statesdb, lmdb::dbi &membersdb) if (res) { try { - StateEvent msg = json::parse(std::string(event.data(), event.size())); + StateEvent msg = + json::parse(std::string_view(event.data(), event.size())); if (!msg.content.name.empty()) return QString::fromStdString(msg.content.name); @@ -1515,7 +1540,7 @@ Cache::getRoomName(lmdb::txn &txn, lmdb::dbi &statesdb, lmdb::dbi &membersdb) if (res) { try { StateEvent msg = - json::parse(std::string(event.data(), event.size())); + json::parse(std::string_view(event.data(), event.size())); if (!msg.content.alias.empty()) return QString::fromStdString(msg.content.alias); @@ -1578,7 +1603,7 @@ Cache::getRoomJoinRule(lmdb::txn &txn, lmdb::dbi &statesdb) if (res) { try { StateEvent msg = - json::parse(std::string(event.data(), event.size())); + json::parse(std::string_view(event.data(), event.size())); return msg.content.join_rule; } catch (const json::exception &e) { nhlog::db()->warn("failed to parse m.room.join_rule event: {}", e.what()); @@ -1600,7 +1625,7 @@ Cache::getRoomGuestAccess(lmdb::txn &txn, lmdb::dbi &statesdb) if (res) { try { StateEvent msg = - json::parse(std::string(event.data(), event.size())); + json::parse(std::string_view(event.data(), event.size())); return msg.content.guest_access == AccessState::CanJoin; } catch (const json::exception &e) { nhlog::db()->warn("failed to parse m.room.guest_access event: {}", @@ -1623,7 +1648,7 @@ Cache::getRoomTopic(lmdb::txn &txn, lmdb::dbi &statesdb) if (res) { try { StateEvent msg = - json::parse(std::string(event.data(), event.size())); + json::parse(std::string_view(event.data(), event.size())); if (!msg.content.topic.empty()) return QString::fromStdString(msg.content.topic); @@ -1648,7 +1673,7 @@ Cache::getRoomVersion(lmdb::txn &txn, lmdb::dbi &statesdb) if (res) { try { StateEvent msg = - json::parse(std::string(event.data(), event.size())); + json::parse(std::string_view(event.data(), event.size())); if (!msg.content.room_version.empty()) return QString::fromStdString(msg.content.room_version); @@ -1674,7 +1699,7 @@ Cache::getInviteRoomName(lmdb::txn &txn, lmdb::dbi &statesdb, lmdb::dbi &members if (res) { try { StrippedEvent msg = - json::parse(std::string(event.data(), event.size())); + json::parse(std::string_view(event.data(), event.size())); return QString::fromStdString(msg.content.name); } catch (const json::exception &e) { nhlog::db()->warn("failed to parse m.room.name event: {}", e.what()); @@ -1716,7 +1741,7 @@ Cache::getInviteRoomAvatarUrl(lmdb::txn &txn, lmdb::dbi &statesdb, lmdb::dbi &me if (res) { try { StrippedEvent msg = - json::parse(std::string(event.data(), event.size())); + json::parse(std::string_view(event.data(), event.size())); return QString::fromStdString(msg.content.url); } catch (const json::exception &e) { nhlog::db()->warn("failed to parse m.room.avatar event: {}", e.what()); @@ -1758,7 +1783,7 @@ Cache::getInviteRoomTopic(lmdb::txn &txn, lmdb::dbi &db) if (res) { try { StrippedEvent msg = - json::parse(std::string(event.data(), event.size())); + json::parse(std::string_view(event.data(), event.size())); return QString::fromStdString(msg.content.topic); } catch (const json::exception &e) { nhlog::db()->warn("failed to parse m.room.topic event: {}", e.what()); @@ -1789,7 +1814,7 @@ Cache::getRoomAvatar(const std::string &room_id) std::string media_url; try { - RoomInfo info = json::parse(std::string(response.data(), response.size())); + RoomInfo info = json::parse(std::string_view(response.data(), response.size())); media_url = std::move(info.avatar_url); if (media_url.empty()) { @@ -2197,7 +2222,7 @@ Cache::deleteOldMessages() while (cursor.get(indexVal, val, MDB_NEXT) && message_count-- < MAX_RESTORED_MESSAGES) { - auto obj = json::parse(std::string(val.data(), val.size())); + auto obj = json::parse(std::string_view(val.data(), val.size())); if (obj.count("event_id") != 0) lmdb::dbi_del( @@ -2239,7 +2264,7 @@ Cache::hasEnoughPowerLevel(const std::vector &eventTypes if (res) { try { StateEvent msg = - json::parse(std::string(event.data(), event.size())); + json::parse(std::string_view(event.data(), event.size())); user_level = msg.content.user_level(user_id); @@ -2354,7 +2379,7 @@ Cache::presenceState(const std::string &user_id) if (res) { mtx::events::presence::Presence presence = - json::parse(std::string(presenceVal.data(), presenceVal.size())); + json::parse(std::string_view(presenceVal.data(), presenceVal.size())); state = presence.presence; } @@ -2376,7 +2401,7 @@ Cache::statusMessage(const std::string &user_id) if (res) { mtx::events::presence::Presence presence = - json::parse(std::string(presenceVal.data(), presenceVal.size())); + json::parse(std::string_view(presenceVal.data(), presenceVal.size())); status_msg = presence.status_msg; } diff --git a/src/Cache_p.h b/src/Cache_p.h index 37486ca0..10839967 100644 --- a/src/Cache_p.h +++ b/src/Cache_p.h @@ -259,6 +259,10 @@ private: int64_t index = std::numeric_limits::max(), bool forward = false); + std::optional getEvent( + const std::string &room_id, + const std::string &event_id); + //! Remove a room from the cache. // void removeLeftRoom(lmdb::txn &txn, const std::string &room_id); template From 0da1a6d5fce0f25162abfb9bcfd4041fb167ebf4 Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Wed, 8 Jul 2020 02:02:14 +0200 Subject: [PATCH 05/50] Add relations and order without hidden events to db --- src/Cache.cpp | 119 ++++++++++++++++++++++++++++++-------------------- src/Cache_p.h | 18 ++++++++ 2 files changed, 90 insertions(+), 47 deletions(-) diff --git a/src/Cache.cpp b/src/Cache.cpp index f07c3855..852d45ec 100644 --- a/src/Cache.cpp +++ b/src/Cache.cpp @@ -1245,23 +1245,23 @@ Cache::Messages Cache::getTimelineMessages(lmdb::txn &txn, const std::string &room_id, int64_t index, bool forward) { // TODO(nico): Limit the messages returned by this maybe? - auto orderDb = getEventOrderDb(txn, room_id); + auto orderDb = getOrderToMessageDb(txn, room_id); auto eventsDb = getEventsDb(txn, room_id); Messages messages{}; - lmdb::val indexVal, val; + lmdb::val indexVal, event_id; auto cursor = lmdb::cursor::open(txn, orderDb); if (index == std::numeric_limits::max()) { - if (cursor.get(indexVal, val, forward ? MDB_FIRST : MDB_LAST)) { + if (cursor.get(indexVal, event_id, forward ? MDB_FIRST : MDB_LAST)) { index = *indexVal.data(); } else { messages.end_of_cache = true; return messages; } } else { - if (cursor.get(indexVal, val, MDB_SET)) { + if (cursor.get(indexVal, event_id, MDB_SET)) { index = *indexVal.data(); } else { messages.end_of_cache = true; @@ -1272,16 +1272,10 @@ Cache::getTimelineMessages(lmdb::txn &txn, const std::string &room_id, int64_t i int counter = 0; bool ret; - while ((ret = cursor.get(indexVal, val, forward ? MDB_NEXT : MDB_LAST)) && + while ((ret = cursor.get(indexVal, event_id, forward ? MDB_NEXT : MDB_LAST)) && counter++ < BATCH_SIZE) { - auto obj = json::parse(std::string_view(val.data(), val.size())); - - if (obj.count("event_id") == 0) - break; - lmdb::val event; - bool success = lmdb::dbi_get( - txn, eventsDb, lmdb::val(obj["event_id"].get()), event); + bool success = lmdb::dbi_get(txn, eventsDb, event_id, event); if (!success) continue; @@ -1290,11 +1284,6 @@ Cache::getTimelineMessages(lmdb::txn &txn, const std::string &room_id, int64_t i json::parse(std::string_view(event.data(), event.size())), te); messages.timeline.events.push_back(std::move(te.data)); - - if (forward && messages.timeline.prev_batch.empty() && obj.contains("prev_batch")) - messages.timeline.prev_batch = obj["prev_batch"]; - else if (!forward && obj.contains("prev_batch")) - messages.timeline.prev_batch = obj["prev_batch"]; } cursor.close(); @@ -1363,7 +1352,7 @@ Cache::roomInfo(bool withInvites) std::string Cache::getLastEventId(lmdb::txn &txn, const std::string &room_id) { - auto orderDb = getEventOrderDb(txn, room_id); + auto orderDb = getOrderToMessageDb(txn, room_id); lmdb::val indexVal, val; @@ -1372,18 +1361,13 @@ Cache::getLastEventId(lmdb::txn &txn, const std::string &room_id) return {}; } - auto obj = json::parse(std::string_view(val.data(), val.size())); - - if (obj.count("event_id") == 0) - return {}; - else - return obj["event_id"]; + return std::string(val.data(), val.size()); } DescInfo Cache::getLastMessageInfo(lmdb::txn &txn, const std::string &room_id) { - auto orderDb = getEventOrderDb(txn, room_id); + auto orderDb = getOrderToMessageDb(txn, room_id); auto eventsDb = getEventsDb(txn, room_id); if (orderDb.size(txn) == 0) return DescInfo{}; @@ -1392,19 +1376,13 @@ Cache::getLastMessageInfo(lmdb::txn &txn, const std::string &room_id) DescInfo fallbackDesc{}; - lmdb::val indexVal, val; + lmdb::val indexVal, event_id; auto cursor = lmdb::cursor::open(txn, orderDb); - cursor.get(indexVal, val, MDB_LAST); - while (cursor.get(indexVal, val, MDB_PREV)) { - auto temp = json::parse(std::string_view(val.data(), val.size())); - - if (temp.count("event_id") == 0) - break; - + cursor.get(indexVal, event_id, MDB_LAST); + while (cursor.get(indexVal, event_id, MDB_PREV)) { lmdb::val event; - bool success = lmdb::dbi_get( - txn, eventsDb, lmdb::val(temp["event_id"].get()), event); + bool success = lmdb::dbi_get(txn, eventsDb, event_id, event); if (!success) continue; @@ -2015,11 +1993,17 @@ Cache::saveTimelineMessages(lmdb::txn &txn, const std::string &room_id, const mtx::responses::Timeline &res) { - auto eventsDb = getEventsDb(txn, room_id); + auto eventsDb = getEventsDb(txn, room_id); + auto relationsDb = getRelationsDb(txn, room_id); - auto orderDb = getEventOrderDb(txn, room_id); - if (res.limited) + auto orderDb = getEventOrderDb(txn, room_id); + auto msg2orderDb = getMessageToOrderDb(txn, room_id); + auto order2msgDb = getOrderToMessageDb(txn, room_id); + if (res.limited) { lmdb::dbi_drop(txn, orderDb, false); + lmdb::dbi_drop(txn, msg2orderDb, false); + lmdb::dbi_drop(txn, order2msgDb, false); + } using namespace mtx::events; using namespace mtx::events::state; @@ -2031,6 +2015,12 @@ Cache::saveTimelineMessages(lmdb::txn &txn, index = *indexVal.data(); } + int64_t msgIndex = 0; + auto msgCursor = lmdb::cursor::open(txn, order2msgDb); + if (msgCursor.get(indexVal, val, MDB_LAST)) { + msgIndex = *indexVal.data(); + } + bool first = true; for (const auto &e : res.events) { auto event = mtx::accessors::serialize_event(e); @@ -2039,17 +2029,17 @@ Cache::saveTimelineMessages(lmdb::txn &txn, lmdb::dbi_put( txn, eventsDb, lmdb::val(redaction->redacts), lmdb::val(event.dump())); } else { - lmdb::dbi_put(txn, - eventsDb, - lmdb::val(event["event_id"].get()), - lmdb::val(event.dump())); + std::string event_id_val = event["event_id"].get(); + lmdb::val event_id = event_id_val; + lmdb::dbi_put(txn, eventsDb, event_id, lmdb::val(event.dump())); ++index; json orderEntry = json::object(); - orderEntry["event_id"] = event["event_id"]; + orderEntry["event_id"] = event_id_val; if (first) orderEntry["prev_batch"] = res.prev_batch; + first = false; nhlog::db()->debug("saving '{}'", orderEntry.dump()); @@ -2057,7 +2047,32 @@ Cache::saveTimelineMessages(lmdb::txn &txn, lmdb::val(&index, sizeof(index)), lmdb::val(orderEntry.dump()), MDB_APPEND); - first = false; + + // TODO(Nico): Allow blacklisting more event types in UI + if (event["type"] != "m.reaction" && event["type"] != "m.dummy") { + ++msgIndex; + lmdb::cursor_put(msgCursor.handle(), + lmdb::val(&msgIndex, sizeof(msgIndex)), + event_id, + MDB_APPEND); + + lmdb::dbi_put(txn, + msg2orderDb, + event_id, + lmdb::val(&msgIndex, sizeof(msgIndex))); + } + + if (event.contains("content") && + event["content"].contains("m.relates_to")) { + auto temp = event["content"]["m.relates_to"]; + std::string relates_to = temp.contains("m.in_reply_to") + ? temp["m.in_reply_to"]["event_id"] + : temp["event_id"]; + + if (!relates_to.empty()) + lmdb::dbi_put( + txn, relationsDb, lmdb::val(relates_to), event_id); + } } } } @@ -2201,6 +2216,8 @@ Cache::deleteOldMessages() for (const auto &room_id : room_ids) { auto orderDb = getEventOrderDb(txn, room_id); + auto o2m = getOrderToMessageDb(txn, room_id); + auto m2o = getMessageToOrderDb(txn, room_id); auto eventsDb = getEventsDb(txn, room_id); auto cursor = lmdb::cursor::open(txn, orderDb); @@ -2224,9 +2241,17 @@ Cache::deleteOldMessages() message_count-- < MAX_RESTORED_MESSAGES) { auto obj = json::parse(std::string_view(val.data(), val.size())); - if (obj.count("event_id") != 0) - lmdb::dbi_del( - txn, eventsDb, lmdb::val(obj["event_id"].get())); + if (obj.count("event_id") != 0) { + lmdb::val event_id = obj["event_id"].get(); + lmdb::dbi_del(txn, eventsDb, event_id); + + lmdb::val order{}; + bool exists = lmdb::dbi_get(txn, m2o, event_id, order); + if (exists) { + lmdb::dbi_del(txn, o2m, order); + lmdb::dbi_del(txn, m2o, event_id); + } + } lmdb::cursor_del(cursor); } cursor.close(); diff --git a/src/Cache_p.h b/src/Cache_p.h index 10839967..3f7b592d 100644 --- a/src/Cache_p.h +++ b/src/Cache_p.h @@ -424,6 +424,24 @@ private: txn, std::string(room_id + "/event_order").c_str(), MDB_CREATE | MDB_INTEGERKEY); } + lmdb::dbi getMessageToOrderDb(lmdb::txn &txn, const std::string &room_id) + { + return lmdb::dbi::open( + txn, std::string(room_id + "/msg2order").c_str(), MDB_CREATE); + } + + lmdb::dbi getOrderToMessageDb(lmdb::txn &txn, const std::string &room_id) + { + return lmdb::dbi::open( + txn, std::string(room_id + "/order2msg").c_str(), MDB_CREATE | MDB_INTEGERKEY); + } + + lmdb::dbi getRelationsDb(lmdb::txn &txn, const std::string &room_id) + { + return lmdb::dbi::open( + txn, std::string(room_id + "/related").c_str(), MDB_CREATE | MDB_DUPSORT); + } + lmdb::dbi getInviteStatesDb(lmdb::txn &txn, const std::string &room_id) { return lmdb::dbi::open( From fe12e63c7c406c2e1b25bf580f2bc73f0cefdb21 Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Wed, 8 Jul 2020 02:02:48 +0200 Subject: [PATCH 06/50] Fix parent undefined warning --- resources/qml/TimelineView.qml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/qml/TimelineView.qml b/resources/qml/TimelineView.qml index 5494c1ba..f81f5986 100644 --- a/resources/qml/TimelineView.qml +++ b/resources/qml/TimelineView.qml @@ -181,7 +181,7 @@ Page { id: wrapper property Item section - anchors.horizontalCenter: parent.horizontalCenter + anchors.horizontalCenter: parent ? parent.horizontalCenter : undefined width: chat.delegateMaxWidth height: section ? section.height + timelinerow.height : timelinerow.height color: "transparent" From 530c531c4b447a0d3599a74731441f2656374f3f Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Thu, 9 Jul 2020 23:15:22 +0200 Subject: [PATCH 07/50] WIP: Event Store split out --- CMakeLists.txt | 2 + resources/qml/TimelineRow.qml | 2 +- resources/qml/TimelineView.qml | 2 +- src/Cache.cpp | 133 ++++++- src/Cache_p.h | 39 +- src/timeline/EventStore.cpp | 259 +++++++++++++ src/timeline/EventStore.h | 98 +++++ src/timeline/TimelineModel.cpp | 647 ++++++++++++--------------------- src/timeline/TimelineModel.h | 13 +- 9 files changed, 756 insertions(+), 439 deletions(-) create mode 100644 src/timeline/EventStore.cpp create mode 100644 src/timeline/EventStore.h diff --git a/CMakeLists.txt b/CMakeLists.txt index f1ccde5f..8d1441c1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -250,6 +250,7 @@ set(SRC_FILES # Timeline + src/timeline/EventStore.cpp src/timeline/ReactionsModel.cpp src/timeline/TimelineViewManager.cpp src/timeline/TimelineModel.cpp @@ -453,6 +454,7 @@ qt5_wrap_cpp(MOC_HEADERS src/emoji/Provider.h # Timeline + src/timeline/EventStore.h src/timeline/ReactionsModel.h src/timeline/TimelineViewManager.h src/timeline/TimelineModel.h diff --git a/resources/qml/TimelineRow.qml b/resources/qml/TimelineRow.qml index dfee62dc..e87590f1 100644 --- a/resources/qml/TimelineRow.qml +++ b/resources/qml/TimelineRow.qml @@ -45,7 +45,7 @@ MouseArea { // fancy reply, if this is a reply Reply { visible: model.replyTo - modelData: chat.model.getDump(model.replyTo) + modelData: chat.model.getDump(model.replyTo, model.id) userColor: timelineManager.userColor(modelData.userId, colors.window) } diff --git a/resources/qml/TimelineView.qml b/resources/qml/TimelineView.qml index f81f5986..fd185bd9 100644 --- a/resources/qml/TimelineView.qml +++ b/resources/qml/TimelineView.qml @@ -353,7 +353,7 @@ Page { anchors.rightMargin: 20 anchors.bottom: parent.bottom - modelData: chat.model ? chat.model.getDump(chat.model.reply) : {} + modelData: chat.model ? chat.model.getDump(chat.model.reply, chat.model.id) : {} userColor: timelineManager.userColor(modelData.userId, colors.window) } diff --git a/src/Cache.cpp b/src/Cache.cpp index 852d45ec..d2c790dd 100644 --- a/src/Cache.cpp +++ b/src/Cache.cpp @@ -1272,7 +1272,10 @@ Cache::getTimelineMessages(lmdb::txn &txn, const std::string &room_id, int64_t i int counter = 0; bool ret; - while ((ret = cursor.get(indexVal, event_id, forward ? MDB_NEXT : MDB_LAST)) && + while ((ret = cursor.get(indexVal, + event_id, + counter == 0 ? (forward ? MDB_FIRST : MDB_LAST) + : (forward ? MDB_NEXT : MDB_PREV))) && counter++ < BATCH_SIZE) { lmdb::val event; bool success = lmdb::dbi_get(txn, eventsDb, event_id, event); @@ -1280,8 +1283,13 @@ Cache::getTimelineMessages(lmdb::txn &txn, const std::string &room_id, int64_t i continue; mtx::events::collections::TimelineEvent te; - mtx::events::collections::from_json( - json::parse(std::string_view(event.data(), event.size())), te); + try { + mtx::events::collections::from_json( + json::parse(std::string_view(event.data(), event.size())), te); + } catch (std::exception &e) { + nhlog::db()->error("Failed to parse message from cache {}", e.what()); + continue; + } messages.timeline.events.push_back(std::move(te.data)); } @@ -1306,8 +1314,13 @@ Cache::getEvent(const std::string &room_id, const std::string &event_id) return {}; mtx::events::collections::TimelineEvent te; - mtx::events::collections::from_json( - json::parse(std::string_view(event.data(), event.size())), te); + try { + mtx::events::collections::from_json( + json::parse(std::string_view(event.data(), event.size())), te); + } catch (std::exception &e) { + nhlog::db()->error("Failed to parse message from cache {}", e.what()); + return std::nullopt; + } return te; } @@ -1364,6 +1377,61 @@ Cache::getLastEventId(lmdb::txn &txn, const std::string &room_id) return std::string(val.data(), val.size()); } +std::optional +Cache::getTimelineRange(const std::string &room_id) +{ + auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + auto orderDb = getOrderToMessageDb(txn, room_id); + + lmdb::val indexVal, val; + + auto cursor = lmdb::cursor::open(txn, orderDb); + if (!cursor.get(indexVal, val, MDB_LAST)) { + return {}; + } + + TimelineRange range{}; + range.last = *indexVal.data(); + + if (!cursor.get(indexVal, val, MDB_FIRST)) { + return {}; + } + range.first = *indexVal.data(); + + return range; +} +std::optional +Cache::getTimelineIndex(const std::string &room_id, std::string_view event_id) +{ + auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + auto orderDb = getMessageToOrderDb(txn, room_id); + + lmdb::val indexVal{event_id.data(), event_id.size()}, val; + + bool success = lmdb::dbi_get(txn, orderDb, indexVal, val); + if (!success) { + return {}; + } + + return *val.data(); +} + +std::optional +Cache::getTimelineEventId(const std::string &room_id, int64_t index) +{ + auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + auto orderDb = getOrderToMessageDb(txn, room_id); + + lmdb::val indexVal{&index, sizeof(index)}, val; + + bool success = lmdb::dbi_get(txn, orderDb, indexVal, val); + if (!success) { + return {}; + } + + return std::string(val.data(), val.size()); +} + DescInfo Cache::getLastMessageInfo(lmdb::txn &txn, const std::string &room_id) { @@ -1379,8 +1447,10 @@ Cache::getLastMessageInfo(lmdb::txn &txn, const std::string &room_id) lmdb::val indexVal, event_id; auto cursor = lmdb::cursor::open(txn, orderDb); - cursor.get(indexVal, event_id, MDB_LAST); - while (cursor.get(indexVal, event_id, MDB_PREV)) { + bool first = true; + while (cursor.get(indexVal, event_id, first ? MDB_LAST : MDB_PREV)) { + first = false; + lmdb::val event; bool success = lmdb::dbi_get(txn, eventsDb, event_id, event); if (!success) @@ -2026,8 +2096,43 @@ Cache::saveTimelineMessages(lmdb::txn &txn, auto event = mtx::accessors::serialize_event(e); if (auto redaction = std::get_if>(&e)) { - lmdb::dbi_put( - txn, eventsDb, lmdb::val(redaction->redacts), lmdb::val(event.dump())); + if (redaction->redacts.empty()) + continue; + + lmdb::val ev{}; + bool success = + lmdb::dbi_get(txn, eventsDb, lmdb::val(redaction->redacts), ev); + if (!success) + continue; + + mtx::events::collections::TimelineEvent te; + + try { + mtx::events::collections::from_json( + json::parse(std::string_view(ev.data(), ev.size())), te); + } catch (std::exception &e) { + nhlog::db()->error("Failed to parse message from cache {}", + e.what()); + continue; + } + + auto redactedEvent = std::visit( + [](const auto &ev) -> mtx::events::RoomEvent { + mtx::events::RoomEvent replacement = + {}; + replacement.event_id = ev.event_id; + replacement.room_id = ev.room_id; + replacement.sender = ev.sender; + replacement.origin_server_ts = ev.origin_server_ts; + replacement.type = ev.type; + return replacement; + }, + te.data); + + lmdb::dbi_put(txn, + eventsDb, + lmdb::val(redaction->redacts), + lmdb::val(json(redactedEvent).dump())); } else { std::string event_id_val = event["event_id"].get(); lmdb::val event_id = event_id_val; @@ -2237,8 +2342,10 @@ Cache::deleteOldMessages() if (message_count < MAX_RESTORED_MESSAGES) continue; - while (cursor.get(indexVal, val, MDB_NEXT) && + bool start = true; + while (cursor.get(indexVal, val, start ? MDB_FIRST : MDB_NEXT) && message_count-- < MAX_RESTORED_MESSAGES) { + start = false; auto obj = json::parse(std::string_view(val.data(), val.size())); if (obj.count("event_id") != 0) { @@ -2394,6 +2501,9 @@ Cache::removeAvatarUrl(const QString &room_id, const QString &user_id) mtx::presence::PresenceState Cache::presenceState(const std::string &user_id) { + if (user_id.empty()) + return {}; + lmdb::val presenceVal; auto txn = lmdb::txn::begin(env_); @@ -2416,6 +2526,9 @@ Cache::presenceState(const std::string &user_id) std::string Cache::statusMessage(const std::string &user_id) { + if (user_id.empty()) + return {}; + lmdb::val presenceVal; auto txn = lmdb::txn::begin(env_); diff --git a/src/Cache_p.h b/src/Cache_p.h index 3f7b592d..40c8e98b 100644 --- a/src/Cache_p.h +++ b/src/Cache_p.h @@ -170,6 +170,30 @@ public: //! Add all notifications containing a user mention to the db. void saveTimelineMentions(const mtx::responses::Notifications &res); + //! retrieve events in timeline and related functions + struct Messages + { + mtx::responses::Timeline timeline; + uint64_t next_index; + bool end_of_cache = false; + }; + Messages getTimelineMessages(lmdb::txn &txn, + const std::string &room_id, + int64_t index = std::numeric_limits::max(), + bool forward = false); + + std::optional getEvent( + const std::string &room_id, + const std::string &event_id); + struct TimelineRange + { + int64_t first, last; + }; + std::optional getTimelineRange(const std::string &room_id); + std::optional getTimelineIndex(const std::string &room_id, + std::string_view event_id); + std::optional getTimelineEventId(const std::string &room_id, int64_t index); + //! Remove old unused data. void deleteOldMessages(); void deleteOldData() noexcept; @@ -248,21 +272,6 @@ private: const std::string &room_id, const mtx::responses::Timeline &res); - struct Messages - { - mtx::responses::Timeline timeline; - uint64_t next_index; - bool end_of_cache = false; - }; - Messages getTimelineMessages(lmdb::txn &txn, - const std::string &room_id, - int64_t index = std::numeric_limits::max(), - bool forward = false); - - std::optional getEvent( - const std::string &room_id, - const std::string &event_id); - //! Remove a room from the cache. // void removeLeftRoom(lmdb::txn &txn, const std::string &room_id); template diff --git a/src/timeline/EventStore.cpp b/src/timeline/EventStore.cpp new file mode 100644 index 00000000..eb54d475 --- /dev/null +++ b/src/timeline/EventStore.cpp @@ -0,0 +1,259 @@ +#include "EventStore.h" + +#include + +#include "Cache_p.h" +#include "EventAccessors.h" +#include "Logging.h" +#include "Olm.h" + +QCache EventStore::decryptedEvents_{ + 1000}; +QCache EventStore::events_by_id_{ + 1000}; +QCache EventStore::events_{1000}; + +EventStore::EventStore(std::string room_id, QObject *) + : room_id_(std::move(room_id)) +{ + auto range = cache::client()->getTimelineRange(room_id_); + + if (range) { + this->first = range->first; + this->last = range->last; + } +} + +void +EventStore::handleSync(const mtx::responses::Timeline &events) +{ + if (this->thread() != QThread::currentThread()) + nhlog::db()->warn("{} called from a different thread!", __func__); + + auto range = cache::client()->getTimelineRange(room_id_); + + if (range && range->last > this->last) { + emit beginInsertRows(toExternalIdx(this->last + 1), toExternalIdx(range->last)); + this->last = range->last; + emit endInsertRows(); + } + + for (const auto &event : events.events) { + std::string relates_to; + if (auto redaction = + std::get_if>( + &event)) { + relates_to = redaction->redacts; + } else if (auto reaction = + std::get_if>( + &event)) { + relates_to = reaction->content.relates_to.event_id; + } else { + relates_to = mtx::accessors::in_reply_to_event(event); + } + + if (!relates_to.empty()) { + auto idx = cache::client()->getTimelineIndex(room_id_, relates_to); + if (idx) + emit dataChanged(toExternalIdx(*idx), toExternalIdx(*idx)); + } + } +} + +mtx::events::collections::TimelineEvents * +EventStore::event(int idx, bool decrypt) +{ + if (this->thread() != QThread::currentThread()) + nhlog::db()->warn("{} called from a different thread!", __func__); + + Index index{room_id_, toInternalIdx(idx)}; + if (index.idx > last || index.idx < first) + return nullptr; + + auto event_ptr = events_.object(index); + if (!event_ptr) { + auto event_id = cache::client()->getTimelineEventId(room_id_, index.idx); + if (!event_id) + return nullptr; + + auto event = cache::client()->getEvent(room_id_, *event_id); + if (!event) + return nullptr; + else + event_ptr = + new mtx::events::collections::TimelineEvents(std::move(event->data)); + events_.insert(index, event_ptr); + } + + if (decrypt) + if (auto encrypted = + std::get_if>( + event_ptr)) + return decryptEvent({room_id_, encrypted->event_id}, *encrypted); + + return event_ptr; +} + +std::optional +EventStore::idToIndex(std::string_view id) const +{ + if (this->thread() != QThread::currentThread()) + nhlog::db()->warn("{} called from a different thread!", __func__); + + auto idx = cache::client()->getTimelineIndex(room_id_, id); + if (idx) + return toExternalIdx(*idx); + else + return std::nullopt; +} +std::optional +EventStore::indexToId(int idx) const +{ + if (this->thread() != QThread::currentThread()) + nhlog::db()->warn("{} called from a different thread!", __func__); + + return cache::client()->getTimelineEventId(room_id_, toInternalIdx(idx)); +} + +mtx::events::collections::TimelineEvents * +EventStore::decryptEvent(const IdIndex &idx, + const mtx::events::EncryptedEvent &e) +{ + if (auto cachedEvent = decryptedEvents_.object(idx)) + return cachedEvent; + + MegolmSessionIndex index; + index.room_id = room_id_; + index.session_id = e.content.session_id; + index.sender_key = e.content.sender_key; + + mtx::events::RoomEvent dummy; + dummy.origin_server_ts = e.origin_server_ts; + dummy.event_id = e.event_id; + dummy.sender = e.sender; + dummy.content.body = + tr("-- Encrypted Event (No keys found for decryption) --", + "Placeholder, when the message was not decrypted yet or can't be decrypted.") + .toStdString(); + + auto asCacheEntry = [&idx](mtx::events::collections::TimelineEvents &&event) { + auto event_ptr = new mtx::events::collections::TimelineEvents(std::move(event)); + decryptedEvents_.insert(idx, event_ptr); + return event_ptr; + }; + + try { + if (!cache::client()->inboundMegolmSessionExists(index)) { + nhlog::crypto()->info("Could not find inbound megolm session ({}, {}, {})", + index.room_id, + index.session_id, + e.sender); + // TODO: request megolm session_id & session_key from the sender. + return asCacheEntry(std::move(dummy)); + } + } catch (const lmdb::error &e) { + nhlog::db()->critical("failed to check megolm session's existence: {}", e.what()); + dummy.content.body = tr("-- Decryption Error (failed to communicate with DB) --", + "Placeholder, when the message can't be decrypted, because " + "the DB access failed when trying to lookup the session.") + .toStdString(); + return asCacheEntry(std::move(dummy)); + } + + std::string msg_str; + try { + auto session = cache::client()->getInboundMegolmSession(index); + auto res = olm::client()->decrypt_group_message(session, e.content.ciphertext); + msg_str = std::string((char *)res.data.data(), res.data.size()); + } catch (const lmdb::error &e) { + nhlog::db()->critical("failed to retrieve megolm session with index ({}, {}, {})", + index.room_id, + index.session_id, + index.sender_key, + e.what()); + dummy.content.body = + tr("-- Decryption Error (failed to retrieve megolm keys from db) --", + "Placeholder, when the message can't be decrypted, because the DB access " + "failed.") + .toStdString(); + return asCacheEntry(std::move(dummy)); + } catch (const mtx::crypto::olm_exception &e) { + nhlog::crypto()->critical("failed to decrypt message with index ({}, {}, {}): {}", + index.room_id, + index.session_id, + index.sender_key, + e.what()); + dummy.content.body = + tr("-- Decryption Error (%1) --", + "Placeholder, when the message can't be decrypted. In this case, the Olm " + "decrytion returned an error, which is passed as %1.") + .arg(e.what()) + .toStdString(); + return asCacheEntry(std::move(dummy)); + } + + // Add missing fields for the event. + json body = json::parse(msg_str); + body["event_id"] = e.event_id; + body["sender"] = e.sender; + body["origin_server_ts"] = e.origin_server_ts; + body["unsigned"] = e.unsigned_data; + + // relations are unencrypted in content... + if (json old_ev = e; old_ev["content"].count("m.relates_to") != 0) + body["content"]["m.relates_to"] = old_ev["content"]["m.relates_to"]; + + json event_array = json::array(); + event_array.push_back(body); + + std::vector temp_events; + mtx::responses::utils::parse_timeline_events(event_array, temp_events); + + if (temp_events.size() == 1) { + auto encInfo = mtx::accessors::file(temp_events[0]); + + if (encInfo) + emit newEncryptedImage(encInfo.value()); + + return asCacheEntry(std::move(temp_events[0])); + } + + dummy.content.body = + tr("-- Encrypted Event (Unknown event type) --", + "Placeholder, when the message was decrypted, but we couldn't parse it, because " + "Nheko/mtxclient don't support that event type yet.") + .toStdString(); + return asCacheEntry(std::move(dummy)); +} + +mtx::events::collections::TimelineEvents * +EventStore::event(std::string_view id, std::string_view related_to, bool decrypt) +{ + if (this->thread() != QThread::currentThread()) + nhlog::db()->warn("{} called from a different thread!", __func__); + + if (id.empty()) + return nullptr; + + IdIndex index{room_id_, std::string(id.data(), id.size())}; + + auto event_ptr = events_by_id_.object(index); + if (!event_ptr) { + auto event = cache::client()->getEvent(room_id_, index.id); + if (!event) { + // fetch + (void)related_to; + return nullptr; + } + event_ptr = new mtx::events::collections::TimelineEvents(std::move(event->data)); + events_by_id_.insert(index, event_ptr); + } + + if (decrypt) + if (auto encrypted = + std::get_if>( + event_ptr)) + return decryptEvent(index, *encrypted); + + return event_ptr; +} diff --git a/src/timeline/EventStore.h b/src/timeline/EventStore.h new file mode 100644 index 00000000..77d73536 --- /dev/null +++ b/src/timeline/EventStore.h @@ -0,0 +1,98 @@ +#pragma once + +#include +#include + +#include +#include +#include + +#include +#include + +class EventStore : public QObject +{ + Q_OBJECT + +public: + EventStore(std::string room_id, QObject *parent); + + struct Index + { + std::string room; + int64_t idx; + + friend uint qHash(const Index &i, uint seed = 0) noexcept + { + QtPrivate::QHashCombine hash; + seed = hash(seed, QByteArray::fromRawData(i.room.data(), i.room.size())); + seed = hash(seed, i.idx); + return seed; + } + + friend bool operator==(const Index &a, const Index &b) noexcept + { + return a.idx == b.idx && a.room == b.room; + } + }; + struct IdIndex + { + std::string room, id; + + friend uint qHash(const IdIndex &i, uint seed = 0) noexcept + { + QtPrivate::QHashCombine hash; + seed = hash(seed, QByteArray::fromRawData(i.room.data(), i.room.size())); + seed = hash(seed, QByteArray::fromRawData(i.id.data(), i.id.size())); + return seed; + } + + friend bool operator==(const IdIndex &a, const IdIndex &b) noexcept + { + return a.id == b.id && a.room == b.room; + } + }; + + void fetchMore(); + void handleSync(const mtx::responses::Timeline &events); + + // optionally returns the event or nullptr and fetches it, after which it emits a + // relatedFetched event + mtx::events::collections::TimelineEvents *event(std::string_view id, + std::string_view related_to, + bool decrypt = true); + // always returns a proper event as long as the idx is valid + mtx::events::collections::TimelineEvents *event(int idx, bool decrypt = true); + + int size() const + { + return last != std::numeric_limits::max() + ? static_cast(last - first) + 1 + : 0; + } + int toExternalIdx(int64_t idx) const { return static_cast(idx - first); } + int64_t toInternalIdx(int idx) const { return first + idx; } + + std::optional idToIndex(std::string_view id) const; + std::optional indexToId(int idx) const; + +signals: + void beginInsertRows(int from, int to); + void endInsertRows(); + void dataChanged(int from, int to); + void newEncryptedImage(mtx::crypto::EncryptedFile encryptionInfo); + +private: + mtx::events::collections::TimelineEvents *decryptEvent( + const IdIndex &idx, + const mtx::events::EncryptedEvent &e); + + std::string room_id_; + + int64_t first = std::numeric_limits::max(), + last = std::numeric_limits::max(); + + static QCache decryptedEvents_; + static QCache events_; + static QCache events_by_id_; +}; diff --git a/src/timeline/TimelineModel.cpp b/src/timeline/TimelineModel.cpp index 504c6dcf..492d4e0a 100644 --- a/src/timeline/TimelineModel.cpp +++ b/src/timeline/TimelineModel.cpp @@ -141,6 +141,7 @@ toRoomEventTypeString(const mtx::events::collections::TimelineEvents &event) TimelineModel::TimelineModel(TimelineViewManager *manager, QString room_id, QObject *parent) : QAbstractListModel(parent) + , events(room_id.toStdString(), this) , room_id_(room_id) , manager_(manager) { @@ -165,41 +166,41 @@ TimelineModel::TimelineModel(TimelineViewManager *manager, QString room_id, QObj this, [this](QString txn_id, QString event_id) { pending.removeOne(txn_id); + (void)event_id; + // auto ev = events.value(txn_id); - auto ev = events.value(txn_id); + // if (auto reaction = + // std::get_if>(&ev)) { + // QString reactedTo = + // QString::fromStdString(reaction->content.relates_to.event_id); + // auto &rModel = reactions[reactedTo]; + // rModel.removeReaction(*reaction); + // auto rCopy = *reaction; + // rCopy.event_id = event_id.toStdString(); + // rModel.addReaction(room_id_.toStdString(), rCopy); + //} - if (auto reaction = - std::get_if>(&ev)) { - QString reactedTo = - QString::fromStdString(reaction->content.relates_to.event_id); - auto &rModel = reactions[reactedTo]; - rModel.removeReaction(*reaction); - auto rCopy = *reaction; - rCopy.event_id = event_id.toStdString(); - rModel.addReaction(room_id_.toStdString(), rCopy); - } + // int idx = idToIndex(txn_id); + // if (idx < 0) { + // // transaction already received via sync + // return; + //} + // eventOrder[idx] = event_id; + // ev = std::visit( + // [event_id](const auto &e) -> mtx::events::collections::TimelineEvents { + // auto eventCopy = e; + // eventCopy.event_id = event_id.toStdString(); + // return eventCopy; + // }, + // ev); - int idx = idToIndex(txn_id); - if (idx < 0) { - // transaction already received via sync - return; - } - eventOrder[idx] = event_id; - ev = std::visit( - [event_id](const auto &e) -> mtx::events::collections::TimelineEvents { - auto eventCopy = e; - eventCopy.event_id = event_id.toStdString(); - return eventCopy; - }, - ev); + // events.remove(txn_id); + // events.insert(event_id, ev); - events.remove(txn_id); - events.insert(event_id, ev); + //// mark our messages as read + // readEvent(event_id.toStdString()); - // mark our messages as read - readEvent(event_id.toStdString()); - - emit dataChanged(index(idx, 0), index(idx, 0)); + // emit dataChanged(index(idx, 0), index(idx, 0)); if (pending.size() > 0) emit nextPendingMessage(); @@ -224,16 +225,24 @@ TimelineModel::TimelineModel(TimelineViewManager *manager, QString room_id, QObj Qt::QueuedConnection); connect( + &events, + &EventStore::dataChanged, this, - &TimelineModel::eventFetched, - this, - [this](QString requestingEvent, mtx::events::collections::TimelineEvents event) { - events.insert(QString::fromStdString(mtx::accessors::event_id(event)), event); - auto idx = idToIndex(requestingEvent); - if (idx >= 0) - emit dataChanged(index(idx, 0), index(idx, 0)); + [this](int from, int to) { + emit dataChanged(index(events.size() - to, 0), index(events.size() - from, 0)); }, Qt::QueuedConnection); + + connect(&events, &EventStore::beginInsertRows, this, [this](int from, int to) { + nhlog::ui()->info("begin insert from {} to {}", + events.size() - to + (to - from), + events.size() - from + (to - from)); + beginInsertRows(QModelIndex(), + events.size() - to + (to - from), + events.size() - from + (to - from)); + }); + connect(&events, &EventStore::endInsertRows, this, [this]() { endInsertRows(); }); + connect(&events, &EventStore::newEncryptedImage, this, &TimelineModel::newEncryptedImage); } QHash @@ -274,28 +283,22 @@ int TimelineModel::rowCount(const QModelIndex &parent) const { Q_UNUSED(parent); - return (int)this->eventOrder.size(); + return this->events.size() + static_cast(pending.size()); } QVariantMap -TimelineModel::getDump(QString eventId) const +TimelineModel::getDump(QString eventId, QString relatedTo) const { - if (events.contains(eventId)) - return data(eventId, Dump).toMap(); + if (auto event = events.event(eventId.toStdString(), relatedTo.toStdString())) + return data(*event, Dump).toMap(); return {}; } QVariant -TimelineModel::data(const QString &id, int role) const +TimelineModel::data(const mtx::events::collections::TimelineEvents &event, int role) const { using namespace mtx::accessors; - namespace acc = mtx::accessors; - mtx::events::collections::TimelineEvents event = events.value(id); - - if (auto e = - std::get_if>(&event)) { - event = decryptEvent(*e).event; - } + namespace acc = mtx::accessors; switch (role) { case UserId: @@ -381,8 +384,9 @@ TimelineModel::data(const QString &id, int role) const return QVariant(prop > 0 ? prop : 1.); } case Id: - return id; + return QVariant(QString::fromStdString(event_id(event))); case State: { + auto id = QString::fromStdString(event_id(event)); auto containsOthers = [](const auto &vec) { for (const auto &e : vec) if (e.second != http::client()->user_id().to_string()) @@ -401,19 +405,22 @@ TimelineModel::data(const QString &id, int role) const return qml_mtx_events::Received; } case IsEncrypted: { - return std::holds_alternative< - mtx::events::EncryptedEvent>(events[id]); + // return std::holds_alternative< + // mtx::events::EncryptedEvent>(events[id]); + return false; } case IsRoomEncrypted: { return cache::isRoomEncrypted(room_id_.toStdString()); } case ReplyTo: return QVariant(QString::fromStdString(in_reply_to_event(event))); - case Reactions: + case Reactions: { + auto id = QString::fromStdString(event_id(event)); if (reactions.count(id)) return QVariant::fromValue((QObject *)&reactions.at(id)); else return {}; + } case RoomId: return QVariant(room_id_); case RoomName: @@ -425,30 +432,31 @@ TimelineModel::data(const QString &id, int role) const auto names = roleNames(); // m.insert(names[Section], data(id, static_cast(Section))); - m.insert(names[Type], data(id, static_cast(Type))); - m.insert(names[TypeString], data(id, static_cast(TypeString))); - m.insert(names[IsOnlyEmoji], data(id, static_cast(IsOnlyEmoji))); - m.insert(names[Body], data(id, static_cast(Body))); - m.insert(names[FormattedBody], data(id, static_cast(FormattedBody))); - m.insert(names[UserId], data(id, static_cast(UserId))); - m.insert(names[UserName], data(id, static_cast(UserName))); - m.insert(names[Timestamp], data(id, static_cast(Timestamp))); - m.insert(names[Url], data(id, static_cast(Url))); - m.insert(names[ThumbnailUrl], data(id, static_cast(ThumbnailUrl))); - m.insert(names[Blurhash], data(id, static_cast(Blurhash))); - m.insert(names[Filename], data(id, static_cast(Filename))); - m.insert(names[Filesize], data(id, static_cast(Filesize))); - m.insert(names[MimeType], data(id, static_cast(MimeType))); - m.insert(names[Height], data(id, static_cast(Height))); - m.insert(names[Width], data(id, static_cast(Width))); - m.insert(names[ProportionalHeight], data(id, static_cast(ProportionalHeight))); - m.insert(names[Id], data(id, static_cast(Id))); - m.insert(names[State], data(id, static_cast(State))); - m.insert(names[IsEncrypted], data(id, static_cast(IsEncrypted))); - m.insert(names[IsRoomEncrypted], data(id, static_cast(IsRoomEncrypted))); - m.insert(names[ReplyTo], data(id, static_cast(ReplyTo))); - m.insert(names[RoomName], data(id, static_cast(RoomName))); - m.insert(names[RoomTopic], data(id, static_cast(RoomTopic))); + m.insert(names[Type], data(event, static_cast(Type))); + m.insert(names[TypeString], data(event, static_cast(TypeString))); + m.insert(names[IsOnlyEmoji], data(event, static_cast(IsOnlyEmoji))); + m.insert(names[Body], data(event, static_cast(Body))); + m.insert(names[FormattedBody], data(event, static_cast(FormattedBody))); + m.insert(names[UserId], data(event, static_cast(UserId))); + m.insert(names[UserName], data(event, static_cast(UserName))); + m.insert(names[Timestamp], data(event, static_cast(Timestamp))); + m.insert(names[Url], data(event, static_cast(Url))); + m.insert(names[ThumbnailUrl], data(event, static_cast(ThumbnailUrl))); + m.insert(names[Blurhash], data(event, static_cast(Blurhash))); + m.insert(names[Filename], data(event, static_cast(Filename))); + m.insert(names[Filesize], data(event, static_cast(Filesize))); + m.insert(names[MimeType], data(event, static_cast(MimeType))); + m.insert(names[Height], data(event, static_cast(Height))); + m.insert(names[Width], data(event, static_cast(Width))); + m.insert(names[ProportionalHeight], + data(event, static_cast(ProportionalHeight))); + m.insert(names[Id], data(event, static_cast(Id))); + m.insert(names[State], data(event, static_cast(State))); + m.insert(names[IsEncrypted], data(event, static_cast(IsEncrypted))); + m.insert(names[IsRoomEncrypted], data(event, static_cast(IsRoomEncrypted))); + m.insert(names[ReplyTo], data(event, static_cast(ReplyTo))); + m.insert(names[RoomName], data(event, static_cast(RoomName))); + m.insert(names[RoomTopic], data(event, static_cast(RoomTopic))); return QVariant(m); } @@ -462,29 +470,33 @@ TimelineModel::data(const QModelIndex &index, int role) const { using namespace mtx::accessors; namespace acc = mtx::accessors; - if (index.row() < 0 && index.row() >= (int)eventOrder.size()) + if (index.row() < 0 && index.row() >= rowCount()) return QVariant(); - QString id = eventOrder[index.row()]; + auto event = events.event(rowCount() - index.row() - 1); - mtx::events::collections::TimelineEvents event = events.value(id); + if (!event) + return ""; if (role == Section) { - QDateTime date = origin_server_ts(event); + QDateTime date = origin_server_ts(*event); date.setTime(QTime()); - std::string userId = acc::sender(event); + std::string userId = acc::sender(*event); - for (size_t r = index.row() + 1; r < eventOrder.size(); r++) { - auto tempEv = events.value(eventOrder[r]); - QDateTime prevDate = origin_server_ts(tempEv); + for (int r = rowCount() - index.row(); r < events.size(); r++) { + auto tempEv = events.event(r); + if (!tempEv) + break; + + QDateTime prevDate = origin_server_ts(*tempEv); prevDate.setTime(QTime()); if (prevDate != date) return QString("%2 %1") .arg(date.toMSecsSinceEpoch()) .arg(QString::fromStdString(userId)); - std::string prevUserId = acc::sender(tempEv); + std::string prevUserId = acc::sender(*tempEv); if (userId != prevUserId) break; } @@ -492,16 +504,16 @@ TimelineModel::data(const QModelIndex &index, int role) const return QString("%1").arg(QString::fromStdString(userId)); } - return data(id, role); + return data(*event, role); } bool TimelineModel::canFetchMore(const QModelIndex &) const { - if (eventOrder.empty()) + if (!events.size()) return true; if (!std::holds_alternative>( - events[eventOrder.back()])) + *events.event(0))) return true; else @@ -562,13 +574,9 @@ TimelineModel::addEvents(const mtx::responses::Timeline &timeline) if (timeline.events.empty()) return; - std::vector ids = internalAddEvents(timeline.events); + internalAddEvents(timeline.events); - if (!ids.empty()) { - beginInsertRows(QModelIndex(), 0, static_cast(ids.size() - 1)); - this->eventOrder.insert(this->eventOrder.begin(), ids.rbegin(), ids.rend()); - endInsertRows(); - } + events.handleSync(timeline); if (!timeline.events.empty()) updateLastMessage(); @@ -613,21 +621,17 @@ isYourJoin(const mtx::events::Event &) void TimelineModel::updateLastMessage() { - for (auto it = eventOrder.begin(); it != eventOrder.end(); ++it) { - auto event = events.value(*it); - if (auto e = std::get_if>( - &event)) { - if (decryptDescription) { - event = decryptEvent(*e).event; - } - } + for (auto it = events.size() - 1; it >= 0; --it) { + auto event = events.event(it, decryptDescription); + if (!event) + continue; - if (std::visit([](const auto &e) -> bool { return isYourJoin(e); }, event)) { - auto time = mtx::accessors::origin_server_ts(event); + if (std::visit([](const auto &e) -> bool { return isYourJoin(e); }, *event)) { + auto time = mtx::accessors::origin_server_ts(*event); uint64_t ts = time.toMSecsSinceEpoch(); emit manager_->updateRoomsLastMessage( room_id_, - DescInfo{QString::fromStdString(mtx::accessors::event_id(event)), + DescInfo{QString::fromStdString(mtx::accessors::event_id(*event)), QString::fromStdString(http::client()->user_id().to_string()), tr("You joined this room."), utils::descriptiveTime(time), @@ -635,54 +639,34 @@ TimelineModel::updateLastMessage() time}); return; } - if (!std::visit([](const auto &e) -> bool { return isMessage(e); }, event)) + if (!std::visit([](const auto &e) -> bool { return isMessage(e); }, *event)) continue; auto description = utils::getMessageDescription( - event, QString::fromStdString(http::client()->user_id().to_string()), room_id_); + *event, QString::fromStdString(http::client()->user_id().to_string()), room_id_); emit manager_->updateRoomsLastMessage(room_id_, description); return; } } -std::vector +void TimelineModel::internalAddEvents( const std::vector &timeline) { - std::vector ids; for (auto e : timeline) { QString id = QString::fromStdString(mtx::accessors::event_id(e)); - if (this->events.contains(id)) { - this->events.insert(id, e); - int idx = idToIndex(id); - emit dataChanged(index(idx, 0), index(idx, 0)); - continue; - } - - QString txid = QString::fromStdString(mtx::accessors::transaction_id(e)); - if (this->pending.removeOne(txid)) { - this->events.insert(id, e); - this->events.remove(txid); - int idx = idToIndex(txid); - if (idx < 0) { - nhlog::ui()->warn("Received index out of range"); - continue; - } - eventOrder[idx] = id; - emit dataChanged(index(idx, 0), index(idx, 0)); - continue; - } - if (auto redaction = std::get_if>(&e)) { QString redacts = QString::fromStdString(redaction->redacts); - auto redacted = std::find(eventOrder.begin(), eventOrder.end(), redacts); - auto event = events.value(redacts); + auto event = events.event(redaction->redacts, redaction->event_id); + if (!event) + continue; + if (auto reaction = std::get_if>( - &event)) { + event)) { QString reactedTo = QString::fromStdString(reaction->content.relates_to.event_id); reactions[reactedTo].removeReaction(*reaction); @@ -691,26 +675,6 @@ TimelineModel::internalAddEvents( emit dataChanged(index(idx, 0), index(idx, 0)); } - if (redacted != eventOrder.end()) { - auto redactedEvent = std::visit( - [](const auto &ev) - -> mtx::events::RoomEvent { - mtx::events::RoomEvent - replacement = {}; - replacement.event_id = ev.event_id; - replacement.room_id = ev.room_id; - replacement.sender = ev.sender; - replacement.origin_server_ts = ev.origin_server_ts; - replacement.type = ev.type; - return replacement; - }, - e); - events.insert(redacts, redactedEvent); - - int row = (int)std::distance(eventOrder.begin(), redacted); - emit dataChanged(index(row, 0), index(row, 0)); - } - continue; // don't insert redaction into timeline } @@ -718,14 +682,13 @@ TimelineModel::internalAddEvents( std::get_if>(&e)) { QString reactedTo = QString::fromStdString(reaction->content.relates_to.event_id); - events.insert(id, e); - // remove local echo - if (!txid.isEmpty()) { - auto rCopy = *reaction; - rCopy.event_id = txid.toStdString(); - reactions[reactedTo].removeReaction(rCopy); - } + // // remove local echo + // if (!txid.isEmpty()) { + // auto rCopy = *reaction; + // rCopy.event_id = txid.toStdString(); + // reactions[reactedTo].removeReaction(rCopy); + // } reactions[reactedTo].addReaction(room_id_.toStdString(), *reaction); int idx = idToIndex(reactedTo); @@ -734,40 +697,27 @@ TimelineModel::internalAddEvents( continue; // don't insert reaction into timeline } - if (auto event = - std::get_if>(&e)) { - auto e_ = decryptEvent(*event).event; - auto encInfo = mtx::accessors::file(e_); - - if (encInfo) - emit newEncryptedImage(encInfo.value()); - } - - this->events.insert(id, e); - ids.push_back(id); - - auto replyTo = mtx::accessors::in_reply_to_event(e); - auto qReplyTo = QString::fromStdString(replyTo); - if (!replyTo.empty() && !events.contains(qReplyTo)) { - http::client()->get_event( - this->room_id_.toStdString(), - replyTo, - [this, id, replyTo]( - const mtx::events::collections::TimelineEvents &timeline, - mtx::http::RequestErr err) { - if (err) { - nhlog::net()->error( - "Failed to retrieve event with id {}, which was " - "requested to show the replyTo for event {}", - replyTo, - id.toStdString()); - return; - } - emit eventFetched(id, timeline); - }); - } + // auto replyTo = mtx::accessors::in_reply_to_event(e); + // auto qReplyTo = QString::fromStdString(replyTo); + // if (!replyTo.empty() && !events.contains(qReplyTo)) { + // http::client()->get_event( + // this->room_id_.toStdString(), + // replyTo, + // [this, id, replyTo]( + // const mtx::events::collections::TimelineEvents &timeline, + // mtx::http::RequestErr err) { + // if (err) { + // nhlog::net()->error( + // "Failed to retrieve event with id {}, which was " + // "requested to show the replyTo for event {}", + // replyTo, + // id.toStdString()); + // return; + // } + // emit eventFetched(id, timeline); + // }); + //} } - return ids; } void @@ -798,22 +748,23 @@ TimelineModel::readEvent(const std::string &id) void TimelineModel::addBackwardsEvents(const mtx::responses::Messages &msgs) { - std::vector ids = internalAddEvents(msgs.chunk); + (void)msgs; + // std::vector ids = internalAddEvents(msgs.chunk); - if (!ids.empty()) { - beginInsertRows(QModelIndex(), - static_cast(this->eventOrder.size()), - static_cast(this->eventOrder.size() + ids.size() - 1)); - this->eventOrder.insert(this->eventOrder.end(), ids.begin(), ids.end()); - endInsertRows(); - } + // if (!ids.empty()) { + // beginInsertRows(QModelIndex(), + // static_cast(this->eventOrder.size()), + // static_cast(this->eventOrder.size() + ids.size() - 1)); + // this->eventOrder.insert(this->eventOrder.end(), ids.begin(), ids.end()); + // endInsertRows(); + //} - prev_batch_token_ = QString::fromStdString(msgs.end); + // prev_batch_token_ = QString::fromStdString(msgs.end); - if (ids.empty() && !msgs.chunk.empty()) { - // no visible events fetched, prevent loading from stopping - fetchMore(QModelIndex()); - } + // if (ids.empty() && !msgs.chunk.empty()) { + // // no visible events fetched, prevent loading from stopping + // fetchMore(QModelIndex()); + //} } QString @@ -852,7 +803,10 @@ TimelineModel::escapeEmoji(QString str) const void TimelineModel::viewRawMessage(QString id) const { - std::string ev = mtx::accessors::serialize_event(events.value(id)).dump(4); + auto e = events.event(id.toStdString(), "", false); + if (!e) + return; + std::string ev = mtx::accessors::serialize_event(*e).dump(4); auto dialog = new dialogs::RawMessage(QString::fromStdString(ev)); Q_UNUSED(dialog); } @@ -860,13 +814,11 @@ TimelineModel::viewRawMessage(QString id) const void TimelineModel::viewDecryptedRawMessage(QString id) const { - auto event = events.value(id); - if (auto e = - std::get_if>(&event)) { - event = decryptEvent(*e).event; - } + auto e = events.event(id.toStdString(), ""); + if (!e) + return; - std::string ev = mtx::accessors::serialize_event(event).dump(4); + std::string ev = mtx::accessors::serialize_event(*e).dump(4); auto dialog = new dialogs::RawMessage(QString::fromStdString(ev)); Q_UNUSED(dialog); } @@ -877,114 +829,6 @@ TimelineModel::openUserProfile(QString userid) const MainWindow::instance()->openUserProfile(userid, room_id_); } -DecryptionResult -TimelineModel::decryptEvent(const mtx::events::EncryptedEvent &e) const -{ - static QCache decryptedEvents{300}; - - if (auto cachedEvent = decryptedEvents.object(e.event_id)) - return *cachedEvent; - - MegolmSessionIndex index; - index.room_id = room_id_.toStdString(); - index.session_id = e.content.session_id; - index.sender_key = e.content.sender_key; - - mtx::events::RoomEvent dummy; - dummy.origin_server_ts = e.origin_server_ts; - dummy.event_id = e.event_id; - dummy.sender = e.sender; - dummy.content.body = - tr("-- Encrypted Event (No keys found for decryption) --", - "Placeholder, when the message was not decrypted yet or can't be decrypted.") - .toStdString(); - - try { - if (!cache::inboundMegolmSessionExists(index)) { - nhlog::crypto()->info("Could not find inbound megolm session ({}, {}, {})", - index.room_id, - index.session_id, - e.sender); - // TODO: request megolm session_id & session_key from the sender. - decryptedEvents.insert( - dummy.event_id, new DecryptionResult{dummy, false}, 1); - return {dummy, false}; - } - } catch (const lmdb::error &e) { - nhlog::db()->critical("failed to check megolm session's existence: {}", e.what()); - dummy.content.body = tr("-- Decryption Error (failed to communicate with DB) --", - "Placeholder, when the message can't be decrypted, because " - "the DB access failed when trying to lookup the session.") - .toStdString(); - decryptedEvents.insert(dummy.event_id, new DecryptionResult{dummy, false}, 1); - return {dummy, false}; - } - - std::string msg_str; - try { - auto session = cache::getInboundMegolmSession(index); - auto res = olm::client()->decrypt_group_message(session, e.content.ciphertext); - msg_str = std::string((char *)res.data.data(), res.data.size()); - } catch (const lmdb::error &e) { - nhlog::db()->critical("failed to retrieve megolm session with index ({}, {}, {})", - index.room_id, - index.session_id, - index.sender_key, - e.what()); - dummy.content.body = - tr("-- Decryption Error (failed to retrieve megolm keys from db) --", - "Placeholder, when the message can't be decrypted, because the DB access " - "failed.") - .toStdString(); - decryptedEvents.insert(dummy.event_id, new DecryptionResult{dummy, false}, 1); - return {dummy, false}; - } catch (const mtx::crypto::olm_exception &e) { - nhlog::crypto()->critical("failed to decrypt message with index ({}, {}, {}): {}", - index.room_id, - index.session_id, - index.sender_key, - e.what()); - dummy.content.body = - tr("-- Decryption Error (%1) --", - "Placeholder, when the message can't be decrypted. In this case, the Olm " - "decrytion returned an error, which is passed ad %1.") - .arg(e.what()) - .toStdString(); - decryptedEvents.insert(dummy.event_id, new DecryptionResult{dummy, false}, 1); - return {dummy, false}; - } - - // Add missing fields for the event. - json body = json::parse(msg_str); - body["event_id"] = e.event_id; - body["sender"] = e.sender; - body["origin_server_ts"] = e.origin_server_ts; - body["unsigned"] = e.unsigned_data; - - // relations are unencrypted in content... - if (json old_ev = e; old_ev["content"].count("m.relates_to") != 0) - body["content"]["m.relates_to"] = old_ev["content"]["m.relates_to"]; - - json event_array = json::array(); - event_array.push_back(body); - - std::vector temp_events; - mtx::responses::utils::parse_timeline_events(event_array, temp_events); - - if (temp_events.size() == 1) { - decryptedEvents.insert(e.event_id, new DecryptionResult{temp_events[0], true}, 1); - return {temp_events[0], true}; - } - - dummy.content.body = - tr("-- Encrypted Event (Unknown event type) --", - "Placeholder, when the message was decrypted, but we couldn't parse it, because " - "Nheko/mtxclient don't support that event type yet.") - .toStdString(); - decryptedEvents.insert(dummy.event_id, new DecryptionResult{dummy, false}, 1); - return {dummy, false}; -} - void TimelineModel::replyAction(QString id) { @@ -995,23 +839,18 @@ TimelineModel::replyAction(QString id) RelatedInfo TimelineModel::relatedInfo(QString id) { - if (!events.contains(id)) + auto event = events.event(id.toStdString(), ""); + if (!event) return {}; - auto event = events.value(id); - if (auto e = - std::get_if>(&event)) { - event = decryptEvent(*e).event; - } - RelatedInfo related = {}; - related.quoted_user = QString::fromStdString(mtx::accessors::sender(event)); - related.related_event = mtx::accessors::event_id(event); - related.type = mtx::accessors::msg_type(event); + related.quoted_user = QString::fromStdString(mtx::accessors::sender(*event)); + related.related_event = mtx::accessors::event_id(*event); + related.type = mtx::accessors::msg_type(*event); // get body, strip reply fallback, then transform the event to text, if it is a media event // etc - related.quoted_body = QString::fromStdString(mtx::accessors::body(event)); + related.quoted_body = QString::fromStdString(mtx::accessors::body(*event)); QRegularExpression plainQuote("^>.*?$\n?", QRegularExpression::MultilineOption); while (related.quoted_body.startsWith(">")) related.quoted_body.remove(plainQuote); @@ -1020,7 +859,7 @@ TimelineModel::relatedInfo(QString id) related.quoted_body = utils::getQuoteBody(related); // get quoted body and strip reply fallback - related.quoted_formatted_body = mtx::accessors::formattedBodyWithFallback(event); + related.quoted_formatted_body = mtx::accessors::formattedBodyWithFallback(*event); related.quoted_formatted_body.remove(QRegularExpression( ".*", QRegularExpression::DotMatchesEverythingOption)); related.room = room_id_; @@ -1058,18 +897,19 @@ TimelineModel::idToIndex(QString id) const { if (id.isEmpty()) return -1; - for (int i = 0; i < (int)eventOrder.size(); i++) - if (id == eventOrder[i]) - return i; - return -1; + + auto idx = events.idToIndex(id.toStdString()); + if (idx) + return events.size() - *idx; + else + return -1; } QString TimelineModel::indexToId(int index) const { - if (index < 0 || index >= (int)eventOrder.size()) - return ""; - return eventOrder[index]; + auto id = events.indexToId(events.size() - index); + return id ? QString::fromStdString(*id) : ""; } // Note: this will only be called for our messages @@ -1477,58 +1317,56 @@ struct SendMessageVisitor void TimelineModel::processOnePendingMessage() { - if (pending.isEmpty()) - return; + // if (pending.isEmpty()) + // return; - QString txn_id_qstr = pending.first(); + // QString txn_id_qstr = pending.first(); - auto event = events.value(txn_id_qstr); - std::visit(SendMessageVisitor{txn_id_qstr, this}, event); + // auto event = events.value(txn_id_qstr); + // std::visit(SendMessageVisitor{txn_id_qstr, this}, event); } void TimelineModel::addPendingMessage(mtx::events::collections::TimelineEvents event) { - std::visit( - [](auto &msg) { - msg.type = mtx::events::EventType::RoomMessage; - msg.event_id = http::client()->generate_txn_id(); - msg.sender = http::client()->user_id().to_string(); - msg.origin_server_ts = QDateTime::currentMSecsSinceEpoch(); - }, - event); + (void)event; + // std::visit( + // [](auto &msg) { + // msg.type = mtx::events::EventType::RoomMessage; + // msg.event_id = http::client()->generate_txn_id(); + // msg.sender = http::client()->user_id().to_string(); + // msg.origin_server_ts = QDateTime::currentMSecsSinceEpoch(); + // }, + // event); - internalAddEvents({event}); + // internalAddEvents({event}); - QString txn_id_qstr = QString::fromStdString(mtx::accessors::event_id(event)); - pending.push_back(txn_id_qstr); - if (!std::get_if>(&event)) { - beginInsertRows(QModelIndex(), 0, 0); - this->eventOrder.insert(this->eventOrder.begin(), txn_id_qstr); - endInsertRows(); - } - updateLastMessage(); + // QString txn_id_qstr = QString::fromStdString(mtx::accessors::event_id(event)); + // pending.push_back(txn_id_qstr); + // if (!std::get_if>(&event)) { + // beginInsertRows(QModelIndex(), 0, 0); + // this->eventOrder.insert(this->eventOrder.begin(), txn_id_qstr); + // endInsertRows(); + //} + // updateLastMessage(); - emit nextPendingMessage(); + // emit nextPendingMessage(); } bool TimelineModel::saveMedia(QString eventId) const { - mtx::events::collections::TimelineEvents event = events.value(eventId); + mtx::events::collections::TimelineEvents *event = events.event(eventId.toStdString(), ""); + if (!event) + return false; - if (auto e = - std::get_if>(&event)) { - event = decryptEvent(*e).event; - } + QString mxcUrl = QString::fromStdString(mtx::accessors::url(*event)); + QString originalFilename = QString::fromStdString(mtx::accessors::filename(*event)); + QString mimeType = QString::fromStdString(mtx::accessors::mimetype(*event)); - QString mxcUrl = QString::fromStdString(mtx::accessors::url(event)); - QString originalFilename = QString::fromStdString(mtx::accessors::filename(event)); - QString mimeType = QString::fromStdString(mtx::accessors::mimetype(event)); + auto encryptionInfo = mtx::accessors::file(*event); - auto encryptionInfo = mtx::accessors::file(event); - - qml_mtx_events::EventType eventType = toRoomEventType(event); + qml_mtx_events::EventType eventType = toRoomEventType(*event); QString dialogTitle; if (eventType == qml_mtx_events::EventType::ImageMessage) { @@ -1593,18 +1431,15 @@ TimelineModel::saveMedia(QString eventId) const void TimelineModel::cacheMedia(QString eventId) { - mtx::events::collections::TimelineEvents event = events.value(eventId); + mtx::events::collections::TimelineEvents *event = events.event(eventId.toStdString(), ""); + if (!event) + return; - if (auto e = - std::get_if>(&event)) { - event = decryptEvent(*e).event; - } + QString mxcUrl = QString::fromStdString(mtx::accessors::url(*event)); + QString originalFilename = QString::fromStdString(mtx::accessors::filename(*event)); + QString mimeType = QString::fromStdString(mtx::accessors::mimetype(*event)); - QString mxcUrl = QString::fromStdString(mtx::accessors::url(event)); - QString originalFilename = QString::fromStdString(mtx::accessors::filename(event)); - QString mimeType = QString::fromStdString(mtx::accessors::mimetype(event)); - - auto encryptionInfo = mtx::accessors::file(event); + auto encryptionInfo = mtx::accessors::file(*event); // If the message is a link to a non mxcUrl, don't download it if (!mxcUrl.startsWith("mxc://")) { @@ -1725,11 +1560,11 @@ TimelineModel::formatTypingUsers(const std::vector &users, QColor bg) QString TimelineModel::formatJoinRuleEvent(QString id) { - if (!events.contains(id)) + mtx::events::collections::TimelineEvents *e = events.event(id.toStdString(), ""); + if (!e) return ""; - auto event = - std::get_if>(&events[id]); + auto event = std::get_if>(e); if (!event) return ""; @@ -1750,11 +1585,11 @@ TimelineModel::formatJoinRuleEvent(QString id) QString TimelineModel::formatGuestAccessEvent(QString id) { - if (!events.contains(id)) + mtx::events::collections::TimelineEvents *e = events.event(id.toStdString(), ""); + if (!e) return ""; - auto event = - std::get_if>(&events[id]); + auto event = std::get_if>(e); if (!event) return ""; @@ -1774,11 +1609,11 @@ TimelineModel::formatGuestAccessEvent(QString id) QString TimelineModel::formatHistoryVisibilityEvent(QString id) { - if (!events.contains(id)) + mtx::events::collections::TimelineEvents *e = events.event(id.toStdString(), ""); + if (!e) return ""; - auto event = - std::get_if>(&events[id]); + auto event = std::get_if>(e); if (!event) return ""; @@ -1808,11 +1643,11 @@ TimelineModel::formatHistoryVisibilityEvent(QString id) QString TimelineModel::formatPowerLevelEvent(QString id) { - if (!events.contains(id)) + mtx::events::collections::TimelineEvents *e = events.event(id.toStdString(), ""); + if (!e) return ""; - auto event = - std::get_if>(&events[id]); + auto event = std::get_if>(e); if (!event) return ""; @@ -1826,28 +1661,30 @@ TimelineModel::formatPowerLevelEvent(QString id) QString TimelineModel::formatMemberEvent(QString id) { - if (!events.contains(id)) + mtx::events::collections::TimelineEvents *e = events.event(id.toStdString(), ""); + if (!e) return ""; - auto event = std::get_if>(&events[id]); + auto event = std::get_if>(e); if (!event) return ""; mtx::events::StateEvent *prevEvent = nullptr; - QString prevEventId = QString::fromStdString(event->unsigned_data.replaces_state); - if (!prevEventId.isEmpty()) { - if (!events.contains(prevEventId)) { + if (!event->unsigned_data.replaces_state.empty()) { + auto tempPrevEvent = + events.event(event->unsigned_data.replaces_state, event->event_id); + if (!tempPrevEvent) { http::client()->get_event( this->room_id_.toStdString(), event->unsigned_data.replaces_state, - [this, id, prevEventId]( + [this, id, prevEventId = event->unsigned_data.replaces_state]( const mtx::events::collections::TimelineEvents &timeline, mtx::http::RequestErr err) { if (err) { nhlog::net()->error( "Failed to retrieve event with id {}, which was " "requested to show the membership for event {}", - prevEventId.toStdString(), + prevEventId, id.toStdString()); return; } @@ -1856,7 +1693,7 @@ TimelineModel::formatMemberEvent(QString id) } else { prevEvent = std::get_if>( - &events[prevEventId]); + tempPrevEvent); } } diff --git a/src/timeline/TimelineModel.h b/src/timeline/TimelineModel.h index a3b92f83..f322b482 100644 --- a/src/timeline/TimelineModel.h +++ b/src/timeline/TimelineModel.h @@ -9,6 +9,7 @@ #include #include "CacheCryptoStructs.h" +#include "EventStore.h" #include "ReactionsModel.h" namespace mtx::http { @@ -170,7 +171,7 @@ public: QHash roleNames() const override; int rowCount(const QModelIndex &parent = QModelIndex()) const override; QVariant data(const QModelIndex &index, int role = Qt::DisplayRole) const override; - QVariant data(const QString &id, int role) const; + QVariant data(const mtx::events::collections::TimelineEvents &event, int role) const; bool canFetchMore(const QModelIndex &) const override; void fetchMore(const QModelIndex &) override; @@ -207,7 +208,7 @@ public slots: void setCurrentIndex(int index); int currentIndex() const { return idToIndex(currentId); } void markEventsAsRead(const std::vector &event_ids); - QVariantMap getDump(QString eventId) const; + QVariantMap getDump(QString eventId, QString relatedTo) const; void updateTypingUsers(const std::vector &users) { if (this->typingUsers_ != users) { @@ -257,9 +258,7 @@ signals: void paginationInProgressChanged(const bool); private: - DecryptionResult decryptEvent( - const mtx::events::EncryptedEvent &e) const; - std::vector internalAddEvents( + void internalAddEvents( const std::vector &timeline); void sendEncryptedMessage(const std::string &txn_id, nlohmann::json content); void handleClaimedKeys(std::shared_ptr keeper, @@ -272,12 +271,12 @@ private: void setPaginationInProgress(const bool paginationInProgress); - QHash events; QSet read; QList pending; - std::vector eventOrder; std::map reactions; + mutable EventStore events; + QString room_id_; QString prev_batch_token_; From 3421728898cd12a39d541ae7bedee4f0e58f47b5 Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Fri, 10 Jul 2020 01:37:55 +0200 Subject: [PATCH 08/50] Fetch missing events --- src/Cache.cpp | 11 ++++++++++ src/Cache_p.h | 3 +++ src/timeline/EventStore.cpp | 37 ++++++++++++++++++++++++++++++++-- src/timeline/EventStore.h | 3 +++ src/timeline/TimelineModel.cpp | 21 ------------------- 5 files changed, 52 insertions(+), 23 deletions(-) diff --git a/src/Cache.cpp b/src/Cache.cpp index d2c790dd..173b2c70 100644 --- a/src/Cache.cpp +++ b/src/Cache.cpp @@ -1324,6 +1324,17 @@ Cache::getEvent(const std::string &room_id, const std::string &event_id) return te; } +void +Cache::storeEvent(const std::string &room_id, + const std::string &event_id, + const mtx::events::collections::TimelineEvent &event) +{ + auto txn = lmdb::txn::begin(env_); + auto eventsDb = getEventsDb(txn, room_id); + auto event_json = mtx::accessors::serialize_event(event.data); + lmdb::dbi_put(txn, eventsDb, lmdb::val(event_id), lmdb::val(event_json.dump())); + txn.commit(); +} QMap Cache::roomInfo(bool withInvites) diff --git a/src/Cache_p.h b/src/Cache_p.h index 40c8e98b..6b4b260e 100644 --- a/src/Cache_p.h +++ b/src/Cache_p.h @@ -185,6 +185,9 @@ public: std::optional getEvent( const std::string &room_id, const std::string &event_id); + void storeEvent(const std::string &room_id, + const std::string &event_id, + const mtx::events::collections::TimelineEvent &event); struct TimelineRange { int64_t first, last; diff --git a/src/timeline/EventStore.cpp b/src/timeline/EventStore.cpp index eb54d475..719743fb 100644 --- a/src/timeline/EventStore.cpp +++ b/src/timeline/EventStore.cpp @@ -5,6 +5,7 @@ #include "Cache_p.h" #include "EventAccessors.h" #include "Logging.h" +#include "MatrixClient.h" #include "Olm.h" QCache EventStore::decryptedEvents_{ @@ -22,6 +23,23 @@ EventStore::EventStore(std::string room_id, QObject *) this->first = range->first; this->last = range->last; } + + connect( + this, + &EventStore::eventFetched, + this, + [this](std::string id, + std::string relatedTo, + mtx::events::collections::TimelineEvents timeline) { + cache::client()->storeEvent(room_id_, id, {timeline}); + + if (!relatedTo.empty()) { + auto idx = idToIndex(id); + if (idx) + emit dataChanged(*idx, *idx); + } + }, + Qt::QueuedConnection); } void @@ -241,8 +259,23 @@ EventStore::event(std::string_view id, std::string_view related_to, bool decrypt if (!event_ptr) { auto event = cache::client()->getEvent(room_id_, index.id); if (!event) { - // fetch - (void)related_to; + http::client()->get_event( + room_id_, + index.id, + [this, + relatedTo = std::string(related_to.data(), related_to.size()), + id = index.id](const mtx::events::collections::TimelineEvents &timeline, + mtx::http::RequestErr err) { + if (err) { + nhlog::net()->error( + "Failed to retrieve event with id {}, which was " + "requested to show the replyTo for event {}", + relatedTo, + id); + return; + } + emit eventFetched(id, relatedTo, timeline); + }); return nullptr; } event_ptr = new mtx::events::collections::TimelineEvents(std::move(event->data)); diff --git a/src/timeline/EventStore.h b/src/timeline/EventStore.h index 77d73536..83c8f7a4 100644 --- a/src/timeline/EventStore.h +++ b/src/timeline/EventStore.h @@ -81,6 +81,9 @@ signals: void endInsertRows(); void dataChanged(int from, int to); void newEncryptedImage(mtx::crypto::EncryptedFile encryptionInfo); + void eventFetched(std::string id, + std::string relatedTo, + mtx::events::collections::TimelineEvents timeline); private: mtx::events::collections::TimelineEvents *decryptEvent( diff --git a/src/timeline/TimelineModel.cpp b/src/timeline/TimelineModel.cpp index 492d4e0a..6df92d7a 100644 --- a/src/timeline/TimelineModel.cpp +++ b/src/timeline/TimelineModel.cpp @@ -696,27 +696,6 @@ TimelineModel::internalAddEvents( emit dataChanged(index(idx, 0), index(idx, 0)); continue; // don't insert reaction into timeline } - - // auto replyTo = mtx::accessors::in_reply_to_event(e); - // auto qReplyTo = QString::fromStdString(replyTo); - // if (!replyTo.empty() && !events.contains(qReplyTo)) { - // http::client()->get_event( - // this->room_id_.toStdString(), - // replyTo, - // [this, id, replyTo]( - // const mtx::events::collections::TimelineEvents &timeline, - // mtx::http::RequestErr err) { - // if (err) { - // nhlog::net()->error( - // "Failed to retrieve event with id {}, which was " - // "requested to show the replyTo for event {}", - // replyTo, - // id.toStdString()); - // return; - // } - // emit eventFetched(id, timeline); - // }); - //} } } From da2f80df60c7ca99efdf88c528fc357a052f4c3f Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Fri, 10 Jul 2020 03:17:23 +0200 Subject: [PATCH 09/50] Fix translation loading Explanation see here: https://www.kdab.com/fixing-a-common-antipattern-when-loading-translations-in-qt/ --- src/main.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main.cpp b/src/main.cpp index 46691e6f..e02ffa36 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -173,11 +173,12 @@ main(int argc, char *argv[]) QString lang = QLocale::system().name(); QTranslator qtTranslator; - qtTranslator.load("qt_" + lang, QLibraryInfo::location(QLibraryInfo::TranslationsPath)); + qtTranslator.load( + QLocale(), "qt", "_", QLibraryInfo::location(QLibraryInfo::TranslationsPath)); app.installTranslator(&qtTranslator); QTranslator appTranslator; - appTranslator.load("nheko_" + lang, ":/translations"); + appTranslator.load(QLocale(), "nheko", "_", ":/translations"); app.installTranslator(&appTranslator); MainWindow w; From 9479fcde0847cffa49217d86f0bf91f7d03440a2 Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Sat, 11 Jul 2020 02:15:53 +0200 Subject: [PATCH 10/50] Initialize Profile later --- src/ChatPage.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ChatPage.cpp b/src/ChatPage.cpp index 0b290927..3ef28c86 100644 --- a/src/ChatPage.cpp +++ b/src/ChatPage.cpp @@ -795,8 +795,6 @@ ChatPage::loadStateFromCache() nhlog::db()->info("restoring state from cache"); - getProfileInfo(); - QtConcurrent::run([this]() { try { cache::restoreSessions(); @@ -829,6 +827,8 @@ ChatPage::loadStateFromCache() nhlog::crypto()->info("ed25519 : {}", olm::client()->identity_keys().ed25519); nhlog::crypto()->info("curve25519: {}", olm::client()->identity_keys().curve25519); + getProfileInfo(); + // Start receiving events. emit trySyncCb(); }); From 9ae7d0dce3d78cefc0498e2322117ef00c6ec2e8 Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Mon, 13 Jul 2020 00:08:58 +0200 Subject: [PATCH 11/50] Readd pagination and fix redactions --- resources/qml/delegates/MessageDelegate.qml | 6 + src/Cache.cpp | 175 ++++++++++++++------ src/Cache_p.h | 15 +- src/ChatPage.cpp | 56 +++---- src/timeline/EventStore.cpp | 55 +++++- src/timeline/EventStore.h | 17 +- src/timeline/TimelineModel.cpp | 53 +++--- 7 files changed, 255 insertions(+), 122 deletions(-) diff --git a/resources/qml/delegates/MessageDelegate.qml b/resources/qml/delegates/MessageDelegate.qml index 17fe7360..9630ae3a 100644 --- a/resources/qml/delegates/MessageDelegate.qml +++ b/resources/qml/delegates/MessageDelegate.qml @@ -66,6 +66,12 @@ Item { text: qsTr("redacted") } } + DelegateChoice { + roleValue: MtxEvent.Redaction + Pill { + text: qsTr("redacted") + } + } DelegateChoice { roleValue: MtxEvent.Encryption Pill { diff --git a/src/Cache.cpp b/src/Cache.cpp index 173b2c70..233ef2b4 100644 --- a/src/Cache.cpp +++ b/src/Cache.cpp @@ -1241,8 +1241,25 @@ Cache::getTimelineMentions() return notifs; } +std::string +Cache::previousBatchToken(const std::string &room_id) +{ + auto txn = lmdb::txn::begin(env_, nullptr); + auto orderDb = getEventOrderDb(txn, room_id); + + auto cursor = lmdb::cursor::open(txn, orderDb); + lmdb::val indexVal, val; + if (!cursor.get(indexVal, val, MDB_FIRST)) { + return ""; + } + + auto j = json::parse(std::string_view(val.data(), val.size())); + + return j.value("prev_batch", ""); +} + Cache::Messages -Cache::getTimelineMessages(lmdb::txn &txn, const std::string &room_id, int64_t index, bool forward) +Cache::getTimelineMessages(lmdb::txn &txn, const std::string &room_id, uint64_t index, bool forward) { // TODO(nico): Limit the messages returned by this maybe? auto orderDb = getOrderToMessageDb(txn, room_id); @@ -1253,16 +1270,16 @@ Cache::getTimelineMessages(lmdb::txn &txn, const std::string &room_id, int64_t i lmdb::val indexVal, event_id; auto cursor = lmdb::cursor::open(txn, orderDb); - if (index == std::numeric_limits::max()) { + if (index == std::numeric_limits::max()) { if (cursor.get(indexVal, event_id, forward ? MDB_FIRST : MDB_LAST)) { - index = *indexVal.data(); + index = *indexVal.data(); } else { messages.end_of_cache = true; return messages; } } else { if (cursor.get(indexVal, event_id, MDB_SET)) { - index = *indexVal.data(); + index = *indexVal.data(); } else { messages.end_of_cache = true; return messages; @@ -1296,7 +1313,7 @@ Cache::getTimelineMessages(lmdb::txn &txn, const std::string &room_id, int64_t i cursor.close(); // std::reverse(timeline.events.begin(), timeline.events.end()); - messages.next_index = *indexVal.data(); + messages.next_index = *indexVal.data(); messages.end_of_cache = !ret; return messages; @@ -1402,16 +1419,16 @@ Cache::getTimelineRange(const std::string &room_id) } TimelineRange range{}; - range.last = *indexVal.data(); + range.last = *indexVal.data(); if (!cursor.get(indexVal, val, MDB_FIRST)) { return {}; } - range.first = *indexVal.data(); + range.first = *indexVal.data(); return range; } -std::optional +std::optional Cache::getTimelineIndex(const std::string &room_id, std::string_view event_id) { auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); @@ -1424,11 +1441,11 @@ Cache::getTimelineIndex(const std::string &room_id, std::string_view event_id) return {}; } - return *val.data(); + return *val.data(); } std::optional -Cache::getTimelineEventId(const std::string &room_id, int64_t index) +Cache::getTimelineEventId(const std::string &room_id, uint64_t index) { auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); auto orderDb = getOrderToMessageDb(txn, room_id); @@ -2074,6 +2091,9 @@ Cache::saveTimelineMessages(lmdb::txn &txn, const std::string &room_id, const mtx::responses::Timeline &res) { + if (res.events.empty()) + return; + auto eventsDb = getEventsDb(txn, room_id); auto relationsDb = getRelationsDb(txn, room_id); @@ -2090,16 +2110,16 @@ Cache::saveTimelineMessages(lmdb::txn &txn, using namespace mtx::events::state; lmdb::val indexVal, val; - int64_t index = 0; - auto cursor = lmdb::cursor::open(txn, orderDb); + uint64_t index = std::numeric_limits::max() / 2; + auto cursor = lmdb::cursor::open(txn, orderDb); if (cursor.get(indexVal, val, MDB_LAST)) { index = *indexVal.data(); } - int64_t msgIndex = 0; - auto msgCursor = lmdb::cursor::open(txn, order2msgDb); + uint64_t msgIndex = std::numeric_limits::max() / 2; + auto msgCursor = lmdb::cursor::open(txn, order2msgDb); if (msgCursor.get(indexVal, val, MDB_LAST)) { - msgIndex = *indexVal.data(); + msgIndex = *indexVal.data(); } bool first = true; @@ -2111,39 +2131,19 @@ Cache::saveTimelineMessages(lmdb::txn &txn, continue; lmdb::val ev{}; - bool success = - lmdb::dbi_get(txn, eventsDb, lmdb::val(redaction->redacts), ev); - if (!success) - continue; + lmdb::dbi_put( + txn, eventsDb, lmdb::val(redaction->redacts), lmdb::val(event.dump())); + lmdb::dbi_put( + txn, eventsDb, lmdb::val(redaction->event_id), lmdb::val(event.dump())); - mtx::events::collections::TimelineEvent te; - - try { - mtx::events::collections::from_json( - json::parse(std::string_view(ev.data(), ev.size())), te); - } catch (std::exception &e) { - nhlog::db()->error("Failed to parse message from cache {}", - e.what()); - continue; + lmdb::val oldIndex{}; + if (lmdb::dbi_get( + txn, msg2orderDb, lmdb::val(redaction->redacts), oldIndex)) { + lmdb::dbi_put( + txn, order2msgDb, oldIndex, lmdb::val(redaction->event_id)); + lmdb::dbi_put( + txn, msg2orderDb, lmdb::val(redaction->event_id), oldIndex); } - - auto redactedEvent = std::visit( - [](const auto &ev) -> mtx::events::RoomEvent { - mtx::events::RoomEvent replacement = - {}; - replacement.event_id = ev.event_id; - replacement.room_id = ev.room_id; - replacement.sender = ev.sender; - replacement.origin_server_ts = ev.origin_server_ts; - replacement.type = ev.type; - return replacement; - }, - te.data); - - lmdb::dbi_put(txn, - eventsDb, - lmdb::val(redaction->redacts), - lmdb::val(json(redactedEvent).dump())); } else { std::string event_id_val = event["event_id"].get(); lmdb::val event_id = event_id_val; @@ -2193,6 +2193,83 @@ Cache::saveTimelineMessages(lmdb::txn &txn, } } +uint64_t +Cache::saveOldMessages(const std::string &room_id, const mtx::responses::Messages &res) +{ + auto txn = lmdb::txn::begin(env_); + auto eventsDb = getEventsDb(txn, room_id); + auto relationsDb = getRelationsDb(txn, room_id); + + auto orderDb = getEventOrderDb(txn, room_id); + auto msg2orderDb = getMessageToOrderDb(txn, room_id); + auto order2msgDb = getOrderToMessageDb(txn, room_id); + + lmdb::val indexVal, val; + uint64_t index = std::numeric_limits::max() / 2; + auto cursor = lmdb::cursor::open(txn, orderDb); + if (cursor.get(indexVal, val, MDB_FIRST)) { + index = *indexVal.data(); + } + + uint64_t msgIndex = std::numeric_limits::max() / 2; + auto msgCursor = lmdb::cursor::open(txn, order2msgDb); + if (msgCursor.get(indexVal, val, MDB_FIRST)) { + msgIndex = *indexVal.data(); + } + + if (res.chunk.empty()) + return index; + + std::string event_id_val; + for (const auto &e : res.chunk) { + auto event = mtx::accessors::serialize_event(e); + event_id_val = event["event_id"].get(); + lmdb::val event_id = event_id_val; + lmdb::dbi_put(txn, eventsDb, event_id, lmdb::val(event.dump())); + + --index; + + json orderEntry = json::object(); + orderEntry["event_id"] = event_id_val; + + nhlog::db()->debug("saving '{}'", orderEntry.dump()); + + lmdb::dbi_put( + txn, orderDb, lmdb::val(&index, sizeof(index)), lmdb::val(orderEntry.dump())); + + // TODO(Nico): Allow blacklisting more event types in UI + if (event["type"] != "m.reaction" && event["type"] != "m.dummy") { + --msgIndex; + lmdb::dbi_put( + txn, order2msgDb, lmdb::val(&msgIndex, sizeof(msgIndex)), event_id); + + lmdb::dbi_put( + txn, msg2orderDb, event_id, lmdb::val(&msgIndex, sizeof(msgIndex))); + } + + if (event.contains("content") && event["content"].contains("m.relates_to")) { + auto temp = event["content"]["m.relates_to"]; + std::string relates_to = temp.contains("m.in_reply_to") + ? temp["m.in_reply_to"]["event_id"] + : temp["event_id"]; + + if (!relates_to.empty()) + lmdb::dbi_put(txn, relationsDb, lmdb::val(relates_to), event_id); + } + } + + json orderEntry = json::object(); + orderEntry["event_id"] = event_id_val; + orderEntry["prev_batch"] = res.end; + lmdb::cursor_put( + cursor.handle(), lmdb::val(&index, sizeof(index)), lmdb::val(orderEntry.dump())); + nhlog::db()->debug("saving '{}'", orderEntry.dump()); + + txn.commit(); + + return msgIndex; +} + mtx::responses::Notifications Cache::getTimelineMentionsForRoom(lmdb::txn &txn, const std::string &room_id) { @@ -2337,14 +2414,14 @@ Cache::deleteOldMessages() auto eventsDb = getEventsDb(txn, room_id); auto cursor = lmdb::cursor::open(txn, orderDb); - int64_t first, last; + uint64_t first, last; if (cursor.get(indexVal, val, MDB_LAST)) { - last = *indexVal.data(); + last = *indexVal.data(); } else { continue; } if (cursor.get(indexVal, val, MDB_FIRST)) { - first = *indexVal.data(); + first = *indexVal.data(); } else { continue; } diff --git a/src/Cache_p.h b/src/Cache_p.h index 6b4b260e..1d6d62dd 100644 --- a/src/Cache_p.h +++ b/src/Cache_p.h @@ -179,8 +179,8 @@ public: }; Messages getTimelineMessages(lmdb::txn &txn, const std::string &room_id, - int64_t index = std::numeric_limits::max(), - bool forward = false); + uint64_t index = std::numeric_limits::max(), + bool forward = false); std::optional getEvent( const std::string &room_id, @@ -190,12 +190,15 @@ public: const mtx::events::collections::TimelineEvent &event); struct TimelineRange { - int64_t first, last; + uint64_t first, last; }; std::optional getTimelineRange(const std::string &room_id); - std::optional getTimelineIndex(const std::string &room_id, - std::string_view event_id); - std::optional getTimelineEventId(const std::string &room_id, int64_t index); + std::optional getTimelineIndex(const std::string &room_id, + std::string_view event_id); + std::optional getTimelineEventId(const std::string &room_id, uint64_t index); + + std::string previousBatchToken(const std::string &room_id); + uint64_t saveOldMessages(const std::string &room_id, const mtx::responses::Messages &res); //! Remove old unused data. void deleteOldMessages(); diff --git a/src/ChatPage.cpp b/src/ChatPage.cpp index 3ef28c86..666912ee 100644 --- a/src/ChatPage.cpp +++ b/src/ChatPage.cpp @@ -795,43 +795,39 @@ ChatPage::loadStateFromCache() nhlog::db()->info("restoring state from cache"); - QtConcurrent::run([this]() { - try { - cache::restoreSessions(); - olm::client()->load(cache::restoreOlmAccount(), STORAGE_SECRET_KEY); + try { + cache::restoreSessions(); + olm::client()->load(cache::restoreOlmAccount(), STORAGE_SECRET_KEY); - cache::populateMembers(); + cache::populateMembers(); - emit initializeEmptyViews(cache::roomMessages()); - emit initializeRoomList(cache::roomInfo()); - emit initializeMentions(cache::getTimelineMentions()); - emit syncTags(cache::roomInfo().toStdMap()); + emit initializeEmptyViews(cache::roomMessages()); + emit initializeRoomList(cache::roomInfo()); + emit initializeMentions(cache::getTimelineMentions()); + emit syncTags(cache::roomInfo().toStdMap()); - cache::calculateRoomReadStatus(); + cache::calculateRoomReadStatus(); - } catch (const mtx::crypto::olm_exception &e) { - nhlog::crypto()->critical("failed to restore olm account: {}", e.what()); - emit dropToLoginPageCb( - tr("Failed to restore OLM account. Please login again.")); - return; - } catch (const lmdb::error &e) { - nhlog::db()->critical("failed to restore cache: {}", e.what()); - emit dropToLoginPageCb( - tr("Failed to restore save data. Please login again.")); - return; - } catch (const json::exception &e) { - nhlog::db()->critical("failed to parse cache data: {}", e.what()); - return; - } + } catch (const mtx::crypto::olm_exception &e) { + nhlog::crypto()->critical("failed to restore olm account: {}", e.what()); + emit dropToLoginPageCb(tr("Failed to restore OLM account. Please login again.")); + return; + } catch (const lmdb::error &e) { + nhlog::db()->critical("failed to restore cache: {}", e.what()); + emit dropToLoginPageCb(tr("Failed to restore save data. Please login again.")); + return; + } catch (const json::exception &e) { + nhlog::db()->critical("failed to parse cache data: {}", e.what()); + return; + } - nhlog::crypto()->info("ed25519 : {}", olm::client()->identity_keys().ed25519); - nhlog::crypto()->info("curve25519: {}", olm::client()->identity_keys().curve25519); + nhlog::crypto()->info("ed25519 : {}", olm::client()->identity_keys().ed25519); + nhlog::crypto()->info("curve25519: {}", olm::client()->identity_keys().curve25519); - getProfileInfo(); + getProfileInfo(); - // Start receiving events. - emit trySyncCb(); - }); + // Start receiving events. + emit trySyncCb(); } void diff --git a/src/timeline/EventStore.cpp b/src/timeline/EventStore.cpp index 719743fb..7f21e1ed 100644 --- a/src/timeline/EventStore.cpp +++ b/src/timeline/EventStore.cpp @@ -34,12 +34,31 @@ EventStore::EventStore(std::string room_id, QObject *) cache::client()->storeEvent(room_id_, id, {timeline}); if (!relatedTo.empty()) { - auto idx = idToIndex(id); + auto idx = idToIndex(relatedTo); if (idx) emit dataChanged(*idx, *idx); } }, Qt::QueuedConnection); + + connect( + this, + &EventStore::oldMessagesRetrieved, + this, + [this](const mtx::responses::Messages &res) { + // + uint64_t newFirst = cache::client()->saveOldMessages(room_id_, res); + if (newFirst == first) + fetchMore(); + else { + emit beginInsertRows(toExternalIdx(newFirst), + toExternalIdx(this->first - 1)); + this->first = newFirst; + emit endInsertRows(); + emit fetchedMore(); + } + }, + Qt::QueuedConnection); } void @@ -49,8 +68,16 @@ EventStore::handleSync(const mtx::responses::Timeline &events) nhlog::db()->warn("{} called from a different thread!", __func__); auto range = cache::client()->getTimelineRange(room_id_); + if (!range) + return; - if (range && range->last > this->last) { + if (events.limited) { + emit beginResetModel(); + this->last = range->last; + this->first = range->first; + emit endResetModel(); + + } else if (range->last > this->last) { emit beginInsertRows(toExternalIdx(this->last + 1), toExternalIdx(range->last)); this->last = range->last; emit endInsertRows(); @@ -290,3 +317,27 @@ EventStore::event(std::string_view id, std::string_view related_to, bool decrypt return event_ptr; } + +void +EventStore::fetchMore() +{ + mtx::http::MessagesOpts opts; + opts.room_id = room_id_; + opts.from = cache::client()->previousBatchToken(room_id_); + + nhlog::ui()->debug("Paginating room {}, token {}", opts.room_id, opts.from); + + http::client()->messages( + opts, [this, opts](const mtx::responses::Messages &res, mtx::http::RequestErr err) { + if (err) { + nhlog::net()->error("failed to call /messages ({}): {} - {} - {}", + opts.room_id, + mtx::errors::to_string(err->matrix_error.errcode), + err->matrix_error.error, + err->parse_error); + return; + } + + emit oldMessagesRetrieved(std::move(res)); + }); +} diff --git a/src/timeline/EventStore.h b/src/timeline/EventStore.h index 83c8f7a4..f2997245 100644 --- a/src/timeline/EventStore.h +++ b/src/timeline/EventStore.h @@ -8,6 +8,7 @@ #include #include +#include #include class EventStore : public QObject @@ -20,7 +21,7 @@ public: struct Index { std::string room; - int64_t idx; + uint64_t idx; friend uint qHash(const Index &i, uint seed = 0) noexcept { @@ -66,12 +67,12 @@ public: int size() const { - return last != std::numeric_limits::max() + return last != std::numeric_limits::max() ? static_cast(last - first) + 1 : 0; } - int toExternalIdx(int64_t idx) const { return static_cast(idx - first); } - int64_t toInternalIdx(int idx) const { return first + idx; } + int toExternalIdx(uint64_t idx) const { return static_cast(idx - first); } + uint64_t toInternalIdx(int idx) const { return first + idx; } std::optional idToIndex(std::string_view id) const; std::optional indexToId(int idx) const; @@ -79,11 +80,15 @@ public: signals: void beginInsertRows(int from, int to); void endInsertRows(); + void beginResetModel(); + void endResetModel(); void dataChanged(int from, int to); void newEncryptedImage(mtx::crypto::EncryptedFile encryptionInfo); void eventFetched(std::string id, std::string relatedTo, mtx::events::collections::TimelineEvents timeline); + void oldMessagesRetrieved(const mtx::responses::Messages &); + void fetchedMore(); private: mtx::events::collections::TimelineEvents *decryptEvent( @@ -92,8 +97,8 @@ private: std::string room_id_; - int64_t first = std::numeric_limits::max(), - last = std::numeric_limits::max(); + uint64_t first = std::numeric_limits::max(), + last = std::numeric_limits::max(); static QCache decryptedEvents_; static QCache events_; diff --git a/src/timeline/TimelineModel.cpp b/src/timeline/TimelineModel.cpp index 6df92d7a..60264e86 100644 --- a/src/timeline/TimelineModel.cpp +++ b/src/timeline/TimelineModel.cpp @@ -229,20 +229,33 @@ TimelineModel::TimelineModel(TimelineViewManager *manager, QString room_id, QObj &EventStore::dataChanged, this, [this](int from, int to) { - emit dataChanged(index(events.size() - to, 0), index(events.size() - from, 0)); + nhlog::ui()->debug( + "data changed {} to {}", events.size() - to - 1, events.size() - from - 1); + emit dataChanged(index(events.size() - to - 1, 0), + index(events.size() - from - 1, 0)); }, Qt::QueuedConnection); connect(&events, &EventStore::beginInsertRows, this, [this](int from, int to) { - nhlog::ui()->info("begin insert from {} to {}", - events.size() - to + (to - from), - events.size() - from + (to - from)); - beginInsertRows(QModelIndex(), - events.size() - to + (to - from), - events.size() - from + (to - from)); + int first = events.size() - to; + int last = events.size() - from; + if (from >= events.size()) { + int batch_size = to - from; + first += batch_size; + last += batch_size; + } else { + first -= 1; + last -= 1; + } + nhlog::ui()->debug("begin insert from {} to {}", first, last); + beginInsertRows(QModelIndex(), first, last); }); connect(&events, &EventStore::endInsertRows, this, [this]() { endInsertRows(); }); + connect(&events, &EventStore::beginResetModel, this, [this]() { beginResetModel(); }); + connect(&events, &EventStore::endResetModel, this, [this]() { endResetModel(); }); connect(&events, &EventStore::newEncryptedImage, this, &TimelineModel::newEncryptedImage); + connect( + &events, &EventStore::fetchedMore, this, [this]() { setPaginationInProgress(false); }); } QHash @@ -512,8 +525,9 @@ TimelineModel::canFetchMore(const QModelIndex &) const { if (!events.size()) return true; - if (!std::holds_alternative>( - *events.event(0))) + if (auto first = events.event(0); + first && + !std::holds_alternative>(*first)) return true; else @@ -540,27 +554,8 @@ TimelineModel::fetchMore(const QModelIndex &) } setPaginationInProgress(true); - mtx::http::MessagesOpts opts; - opts.room_id = room_id_.toStdString(); - opts.from = prev_batch_token_.toStdString(); - nhlog::ui()->debug("Paginating room {}", opts.room_id); - - http::client()->messages( - opts, [this, opts](const mtx::responses::Messages &res, mtx::http::RequestErr err) { - if (err) { - nhlog::net()->error("failed to call /messages ({}): {} - {} - {}", - opts.room_id, - mtx::errors::to_string(err->matrix_error.errcode), - err->matrix_error.error, - err->parse_error); - setPaginationInProgress(false); - return; - } - - emit oldMessagesRetrieved(std::move(res)); - setPaginationInProgress(false); - }); + events.fetchMore(); } void From 56ea89aa1133f01e356b1e7dce4322b883600e53 Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Sat, 18 Jul 2020 17:43:49 +0200 Subject: [PATCH 12/50] Reenable sending messages --- src/Cache.cpp | 140 ++++++++++++- src/Cache_p.h | 18 ++ src/ChatPage.cpp | 20 +- src/dialogs/RoomSettings.cpp | 10 +- src/timeline/EventStore.cpp | 109 ++++++++++ src/timeline/EventStore.h | 10 + src/timeline/TimelineModel.cpp | 360 +++++---------------------------- src/timeline/TimelineModel.h | 20 +- 8 files changed, 340 insertions(+), 347 deletions(-) diff --git a/src/Cache.cpp b/src/Cache.cpp index 233ef2b4..8fa94d1e 100644 --- a/src/Cache.cpp +++ b/src/Cache.cpp @@ -2086,6 +2086,77 @@ Cache::isRoomMember(const std::string &user_id, const std::string &room_id) return res; } +void +Cache::savePendingMessage(const std::string &room_id, + const mtx::events::collections::TimelineEvent &message) +{ + auto txn = lmdb::txn::begin(env_); + + mtx::responses::Timeline timeline; + timeline.events.push_back(message.data); + saveTimelineMessages(txn, room_id, timeline); + + auto pending = getPendingMessagesDb(txn, room_id); + + int64_t now = QDateTime::currentMSecsSinceEpoch(); + lmdb::dbi_put(txn, + pending, + lmdb::val(&now, sizeof(now)), + lmdb::val(mtx::accessors::event_id(message.data))); + + txn.commit(); +} + +std::optional +Cache::firstPendingMessage(const std::string &room_id) +{ + auto txn = lmdb::txn::begin(env_); + auto pending = getPendingMessagesDb(txn, room_id); + + auto pendingCursor = lmdb::cursor::open(txn, pending); + lmdb::val tsIgnored, pendingTxn; + while (pendingCursor.get(tsIgnored, pendingTxn, MDB_NEXT)) { + auto eventsDb = getEventsDb(txn, room_id); + lmdb::val event; + if (!lmdb::dbi_get(txn, eventsDb, pendingTxn, event)) { + lmdb::dbi_del(txn, pending, tsIgnored, pendingTxn); + continue; + } + + try { + mtx::events::collections::TimelineEvent te; + mtx::events::collections::from_json( + json::parse(std::string_view(event.data(), event.size())), te); + + txn.commit(); + return te; + } catch (std::exception &e) { + nhlog::db()->error("Failed to parse message from cache {}", e.what()); + lmdb::dbi_del(txn, pending, tsIgnored, pendingTxn); + continue; + } + } + + txn.commit(); + + return std::nullopt; +} + +void +Cache::removePendingStatus(const std::string &room_id, const std::string &txn_id) +{ + auto txn = lmdb::txn::begin(env_); + auto pending = getPendingMessagesDb(txn, room_id); + auto pendingCursor = lmdb::cursor::open(txn, pending); + lmdb::val tsIgnored, pendingTxn; + while (pendingCursor.get(tsIgnored, pendingTxn, MDB_NEXT)) { + if (std::string_view(pendingTxn.data(), pendingTxn.size()) == txn_id) + lmdb::cursor_del(pendingCursor); + } + + txn.commit(); +} + void Cache::saveTimelineMessages(lmdb::txn &txn, const std::string &room_id, @@ -2098,12 +2169,17 @@ Cache::saveTimelineMessages(lmdb::txn &txn, auto relationsDb = getRelationsDb(txn, room_id); auto orderDb = getEventOrderDb(txn, room_id); + auto evToOrderDb = getEventToOrderDb(txn, room_id); auto msg2orderDb = getMessageToOrderDb(txn, room_id); auto order2msgDb = getOrderToMessageDb(txn, room_id); + auto pending = getPendingMessagesDb(txn, room_id); + if (res.limited) { lmdb::dbi_drop(txn, orderDb, false); + lmdb::dbi_drop(txn, evToOrderDb, false); lmdb::dbi_drop(txn, msg2orderDb, false); lmdb::dbi_drop(txn, order2msgDb, false); + lmdb::dbi_drop(txn, pending, true); } using namespace mtx::events; @@ -2124,9 +2200,55 @@ Cache::saveTimelineMessages(lmdb::txn &txn, bool first = true; for (const auto &e : res.events) { - auto event = mtx::accessors::serialize_event(e); - if (auto redaction = - std::get_if>(&e)) { + auto event = mtx::accessors::serialize_event(e); + auto txn_id = mtx::accessors::transaction_id(e); + + lmdb::val txn_order; + if (!txn_id.empty() && + lmdb::dbi_get(txn, evToOrderDb, lmdb::val(txn_id), txn_order)) { + std::string event_id_val = event["event_id"].get(); + lmdb::val event_id = event_id_val; + lmdb::dbi_put(txn, eventsDb, event_id, lmdb::val(event.dump())); + lmdb::dbi_del(txn, eventsDb, lmdb::val(txn_id)); + + lmdb::val msg_txn_order; + if (lmdb::dbi_get(txn, msg2orderDb, lmdb::val(txn_id), msg_txn_order)) { + lmdb::dbi_put(txn, order2msgDb, msg_txn_order, event_id); + lmdb::dbi_put(txn, msg2orderDb, event_id, msg_txn_order); + lmdb::dbi_del(txn, msg2orderDb, lmdb::val(txn_id)); + } + + lmdb::dbi_put(txn, orderDb, txn_order, event_id); + lmdb::dbi_put(txn, evToOrderDb, event_id, txn_order); + lmdb::dbi_del(txn, evToOrderDb, lmdb::val(txn_id)); + + if (event.contains("content") && + event["content"].contains("m.relates_to")) { + auto temp = event["content"]["m.relates_to"]; + std::string relates_to = temp.contains("m.in_reply_to") + ? temp["m.in_reply_to"]["event_id"] + : temp["event_id"]; + + if (!relates_to.empty()) { + lmdb::dbi_del(txn, + relationsDb, + lmdb::val(relates_to), + lmdb::val(txn_id)); + lmdb::dbi_put( + txn, relationsDb, lmdb::val(relates_to), event_id); + } + } + + auto pendingCursor = lmdb::cursor::open(txn, pending); + lmdb::val tsIgnored, pendingTxn; + while (pendingCursor.get(tsIgnored, pendingTxn, MDB_NEXT)) { + if (std::string_view(pendingTxn.data(), pendingTxn.size()) == + txn_id) + lmdb::cursor_del(pendingCursor); + } + } else if (auto redaction = + std::get_if>( + &e)) { if (redaction->redacts.empty()) continue; @@ -2145,15 +2267,20 @@ Cache::saveTimelineMessages(lmdb::txn &txn, txn, msg2orderDb, lmdb::val(redaction->event_id), oldIndex); } } else { - std::string event_id_val = event["event_id"].get(); - lmdb::val event_id = event_id_val; + std::string event_id_val = event.value("event_id", ""); + if (event_id_val.empty()) { + nhlog::db()->error("Event without id!"); + continue; + } + + lmdb::val event_id = event_id_val; lmdb::dbi_put(txn, eventsDb, event_id, lmdb::val(event.dump())); ++index; json orderEntry = json::object(); orderEntry["event_id"] = event_id_val; - if (first) + if (first && !res.prev_batch.empty()) orderEntry["prev_batch"] = res.prev_batch; first = false; @@ -2163,6 +2290,7 @@ Cache::saveTimelineMessages(lmdb::txn &txn, lmdb::val(&index, sizeof(index)), lmdb::val(orderEntry.dump()), MDB_APPEND); + lmdb::dbi_put(txn, evToOrderDb, event_id, lmdb::val(&index, sizeof(index))); // TODO(Nico): Allow blacklisting more event types in UI if (event["type"] != "m.reaction" && event["type"] != "m.dummy") { diff --git a/src/Cache_p.h b/src/Cache_p.h index 1d6d62dd..88308e45 100644 --- a/src/Cache_p.h +++ b/src/Cache_p.h @@ -199,6 +199,11 @@ public: std::string previousBatchToken(const std::string &room_id); uint64_t saveOldMessages(const std::string &room_id, const mtx::responses::Messages &res); + void savePendingMessage(const std::string &room_id, + const mtx::events::collections::TimelineEvent &message); + std::optional firstPendingMessage( + const std::string &room_id); + void removePendingStatus(const std::string &room_id, const std::string &txn_id); //! Remove old unused data. void deleteOldMessages(); @@ -439,6 +444,13 @@ private: txn, std::string(room_id + "/event_order").c_str(), MDB_CREATE | MDB_INTEGERKEY); } + // inverse of EventOrderDb + lmdb::dbi getEventToOrderDb(lmdb::txn &txn, const std::string &room_id) + { + return lmdb::dbi::open( + txn, std::string(room_id + "/event2order").c_str(), MDB_CREATE); + } + lmdb::dbi getMessageToOrderDb(lmdb::txn &txn, const std::string &room_id) { return lmdb::dbi::open( @@ -451,6 +463,12 @@ private: txn, std::string(room_id + "/order2msg").c_str(), MDB_CREATE | MDB_INTEGERKEY); } + lmdb::dbi getPendingMessagesDb(lmdb::txn &txn, const std::string &room_id) + { + return lmdb::dbi::open( + txn, std::string(room_id + "/pending").c_str(), MDB_CREATE | MDB_INTEGERKEY); + } + lmdb::dbi getRelationsDb(lmdb::txn &txn, const std::string &room_id) { return lmdb::dbi::open( diff --git a/src/ChatPage.cpp b/src/ChatPage.cpp index 666912ee..813b0c2a 100644 --- a/src/ChatPage.cpp +++ b/src/ChatPage.cpp @@ -313,17 +313,15 @@ ChatPage::ChatPage(QSharedPointer userSettings, QWidget *parent) .toStdString(); member.membership = mtx::events::state::Membership::Join; - http::client() - ->send_state_event( - currentRoom().toStdString(), - http::client()->user_id().to_string(), - member, - [](mtx::responses::EventId, mtx::http::RequestErr err) { - if (err) - nhlog::net()->error("Failed to set room displayname: {}", - err->matrix_error.error); - }); + http::client()->send_state_event( + currentRoom().toStdString(), + http::client()->user_id().to_string(), + member, + [](mtx::responses::EventId, mtx::http::RequestErr err) { + if (err) + nhlog::net()->error("Failed to set room displayname: {}", + err->matrix_error.error); + }); }); connect( diff --git a/src/dialogs/RoomSettings.cpp b/src/dialogs/RoomSettings.cpp index 26aece32..822b7218 100644 --- a/src/dialogs/RoomSettings.cpp +++ b/src/dialogs/RoomSettings.cpp @@ -151,7 +151,7 @@ EditModal::applyClicked() state::Name body; body.name = newName.toStdString(); - http::client()->send_state_event( + http::client()->send_state_event( roomId_.toStdString(), body, [proxy, newName](const mtx::responses::EventId &, mtx::http::RequestErr err) { @@ -169,7 +169,7 @@ EditModal::applyClicked() state::Topic body; body.topic = newTopic.toStdString(); - http::client()->send_state_event( + http::client()->send_state_event( roomId_.toStdString(), body, [proxy](const mtx::responses::EventId &, mtx::http::RequestErr err) { @@ -694,7 +694,7 @@ RoomSettings::updateAccessRules(const std::string &room_id, startLoadingSpinner(); resetErrorLabel(); - http::client()->send_state_event( + http::client()->send_state_event( room_id, join_rule, [this, room_id, guest_access](const mtx::responses::EventId &, @@ -708,7 +708,7 @@ RoomSettings::updateAccessRules(const std::string &room_id, return; } - http::client()->send_state_event( + http::client()->send_state_event( room_id, guest_access, [this](const mtx::responses::EventId &, mtx::http::RequestErr err) { @@ -843,7 +843,7 @@ RoomSettings::updateAvatar() avatar_event.image_info.size = size; avatar_event.url = res.content_uri; - http::client()->send_state_event( + http::client()->send_state_event( room_id, avatar_event, [content = std::move(content), proxy = std::move(proxy)]( diff --git a/src/timeline/EventStore.cpp b/src/timeline/EventStore.cpp index 7f21e1ed..b7cf4f96 100644 --- a/src/timeline/EventStore.cpp +++ b/src/timeline/EventStore.cpp @@ -1,6 +1,7 @@ #include "EventStore.h" #include +#include #include "Cache_p.h" #include "EventAccessors.h" @@ -59,6 +60,104 @@ EventStore::EventStore(std::string room_id, QObject *) } }, Qt::QueuedConnection); + + connect(this, &EventStore::processPending, this, [this]() { + if (!current_txn.empty()) { + nhlog::ui()->debug("Already processing {}", current_txn); + return; + } + + auto event = cache::client()->firstPendingMessage(room_id_); + + if (!event) { + nhlog::ui()->debug("No event to send"); + return; + } + + std::visit( + [this](auto e) { + auto txn_id = e.event_id; + this->current_txn = txn_id; + + if (txn_id.empty() || txn_id[0] != 'm') { + nhlog::ui()->debug("Invalid txn id '{}'", txn_id); + cache::client()->removePendingStatus(room_id_, txn_id); + return; + } + + if constexpr (mtx::events::message_content_to_type != + mtx::events::EventType::Unsupported) + http::client()->send_room_message( + room_id_, + txn_id, + e.content, + [this, txn_id](const mtx::responses::EventId &, + mtx::http::RequestErr err) { + if (err) { + const int status_code = + static_cast(err->status_code); + nhlog::net()->warn( + "[{}] failed to send message: {} {}", + txn_id, + err->matrix_error.error, + status_code); + emit messageFailed(txn_id); + return; + } + emit messageSent(txn_id); + }); + }, + event->data); + }); + + connect( + this, + &EventStore::messageFailed, + this, + [this](std::string txn_id) { + if (current_txn == txn_id) { + current_txn_error_count++; + if (current_txn_error_count > 10) { + nhlog::ui()->debug("failing txn id '{}'", txn_id); + cache::client()->removePendingStatus(room_id_, txn_id); + current_txn_error_count = 0; + } + } + QTimer::singleShot(1000, this, [this]() { + nhlog::ui()->debug("timeout"); + this->current_txn = ""; + emit processPending(); + }); + }, + Qt::QueuedConnection); + + connect( + this, + &EventStore::messageSent, + this, + [this](std::string txn_id) { + nhlog::ui()->debug("sent {}", txn_id); + cache::client()->removePendingStatus(room_id_, txn_id); + this->current_txn = ""; + this->current_txn_error_count = 0; + emit processPending(); + }, + Qt::QueuedConnection); +} + +void +EventStore::addPending(mtx::events::collections::TimelineEvents event) +{ + if (this->thread() != QThread::currentThread()) + nhlog::db()->warn("{} called from a different thread!", __func__); + + cache::client()->savePendingMessage(this->room_id_, {event}); + mtx::responses::Timeline events; + events.limited = false; + events.events.emplace_back(event); + handleSync(events); + + emit processPending(); } void @@ -102,6 +201,16 @@ EventStore::handleSync(const mtx::responses::Timeline &events) if (idx) emit dataChanged(toExternalIdx(*idx), toExternalIdx(*idx)); } + + if (auto txn_id = mtx::accessors::transaction_id(event); !txn_id.empty()) { + auto idx = cache::client()->getTimelineIndex( + room_id_, mtx::accessors::event_id(event)); + if (idx) { + Index index{room_id_, *idx}; + events_.remove(index); + emit dataChanged(toExternalIdx(*idx), toExternalIdx(*idx)); + } + } } } diff --git a/src/timeline/EventStore.h b/src/timeline/EventStore.h index f2997245..b4d5bb23 100644 --- a/src/timeline/EventStore.h +++ b/src/timeline/EventStore.h @@ -90,6 +90,13 @@ signals: void oldMessagesRetrieved(const mtx::responses::Messages &); void fetchedMore(); + void processPending(); + void messageSent(std::string txn_id); + void messageFailed(std::string txn_id); + +public slots: + void addPending(mtx::events::collections::TimelineEvents event); + private: mtx::events::collections::TimelineEvents *decryptEvent( const IdIndex &idx, @@ -103,4 +110,7 @@ private: static QCache decryptedEvents_; static QCache events_; static QCache events_by_id_; + + std::string current_txn; + int current_txn_error_count = 0; }; diff --git a/src/timeline/TimelineModel.cpp b/src/timeline/TimelineModel.cpp index 60264e86..aa6cea4f 100644 --- a/src/timeline/TimelineModel.cpp +++ b/src/timeline/TimelineModel.cpp @@ -145,67 +145,6 @@ TimelineModel::TimelineModel(TimelineViewManager *manager, QString room_id, QObj , room_id_(room_id) , manager_(manager) { - connect(this, - &TimelineModel::oldMessagesRetrieved, - this, - &TimelineModel::addBackwardsEvents, - Qt::QueuedConnection); - connect( - this, - &TimelineModel::messageFailed, - this, - [this](QString txn_id) { - nhlog::ui()->error("Failed to send {}, retrying", txn_id.toStdString()); - - QTimer::singleShot(5000, this, [this]() { emit nextPendingMessage(); }); - }, - Qt::QueuedConnection); - connect( - this, - &TimelineModel::messageSent, - this, - [this](QString txn_id, QString event_id) { - pending.removeOne(txn_id); - (void)event_id; - // auto ev = events.value(txn_id); - - // if (auto reaction = - // std::get_if>(&ev)) { - // QString reactedTo = - // QString::fromStdString(reaction->content.relates_to.event_id); - // auto &rModel = reactions[reactedTo]; - // rModel.removeReaction(*reaction); - // auto rCopy = *reaction; - // rCopy.event_id = event_id.toStdString(); - // rModel.addReaction(room_id_.toStdString(), rCopy); - //} - - // int idx = idToIndex(txn_id); - // if (idx < 0) { - // // transaction already received via sync - // return; - //} - // eventOrder[idx] = event_id; - // ev = std::visit( - // [event_id](const auto &e) -> mtx::events::collections::TimelineEvents { - // auto eventCopy = e; - // eventCopy.event_id = event_id.toStdString(); - // return eventCopy; - // }, - // ev); - - // events.remove(txn_id); - // events.insert(event_id, ev); - - //// mark our messages as read - // readEvent(event_id.toStdString()); - - // emit dataChanged(index(idx, 0), index(idx, 0)); - - if (pending.size() > 0) - emit nextPendingMessage(); - }, - Qt::QueuedConnection); connect( this, &TimelineModel::redactionFailed, @@ -213,16 +152,12 @@ TimelineModel::TimelineModel(TimelineViewManager *manager, QString room_id, QObj [](const QString &msg) { emit ChatPage::instance()->showNotification(msg); }, Qt::QueuedConnection); - connect(this, - &TimelineModel::nextPendingMessage, - this, - &TimelineModel::processOnePendingMessage, - Qt::QueuedConnection); connect(this, &TimelineModel::newMessageToSend, this, &TimelineModel::addPendingMessage, Qt::QueuedConnection); + connect(this, &TimelineModel::addPendingMessageToStore, &events, &EventStore::addPending); connect( &events, @@ -296,7 +231,7 @@ int TimelineModel::rowCount(const QModelIndex &parent) const { Q_UNUSED(parent); - return this->events.size() + static_cast(pending.size()); + return this->events.size(); } QVariantMap @@ -410,7 +345,7 @@ TimelineModel::data(const mtx::events::collections::TimelineEvents &event, int r // only show read receipts for messages not from us if (acc::sender(event) != http::client()->user_id().to_string()) return qml_mtx_events::Empty; - else if (pending.contains(id)) + else if (!id.isEmpty() && id[0] == "m") return qml_mtx_events::Sent; else if (read.contains(id) || containsOthers(cache::readReceipts(id, room_id_))) return qml_mtx_events::Read; @@ -428,11 +363,7 @@ TimelineModel::data(const mtx::events::collections::TimelineEvents &event, int r case ReplyTo: return QVariant(QString::fromStdString(in_reply_to_event(event))); case Reactions: { - auto id = QString::fromStdString(event_id(event)); - if (reactions.count(id)) - return QVariant::fromValue((QObject *)&reactions.at(id)); - else - return {}; + return {}; } case RoomId: return QVariant(room_id_); @@ -561,16 +492,9 @@ TimelineModel::fetchMore(const QModelIndex &) void TimelineModel::addEvents(const mtx::responses::Timeline &timeline) { - if (isInitialSync) { - prev_batch_token_ = QString::fromStdString(timeline.prev_batch); - isInitialSync = false; - } - if (timeline.events.empty()) return; - internalAddEvents(timeline.events); - events.handleSync(timeline); if (!timeline.events.empty()) @@ -644,56 +568,6 @@ TimelineModel::updateLastMessage() } } -void -TimelineModel::internalAddEvents( - const std::vector &timeline) -{ - for (auto e : timeline) { - QString id = QString::fromStdString(mtx::accessors::event_id(e)); - - if (auto redaction = - std::get_if>(&e)) { - QString redacts = QString::fromStdString(redaction->redacts); - - auto event = events.event(redaction->redacts, redaction->event_id); - if (!event) - continue; - - if (auto reaction = - std::get_if>( - event)) { - QString reactedTo = - QString::fromStdString(reaction->content.relates_to.event_id); - reactions[reactedTo].removeReaction(*reaction); - int idx = idToIndex(reactedTo); - if (idx >= 0) - emit dataChanged(index(idx, 0), index(idx, 0)); - } - - continue; // don't insert redaction into timeline - } - - if (auto reaction = - std::get_if>(&e)) { - QString reactedTo = - QString::fromStdString(reaction->content.relates_to.event_id); - - // // remove local echo - // if (!txid.isEmpty()) { - // auto rCopy = *reaction; - // rCopy.event_id = txid.toStdString(); - // reactions[reactedTo].removeReaction(rCopy); - // } - - reactions[reactedTo].addReaction(room_id_.toStdString(), *reaction); - int idx = idToIndex(reactedTo); - if (idx >= 0) - emit dataChanged(index(idx, 0), index(idx, 0)); - continue; // don't insert reaction into timeline - } - } -} - void TimelineModel::setCurrentIndex(int index) { @@ -701,7 +575,7 @@ TimelineModel::setCurrentIndex(int index) currentId = indexToId(index); emit currentIndexChanged(index); - if ((oldIndex > index || oldIndex == -1) && !pending.contains(currentId) && + if ((oldIndex > index || oldIndex == -1) && !currentId.startsWith("m") && ChatPage::instance()->isActiveWindow()) { readEvent(currentId.toStdString()); } @@ -719,28 +593,6 @@ TimelineModel::readEvent(const std::string &id) }); } -void -TimelineModel::addBackwardsEvents(const mtx::responses::Messages &msgs) -{ - (void)msgs; - // std::vector ids = internalAddEvents(msgs.chunk); - - // if (!ids.empty()) { - // beginInsertRows(QModelIndex(), - // static_cast(this->eventOrder.size()), - // static_cast(this->eventOrder.size() + ids.size() - 1)); - // this->eventOrder.insert(this->eventOrder.end(), ids.begin(), ids.end()); - // endInsertRows(); - //} - - // prev_batch_token_ = QString::fromStdString(msgs.end); - - // if (ids.empty() && !msgs.chunk.empty()) { - // // no visible events fetched, prevent loading from stopping - // fetchMore(QModelIndex()); - //} -} - QString TimelineModel::displayName(QString id) const { @@ -902,7 +754,7 @@ TimelineModel::markEventsAsRead(const std::vector &event_ids) } void -TimelineModel::sendEncryptedMessage(const std::string &txn_id, nlohmann::json content) +TimelineModel::sendEncryptedMessage(const std::string txn_id, nlohmann::json content) { const auto room_id = room_id_.toStdString(); @@ -914,28 +766,15 @@ TimelineModel::sendEncryptedMessage(const std::string &txn_id, nlohmann::json co try { // Check if we have already an outbound megolm session then we can use. if (cache::outboundMegolmSessionExists(room_id)) { - auto data = + mtx::events::EncryptedEvent event; + event.content = olm::encrypt_group_message(room_id, http::client()->device_id(), doc); + event.event_id = txn_id; + event.room_id = room_id; + event.sender = http::client()->user_id().to_string(); + event.type = mtx::events::EventType::RoomEncrypted; - http::client()->send_room_message( - room_id, - txn_id, - data, - [this, txn_id](const mtx::responses::EventId &res, - mtx::http::RequestErr err) { - if (err) { - const int status_code = - static_cast(err->status_code); - nhlog::net()->warn("[{}] failed to send message: {} {}", - txn_id, - err->matrix_error.error, - status_code); - emit messageFailed(QString::fromStdString(txn_id)); - } - emit messageSent( - QString::fromStdString(txn_id), - QString::fromStdString(res.event_id.to_string())); - }); + emit this->addPendingMessageToStore(event); return; } @@ -964,40 +803,24 @@ TimelineModel::sendEncryptedMessage(const std::string &txn_id, nlohmann::json co const auto members = cache::roomMembers(room_id); nhlog::ui()->info("retrieved {} members for {}", members.size(), room_id); - auto keeper = - std::make_shared([megolm_payload, room_id, doc, txn_id, this]() { - try { - auto data = olm::encrypt_group_message( - room_id, http::client()->device_id(), doc); + auto keeper = std::make_shared([room_id, doc, txn_id, this]() { + try { + mtx::events::EncryptedEvent event; + event.content = olm::encrypt_group_message( + room_id, http::client()->device_id(), doc); + event.event_id = txn_id; + event.room_id = room_id; + event.sender = http::client()->user_id().to_string(); + event.type = mtx::events::EventType::RoomEncrypted; - http::client() - ->send_room_message( - room_id, - txn_id, - data, - [this, txn_id](const mtx::responses::EventId &res, - mtx::http::RequestErr err) { - if (err) { - const int status_code = - static_cast(err->status_code); - nhlog::net()->warn( - "[{}] failed to send message: {} {}", - txn_id, - err->matrix_error.error, - status_code); - emit messageFailed( - QString::fromStdString(txn_id)); - } - emit messageSent( - QString::fromStdString(txn_id), - QString::fromStdString(res.event_id.to_string())); - }); - } catch (const lmdb::error &e) { - nhlog::db()->critical( - "failed to save megolm outbound session: {}", e.what()); - emit messageFailed(QString::fromStdString(txn_id)); - } - }); + emit this->addPendingMessageToStore(event); + } catch (const lmdb::error &e) { + nhlog::db()->critical("failed to save megolm outbound session: {}", + e.what()); + emit ChatPage::instance()->showNotification( + tr("Failed to encrypt event, sending aborted!")); + } + }); mtx::requests::QueryKeys req; for (const auto &member : members) @@ -1011,8 +834,8 @@ TimelineModel::sendEncryptedMessage(const std::string &txn_id, nlohmann::json co nhlog::net()->warn("failed to query device keys: {} {}", err->matrix_error.error, static_cast(err->status_code)); - // TODO: Mark the event as failed. Communicate with the UI. - emit messageFailed(QString::fromStdString(txn_id)); + emit ChatPage::instance()->showNotification( + tr("Failed to encrypt event, sending aborted!")); return; } @@ -1112,11 +935,13 @@ TimelineModel::sendEncryptedMessage(const std::string &txn_id, nlohmann::json co } catch (const lmdb::error &e) { nhlog::db()->critical( "failed to open outbound megolm session ({}): {}", room_id, e.what()); - emit messageFailed(QString::fromStdString(txn_id)); + emit ChatPage::instance()->showNotification( + tr("Failed to encrypt event, sending aborted!")); } catch (const mtx::crypto::olm_exception &e) { nhlog::crypto()->critical( "failed to open outbound megolm session ({}): {}", room_id, e.what()); - emit messageFailed(QString::fromStdString(txn_id)); + emit ChatPage::instance()->showNotification( + tr("Failed to encrypt event, sending aborted!")); } } @@ -1208,9 +1033,8 @@ TimelineModel::handleClaimedKeys(std::shared_ptr keeper, struct SendMessageVisitor { - SendMessageVisitor(const QString &txn_id, TimelineModel *model) - : txn_id_qstr_(txn_id) - , model_(model) + explicit SendMessageVisitor(TimelineModel *model) + : model_(model) {} // Do-nothing operator for all unhandled events @@ -1228,29 +1052,9 @@ struct SendMessageVisitor if (encInfo) emit model_->newEncryptedImage(encInfo.value()); - model_->sendEncryptedMessage(txn_id_qstr_.toStdString(), - nlohmann::json(msg.content)); + model_->sendEncryptedMessage(msg.event_id, nlohmann::json(msg.content)); } else { - QString txn_id_qstr = txn_id_qstr_; - TimelineModel *model = model_; - http::client()->send_room_message( - model->room_id_.toStdString(), - txn_id_qstr.toStdString(), - msg.content, - [txn_id_qstr, model](const mtx::responses::EventId &res, - mtx::http::RequestErr err) { - if (err) { - const int status_code = - static_cast(err->status_code); - nhlog::net()->warn("[{}] failed to send message: {} {}", - txn_id_qstr.toStdString(), - err->matrix_error.error, - status_code); - emit model->messageFailed(txn_id_qstr); - } - emit model->messageSent( - txn_id_qstr, QString::fromStdString(res.event_id.to_string())); - }); + emit model_->addPendingMessageToStore(msg); } } @@ -1260,71 +1064,26 @@ struct SendMessageVisitor // cannot handle it correctly. See the MSC for more details: // https://github.com/matrix-org/matrix-doc/blob/matthew/msc1849/proposals/1849-aggregations.md#end-to-end-encryption void operator()(const mtx::events::RoomEvent &msg) - { - QString txn_id_qstr = txn_id_qstr_; - TimelineModel *model = model_; - http::client() - ->send_room_message( - model->room_id_.toStdString(), - txn_id_qstr.toStdString(), - msg.content, - [txn_id_qstr, model](const mtx::responses::EventId &res, - mtx::http::RequestErr err) { - if (err) { - const int status_code = static_cast(err->status_code); - nhlog::net()->warn("[{}] failed to send message: {} {}", - txn_id_qstr.toStdString(), - err->matrix_error.error, - status_code); - emit model->messageFailed(txn_id_qstr); - } - emit model->messageSent( - txn_id_qstr, QString::fromStdString(res.event_id.to_string())); - }); + emit model_->addPendingMessageToStore(msg); } - QString txn_id_qstr_; TimelineModel *model_; }; -void -TimelineModel::processOnePendingMessage() -{ - // if (pending.isEmpty()) - // return; - - // QString txn_id_qstr = pending.first(); - - // auto event = events.value(txn_id_qstr); - // std::visit(SendMessageVisitor{txn_id_qstr, this}, event); -} - void TimelineModel::addPendingMessage(mtx::events::collections::TimelineEvents event) { - (void)event; - // std::visit( - // [](auto &msg) { - // msg.type = mtx::events::EventType::RoomMessage; - // msg.event_id = http::client()->generate_txn_id(); - // msg.sender = http::client()->user_id().to_string(); - // msg.origin_server_ts = QDateTime::currentMSecsSinceEpoch(); - // }, - // event); + std::visit( + [](auto &msg) { + msg.type = mtx::events::EventType::RoomMessage; + msg.event_id = "m" + http::client()->generate_txn_id(); + msg.sender = http::client()->user_id().to_string(); + msg.origin_server_ts = QDateTime::currentMSecsSinceEpoch(); + }, + event); - // internalAddEvents({event}); - - // QString txn_id_qstr = QString::fromStdString(mtx::accessors::event_id(event)); - // pending.push_back(txn_id_qstr); - // if (!std::get_if>(&event)) { - // beginInsertRows(QModelIndex(), 0, 0); - // this->eventOrder.insert(this->eventOrder.begin(), txn_id_qstr); - // endInsertRows(); - //} - // updateLastMessage(); - - // emit nextPendingMessage(); + std::visit(SendMessageVisitor{this}, event); } bool @@ -1647,24 +1406,7 @@ TimelineModel::formatMemberEvent(QString id) if (!event->unsigned_data.replaces_state.empty()) { auto tempPrevEvent = events.event(event->unsigned_data.replaces_state, event->event_id); - if (!tempPrevEvent) { - http::client()->get_event( - this->room_id_.toStdString(), - event->unsigned_data.replaces_state, - [this, id, prevEventId = event->unsigned_data.replaces_state]( - const mtx::events::collections::TimelineEvents &timeline, - mtx::http::RequestErr err) { - if (err) { - nhlog::net()->error( - "Failed to retrieve event with id {}, which was " - "requested to show the membership for event {}", - prevEventId, - id.toStdString()); - return; - } - emit eventFetched(id, timeline); - }); - } else { + if (tempPrevEvent) { prevEvent = std::get_if>( tempPrevEvent); diff --git a/src/timeline/TimelineModel.h b/src/timeline/TimelineModel.h index f322b482..9f9717df 100644 --- a/src/timeline/TimelineModel.h +++ b/src/timeline/TimelineModel.h @@ -236,31 +236,23 @@ public slots: void setDecryptDescription(bool decrypt) { decryptDescription = decrypt; } private slots: - // Add old events at the top of the timeline. - void addBackwardsEvents(const mtx::responses::Messages &msgs); - void processOnePendingMessage(); void addPendingMessage(mtx::events::collections::TimelineEvents event); signals: - void oldMessagesRetrieved(const mtx::responses::Messages &res); - void messageFailed(QString txn_id); - void messageSent(QString txn_id, QString event_id); void currentIndexChanged(int index); void redactionFailed(QString id); void eventRedacted(QString id); - void nextPendingMessage(); - void newMessageToSend(mtx::events::collections::TimelineEvents event); void mediaCached(QString mxcUrl, QString cacheUrl); void newEncryptedImage(mtx::crypto::EncryptedFile encryptionInfo); - void eventFetched(QString requestingEvent, mtx::events::collections::TimelineEvents event); void typingUsersChanged(std::vector users); void replyChanged(QString reply); void paginationInProgressChanged(const bool); + void newMessageToSend(mtx::events::collections::TimelineEvents event); + void addPendingMessageToStore(mtx::events::collections::TimelineEvents event); + private: - void internalAddEvents( - const std::vector &timeline); - void sendEncryptedMessage(const std::string &txn_id, nlohmann::json content); + void sendEncryptedMessage(const std::string txn_id, nlohmann::json content); void handleClaimedKeys(std::shared_ptr keeper, const std::map &room_key, const std::map &pks, @@ -272,15 +264,11 @@ private: void setPaginationInProgress(const bool paginationInProgress); QSet read; - QList pending; - std::map reactions; mutable EventStore events; QString room_id_; - QString prev_batch_token_; - bool isInitialSync = true; bool decryptDescription = true; bool m_paginationInProgress = false; From a5dda86a6cc249fb6e795066187bcb34305162a3 Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Sat, 18 Jul 2020 17:59:59 +0200 Subject: [PATCH 13/50] Fix encryption indicator --- src/timeline/TimelineModel.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/timeline/TimelineModel.cpp b/src/timeline/TimelineModel.cpp index aa6cea4f..b1cb2d5e 100644 --- a/src/timeline/TimelineModel.cpp +++ b/src/timeline/TimelineModel.cpp @@ -353,9 +353,12 @@ TimelineModel::data(const mtx::events::collections::TimelineEvents &event, int r return qml_mtx_events::Received; } case IsEncrypted: { - // return std::holds_alternative< - // mtx::events::EncryptedEvent>(events[id]); - return false; + auto id = event_id(event); + auto encrypted_event = events.event(id, id, false); + return encrypted_event && + std::holds_alternative< + mtx::events::EncryptedEvent>( + *encrypted_event); } case IsRoomEncrypted: { return cache::isRoomEncrypted(room_id_.toStdString()); From 36e4405f259f4043e0e88a967462f5d1641649be Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Sat, 18 Jul 2020 19:15:50 +0200 Subject: [PATCH 14/50] Fix flickering of encrypted messages when sending using new store --- src/timeline/TimelineModel.cpp | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/timeline/TimelineModel.cpp b/src/timeline/TimelineModel.cpp index b1cb2d5e..8e5b245b 100644 --- a/src/timeline/TimelineModel.cpp +++ b/src/timeline/TimelineModel.cpp @@ -772,10 +772,11 @@ TimelineModel::sendEncryptedMessage(const std::string txn_id, nlohmann::json con mtx::events::EncryptedEvent event; event.content = olm::encrypt_group_message(room_id, http::client()->device_id(), doc); - event.event_id = txn_id; - event.room_id = room_id; - event.sender = http::client()->user_id().to_string(); - event.type = mtx::events::EventType::RoomEncrypted; + event.event_id = txn_id; + event.room_id = room_id; + event.sender = http::client()->user_id().to_string(); + event.type = mtx::events::EventType::RoomEncrypted; + event.origin_server_ts = QDateTime::currentMSecsSinceEpoch(); emit this->addPendingMessageToStore(event); return; @@ -811,10 +812,11 @@ TimelineModel::sendEncryptedMessage(const std::string txn_id, nlohmann::json con mtx::events::EncryptedEvent event; event.content = olm::encrypt_group_message( room_id, http::client()->device_id(), doc); - event.event_id = txn_id; - event.room_id = room_id; - event.sender = http::client()->user_id().to_string(); - event.type = mtx::events::EventType::RoomEncrypted; + event.event_id = txn_id; + event.room_id = room_id; + event.sender = http::client()->user_id().to_string(); + event.type = mtx::events::EventType::RoomEncrypted; + event.origin_server_ts = QDateTime::currentMSecsSinceEpoch(); emit this->addPendingMessageToStore(event); } catch (const lmdb::error &e) { From 046b3f4da6e5b8eec92bd0895048a4df2e916285 Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Sat, 18 Jul 2020 20:39:31 +0200 Subject: [PATCH 15/50] Mark own events as read again after sending --- src/timeline/EventStore.cpp | 15 ++++++++++++--- src/timeline/EventStore.h | 2 +- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/timeline/EventStore.cpp b/src/timeline/EventStore.cpp index b7cf4f96..80e8d474 100644 --- a/src/timeline/EventStore.cpp +++ b/src/timeline/EventStore.cpp @@ -91,7 +91,7 @@ EventStore::EventStore(std::string room_id, QObject *) room_id_, txn_id, e.content, - [this, txn_id](const mtx::responses::EventId &, + [this, txn_id](const mtx::responses::EventId &event_id, mtx::http::RequestErr err) { if (err) { const int status_code = @@ -104,7 +104,7 @@ EventStore::EventStore(std::string room_id, QObject *) emit messageFailed(txn_id); return; } - emit messageSent(txn_id); + emit messageSent(txn_id, event_id.event_id.to_string()); }); }, event->data); @@ -135,8 +135,17 @@ EventStore::EventStore(std::string room_id, QObject *) this, &EventStore::messageSent, this, - [this](std::string txn_id) { + [this](std::string txn_id, std::string event_id) { nhlog::ui()->debug("sent {}", txn_id); + + http::client()->read_event( + room_id_, event_id, [this, event_id](mtx::http::RequestErr err) { + if (err) { + nhlog::net()->warn( + "failed to read_event ({}, {})", room_id_, event_id); + } + }); + cache::client()->removePendingStatus(room_id_, txn_id); this->current_txn = ""; this->current_txn_error_count = 0; diff --git a/src/timeline/EventStore.h b/src/timeline/EventStore.h index b4d5bb23..3a78cba8 100644 --- a/src/timeline/EventStore.h +++ b/src/timeline/EventStore.h @@ -91,7 +91,7 @@ signals: void fetchedMore(); void processPending(); - void messageSent(std::string txn_id); + void messageSent(std::string txn_id, std::string event_id); void messageFailed(std::string txn_id); public slots: From 5695f004a25649f8955659ea57769af8e311be9b Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Sat, 18 Jul 2020 21:00:36 +0200 Subject: [PATCH 16/50] Fix race condition between /messages and /sync --- src/ChatPage.cpp | 80 ++++++++++++++++++++++++++++-------------------- src/ChatPage.h | 2 ++ 2 files changed, 48 insertions(+), 34 deletions(-) diff --git a/src/ChatPage.cpp b/src/ChatPage.cpp index 813b0c2a..c4376905 100644 --- a/src/ChatPage.cpp +++ b/src/ChatPage.cpp @@ -573,6 +573,12 @@ ChatPage::ChatPage(QSharedPointer userSettings, QWidget *parent) [this]() { QTimer::singleShot(RETRY_TIMEOUT, this, &ChatPage::trySync); }, Qt::QueuedConnection); + connect(this, + &ChatPage::newSyncResponse, + this, + &ChatPage::handleSyncResponse, + Qt::QueuedConnection); + connect(this, &ChatPage::dropToLoginPageCb, this, &ChatPage::dropToLoginPage); instance_ = this; @@ -1003,6 +1009,45 @@ ChatPage::startInitialSync() &ChatPage::initialSyncHandler, this, std::placeholders::_1, std::placeholders::_2)); } +void +ChatPage::handleSyncResponse(mtx::responses::Sync res) +{ + nhlog::net()->debug("sync completed: {}", res.next_batch); + + // Ensure that we have enough one-time keys available. + ensureOneTimeKeyCount(res.device_one_time_keys_count); + + // TODO: fine grained error handling + try { + cache::saveState(res); + olm::handle_to_device_messages(res.to_device.events); + + auto updates = cache::roomUpdates(res); + + emit syncTopBar(updates); + emit syncRoomlist(updates); + + emit syncUI(res.rooms); + + emit syncTags(cache::roomTagUpdates(res)); + + // if we process a lot of syncs (1 every 200ms), this means we clean the + // db every 100s + static int syncCounter = 0; + if (syncCounter++ >= 500) { + cache::deleteOldData(); + syncCounter = 0; + } + } catch (const lmdb::map_full_error &e) { + nhlog::db()->error("lmdb is full: {}", e.what()); + cache::deleteOldData(); + } catch (const lmdb::error &e) { + nhlog::db()->error("saving sync response: {}", e.what()); + } + + emit trySyncCb(); +} + void ChatPage::trySync() { @@ -1042,40 +1087,7 @@ ChatPage::trySync() return; } - nhlog::net()->debug("sync completed: {}", res.next_batch); - - // Ensure that we have enough one-time keys available. - ensureOneTimeKeyCount(res.device_one_time_keys_count); - - // TODO: fine grained error handling - try { - cache::saveState(res); - olm::handle_to_device_messages(res.to_device.events); - - auto updates = cache::roomUpdates(res); - - emit syncTopBar(updates); - emit syncRoomlist(updates); - - emit syncUI(res.rooms); - - emit syncTags(cache::roomTagUpdates(res)); - - // if we process a lot of syncs (1 every 200ms), this means we clean the - // db every 100s - static int syncCounter = 0; - if (syncCounter++ >= 500) { - cache::deleteOldData(); - syncCounter = 0; - } - } catch (const lmdb::map_full_error &e) { - nhlog::db()->error("lmdb is full: {}", e.what()); - cache::deleteOldData(); - } catch (const lmdb::error &e) { - nhlog::db()->error("saving sync response: {}", e.what()); - } - - emit trySyncCb(); + emit newSyncResponse(res); }); } diff --git a/src/ChatPage.h b/src/ChatPage.h index c38d7717..18bed289 100644 --- a/src/ChatPage.h +++ b/src/ChatPage.h @@ -139,6 +139,7 @@ signals: void trySyncCb(); void tryDelayedSyncCb(); void tryInitialSyncCb(); + void newSyncResponse(mtx::responses::Sync res); void leftRoom(const QString &room_id); void initializeRoomList(QMap); @@ -173,6 +174,7 @@ private slots: void joinRoom(const QString &room); void sendTypingNotifications(); + void handleSyncResponse(mtx::responses::Sync res); private: static ChatPage *instance_; From 8261446f839315ee683ae23f6c5cbc4a9a025e0a Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Sat, 18 Jul 2020 21:14:45 +0200 Subject: [PATCH 17/50] Fix reply scrolling --- src/timeline/TimelineModel.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/timeline/TimelineModel.cpp b/src/timeline/TimelineModel.cpp index 8e5b245b..470e3988 100644 --- a/src/timeline/TimelineModel.cpp +++ b/src/timeline/TimelineModel.cpp @@ -729,7 +729,7 @@ TimelineModel::idToIndex(QString id) const auto idx = events.idToIndex(id.toStdString()); if (idx) - return events.size() - *idx; + return events.size() - *idx - 1; else return -1; } @@ -737,7 +737,7 @@ TimelineModel::idToIndex(QString id) const QString TimelineModel::indexToId(int index) const { - auto id = events.indexToId(events.size() - index); + auto id = events.indexToId(events.size() - index - 1); return id ? QString::fromStdString(*id) : ""; } From d467568a65235093c5ea4591486818ab5de42119 Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Sat, 18 Jul 2020 22:59:03 +0200 Subject: [PATCH 18/50] Close cursor we don't need and where we overwrite the contents --- src/Cache.cpp | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/Cache.cpp b/src/Cache.cpp index 8fa94d1e..9464a546 100644 --- a/src/Cache.cpp +++ b/src/Cache.cpp @@ -2334,15 +2334,19 @@ Cache::saveOldMessages(const std::string &room_id, const mtx::responses::Message lmdb::val indexVal, val; uint64_t index = std::numeric_limits::max() / 2; - auto cursor = lmdb::cursor::open(txn, orderDb); - if (cursor.get(indexVal, val, MDB_FIRST)) { - index = *indexVal.data(); + { + auto cursor = lmdb::cursor::open(txn, orderDb); + if (cursor.get(indexVal, val, MDB_FIRST)) { + index = *indexVal.data(); + } } uint64_t msgIndex = std::numeric_limits::max() / 2; - auto msgCursor = lmdb::cursor::open(txn, order2msgDb); - if (msgCursor.get(indexVal, val, MDB_FIRST)) { - msgIndex = *indexVal.data(); + { + auto msgCursor = lmdb::cursor::open(txn, order2msgDb); + if (msgCursor.get(indexVal, val, MDB_FIRST)) { + msgIndex = *indexVal.data(); + } } if (res.chunk.empty()) @@ -2389,8 +2393,7 @@ Cache::saveOldMessages(const std::string &room_id, const mtx::responses::Message json orderEntry = json::object(); orderEntry["event_id"] = event_id_val; orderEntry["prev_batch"] = res.end; - lmdb::cursor_put( - cursor.handle(), lmdb::val(&index, sizeof(index)), lmdb::val(orderEntry.dump())); + lmdb::dbi_put(txn, orderDb, lmdb::val(&index, sizeof(index)), lmdb::val(orderEntry.dump())); nhlog::db()->debug("saving '{}'", orderEntry.dump()); txn.commit(); From 6f2bc908badc207754ff55d543d41d9e2b847c97 Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Sun, 19 Jul 2020 12:22:54 +0200 Subject: [PATCH 19/50] Fix reaction display --- CMakeLists.txt | 4 +- resources/qml/Reactions.qml | 18 +++--- src/Cache.cpp | 35 ++++++++++++ src/Cache_p.h | 3 + src/timeline/EventStore.cpp | 70 +++++++++++++++++++++++ src/timeline/EventStore.h | 5 ++ src/timeline/Reaction.cpp | 1 + src/timeline/Reaction.h | 24 ++++++++ src/timeline/ReactionsModel.cpp | 98 --------------------------------- src/timeline/ReactionsModel.h | 41 -------------- src/timeline/TimelineModel.cpp | 3 +- src/timeline/TimelineModel.h | 1 - 12 files changed, 151 insertions(+), 152 deletions(-) create mode 100644 src/timeline/Reaction.cpp create mode 100644 src/timeline/Reaction.h delete mode 100644 src/timeline/ReactionsModel.cpp delete mode 100644 src/timeline/ReactionsModel.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 8d1441c1..658232e8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -251,7 +251,7 @@ set(SRC_FILES # Timeline src/timeline/EventStore.cpp - src/timeline/ReactionsModel.cpp + src/timeline/Reaction.cpp src/timeline/TimelineViewManager.cpp src/timeline/TimelineModel.cpp src/timeline/DelegateChooser.cpp @@ -455,7 +455,7 @@ qt5_wrap_cpp(MOC_HEADERS # Timeline src/timeline/EventStore.h - src/timeline/ReactionsModel.h + src/timeline/Reaction.h src/timeline/TimelineViewManager.h src/timeline/TimelineModel.h src/timeline/DelegateChooser.h diff --git a/resources/qml/Reactions.qml b/resources/qml/Reactions.qml index c06dc826..5b3bbc20 100644 --- a/resources/qml/Reactions.qml +++ b/resources/qml/Reactions.qml @@ -30,11 +30,11 @@ Flow { implicitHeight: contentItem.childrenRect.height ToolTip.visible: hovered - ToolTip.text: model.users + ToolTip.text: modelData.users onClicked: { - console.debug("Picked " + model.key + "in response to " + reactionFlow.eventId + " in room " + reactionFlow.roomId + ". selfReactedEvent: " + model.selfReactedEvent) - timelineManager.reactToMessage(reactionFlow.roomId, reactionFlow.eventId, model.key, model.selfReactedEvent) + console.debug("Picked " + modelData.key + "in response to " + reactionFlow.eventId + " in room " + reactionFlow.roomId + ". selfReactedEvent: " + modelData.selfReactedEvent) + timelineManager.reactToMessage(reactionFlow.roomId, reactionFlow.eventId, modelData.key, modelData.selfReactedEvent) } @@ -49,13 +49,13 @@ Flow { font.family: settings.emojiFont elide: Text.ElideRight elideWidth: 150 - text: model.key + text: modelData.key } Text { anchors.baseline: reactionCounter.baseline id: reactionText - text: textMetrics.elidedText + (textMetrics.elidedText == model.key ? "" : "…") + text: textMetrics.elidedText + (textMetrics.elidedText == modelData.key ? "" : "…") font.family: settings.emojiFont color: reaction.hovered ? colors.highlight : colors.text maximumLineCount: 1 @@ -65,13 +65,13 @@ Flow { id: divider height: Math.floor(reactionCounter.implicitHeight * 1.4) width: 1 - color: (reaction.hovered || model.selfReactedEvent !== '') ? colors.highlight : colors.text + color: (reaction.hovered || modelData.selfReactedEvent !== '') ? colors.highlight : colors.text } Text { anchors.verticalCenter: divider.verticalCenter id: reactionCounter - text: model.counter + text: modelData.count font: reaction.font color: reaction.hovered ? colors.highlight : colors.text } @@ -82,8 +82,8 @@ Flow { implicitWidth: reaction.implicitWidth implicitHeight: reaction.implicitHeight - border.color: (reaction.hovered || model.selfReactedEvent !== '') ? colors.highlight : colors.text - color: model.selfReactedEvent !== '' ? Qt.hsla(highlightHue, highlightSat, highlightLight, 0.20) : colors.base + border.color: (reaction.hovered || modelData.selfReactedEvent !== '') ? colors.highlight : colors.text + color: modelData.selfReactedEvent !== '' ? Qt.hsla(highlightHue, highlightSat, highlightLight, 0.20) : colors.base border.width: 1 radius: reaction.height / 2.0 } diff --git a/src/Cache.cpp b/src/Cache.cpp index 9464a546..0307bee1 100644 --- a/src/Cache.cpp +++ b/src/Cache.cpp @@ -1353,6 +1353,37 @@ Cache::storeEvent(const std::string &room_id, txn.commit(); } +std::vector +Cache::relatedEvents(const std::string &room_id, const std::string &event_id) +{ + auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + auto relationsDb = getRelationsDb(txn, room_id); + + std::vector related_ids; + + auto related_cursor = lmdb::cursor::open(txn, relationsDb); + lmdb::val related_to = event_id, related_event; + bool first = true; + + try { + if (!related_cursor.get(related_to, related_event, MDB_SET)) + return {}; + + while (related_cursor.get( + related_to, related_event, first ? MDB_FIRST_DUP : MDB_NEXT_DUP)) { + first = false; + if (event_id != std::string_view(related_to.data(), related_to.size())) + break; + + related_ids.emplace_back(related_event.data(), related_event.size()); + } + } catch (const lmdb::error &e) { + nhlog::db()->error("related events error: {}", e.what()); + } + + return related_ids; +} + QMap Cache::roomInfo(bool withInvites) { @@ -2354,6 +2385,10 @@ Cache::saveOldMessages(const std::string &room_id, const mtx::responses::Message std::string event_id_val; for (const auto &e : res.chunk) { + if (std::holds_alternative< + mtx::events::RedactionEvent>(e)) + continue; + auto event = mtx::accessors::serialize_event(e); event_id_val = event["event_id"].get(); lmdb::val event_id = event_id_val; diff --git a/src/Cache_p.h b/src/Cache_p.h index 88308e45..61d91b0c 100644 --- a/src/Cache_p.h +++ b/src/Cache_p.h @@ -188,6 +188,9 @@ public: void storeEvent(const std::string &room_id, const std::string &event_id, const mtx::events::collections::TimelineEvent &event); + std::vector relatedEvents(const std::string &room_id, + const std::string &event_id); + struct TimelineRange { uint64_t first, last; diff --git a/src/timeline/EventStore.cpp b/src/timeline/EventStore.cpp index 80e8d474..0bd7a97e 100644 --- a/src/timeline/EventStore.cpp +++ b/src/timeline/EventStore.cpp @@ -3,12 +3,15 @@ #include #include +#include "Cache.h" #include "Cache_p.h" #include "EventAccessors.h" #include "Logging.h" #include "MatrixClient.h" #include "Olm.h" +Q_DECLARE_METATYPE(Reaction) + QCache EventStore::decryptedEvents_{ 1000}; QCache EventStore::events_by_id_{ @@ -18,6 +21,9 @@ QCache EventStore:: EventStore::EventStore(std::string room_id, QObject *) : room_id_(std::move(room_id)) { + static auto reactionType = qRegisterMetaType(); + (void)reactionType; + auto range = cache::client()->getTimelineRange(room_id_); if (range) { @@ -223,6 +229,70 @@ EventStore::handleSync(const mtx::responses::Timeline &events) } } +QVariantList +EventStore::reactions(const std::string &event_id) +{ + auto event_ids = cache::client()->relatedEvents(room_id_, event_id); + + struct TempReaction + { + int count = 0; + std::vector users; + std::string reactedBySelf; + }; + std::map aggregation; + std::vector reactions; + + auto self = http::client()->user_id().to_string(); + for (const auto &id : event_ids) { + auto related_event = event(id, event_id); + if (!related_event) + continue; + + if (auto reaction = std::get_if>( + related_event)) { + auto &agg = aggregation[reaction->content.relates_to.key]; + + if (agg.count == 0) { + Reaction temp{}; + temp.key_ = + QString::fromStdString(reaction->content.relates_to.key); + reactions.push_back(temp); + } + + agg.count++; + agg.users.push_back(cache::displayName(room_id_, reaction->sender)); + if (reaction->sender == self) + agg.reactedBySelf = reaction->event_id; + } + } + + QVariantList temp; + for (auto &reaction : reactions) { + const auto &agg = aggregation[reaction.key_.toStdString()]; + reaction.count_ = agg.count; + reaction.selfReactedEvent_ = QString::fromStdString(agg.reactedBySelf); + + bool first = true; + for (const auto &user : agg.users) { + if (first) + first = false; + else + reaction.users_ += ", "; + + reaction.users_ += QString::fromStdString(user); + } + + nhlog::db()->debug("key: {}, count: {}, users: {}", + reaction.key_.toStdString(), + reaction.count_, + reaction.users_.toStdString()); + temp.append(QVariant::fromValue(reaction)); + } + + return temp; +} + mtx::events::collections::TimelineEvents * EventStore::event(int idx, bool decrypt) { diff --git a/src/timeline/EventStore.h b/src/timeline/EventStore.h index 3a78cba8..5a792040 100644 --- a/src/timeline/EventStore.h +++ b/src/timeline/EventStore.h @@ -5,12 +5,15 @@ #include #include +#include #include #include #include #include +#include "Reaction.h" + class EventStore : public QObject { Q_OBJECT @@ -65,6 +68,8 @@ public: // always returns a proper event as long as the idx is valid mtx::events::collections::TimelineEvents *event(int idx, bool decrypt = true); + QVariantList reactions(const std::string &event_id); + int size() const { return last != std::numeric_limits::max() diff --git a/src/timeline/Reaction.cpp b/src/timeline/Reaction.cpp new file mode 100644 index 00000000..343c4649 --- /dev/null +++ b/src/timeline/Reaction.cpp @@ -0,0 +1 @@ +#include "Reaction.h" diff --git a/src/timeline/Reaction.h b/src/timeline/Reaction.h new file mode 100644 index 00000000..5f122e0a --- /dev/null +++ b/src/timeline/Reaction.h @@ -0,0 +1,24 @@ +#pragma once + +#include +#include + +struct Reaction +{ + Q_GADGET + Q_PROPERTY(QString key READ key) + Q_PROPERTY(QString users READ users) + Q_PROPERTY(QString selfReactedEvent READ selfReactedEvent) + Q_PROPERTY(int count READ count) + +public: + QString key() const { return key_; } + QString users() const { return users_; } + QString selfReactedEvent() const { return selfReactedEvent_; } + int count() const { return count_; } + + QString key_; + QString users_; + QString selfReactedEvent_; + int count_; +}; diff --git a/src/timeline/ReactionsModel.cpp b/src/timeline/ReactionsModel.cpp deleted file mode 100644 index 1200e2ba..00000000 --- a/src/timeline/ReactionsModel.cpp +++ /dev/null @@ -1,98 +0,0 @@ -#include "ReactionsModel.h" - -#include -#include - -QHash -ReactionsModel::roleNames() const -{ - return { - {Key, "key"}, - {Count, "counter"}, - {Users, "users"}, - {SelfReactedEvent, "selfReactedEvent"}, - }; -} - -int -ReactionsModel::rowCount(const QModelIndex &) const -{ - return static_cast(reactions.size()); -} - -QVariant -ReactionsModel::data(const QModelIndex &index, int role) const -{ - const int i = index.row(); - if (i < 0 || i >= static_cast(reactions.size())) - return {}; - - switch (role) { - case Key: - return QString::fromStdString(reactions[i].key); - case Count: - return static_cast(reactions[i].reactions.size()); - case Users: { - QString users; - bool first = true; - for (const auto &reaction : reactions[i].reactions) { - if (!first) - users += ", "; - else - first = false; - users += QString::fromStdString( - cache::displayName(room_id_, reaction.second.sender)); - } - return users; - } - case SelfReactedEvent: - for (const auto &reaction : reactions[i].reactions) - if (reaction.second.sender == http::client()->user_id().to_string()) - return QString::fromStdString(reaction.second.event_id); - return QStringLiteral(""); - default: - return {}; - } -} - -void -ReactionsModel::addReaction(const std::string &room_id, - const mtx::events::RoomEvent &reaction) -{ - room_id_ = room_id; - - int idx = 0; - for (auto &storedReactions : reactions) { - if (storedReactions.key == reaction.content.relates_to.key) { - storedReactions.reactions[reaction.event_id] = reaction; - emit dataChanged(index(idx, 0), index(idx, 0)); - return; - } - idx++; - } - - beginInsertRows(QModelIndex(), idx, idx); - reactions.push_back( - KeyReaction{reaction.content.relates_to.key, {{reaction.event_id, reaction}}}); - endInsertRows(); -} - -void -ReactionsModel::removeReaction(const mtx::events::RoomEvent &reaction) -{ - int idx = 0; - for (auto &storedReactions : reactions) { - if (storedReactions.key == reaction.content.relates_to.key) { - storedReactions.reactions.erase(reaction.event_id); - - if (storedReactions.reactions.size() == 0) { - beginRemoveRows(QModelIndex(), idx, idx); - reactions.erase(reactions.begin() + idx); - endRemoveRows(); - } else - emit dataChanged(index(idx, 0), index(idx, 0)); - return; - } - idx++; - } -} diff --git a/src/timeline/ReactionsModel.h b/src/timeline/ReactionsModel.h deleted file mode 100644 index c839afc8..00000000 --- a/src/timeline/ReactionsModel.h +++ /dev/null @@ -1,41 +0,0 @@ -#pragma once - -#include -#include - -#include -#include - -#include - -class ReactionsModel : public QAbstractListModel -{ - Q_OBJECT -public: - explicit ReactionsModel(QObject *parent = nullptr) { Q_UNUSED(parent); } - enum Roles - { - Key, - Count, - Users, - SelfReactedEvent, - }; - - QHash roleNames() const override; - int rowCount(const QModelIndex &parent = QModelIndex()) const override; - QVariant data(const QModelIndex &index, int role = Qt::DisplayRole) const override; - -public slots: - void addReaction(const std::string &room_id, - const mtx::events::RoomEvent &reaction); - void removeReaction(const mtx::events::RoomEvent &reaction); - -private: - struct KeyReaction - { - std::string key; - std::map> reactions; - }; - std::string room_id_; - std::vector reactions; -}; diff --git a/src/timeline/TimelineModel.cpp b/src/timeline/TimelineModel.cpp index 470e3988..85d2eb4e 100644 --- a/src/timeline/TimelineModel.cpp +++ b/src/timeline/TimelineModel.cpp @@ -366,7 +366,8 @@ TimelineModel::data(const mtx::events::collections::TimelineEvents &event, int r case ReplyTo: return QVariant(QString::fromStdString(in_reply_to_event(event))); case Reactions: { - return {}; + auto id = event_id(event); + return QVariant::fromValue(events.reactions(id)); } case RoomId: return QVariant(room_id_); diff --git a/src/timeline/TimelineModel.h b/src/timeline/TimelineModel.h index 9f9717df..cbe88fd2 100644 --- a/src/timeline/TimelineModel.h +++ b/src/timeline/TimelineModel.h @@ -10,7 +10,6 @@ #include "CacheCryptoStructs.h" #include "EventStore.h" -#include "ReactionsModel.h" namespace mtx::http { using RequestErr = const std::optional &; From 19f27236ea82b1927c83e4e24c71b30061674ee7 Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Mon, 20 Jul 2020 00:42:48 +0200 Subject: [PATCH 20/50] Fix reactions --- resources/qml/Reactions.qml | 2 +- resources/qml/TimelineRow.qml | 1 - resources/qml/TimelineView.qml | 7 +++-- resources/qml/emoji/EmojiButton.qml | 3 +- resources/qml/emoji/EmojiPicker.qml | 24 +++++++-------- src/EventAccessors.cpp | 19 ++++++++++++ src/EventAccessors.h | 2 ++ src/timeline/EventStore.cpp | 14 +++++++++ src/timeline/TimelineModel.cpp | 3 +- src/timeline/TimelineModel.h | 9 ++++++ src/timeline/TimelineViewManager.cpp | 45 +++++++++++++++------------- src/timeline/TimelineViewManager.h | 8 +---- 12 files changed, 89 insertions(+), 48 deletions(-) diff --git a/resources/qml/Reactions.qml b/resources/qml/Reactions.qml index 5b3bbc20..c1091756 100644 --- a/resources/qml/Reactions.qml +++ b/resources/qml/Reactions.qml @@ -34,7 +34,7 @@ Flow { onClicked: { console.debug("Picked " + modelData.key + "in response to " + reactionFlow.eventId + " in room " + reactionFlow.roomId + ". selfReactedEvent: " + modelData.selfReactedEvent) - timelineManager.reactToMessage(reactionFlow.roomId, reactionFlow.eventId, modelData.key, modelData.selfReactedEvent) + timelineManager.queueReactionMessage(reactionFlow.eventId, modelData.key) } diff --git a/resources/qml/TimelineRow.qml b/resources/qml/TimelineRow.qml index e87590f1..8186db8a 100644 --- a/resources/qml/TimelineRow.qml +++ b/resources/qml/TimelineRow.qml @@ -90,7 +90,6 @@ MouseArea { ToolTip.visible: hovered ToolTip.text: qsTr("React") emojiPicker: emojiPopup - room_id: model.roomId event_id: model.id } ImageButton { diff --git a/resources/qml/TimelineView.qml b/resources/qml/TimelineView.qml index fd185bd9..1d7b4a4a 100644 --- a/resources/qml/TimelineView.qml +++ b/resources/qml/TimelineView.qml @@ -40,19 +40,20 @@ Page { id: messageContextMenu modal: true - function show(eventId_, eventType_, isEncrypted_, showAt) { + function show(eventId_, eventType_, isEncrypted_, showAt_) { eventId = eventId_ eventType = eventType_ isEncrypted = isEncrypted_ - popup(showAt) + popup(showAt_) } property string eventId property int eventType property bool isEncrypted + MenuItem { text: qsTr("React") - onClicked: chat.model.reactAction(messageContextMenu.eventId) + onClicked: emojiPopup.show(messageContextMenu.parent, messageContextMenu.eventId) } MenuItem { text: qsTr("Reply") diff --git a/resources/qml/emoji/EmojiButton.qml b/resources/qml/emoji/EmojiButton.qml index f8f75e3e..c5eee4e4 100644 --- a/resources/qml/emoji/EmojiButton.qml +++ b/resources/qml/emoji/EmojiButton.qml @@ -8,11 +8,10 @@ import "../" ImageButton { property var colors: currentActivePalette property var emojiPicker - property string room_id property string event_id image: ":/icons/icons/ui/smile.png" id: emojiButton - onClicked: emojiPicker.visible ? emojiPicker.close() : emojiPicker.show(emojiButton, room_id, event_id) + onClicked: emojiPicker.visible ? emojiPicker.close() : emojiPicker.show(emojiButton, event_id) } diff --git a/resources/qml/emoji/EmojiPicker.qml b/resources/qml/emoji/EmojiPicker.qml index ac67af2a..f75221d5 100644 --- a/resources/qml/emoji/EmojiPicker.qml +++ b/resources/qml/emoji/EmojiPicker.qml @@ -10,17 +10,17 @@ import "../" Popup { - function show(showAt, room_id, event_id) { - console.debug("Showing emojiPicker for " + event_id + "in room " + room_id) - parent = showAt - x = Math.round((showAt.width - width) / 2) - y = showAt.height - emojiPopup.room_id = room_id - emojiPopup.event_id = event_id - open() - } + function show(showAt, event_id) { + console.debug("Showing emojiPicker for " + event_id) + if (showAt){ + parent = showAt + x = Math.round((showAt.width - width) / 2) + y = showAt.height + } + emojiPopup.event_id = event_id + open() + } - property string room_id property string event_id property var colors property alias model: gridView.model @@ -102,9 +102,9 @@ Popup { } // TODO: maybe add favorites at some point? onClicked: { - console.debug("Picked " + model.unicode + "in response to " + emojiPopup.event_id + " in room " + emojiPopup.room_id) + console.debug("Picked " + model.unicode + "in response to " + emojiPopup.event_id) emojiPopup.close() - timelineManager.queueReactionMessage(emojiPopup.room_id, emojiPopup.event_id, model.unicode) + timelineManager.queueReactionMessage(emojiPopup.event_id, model.unicode) } } diff --git a/src/EventAccessors.cpp b/src/EventAccessors.cpp index 7071819b..0618206c 100644 --- a/src/EventAccessors.cpp +++ b/src/EventAccessors.cpp @@ -223,6 +223,20 @@ struct EventInReplyTo } }; +struct EventRelatesTo +{ + template + using related_ev_id_t = decltype(Content::relates_to.event_id); + template + std::string operator()(const mtx::events::Event &e) + { + if constexpr (is_detected::value) { + return e.content.relates_to.event_id; + } + return ""; + } +}; + struct EventTransactionId { template @@ -378,6 +392,11 @@ mtx::accessors::in_reply_to_event(const mtx::events::collections::TimelineEvents { return std::visit(EventInReplyTo{}, event); } +std::string +mtx::accessors::relates_to_event_id(const mtx::events::collections::TimelineEvents &event) +{ + return std::visit(EventRelatesTo{}, event); +} std::string mtx::accessors::transaction_id(const mtx::events::collections::TimelineEvents &event) diff --git a/src/EventAccessors.h b/src/EventAccessors.h index a7577d86..8f08ef1c 100644 --- a/src/EventAccessors.h +++ b/src/EventAccessors.h @@ -53,6 +53,8 @@ mimetype(const mtx::events::collections::TimelineEvents &event); std::string in_reply_to_event(const mtx::events::collections::TimelineEvents &event); std::string +relates_to_event_id(const mtx::events::collections::TimelineEvents &event); +std::string transaction_id(const mtx::events::collections::TimelineEvents &event); int64_t diff --git a/src/timeline/EventStore.cpp b/src/timeline/EventStore.cpp index 0bd7a97e..eb1162cc 100644 --- a/src/timeline/EventStore.cpp +++ b/src/timeline/EventStore.cpp @@ -202,6 +202,20 @@ EventStore::handleSync(const mtx::responses::Timeline &events) if (auto redaction = std::get_if>( &event)) { + // fixup reactions + auto redacted = events_by_id_.object({room_id_, redaction->redacts}); + if (redacted) { + auto id = mtx::accessors::relates_to_event_id(*redacted); + if (!id.empty()) { + auto idx = idToIndex(id); + if (idx) { + events_by_id_.remove( + {room_id_, redaction->redacts}); + emit dataChanged(*idx, *idx); + } + } + } + relates_to = redaction->redacts; } else if (auto reaction = std::get_if>( diff --git a/src/timeline/TimelineModel.cpp b/src/timeline/TimelineModel.cpp index 85d2eb4e..8631eb83 100644 --- a/src/timeline/TimelineModel.cpp +++ b/src/timeline/TimelineModel.cpp @@ -1069,8 +1069,9 @@ struct SendMessageVisitor // reactions need to have the relation outside of ciphertext, or synapse / the homeserver // cannot handle it correctly. See the MSC for more details: // https://github.com/matrix-org/matrix-doc/blob/matthew/msc1849/proposals/1849-aggregations.md#end-to-end-encryption - void operator()(const mtx::events::RoomEvent &msg) + void operator()(mtx::events::RoomEvent msg) { + msg.type = mtx::events::EventType::Reaction; emit model_->addPendingMessageToStore(msg); } diff --git a/src/timeline/TimelineModel.h b/src/timeline/TimelineModel.h index cbe88fd2..f8a84f17 100644 --- a/src/timeline/TimelineModel.h +++ b/src/timeline/TimelineModel.h @@ -197,6 +197,15 @@ public: Q_INVOKABLE void cacheMedia(QString eventId); Q_INVOKABLE bool saveMedia(QString eventId) const; + std::vector<::Reaction> reactions(const std::string &event_id) + { + auto list = events.reactions(event_id); + std::vector<::Reaction> vec; + for (const auto &r : list) + vec.push_back(r.value()); + return vec; + } + void updateLastMessage(); void addEvents(const mtx::responses::Timeline &events); template diff --git a/src/timeline/TimelineViewManager.cpp b/src/timeline/TimelineViewManager.cpp index 64af8afb..8cb72edd 100644 --- a/src/timeline/TimelineViewManager.cpp +++ b/src/timeline/TimelineViewManager.cpp @@ -314,35 +314,38 @@ TimelineViewManager::queueEmoteMessage(const QString &msg) } void -TimelineViewManager::reactToMessage(const QString &roomId, - const QString &reactedEvent, - const QString &reactionKey, - const QString &selfReactedEvent) +TimelineViewManager::queueReactionMessage(const QString &reactedEvent, const QString &reactionKey) { + if (!timeline_) + return; + + auto reactions = timeline_->reactions(reactedEvent.toStdString()); + + QString selfReactedEvent; + for (const auto &reaction : reactions) { + if (reactionKey == reaction.key_) { + selfReactedEvent = reaction.selfReactedEvent_; + break; + } + } + + if (selfReactedEvent.startsWith("m")) + return; + // If selfReactedEvent is empty, that means we haven't previously reacted if (selfReactedEvent.isEmpty()) { - queueReactionMessage(roomId, reactedEvent, reactionKey); + mtx::events::msg::Reaction reaction; + reaction.relates_to.rel_type = mtx::common::RelationType::Annotation; + reaction.relates_to.event_id = reactedEvent.toStdString(); + reaction.relates_to.key = reactionKey.toStdString(); + + timeline_->sendMessage(reaction); // Otherwise, we have previously reacted and the reaction should be redacted } else { - auto model = models.value(roomId); - model->redactEvent(selfReactedEvent); + timeline_->redactEvent(selfReactedEvent); } } -void -TimelineViewManager::queueReactionMessage(const QString &roomId, - const QString &reactedEvent, - const QString &reactionKey) -{ - mtx::events::msg::Reaction reaction; - reaction.relates_to.rel_type = mtx::common::RelationType::Annotation; - reaction.relates_to.event_id = reactedEvent.toStdString(); - reaction.relates_to.key = reactionKey.toStdString(); - - auto model = models.value(roomId); - model->sendMessage(reaction); -} - void TimelineViewManager::queueImageMessage(const QString &roomid, const QString &filename, diff --git a/src/timeline/TimelineViewManager.h b/src/timeline/TimelineViewManager.h index ed095058..63106916 100644 --- a/src/timeline/TimelineViewManager.h +++ b/src/timeline/TimelineViewManager.h @@ -61,13 +61,7 @@ public slots: void setHistoryView(const QString &room_id); void updateColorPalette(); - void queueReactionMessage(const QString &roomId, - const QString &reactedEvent, - const QString &reaction); - void reactToMessage(const QString &roomId, - const QString &reactedEvent, - const QString &reactionKey, - const QString &selfReactedEvent); + void queueReactionMessage(const QString &reactedEvent, const QString &reactionKey); void queueTextMessage(const QString &msg); void queueEmoteMessage(const QString &msg); void queueImageMessage(const QString &roomid, From f23d733cffc9cfbc806631fe670d3ca28c40417a Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Mon, 20 Jul 2020 18:25:22 +0200 Subject: [PATCH 21/50] Fix room joins --- src/Cache.cpp | 12 ++++++++++-- src/ChatPage.cpp | 9 ++------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/Cache.cpp b/src/Cache.cpp index 0307bee1..3aec445a 100644 --- a/src/Cache.cpp +++ b/src/Cache.cpp @@ -1439,8 +1439,16 @@ Cache::getLastEventId(lmdb::txn &txn, const std::string &room_id) std::optional Cache::getTimelineRange(const std::string &room_id) { - auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); - auto orderDb = getOrderToMessageDb(txn, room_id); + auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + lmdb::dbi orderDb{0}; + try { + orderDb = getOrderToMessageDb(txn, room_id); + } catch (lmdb::runtime_error &e) { + nhlog::db()->error("Can't open db for room '{}', probably doesn't exist yet. ({})", + room_id, + e.what()); + return {}; + } lmdb::val indexVal, val; diff --git a/src/ChatPage.cpp b/src/ChatPage.cpp index c4376905..012f1e69 100644 --- a/src/ChatPage.cpp +++ b/src/ChatPage.cpp @@ -244,7 +244,6 @@ ChatPage::ChatPage(QSharedPointer userSettings, QWidget *parent) room_list_, &RoomList::roomChanged, view_manager_, &TimelineViewManager::setHistoryView); connect(room_list_, &RoomList::acceptInvite, this, [this](const QString &room_id) { - view_manager_->addRoom(room_id); joinRoom(room_id); room_list_->removeRoom(room_id, currentRoom() == room_id); }); @@ -543,12 +542,8 @@ ChatPage::ChatPage(QSharedPointer userSettings, QWidget *parent) emit notificationsRetrieved(std::move(res)); }); }); - connect(this, &ChatPage::syncRoomlist, room_list_, &RoomList::sync, Qt::QueuedConnection); - connect(this, - &ChatPage::syncTags, - communitiesList_, - &CommunitiesList::syncTags, - Qt::QueuedConnection); + connect(this, &ChatPage::syncRoomlist, room_list_, &RoomList::sync); + connect(this, &ChatPage::syncTags, communitiesList_, &CommunitiesList::syncTags); connect( this, &ChatPage::syncTopBar, this, [this](const std::map &updates) { if (updates.find(currentRoom()) != updates.end()) From 147ae68c31e82828b8cf4aa870de3b2a4903e4ee Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Thu, 23 Jul 2020 14:33:04 +0200 Subject: [PATCH 22/50] Don't send url, if we send an encrypted file Fix issues when sending images to some clients. --- src/timeline/TimelineViewManager.cpp | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/src/timeline/TimelineViewManager.cpp b/src/timeline/TimelineViewManager.cpp index 8cb72edd..975dd5fb 100644 --- a/src/timeline/TimelineViewManager.cpp +++ b/src/timeline/TimelineViewManager.cpp @@ -361,10 +361,13 @@ TimelineViewManager::queueImageMessage(const QString &roomid, image.info.size = dsize; image.info.blurhash = blurhash.toStdString(); image.body = filename.toStdString(); - image.url = url.toStdString(); image.info.h = dimensions.height(); image.info.w = dimensions.width(); - image.file = file; + + if (file) + image.file = file; + else + image.url = url.toStdString(); auto model = models.value(roomid); if (!model->reply().isEmpty()) { @@ -388,8 +391,11 @@ TimelineViewManager::queueFileMessage( file.info.mimetype = mime.toStdString(); file.info.size = dsize; file.body = filename.toStdString(); - file.url = url.toStdString(); - file.file = encryptedFile; + + if (encryptedFile) + file.file = encryptedFile; + else + file.url = url.toStdString(); auto model = models.value(roomid); if (!model->reply().isEmpty()) { @@ -413,7 +419,11 @@ TimelineViewManager::queueAudioMessage(const QString &roomid, audio.info.size = dsize; audio.body = filename.toStdString(); audio.url = url.toStdString(); - audio.file = file; + + if (file) + audio.file = file; + else + audio.url = url.toStdString(); auto model = models.value(roomid); if (!model->reply().isEmpty()) { @@ -436,8 +446,11 @@ TimelineViewManager::queueVideoMessage(const QString &roomid, video.info.mimetype = mime.toStdString(); video.info.size = dsize; video.body = filename.toStdString(); - video.url = url.toStdString(); - video.file = file; + + if (file) + video.file = file; + else + video.url = url.toStdString(); auto model = models.value(roomid); if (!model->reply().isEmpty()) { From b294430fe5814d56dcfb32b9db22a8d5f52bfdce Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Sat, 25 Jul 2020 14:08:13 +0200 Subject: [PATCH 23/50] Return to redacted messages instead of just storing the redaction --- src/Cache.cpp | 58 ++++++++++++++++++++++++------------- src/timeline/EventStore.cpp | 7 ++++- 2 files changed, 44 insertions(+), 21 deletions(-) diff --git a/src/Cache.cpp b/src/Cache.cpp index 3aec445a..59755e1e 100644 --- a/src/Cache.cpp +++ b/src/Cache.cpp @@ -2242,11 +2242,17 @@ Cache::saveTimelineMessages(lmdb::txn &txn, auto event = mtx::accessors::serialize_event(e); auto txn_id = mtx::accessors::transaction_id(e); + std::string event_id_val = event.value("event_id", ""); + if (event_id_val.empty()) { + nhlog::db()->error("Event without id!"); + continue; + } + + lmdb::val event_id = event_id_val; + lmdb::val txn_order; if (!txn_id.empty() && lmdb::dbi_get(txn, evToOrderDb, lmdb::val(txn_id), txn_order)) { - std::string event_id_val = event["event_id"].get(); - lmdb::val event_id = event_id_val; lmdb::dbi_put(txn, eventsDb, event_id, lmdb::val(event.dump())); lmdb::dbi_del(txn, eventsDb, lmdb::val(txn_id)); @@ -2291,28 +2297,40 @@ Cache::saveTimelineMessages(lmdb::txn &txn, if (redaction->redacts.empty()) continue; - lmdb::val ev{}; - lmdb::dbi_put( - txn, eventsDb, lmdb::val(redaction->redacts), lmdb::val(event.dump())); - lmdb::dbi_put( - txn, eventsDb, lmdb::val(redaction->event_id), lmdb::val(event.dump())); + lmdb::val oldEvent; + bool success = + lmdb::dbi_get(txn, eventsDb, lmdb::val(redaction->redacts), oldEvent); + if (!success) + continue; - lmdb::val oldIndex{}; - if (lmdb::dbi_get( - txn, msg2orderDb, lmdb::val(redaction->redacts), oldIndex)) { - lmdb::dbi_put( - txn, order2msgDb, oldIndex, lmdb::val(redaction->event_id)); - lmdb::dbi_put( - txn, msg2orderDb, lmdb::val(redaction->event_id), oldIndex); - } - } else { - std::string event_id_val = event.value("event_id", ""); - if (event_id_val.empty()) { - nhlog::db()->error("Event without id!"); + mtx::events::collections::TimelineEvent te; + try { + mtx::events::collections::from_json( + json::parse(std::string_view(oldEvent.data(), oldEvent.size())), + te); + // overwrite the content and add redation data + std::visit( + [redaction](auto &ev) { + ev.unsigned_data.redacted_because = *redaction; + ev.unsigned_data.redacted_by = redaction->event_id; + }, + te.data); + event = mtx::accessors::serialize_event(te.data); + event["content"].clear(); + + } catch (std::exception &e) { + nhlog::db()->error("Failed to parse message from cache {}", + e.what()); continue; } - lmdb::val event_id = event_id_val; + lmdb::dbi_put( + txn, eventsDb, lmdb::val(redaction->redacts), lmdb::val(event.dump())); + lmdb::dbi_put(txn, + eventsDb, + lmdb::val(redaction->event_id), + lmdb::val(json(*redaction).dump())); + } else { lmdb::dbi_put(txn, eventsDb, event_id, lmdb::val(event.dump())); ++index; diff --git a/src/timeline/EventStore.cpp b/src/timeline/EventStore.cpp index eb1162cc..704402c8 100644 --- a/src/timeline/EventStore.cpp +++ b/src/timeline/EventStore.cpp @@ -211,6 +211,7 @@ EventStore::handleSync(const mtx::responses::Timeline &events) if (idx) { events_by_id_.remove( {room_id_, redaction->redacts}); + events_.remove({room_id_, toInternalIdx(*idx)}); emit dataChanged(*idx, *idx); } } @@ -227,8 +228,12 @@ EventStore::handleSync(const mtx::responses::Timeline &events) if (!relates_to.empty()) { auto idx = cache::client()->getTimelineIndex(room_id_, relates_to); - if (idx) + if (idx) { + events_by_id_.remove({room_id_, relates_to}); + decryptedEvents_.remove({room_id_, relates_to}); + events_.remove({room_id_, *idx}); emit dataChanged(toExternalIdx(*idx), toExternalIdx(*idx)); + } } if (auto txn_id = mtx::accessors::transaction_id(event); !txn_id.empty()) { From 4e7bd20e0cd38c014824626d2f106c00ec12c31a Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Sat, 25 Jul 2020 19:38:56 +0200 Subject: [PATCH 24/50] Reset fetch in progress when fetch failed --- src/timeline/EventStore.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/timeline/EventStore.cpp b/src/timeline/EventStore.cpp index 704402c8..4f8ffb80 100644 --- a/src/timeline/EventStore.cpp +++ b/src/timeline/EventStore.cpp @@ -542,6 +542,7 @@ EventStore::fetchMore() mtx::errors::to_string(err->matrix_error.errcode), err->matrix_error.error, err->parse_error); + emit fetchedMore(); return; } From fdcf91f5eb550dd5b40e37a8a61099fcfe249060 Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Sat, 25 Jul 2020 22:08:02 +0200 Subject: [PATCH 25/50] Fix binding loop and non integer text height --- resources/qml/MatrixText.qml | 3 +++ resources/qml/TimelineView.qml | 6 +++--- resources/qml/delegates/TextMessage.qml | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/resources/qml/MatrixText.qml b/resources/qml/MatrixText.qml index 9a4f7348..c83069ec 100644 --- a/resources/qml/MatrixText.qml +++ b/resources/qml/MatrixText.qml @@ -8,6 +8,9 @@ TextEdit { selectByMouse: true color: colors.text + font.hintingPreference: Font.PreferFullHinting + renderType: Text.NativeRendering + onLinkActivated: { if (/^https:\/\/matrix.to\/#\/(@.*)$/.test(link)) chat.model.openUserProfile(/^https:\/\/matrix.to\/#\/(@.*)$/.exec(link)[1]) else if (/^https:\/\/matrix.to\/#\/(![^\/]*)$/.test(link)) timelineManager.setHistoryView(/^https:\/\/matrix.to\/#\/(!.*)$/.exec(link)[1]) diff --git a/resources/qml/TimelineView.qml b/resources/qml/TimelineView.qml index 1d7b4a4a..da783a7b 100644 --- a/resources/qml/TimelineView.qml +++ b/resources/qml/TimelineView.qml @@ -11,6 +11,8 @@ import "./delegates" import "./emoji" Page { + id: timelineRoot + property var colors: currentActivePalette property var systemInactive: SystemPalette { colorGroup: SystemPalette.Disabled } property var inactiveColors: currentInactivePalette ? currentInactivePalette : systemInactive @@ -88,8 +90,6 @@ Page { } } - id: timelineRoot - Rectangle { anchors.fill: parent color: colors.window @@ -114,7 +114,7 @@ Page { ListView { id: chat - visible: timelineManager.timeline != null + visible: !!timelineManager.timeline cacheBuffer: 400 diff --git a/resources/qml/delegates/TextMessage.qml b/resources/qml/delegates/TextMessage.qml index b3c45c36..cc2d2da0 100644 --- a/resources/qml/delegates/TextMessage.qml +++ b/resources/qml/delegates/TextMessage.qml @@ -4,7 +4,7 @@ MatrixText { property string formatted: model.data.formattedBody text: "" + formatted.replace("
", "
")
 	width: parent ? parent.width : undefined
-	height: isReply ? Math.min(chat.height / 8, implicitHeight) : undefined
+	height: isReply ? Math.round(Math.min(timelineRoot.height / 8, implicitHeight)) : undefined
 	clip: true
 	font.pointSize: (settings.enlargeEmojiOnlyMessages && model.data.isOnlyEmoji > 0 && model.data.isOnlyEmoji < 4) ? settings.fontSize * 3 : settings.fontSize
 }

From cbb4356b192d8401153d207ad14dbdbdaac2d6da Mon Sep 17 00:00:00 2001
From: Nicolas Werner 
Date: Sat, 25 Jul 2020 22:10:30 +0200
Subject: [PATCH 26/50] Fix more non integer heights

---
 resources/qml/delegates/ImageMessage.qml         | 4 ++--
 resources/qml/delegates/PlayableMediaMessage.qml | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/resources/qml/delegates/ImageMessage.qml b/resources/qml/delegates/ImageMessage.qml
index 62d9de60..3885ddae 100644
--- a/resources/qml/delegates/ImageMessage.qml
+++ b/resources/qml/delegates/ImageMessage.qml
@@ -9,8 +9,8 @@ Item {
 	property double divisor: model.isReply ? 4 : 2
 	property bool tooHigh: tempHeight > timelineRoot.height / divisor
 
-	height: tooHigh ? timelineRoot.height / divisor : tempHeight
-	width: tooHigh ? (timelineRoot.height / divisor) / model.data.proportionalHeight : tempWidth
+	height: Math.round(tooHigh ? timelineRoot.height / divisor : tempHeight)
+	width: Math.round(tooHigh ? (timelineRoot.height / divisor) / model.data.proportionalHeight : tempWidth)
 
 	Image {
 		id: blurhash
diff --git a/resources/qml/delegates/PlayableMediaMessage.qml b/resources/qml/delegates/PlayableMediaMessage.qml
index bab524eb..8d2fa8a8 100644
--- a/resources/qml/delegates/PlayableMediaMessage.qml
+++ b/resources/qml/delegates/PlayableMediaMessage.qml
@@ -9,7 +9,7 @@ Rectangle {
 	id: bg
 	radius: 10
 	color: colors.dark
-	height: content.height + 24
+	height: Math.round(content.height + 24)
 	width: parent ? parent.width : undefined
 
 	Column { 

From 6f557c19a1785d698f42ec2e4a68b15f6ceac7aa Mon Sep 17 00:00:00 2001
From: Nicolas Werner 
Date: Sat, 25 Jul 2020 22:57:54 +0200
Subject: [PATCH 27/50] Optimize scrolling a little bit

---
 resources/qml/MatrixText.qml   |  3 ---
 resources/qml/ScrollHelper.qml |  2 +-
 resources/qml/TimelineView.qml | 49 +++++++++++++++++-----------------
 3 files changed, 25 insertions(+), 29 deletions(-)

diff --git a/resources/qml/MatrixText.qml b/resources/qml/MatrixText.qml
index c83069ec..9a4f7348 100644
--- a/resources/qml/MatrixText.qml
+++ b/resources/qml/MatrixText.qml
@@ -8,9 +8,6 @@ TextEdit {
 	selectByMouse: true
 	color: colors.text
 
-	font.hintingPreference: Font.PreferFullHinting
-	renderType: Text.NativeRendering
-
 	onLinkActivated: {
 		if (/^https:\/\/matrix.to\/#\/(@.*)$/.test(link)) chat.model.openUserProfile(/^https:\/\/matrix.to\/#\/(@.*)$/.exec(link)[1])
 		else if (/^https:\/\/matrix.to\/#\/(![^\/]*)$/.test(link)) timelineManager.setHistoryView(/^https:\/\/matrix.to\/#\/(!.*)$/.exec(link)[1])
diff --git a/resources/qml/ScrollHelper.qml b/resources/qml/ScrollHelper.qml
index e72fcbd2..ba7c2648 100644
--- a/resources/qml/ScrollHelper.qml
+++ b/resources/qml/ScrollHelper.qml
@@ -106,6 +106,6 @@ MouseArea {
         //How long the scrollbar will remain visible
         interval: 500
         // Hide the scrollbars
-        onTriggered: flickable.cancelFlick();
+        onTriggered: { flickable.cancelFlick(); flickable.movementEnded(); }
     }
 }
diff --git a/resources/qml/TimelineView.qml b/resources/qml/TimelineView.qml
index da783a7b..7bdeb01f 100644
--- a/resources/qml/TimelineView.qml
+++ b/resources/qml/TimelineView.qml
@@ -27,16 +27,16 @@ Page {
 		id: fontMetrics
 	}
 
-    EmojiPicker {
-        id: emojiPopup
-        width: 7 * 52 + 20
-        height: 6 * 52 
-        colors: palette
-        model: EmojiProxyModel {
-            category: EmojiCategory.People
-            sourceModel: EmojiModel {}
-        }
-    }
+	EmojiPicker {
+		id: emojiPopup
+		width: 7 * 52 + 20
+		height: 6 * 52 
+		colors: palette
+		model: EmojiProxyModel {
+			category: EmojiCategory.People
+			sourceModel: EmojiModel {}
+		}
+	}
 
 	Menu {
 		id: messageContextMenu
@@ -114,7 +114,7 @@ Page {
 		ListView {
 			id: chat
 
-            visible: !!timelineManager.timeline
+			visible: !!timelineManager.timeline
 
 			cacheBuffer: 400
 
@@ -206,14 +206,13 @@ Page {
 					}
 				}
 
-				Binding {
-					target: chat.model
-					property: "currentIndex"
-					when: y + height + 2 * chat.spacing > chat.contentY + chat.height && y < chat.contentY + chat.height
-					value: index
-					delayed: true
+				Connections {
+					target: chat
+					function onMovementEnded() {
+						if (y + height + 2 * chat.spacing > chat.contentY + chat.height && y < chat.contentY + chat.height)
+							chat.model.currentIndex = index;
+					}
 				}
-
 			}
 
 			section {
@@ -296,13 +295,13 @@ Page {
 				}
 			}
 
-            footer:  BusyIndicator {
-                anchors.horizontalCenter: parent.horizontalCenter
-                running: chat.model && chat.model.paginationInProgress
-                height: 50
-                width: 50
-                z: 3
-            }
+			footer:  BusyIndicator {
+				anchors.horizontalCenter: parent.horizontalCenter
+				running: chat.model && chat.model.paginationInProgress
+				height: 50
+				width: 50
+				z: 3
+			}
 		}
 
 		Rectangle {

From 8bf26917ad0f9f8d74128fbb8d9ad2c8fd495068 Mon Sep 17 00:00:00 2001
From: Nicolas Werner 
Date: Sun, 26 Jul 2020 02:06:38 +0200
Subject: [PATCH 28/50] Make long press menu actually work

---
 resources/qml/MatrixText.qml   |  3 ++-
 resources/qml/TimelineRow.qml  | 27 +++++++++++++++------------
 resources/qml/TimelineView.qml |  6 +++++-
 3 files changed, 22 insertions(+), 14 deletions(-)

diff --git a/resources/qml/MatrixText.qml b/resources/qml/MatrixText.qml
index 9a4f7348..cbb55bea 100644
--- a/resources/qml/MatrixText.qml
+++ b/resources/qml/MatrixText.qml
@@ -5,7 +5,7 @@ TextEdit {
 	textFormat: TextEdit.RichText
 	readOnly: true
 	wrapMode: Text.Wrap
-	selectByMouse: true
+	selectByMouse: ma.containsMouse // try to make scrollable by finger but selectable by mouse
 	color: colors.text
 
 	onLinkActivated: {
@@ -23,6 +23,7 @@ TextEdit {
 		id: ma
 		anchors.fill: parent
 		propagateComposedEvents: true
+		hoverEnabled: true
 		acceptedButtons: Qt.NoButton
 		cursorShape: parent.hoveredLink ? Qt.PointingHandCursor : Qt.ArrowCursor
 	}
diff --git a/resources/qml/TimelineRow.qml b/resources/qml/TimelineRow.qml
index 8186db8a..d1c20278 100644
--- a/resources/qml/TimelineRow.qml
+++ b/resources/qml/TimelineRow.qml
@@ -8,22 +8,25 @@ import im.nheko 1.0
 import "./delegates"
 import "./emoji"
 
-MouseArea {
+Item {
 	anchors.left: parent.left
 	anchors.right: parent.right
 	height: row.height
-	propagateComposedEvents: true
-	preventStealing: true
-	hoverEnabled: true
 
-	acceptedButtons: Qt.LeftButton | Qt.RightButton
-	onClicked: {
-		if (mouse.button === Qt.RightButton)
-		messageContextMenu.show(model.id, model.type, model.isEncrypted, row)
-	}
-	onPressAndHold: {
-		if (mouse.source === Qt.MouseEventNotSynthesized)
-		messageContextMenu.show(model.id, model.type, model.isEncrypted, row)
+	MouseArea {
+		anchors.fill: parent
+		propagateComposedEvents: true
+		preventStealing: true
+		hoverEnabled: true
+
+		acceptedButtons: Qt.AllButtons
+		onClicked: {
+			if (mouse.button === Qt.RightButton)
+			messageContextMenu.show(model.id, model.type, model.isEncrypted, row)
+		}
+		onPressAndHold: {
+			messageContextMenu.show(model.id, model.type, model.isEncrypted, row, mapToItem(timelineRoot, mouse.x, mouse.y))
+		}
 	}
 	Rectangle {
 		color: (settings.messageHoverHighlight && parent.containsMouse) ? colors.base : "transparent"
diff --git a/resources/qml/TimelineView.qml b/resources/qml/TimelineView.qml
index 7bdeb01f..8a5612d2 100644
--- a/resources/qml/TimelineView.qml
+++ b/resources/qml/TimelineView.qml
@@ -42,10 +42,14 @@ Page {
 		id: messageContextMenu
 		modal: true
 
-		function show(eventId_, eventType_, isEncrypted_, showAt_) {
+		function show(eventId_, eventType_, isEncrypted_, showAt_, position) {
 			eventId = eventId_
 			eventType = eventType_
 			isEncrypted = isEncrypted_
+
+			if (position)
+			popup(position, showAt_)
+			else
 			popup(showAt_)
 		}
 

From 28e7ea40cbbc52879dd0f3a1c648b416905e002f Mon Sep 17 00:00:00 2001
From: Nicolas Werner 
Date: Sun, 26 Jul 2020 12:12:19 +0200
Subject: [PATCH 29/50] Bump mtxclient and use git dependency in flatpak

---
 CMakeLists.txt                   | 2 +-
 io.github.NhekoReborn.Nheko.json | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 658232e8..69261046 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -336,7 +336,7 @@ if(USE_BUNDLED_MTXCLIENT)
 	FetchContent_Declare(
 		MatrixClient
 		GIT_REPOSITORY https://github.com/Nheko-Reborn/mtxclient.git
-		GIT_TAG        eddd95a896fad0c51fc800741d82bbc43fc6d41e
+		GIT_TAG        fa6e36dbcd922c1920873b3fcdfe0a9d283f082e
 		)
 	FetchContent_MakeAvailable(MatrixClient)
 else()
diff --git a/io.github.NhekoReborn.Nheko.json b/io.github.NhekoReborn.Nheko.json
index 8e4dbbe6..8cdd7b90 100644
--- a/io.github.NhekoReborn.Nheko.json
+++ b/io.github.NhekoReborn.Nheko.json
@@ -146,9 +146,9 @@
       "name": "mtxclient",
       "sources": [
         {
-          "sha256": "6334bb71821a0fde54fe24f02ad393cdb6836633557ffdd239b29c5d5108daaf",
-          "type": "archive",
-          "url": "https://github.com/Nheko-Reborn/mtxclient/archive/eddd95a896fad0c51fc800741d82bbc43fc6d41e.tar.gz"
+          "commit": "fa6e36dbcd922c1920873b3fcdfe0a9d283f082e",
+          "type": "git",
+          "url": "https://github.com/Nheko-Reborn/mtxclient.git"
         }
       ]
     },

From a00b11def7728320782753bbda09c46582670ddc Mon Sep 17 00:00:00 2001
From: Nicolas Werner 
Date: Sun, 26 Jul 2020 12:33:30 +0200
Subject: [PATCH 30/50] Rename EventStore::event to get to remove ambiguity
 with QObject::event

---
 src/timeline/EventStore.cpp    |  6 +++---
 src/timeline/EventStore.h      |  8 ++++----
 src/timeline/TimelineModel.cpp | 34 +++++++++++++++++-----------------
 3 files changed, 24 insertions(+), 24 deletions(-)

diff --git a/src/timeline/EventStore.cpp b/src/timeline/EventStore.cpp
index 4f8ffb80..94538672 100644
--- a/src/timeline/EventStore.cpp
+++ b/src/timeline/EventStore.cpp
@@ -264,7 +264,7 @@ EventStore::reactions(const std::string &event_id)
 
         auto self = http::client()->user_id().to_string();
         for (const auto &id : event_ids) {
-                auto related_event = event(id, event_id);
+                auto related_event = get(id, event_id);
                 if (!related_event)
                         continue;
 
@@ -313,7 +313,7 @@ EventStore::reactions(const std::string &event_id)
 }
 
 mtx::events::collections::TimelineEvents *
-EventStore::event(int idx, bool decrypt)
+EventStore::get(int idx, bool decrypt)
 {
         if (this->thread() != QThread::currentThread())
                 nhlog::db()->warn("{} called from a different thread!", __func__);
@@ -479,7 +479,7 @@ EventStore::decryptEvent(const IdIndex &idx,
 }
 
 mtx::events::collections::TimelineEvents *
-EventStore::event(std::string_view id, std::string_view related_to, bool decrypt)
+EventStore::get(std::string_view id, std::string_view related_to, bool decrypt)
 {
         if (this->thread() != QThread::currentThread())
                 nhlog::db()->warn("{} called from a different thread!", __func__);
diff --git a/src/timeline/EventStore.h b/src/timeline/EventStore.h
index 5a792040..b5c17d10 100644
--- a/src/timeline/EventStore.h
+++ b/src/timeline/EventStore.h
@@ -62,11 +62,11 @@ public:
 
         // optionally returns the event or nullptr and fetches it, after which it emits a
         // relatedFetched event
-        mtx::events::collections::TimelineEvents *event(std::string_view id,
-                                                        std::string_view related_to,
-                                                        bool decrypt = true);
+        mtx::events::collections::TimelineEvents *get(std::string_view id,
+                                                      std::string_view related_to,
+                                                      bool decrypt = true);
         // always returns a proper event as long as the idx is valid
-        mtx::events::collections::TimelineEvents *event(int idx, bool decrypt = true);
+        mtx::events::collections::TimelineEvents *get(int idx, bool decrypt = true);
 
         QVariantList reactions(const std::string &event_id);
 
diff --git a/src/timeline/TimelineModel.cpp b/src/timeline/TimelineModel.cpp
index 8631eb83..f41e7712 100644
--- a/src/timeline/TimelineModel.cpp
+++ b/src/timeline/TimelineModel.cpp
@@ -237,7 +237,7 @@ TimelineModel::rowCount(const QModelIndex &parent) const
 QVariantMap
 TimelineModel::getDump(QString eventId, QString relatedTo) const
 {
-        if (auto event = events.event(eventId.toStdString(), relatedTo.toStdString()))
+        if (auto event = events.get(eventId.toStdString(), relatedTo.toStdString()))
                 return data(*event, Dump).toMap();
         return {};
 }
@@ -354,7 +354,7 @@ TimelineModel::data(const mtx::events::collections::TimelineEvents &event, int r
         }
         case IsEncrypted: {
                 auto id              = event_id(event);
-                auto encrypted_event = events.event(id, id, false);
+                auto encrypted_event = events.get(id, id, false);
                 return encrypted_event &&
                        std::holds_alternative<
                          mtx::events::EncryptedEvent>(
@@ -421,7 +421,7 @@ TimelineModel::data(const QModelIndex &index, int role) const
         if (index.row() < 0 && index.row() >= rowCount())
                 return QVariant();
 
-        auto event = events.event(rowCount() - index.row() - 1);
+        auto event = events.get(rowCount() - index.row() - 1);
 
         if (!event)
                 return "";
@@ -433,7 +433,7 @@ TimelineModel::data(const QModelIndex &index, int role) const
                 std::string userId = acc::sender(*event);
 
                 for (int r = rowCount() - index.row(); r < events.size(); r++) {
-                        auto tempEv = events.event(r);
+                        auto tempEv = events.get(r);
                         if (!tempEv)
                                 break;
 
@@ -460,7 +460,7 @@ TimelineModel::canFetchMore(const QModelIndex &) const
 {
         if (!events.size())
                 return true;
-        if (auto first = events.event(0);
+        if (auto first = events.get(0);
             first &&
             !std::holds_alternative>(*first))
                 return true;
@@ -545,7 +545,7 @@ void
 TimelineModel::updateLastMessage()
 {
         for (auto it = events.size() - 1; it >= 0; --it) {
-                auto event = events.event(it, decryptDescription);
+                auto event = events.get(it, decryptDescription);
                 if (!event)
                         continue;
 
@@ -633,7 +633,7 @@ TimelineModel::escapeEmoji(QString str) const
 void
 TimelineModel::viewRawMessage(QString id) const
 {
-        auto e = events.event(id.toStdString(), "", false);
+        auto e = events.get(id.toStdString(), "", false);
         if (!e)
                 return;
         std::string ev = mtx::accessors::serialize_event(*e).dump(4);
@@ -644,7 +644,7 @@ TimelineModel::viewRawMessage(QString id) const
 void
 TimelineModel::viewDecryptedRawMessage(QString id) const
 {
-        auto e = events.event(id.toStdString(), "");
+        auto e = events.get(id.toStdString(), "");
         if (!e)
                 return;
 
@@ -669,7 +669,7 @@ TimelineModel::replyAction(QString id)
 RelatedInfo
 TimelineModel::relatedInfo(QString id)
 {
-        auto event = events.event(id.toStdString(), "");
+        auto event = events.get(id.toStdString(), "");
         if (!event)
                 return {};
 
@@ -1096,7 +1096,7 @@ TimelineModel::addPendingMessage(mtx::events::collections::TimelineEvents event)
 bool
 TimelineModel::saveMedia(QString eventId) const
 {
-        mtx::events::collections::TimelineEvents *event = events.event(eventId.toStdString(), "");
+        mtx::events::collections::TimelineEvents *event = events.get(eventId.toStdString(), "");
         if (!event)
                 return false;
 
@@ -1171,7 +1171,7 @@ TimelineModel::saveMedia(QString eventId) const
 void
 TimelineModel::cacheMedia(QString eventId)
 {
-        mtx::events::collections::TimelineEvents *event = events.event(eventId.toStdString(), "");
+        mtx::events::collections::TimelineEvents *event = events.get(eventId.toStdString(), "");
         if (!event)
                 return;
 
@@ -1300,7 +1300,7 @@ TimelineModel::formatTypingUsers(const std::vector &users, QColor bg)
 QString
 TimelineModel::formatJoinRuleEvent(QString id)
 {
-        mtx::events::collections::TimelineEvents *e = events.event(id.toStdString(), "");
+        mtx::events::collections::TimelineEvents *e = events.get(id.toStdString(), "");
         if (!e)
                 return "";
 
@@ -1325,7 +1325,7 @@ TimelineModel::formatJoinRuleEvent(QString id)
 QString
 TimelineModel::formatGuestAccessEvent(QString id)
 {
-        mtx::events::collections::TimelineEvents *e = events.event(id.toStdString(), "");
+        mtx::events::collections::TimelineEvents *e = events.get(id.toStdString(), "");
         if (!e)
                 return "";
 
@@ -1349,7 +1349,7 @@ TimelineModel::formatGuestAccessEvent(QString id)
 QString
 TimelineModel::formatHistoryVisibilityEvent(QString id)
 {
-        mtx::events::collections::TimelineEvents *e = events.event(id.toStdString(), "");
+        mtx::events::collections::TimelineEvents *e = events.get(id.toStdString(), "");
         if (!e)
                 return "";
 
@@ -1383,7 +1383,7 @@ TimelineModel::formatHistoryVisibilityEvent(QString id)
 QString
 TimelineModel::formatPowerLevelEvent(QString id)
 {
-        mtx::events::collections::TimelineEvents *e = events.event(id.toStdString(), "");
+        mtx::events::collections::TimelineEvents *e = events.get(id.toStdString(), "");
         if (!e)
                 return "";
 
@@ -1401,7 +1401,7 @@ TimelineModel::formatPowerLevelEvent(QString id)
 QString
 TimelineModel::formatMemberEvent(QString id)
 {
-        mtx::events::collections::TimelineEvents *e = events.event(id.toStdString(), "");
+        mtx::events::collections::TimelineEvents *e = events.get(id.toStdString(), "");
         if (!e)
                 return "";
 
@@ -1412,7 +1412,7 @@ TimelineModel::formatMemberEvent(QString id)
         mtx::events::StateEvent *prevEvent = nullptr;
         if (!event->unsigned_data.replaces_state.empty()) {
                 auto tempPrevEvent =
-                  events.event(event->unsigned_data.replaces_state, event->event_id);
+                  events.get(event->unsigned_data.replaces_state, event->event_id);
                 if (tempPrevEvent) {
                         prevEvent =
                           std::get_if>(

From ade905c881a0e33fd744a4803f327052fce6a699 Mon Sep 17 00:00:00 2001
From: Nicolas Werner 
Date: Sun, 26 Jul 2020 13:07:36 +0200
Subject: [PATCH 31/50] Fix shadowing variable

---
 src/timeline/EventStore.cpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/timeline/EventStore.cpp b/src/timeline/EventStore.cpp
index 94538672..639cae0f 100644
--- a/src/timeline/EventStore.cpp
+++ b/src/timeline/EventStore.cpp
@@ -292,10 +292,10 @@ EventStore::reactions(const std::string &event_id)
                 reaction.count_            = agg.count;
                 reaction.selfReactedEvent_ = QString::fromStdString(agg.reactedBySelf);
 
-                bool first = true;
+                bool firstReaction = true;
                 for (const auto &user : agg.users) {
-                        if (first)
-                                first = false;
+                        if (firstReaction)
+                                firstReaction = false;
                         else
                                 reaction.users_ += ", ";
 

From 720bb164f7051949f9f47b33d2793bd6e5c13f19 Mon Sep 17 00:00:00 2001
From: Nicolas Werner 
Date: Sun, 26 Jul 2020 19:04:36 +0200
Subject: [PATCH 32/50] Fix migration (hopefully)

---
 src/Cache.cpp | 53 ++++++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 50 insertions(+), 3 deletions(-)

diff --git a/src/Cache.cpp b/src/Cache.cpp
index 59755e1e..628062a1 100644
--- a/src/Cache.cpp
+++ b/src/Cache.cpp
@@ -693,9 +693,56 @@ Cache::runMigrations()
                            auto room_ids = getRoomIds(txn);
 
                            for (const auto &room_id : room_ids) {
-                                   auto messagesDb = lmdb::dbi::open(
-                                     txn, std::string(room_id + "/messages").c_str(), MDB_CREATE);
-                                   lmdb::dbi_drop(txn, messagesDb, true);
+                                   try {
+                                           auto messagesDb = lmdb::dbi::open(
+                                             txn, std::string(room_id + "/messages").c_str());
+
+                                           // keep some old messages and batch token
+                                           {
+                                                   auto roomsCursor =
+                                                     lmdb::cursor::open(txn, messagesDb);
+                                                   lmdb::val ts, stored_message;
+                                                   bool start = true;
+                                                   mtx::responses::Timeline oldMessages;
+                                                   while (roomsCursor.get(ts,
+                                                                          stored_message,
+                                                                          start ? MDB_FIRST
+                                                                                : MDB_NEXT)) {
+                                                           start = false;
+
+                                                           auto j = json::parse(std::string_view(
+                                                             stored_message.data(),
+                                                             stored_message.size()));
+
+                                                           if (oldMessages.prev_batch.empty())
+                                                                   oldMessages.prev_batch =
+                                                                     j["token"].get();
+                                                           else if (j["token"] !=
+                                                                    oldMessages.prev_batch)
+                                                                   break;
+
+                                                           mtx::events::collections::TimelineEvent
+                                                             te;
+                                                           mtx::events::collections::from_json(
+                                                             j["event"], te);
+                                                           oldMessages.events.push_back(te.data);
+                                                   }
+                                                   // messages were stored in reverse order, so we
+                                                   // need to reverse them
+                                                   std::reverse(oldMessages.events.begin(),
+                                                                oldMessages.events.end());
+                                                   // save messages using the new method
+                                                   saveTimelineMessages(txn, room_id, oldMessages);
+                                           }
+
+                                           // delete old messages db
+                                           lmdb::dbi_drop(txn, messagesDb, true);
+                                   } catch (std::exception &e) {
+                                           nhlog::db()->error(
+                                             "While migrating messages from {}, ignoring error {}",
+                                             room_id,
+                                             e.what());
+                                   }
                            }
                            txn.commit();
                    } catch (const lmdb::error &) {

From 12090c0a06403d7c21e6dc2df03d9c526c0b3768 Mon Sep 17 00:00:00 2001
From: Nicolas Werner 
Date: Mon, 27 Jul 2020 16:37:29 +0200
Subject: [PATCH 33/50] Add workaround for duplicate syncs

---
 src/ChatPage.cpp | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/ChatPage.cpp b/src/ChatPage.cpp
index 012f1e69..518be31c 100644
--- a/src/ChatPage.cpp
+++ b/src/ChatPage.cpp
@@ -1060,7 +1060,14 @@ ChatPage::trySync()
         }
 
         http::client()->sync(
-          opts, [this](const mtx::responses::Sync &res, mtx::http::RequestErr err) {
+          opts,
+          [this, since = cache::nextBatchToken()](const mtx::responses::Sync &res,
+                                                  mtx::http::RequestErr err) {
+                  if (since != cache::nextBatchToken()) {
+                          nhlog::net()->warn("Duplicate sync, dropping");
+                          return;
+                  }
+
                   if (err) {
                           const auto error      = QString::fromStdString(err->matrix_error.error);
                           const auto msg        = tr("Please try to login again: %1").arg(error);

From 7f3d97517f334cbc9b07100d20acb612a3293bfd Mon Sep 17 00:00:00 2001
From: Nicolas Werner 
Date: Thu, 30 Jul 2020 18:13:19 +0200
Subject: [PATCH 34/50] Fix double free by closing cursor at the right time

---
 src/Cache.cpp | 59 ++++++++++++++++++++++++++++-----------------------
 1 file changed, 33 insertions(+), 26 deletions(-)

diff --git a/src/Cache.cpp b/src/Cache.cpp
index 628062a1..0c692d07 100644
--- a/src/Cache.cpp
+++ b/src/Cache.cpp
@@ -2199,27 +2199,31 @@ Cache::firstPendingMessage(const std::string &room_id)
         auto txn     = lmdb::txn::begin(env_);
         auto pending = getPendingMessagesDb(txn, room_id);
 
-        auto pendingCursor = lmdb::cursor::open(txn, pending);
-        lmdb::val tsIgnored, pendingTxn;
-        while (pendingCursor.get(tsIgnored, pendingTxn, MDB_NEXT)) {
-                auto eventsDb = getEventsDb(txn, room_id);
-                lmdb::val event;
-                if (!lmdb::dbi_get(txn, eventsDb, pendingTxn, event)) {
-                        lmdb::dbi_del(txn, pending, tsIgnored, pendingTxn);
-                        continue;
-                }
+        {
+                auto pendingCursor = lmdb::cursor::open(txn, pending);
+                lmdb::val tsIgnored, pendingTxn;
+                while (pendingCursor.get(tsIgnored, pendingTxn, MDB_NEXT)) {
+                        auto eventsDb = getEventsDb(txn, room_id);
+                        lmdb::val event;
+                        if (!lmdb::dbi_get(txn, eventsDb, pendingTxn, event)) {
+                                lmdb::dbi_del(txn, pending, tsIgnored, pendingTxn);
+                                continue;
+                        }
 
-                try {
-                        mtx::events::collections::TimelineEvent te;
-                        mtx::events::collections::from_json(
-                          json::parse(std::string_view(event.data(), event.size())), te);
+                        try {
+                                mtx::events::collections::TimelineEvent te;
+                                mtx::events::collections::from_json(
+                                  json::parse(std::string_view(event.data(), event.size())), te);
 
-                        txn.commit();
-                        return te;
-                } catch (std::exception &e) {
-                        nhlog::db()->error("Failed to parse message from cache {}", e.what());
-                        lmdb::dbi_del(txn, pending, tsIgnored, pendingTxn);
-                        continue;
+                                pendingCursor.close();
+                                txn.commit();
+                                return te;
+                        } catch (std::exception &e) {
+                                nhlog::db()->error("Failed to parse message from cache {}",
+                                                   e.what());
+                                lmdb::dbi_del(txn, pending, tsIgnored, pendingTxn);
+                                continue;
+                        }
                 }
         }
 
@@ -2231,13 +2235,16 @@ Cache::firstPendingMessage(const std::string &room_id)
 void
 Cache::removePendingStatus(const std::string &room_id, const std::string &txn_id)
 {
-        auto txn           = lmdb::txn::begin(env_);
-        auto pending       = getPendingMessagesDb(txn, room_id);
-        auto pendingCursor = lmdb::cursor::open(txn, pending);
-        lmdb::val tsIgnored, pendingTxn;
-        while (pendingCursor.get(tsIgnored, pendingTxn, MDB_NEXT)) {
-                if (std::string_view(pendingTxn.data(), pendingTxn.size()) == txn_id)
-                        lmdb::cursor_del(pendingCursor);
+        auto txn     = lmdb::txn::begin(env_);
+        auto pending = getPendingMessagesDb(txn, room_id);
+
+        {
+                auto pendingCursor = lmdb::cursor::open(txn, pending);
+                lmdb::val tsIgnored, pendingTxn;
+                while (pendingCursor.get(tsIgnored, pendingTxn, MDB_NEXT)) {
+                        if (std::string_view(pendingTxn.data(), pendingTxn.size()) == txn_id)
+                                lmdb::cursor_del(pendingCursor);
+                }
         }
 
         txn.commit();

From dbaddb01658e67a69db8121bcd24bf82754855c6 Mon Sep 17 00:00:00 2001
From: Nicolas Werner 
Date: Thu, 6 Aug 2020 19:19:13 +0200
Subject: [PATCH 35/50] Further tweak text element

---
 resources/qml/MatrixText.qml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/resources/qml/MatrixText.qml b/resources/qml/MatrixText.qml
index cbb55bea..5762caae 100644
--- a/resources/qml/MatrixText.qml
+++ b/resources/qml/MatrixText.qml
@@ -5,7 +5,8 @@ TextEdit {
 	textFormat: TextEdit.RichText
 	readOnly: true
 	wrapMode: Text.Wrap
-	selectByMouse: ma.containsMouse // try to make scrollable by finger but selectable by mouse
+	selectByMouse: true
+	activeFocusOnPress: false
 	color: colors.text
 
 	onLinkActivated: {
@@ -23,7 +24,6 @@ TextEdit {
 		id: ma
 		anchors.fill: parent
 		propagateComposedEvents: true
-		hoverEnabled: true
 		acceptedButtons: Qt.NoButton
 		cursorShape: parent.hoveredLink ? Qt.PointingHandCursor : Qt.ArrowCursor
 	}

From 1f9215a5be038f28d95b9b90798dafaa800d4425 Mon Sep 17 00:00:00 2001
From: Nicolas Werner 
Date: Thu, 6 Aug 2020 21:46:16 +0200
Subject: [PATCH 36/50] Split error messages from event decryption

---
 src/Olm.cpp                 |  47 +++++++++++
 src/Olm.h                   |  24 ++++++
 src/timeline/EventStore.cpp | 157 +++++++++++++++++-------------------
 3 files changed, 146 insertions(+), 82 deletions(-)

diff --git a/src/Olm.cpp b/src/Olm.cpp
index 994a3a67..466fe940 100644
--- a/src/Olm.cpp
+++ b/src/Olm.cpp
@@ -3,6 +3,7 @@
 #include "Olm.h"
 
 #include "Cache.h"
+#include "Cache_p.h"
 #include "Logging.h"
 #include "MatrixClient.h"
 #include "Utils.h"
@@ -551,4 +552,50 @@ send_megolm_key_to_device(const std::string &user_id,
           });
 }
 
+DecryptionResult
+decryptEvent(const MegolmSessionIndex &index,
+             const mtx::events::EncryptedEvent &event)
+{
+        try {
+                if (!cache::client()->inboundMegolmSessionExists(index)) {
+                        return {DecryptionErrorCode::MissingSession, std::nullopt, std::nullopt};
+                }
+        } catch (const lmdb::error &e) {
+                return {DecryptionErrorCode::DbError, e.what(), std::nullopt};
+        }
+
+        // TODO: Lookup index,event_id,origin_server_ts tuple for replay attack errors
+        // TODO: Verify sender_key
+
+        std::string msg_str;
+        try {
+                auto session = cache::client()->getInboundMegolmSession(index);
+                auto res = olm::client()->decrypt_group_message(session, event.content.ciphertext);
+                msg_str  = std::string((char *)res.data.data(), res.data.size());
+        } catch (const lmdb::error &e) {
+                return {DecryptionErrorCode::DbError, e.what(), std::nullopt};
+        } catch (const mtx::crypto::olm_exception &e) {
+                return {DecryptionErrorCode::DecryptionFailed, e.what(), std::nullopt};
+        }
+
+        // Add missing fields for the event.
+        json body                = json::parse(msg_str);
+        body["event_id"]         = event.event_id;
+        body["sender"]           = event.sender;
+        body["origin_server_ts"] = event.origin_server_ts;
+        body["unsigned"]         = event.unsigned_data;
+
+        // relations are unencrypted in content...
+        if (json old_ev = event; old_ev["content"].count("m.relates_to") != 0)
+                body["content"]["m.relates_to"] = old_ev["content"]["m.relates_to"];
+
+        mtx::events::collections::TimelineEvent te;
+        try {
+                mtx::events::collections::from_json(body, te);
+        } catch (std::exception &e) {
+                return {DecryptionErrorCode::ParsingFailed, e.what(), std::nullopt};
+        }
+
+        return {std::nullopt, std::nullopt, std::move(te.data)};
+}
 } // namespace olm
diff --git a/src/Olm.h b/src/Olm.h
index 09038ad1..87f4e3ec 100644
--- a/src/Olm.h
+++ b/src/Olm.h
@@ -7,10 +7,30 @@
 #include 
 #include 
 
+#include 
+
 constexpr auto OLM_ALGO = "m.olm.v1.curve25519-aes-sha2";
 
 namespace olm {
 
+enum class DecryptionErrorCode
+{
+        MissingSession, // Session was not found, retrieve from backup or request from other devices
+                        // and try again
+        DbError,        // DB read failed
+        DecryptionFailed,   // libolm error
+        ParsingFailed,      // Failed to parse the actual event
+        ReplayAttack,       // Megolm index reused
+        UnknownFingerprint, // Unknown device Fingerprint
+};
+
+struct DecryptionResult
+{
+        std::optional error;
+        std::optional error_message;
+        std::optional event;
+};
+
 struct OlmMessage
 {
         std::string sender_key;
@@ -65,6 +85,10 @@ encrypt_group_message(const std::string &room_id,
                       const std::string &device_id,
                       nlohmann::json body);
 
+DecryptionResult
+decryptEvent(const MegolmSessionIndex &index,
+             const mtx::events::EncryptedEvent &event);
+
 void
 mark_keys_as_published();
 
diff --git a/src/timeline/EventStore.cpp b/src/timeline/EventStore.cpp
index 639cae0f..0e4c8b05 100644
--- a/src/timeline/EventStore.cpp
+++ b/src/timeline/EventStore.cpp
@@ -379,103 +379,96 @@ EventStore::decryptEvent(const IdIndex &idx,
         index.session_id = e.content.session_id;
         index.sender_key = e.content.sender_key;
 
-        mtx::events::RoomEvent dummy;
-        dummy.origin_server_ts = e.origin_server_ts;
-        dummy.event_id         = e.event_id;
-        dummy.sender           = e.sender;
-        dummy.content.body =
-          tr("-- Encrypted Event (No keys found for decryption) --",
-             "Placeholder, when the message was not decrypted yet or can't be decrypted.")
-            .toStdString();
-
         auto asCacheEntry = [&idx](mtx::events::collections::TimelineEvents &&event) {
                 auto event_ptr = new mtx::events::collections::TimelineEvents(std::move(event));
                 decryptedEvents_.insert(idx, event_ptr);
                 return event_ptr;
         };
 
-        try {
-                if (!cache::client()->inboundMegolmSessionExists(index)) {
+        auto decryptionResult = olm::decryptEvent(index, e);
+
+        if (decryptionResult.error) {
+                mtx::events::RoomEvent dummy;
+                dummy.origin_server_ts = e.origin_server_ts;
+                dummy.event_id         = e.event_id;
+                dummy.sender           = e.sender;
+                switch (*decryptionResult.error) {
+                case olm::DecryptionErrorCode::MissingSession:
+                        dummy.content.body =
+                          tr("-- Encrypted Event (No keys found for decryption) --",
+                             "Placeholder, when the message was not decrypted yet or can't be "
+                             "decrypted.")
+                            .toStdString();
                         nhlog::crypto()->info("Could not find inbound megolm session ({}, {}, {})",
                                               index.room_id,
                                               index.session_id,
                                               e.sender);
-                        // TODO: request megolm session_id & session_key from the sender.
-                        return asCacheEntry(std::move(dummy));
+                        // TODO: Check if this actually works and look in key backup
+                        olm::send_key_request_for(room_id_, e);
+                        break;
+                case olm::DecryptionErrorCode::DbError:
+                        nhlog::db()->critical(
+                          "failed to retrieve megolm session with index ({}, {}, {})",
+                          index.room_id,
+                          index.session_id,
+                          index.sender_key,
+                          decryptionResult.error_message.value_or(""));
+                        dummy.content.body =
+                          tr("-- Decryption Error (failed to retrieve megolm keys from db) --",
+                             "Placeholder, when the message can't be decrypted, because the DB "
+                             "access "
+                             "failed.")
+                            .toStdString();
+                        break;
+                case olm::DecryptionErrorCode::DecryptionFailed:
+                        nhlog::crypto()->critical(
+                          "failed to decrypt message with index ({}, {}, {}): {}",
+                          index.room_id,
+                          index.session_id,
+                          index.sender_key,
+                          decryptionResult.error_message.value_or(""));
+                        dummy.content.body =
+                          tr("-- Decryption Error (%1) --",
+                             "Placeholder, when the message can't be decrypted. In this case, the "
+                             "Olm "
+                             "decrytion returned an error, which is passed as %1.")
+                            .arg(
+                              QString::fromStdString(decryptionResult.error_message.value_or("")))
+                            .toStdString();
+                        break;
+                case olm::DecryptionErrorCode::ParsingFailed:
+                        dummy.content.body =
+                          tr("-- Encrypted Event (Unknown event type) --",
+                             "Placeholder, when the message was decrypted, but we couldn't parse "
+                             "it, because "
+                             "Nheko/mtxclient don't support that event type yet.")
+                            .toStdString();
+                        break;
+                case olm::DecryptionErrorCode::ReplayAttack:
+                        nhlog::crypto()->critical(
+                          "Reply attack while decryptiong event {} in room {} from {}!",
+                          e.event_id,
+                          room_id_,
+                          index.sender_key);
+                        dummy.content.body =
+                          tr("-- Reply attack! This message index was reused! --").toStdString();
+                        break;
+                case olm::DecryptionErrorCode::UnknownFingerprint:
+                        // TODO: don't fail, just show in UI.
+                        nhlog::crypto()->critical("Message by unverified fingerprint {}",
+                                                  index.sender_key);
+                        dummy.content.body =
+                          tr("-- Message by unverified device! --").toStdString();
+                        break;
                 }
-        } catch (const lmdb::error &e) {
-                nhlog::db()->critical("failed to check megolm session's existence: {}", e.what());
-                dummy.content.body = tr("-- Decryption Error (failed to communicate with DB) --",
-                                        "Placeholder, when the message can't be decrypted, because "
-                                        "the DB access failed when trying to lookup the session.")
-                                       .toStdString();
                 return asCacheEntry(std::move(dummy));
         }
 
-        std::string msg_str;
-        try {
-                auto session = cache::client()->getInboundMegolmSession(index);
-                auto res     = olm::client()->decrypt_group_message(session, e.content.ciphertext);
-                msg_str      = std::string((char *)res.data.data(), res.data.size());
-        } catch (const lmdb::error &e) {
-                nhlog::db()->critical("failed to retrieve megolm session with index ({}, {}, {})",
-                                      index.room_id,
-                                      index.session_id,
-                                      index.sender_key,
-                                      e.what());
-                dummy.content.body =
-                  tr("-- Decryption Error (failed to retrieve megolm keys from db) --",
-                     "Placeholder, when the message can't be decrypted, because the DB access "
-                     "failed.")
-                    .toStdString();
-                return asCacheEntry(std::move(dummy));
-        } catch (const mtx::crypto::olm_exception &e) {
-                nhlog::crypto()->critical("failed to decrypt message with index ({}, {}, {}): {}",
-                                          index.room_id,
-                                          index.session_id,
-                                          index.sender_key,
-                                          e.what());
-                dummy.content.body =
-                  tr("-- Decryption Error (%1) --",
-                     "Placeholder, when the message can't be decrypted. In this case, the Olm "
-                     "decrytion returned an error, which is passed as %1.")
-                    .arg(e.what())
-                    .toStdString();
-                return asCacheEntry(std::move(dummy));
-        }
+        auto encInfo = mtx::accessors::file(decryptionResult.event.value());
+        if (encInfo)
+                emit newEncryptedImage(encInfo.value());
 
-        // Add missing fields for the event.
-        json body                = json::parse(msg_str);
-        body["event_id"]         = e.event_id;
-        body["sender"]           = e.sender;
-        body["origin_server_ts"] = e.origin_server_ts;
-        body["unsigned"]         = e.unsigned_data;
-
-        // relations are unencrypted in content...
-        if (json old_ev = e; old_ev["content"].count("m.relates_to") != 0)
-                body["content"]["m.relates_to"] = old_ev["content"]["m.relates_to"];
-
-        json event_array = json::array();
-        event_array.push_back(body);
-
-        std::vector temp_events;
-        mtx::responses::utils::parse_timeline_events(event_array, temp_events);
-
-        if (temp_events.size() == 1) {
-                auto encInfo = mtx::accessors::file(temp_events[0]);
-
-                if (encInfo)
-                        emit newEncryptedImage(encInfo.value());
-
-                return asCacheEntry(std::move(temp_events[0]));
-        }
-
-        dummy.content.body =
-          tr("-- Encrypted Event (Unknown event type) --",
-             "Placeholder, when the message was decrypted, but we couldn't parse it, because "
-             "Nheko/mtxclient don't support that event type yet.")
-            .toStdString();
-        return asCacheEntry(std::move(dummy));
+        return asCacheEntry(std::move(decryptionResult.event.value()));
 }
 
 mtx::events::collections::TimelineEvents *

From 7eb0c4e09cebc339d826068f52d9ab2f35665721 Mon Sep 17 00:00:00 2001
From: Nicolas Werner 
Date: Thu, 6 Aug 2020 22:18:52 +0200
Subject: [PATCH 37/50] Also request keys from own devices

---
 src/Olm.cpp | 48 ++++++++++++++++++++++++++----------------------
 1 file changed, 26 insertions(+), 22 deletions(-)

diff --git a/src/Olm.cpp b/src/Olm.cpp
index 466fe940..e38e9ef7 100644
--- a/src/Olm.cpp
+++ b/src/Olm.cpp
@@ -317,32 +317,36 @@ send_key_request_for(const std::string &room_id,
         using namespace mtx::events;
 
         nhlog::crypto()->debug("sending key request: {}", json(e).dump(2));
-        auto payload = json{{"action", "request"},
-                            {"request_id", http::client()->generate_txn_id()},
-                            {"requesting_device_id", http::client()->device_id()},
-                            {"body",
-                             {{"algorithm", MEGOLM_ALGO},
-                              {"room_id", room_id},
-                              {"sender_key", e.content.sender_key},
-                              {"session_id", e.content.session_id}}}};
 
-        json body;
-        body["messages"][e.sender]                      = json::object();
-        body["messages"][e.sender][e.content.device_id] = payload;
+        mtx::events::msg::KeyRequest request;
+        request.action               = mtx::events::msg::RequestAction::Request;
+        request.algorithm            = MEGOLM_ALGO;
+        request.room_id              = room_id;
+        request.sender_key           = e.content.sender_key;
+        request.session_id           = e.content.session_id;
+        request.request_id           = "key_request." + http::client()->generate_txn_id();
+        request.requesting_device_id = http::client()->device_id();
 
-        nhlog::crypto()->debug("m.room_key_request: {}", body.dump(2));
+        nhlog::crypto()->debug("m.room_key_request: {}", json(request).dump(2));
 
-        http::client()->send_to_device("m.room_key_request", body, [e](mtx::http::RequestErr err) {
-                if (err) {
-                        nhlog::net()->warn("failed to send "
-                                           "send_to_device "
-                                           "message: {}",
-                                           err->matrix_error.error);
-                }
+        std::map> body;
+        body[mtx::identifiers::parse(e.sender)][e.content.device_id] =
+          request;
+        body[http::client()->user_id()]["*"] = request;
 
-                nhlog::net()->info(
-                  "m.room_key_request sent to {}:{}", e.sender, e.content.device_id);
-        });
+        http::client()->send_to_device(
+          http::client()->generate_txn_id(), body, [e](mtx::http::RequestErr err) {
+                  if (err) {
+                          nhlog::net()->warn("failed to send "
+                                             "send_to_device "
+                                             "message: {}",
+                                             err->matrix_error.error);
+                  }
+
+                  nhlog::net()->info("m.room_key_request sent to {}:{} and your own devices",
+                                     e.sender,
+                                     e.content.device_id);
+          });
 }
 
 void

From b972d827cb1d3c35e8c561d1245204bd6f4b21f9 Mon Sep 17 00:00:00 2001
From: Nicolas Werner 
Date: Fri, 7 Aug 2020 13:12:45 +0200
Subject: [PATCH 38/50] Try to fix issue of pagination interfering with
 limited: true

---
 src/timeline/EventStore.cpp | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/timeline/EventStore.cpp b/src/timeline/EventStore.cpp
index 0e4c8b05..a983fe01 100644
--- a/src/timeline/EventStore.cpp
+++ b/src/timeline/EventStore.cpp
@@ -529,6 +529,12 @@ EventStore::fetchMore()
 
         http::client()->messages(
           opts, [this, opts](const mtx::responses::Messages &res, mtx::http::RequestErr err) {
+                  if (cache::client()->previousBatchToken(room_id_) != opts.from) {
+                          nhlog::net()->warn("Cache cleared while fetching more messages, dropping "
+                                             "/messages response");
+                          emit fetchedMore();
+                          return;
+                  }
                   if (err) {
                           nhlog::net()->error("failed to call /messages ({}): {} - {} - {}",
                                               opts.room_id,

From 1e9efa30728fa13474d888abf35b2c01d935c679 Mon Sep 17 00:00:00 2001
From: Nicolas Werner 
Date: Fri, 7 Aug 2020 17:31:58 +0200
Subject: [PATCH 39/50] Try to fix variable timestamp width

---
 resources/qml/TimelineRow.qml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/resources/qml/TimelineRow.qml b/resources/qml/TimelineRow.qml
index d1c20278..a38a4d34 100644
--- a/resources/qml/TimelineRow.qml
+++ b/resources/qml/TimelineRow.qml
@@ -130,6 +130,7 @@ Item {
 		Label {
 			Layout.alignment: Qt.AlignRight | Qt.AlignTop
 			text: model.timestamp.toLocaleTimeString("HH:mm")
+			width: Math.max(implicitWidth, text.length*fontMetrics.maximumCharacterWidth)
 			color: inactiveColors.text
 
 			MouseArea{

From 14a0aac74873c27c0454d206848f27b4eec123ae Mon Sep 17 00:00:00 2001
From: Nicolas Werner 
Date: Sun, 9 Aug 2020 23:36:47 +0200
Subject: [PATCH 40/50] Add /clear-timeline command

---
 src/Cache.cpp                      | 118 ++++++++++++++++++++++++++---
 src/Cache_p.h                      |   3 +
 src/ChatPage.cpp                   |   5 ++
 src/TextInputWidget.cpp            |  24 +++---
 src/TextInputWidget.h              |   1 +
 src/timeline/EventStore.cpp        |  20 +++++
 src/timeline/EventStore.h          |   1 +
 src/timeline/TimelineModel.h       |   1 +
 src/timeline/TimelineViewManager.h |   6 ++
 9 files changed, 157 insertions(+), 22 deletions(-)

diff --git a/src/Cache.cpp b/src/Cache.cpp
index 0c692d07..0d879584 100644
--- a/src/Cache.cpp
+++ b/src/Cache.cpp
@@ -2304,6 +2304,11 @@ Cache::saveTimelineMessages(lmdb::txn &txn,
 
                 lmdb::val event_id = event_id_val;
 
+                json orderEntry        = json::object();
+                orderEntry["event_id"] = event_id_val;
+                if (first && !res.prev_batch.empty())
+                        orderEntry["prev_batch"] = res.prev_batch;
+
                 lmdb::val txn_order;
                 if (!txn_id.empty() &&
                     lmdb::dbi_get(txn, evToOrderDb, lmdb::val(txn_id), txn_order)) {
@@ -2317,7 +2322,7 @@ Cache::saveTimelineMessages(lmdb::txn &txn,
                                 lmdb::dbi_del(txn, msg2orderDb, lmdb::val(txn_id));
                         }
 
-                        lmdb::dbi_put(txn, orderDb, txn_order, event_id);
+                        lmdb::dbi_put(txn, orderDb, txn_order, lmdb::val(orderEntry.dump()));
                         lmdb::dbi_put(txn, evToOrderDb, event_id, txn_order);
                         lmdb::dbi_del(txn, evToOrderDb, lmdb::val(txn_id));
 
@@ -2389,10 +2394,6 @@ Cache::saveTimelineMessages(lmdb::txn &txn,
 
                         ++index;
 
-                        json orderEntry        = json::object();
-                        orderEntry["event_id"] = event_id_val;
-                        if (first && !res.prev_batch.empty())
-                                orderEntry["prev_batch"] = res.prev_batch;
                         first = false;
 
                         nhlog::db()->debug("saving '{}'", orderEntry.dump());
@@ -2440,6 +2441,7 @@ Cache::saveOldMessages(const std::string &room_id, const mtx::responses::Message
         auto relationsDb = getRelationsDb(txn, room_id);
 
         auto orderDb     = getEventOrderDb(txn, room_id);
+        auto evToOrderDb = getEventToOrderDb(txn, room_id);
         auto msg2orderDb = getMessageToOrderDb(txn, room_id);
         auto order2msgDb = getOrderToMessageDb(txn, room_id);
 
@@ -2483,6 +2485,7 @@ Cache::saveOldMessages(const std::string &room_id, const mtx::responses::Message
 
                 lmdb::dbi_put(
                   txn, orderDb, lmdb::val(&index, sizeof(index)), lmdb::val(orderEntry.dump()));
+                lmdb::dbi_put(txn, evToOrderDb, event_id, lmdb::val(&index, sizeof(index)));
 
                 // TODO(Nico): Allow blacklisting more event types in UI
                 if (event["type"] != "m.reaction" && event["type"] != "m.dummy") {
@@ -2516,6 +2519,94 @@ Cache::saveOldMessages(const std::string &room_id, const mtx::responses::Message
         return msgIndex;
 }
 
+void
+Cache::clearTimeline(const std::string &room_id)
+{
+        auto txn         = lmdb::txn::begin(env_);
+        auto eventsDb    = getEventsDb(txn, room_id);
+        auto relationsDb = getRelationsDb(txn, room_id);
+
+        auto orderDb     = getEventOrderDb(txn, room_id);
+        auto evToOrderDb = getEventToOrderDb(txn, room_id);
+        auto msg2orderDb = getMessageToOrderDb(txn, room_id);
+        auto order2msgDb = getOrderToMessageDb(txn, room_id);
+
+        lmdb::val indexVal, val;
+        auto cursor = lmdb::cursor::open(txn, orderDb);
+
+        bool start                   = true;
+        bool passed_pagination_token = false;
+        while (cursor.get(indexVal, val, start ? MDB_LAST : MDB_PREV)) {
+                start = false;
+                json obj;
+
+                try {
+                        obj = json::parse(std::string_view(val.data(), val.size()));
+                } catch (std::exception &) {
+                        // workaround bug in the initial db format, where we sometimes didn't store
+                        // json...
+                        obj = {{"event_id", std::string(val.data(), val.size())}};
+                }
+
+                if (passed_pagination_token) {
+                        if (obj.count("event_id") != 0) {
+                                lmdb::val event_id = obj["event_id"].get();
+                                lmdb::dbi_del(txn, evToOrderDb, event_id);
+                                lmdb::dbi_del(txn, eventsDb, event_id);
+
+                                lmdb::dbi_del(txn, relationsDb, event_id);
+
+                                lmdb::val order{};
+                                bool exists = lmdb::dbi_get(txn, msg2orderDb, event_id, order);
+                                if (exists) {
+                                        lmdb::dbi_del(txn, order2msgDb, order);
+                                        lmdb::dbi_del(txn, msg2orderDb, event_id);
+                                }
+                        }
+                        lmdb::cursor_del(cursor);
+                } else {
+                        if (obj.count("prev_batch") != 0)
+                                passed_pagination_token = true;
+                }
+        }
+
+        auto msgCursor = lmdb::cursor::open(txn, order2msgDb);
+        start          = true;
+        while (msgCursor.get(indexVal, val, start ? MDB_LAST : MDB_PREV)) {
+                start = false;
+
+                lmdb::val eventId;
+                bool innerStart = true;
+                bool found      = false;
+                while (cursor.get(indexVal, eventId, innerStart ? MDB_LAST : MDB_PREV)) {
+                        innerStart = false;
+
+                        json obj;
+                        try {
+                                obj = json::parse(std::string_view(eventId.data(), eventId.size()));
+                        } catch (std::exception &) {
+                                obj = {{"event_id", std::string(eventId.data(), eventId.size())}};
+                        }
+
+                        if (obj["event_id"] == std::string(val.data(), val.size())) {
+                                found = true;
+                                break;
+                        }
+                }
+
+                if (!found)
+                        break;
+        }
+
+        do {
+                lmdb::cursor_del(msgCursor);
+        } while (msgCursor.get(indexVal, val, MDB_PREV));
+
+        cursor.close();
+        msgCursor.close();
+        txn.commit();
+}
+
 mtx::responses::Notifications
 Cache::getTimelineMentionsForRoom(lmdb::txn &txn, const std::string &room_id)
 {
@@ -2654,11 +2745,13 @@ Cache::deleteOldMessages()
         auto room_ids = getRoomIds(txn);
 
         for (const auto &room_id : room_ids) {
-                auto orderDb  = getEventOrderDb(txn, room_id);
-                auto o2m      = getOrderToMessageDb(txn, room_id);
-                auto m2o      = getMessageToOrderDb(txn, room_id);
-                auto eventsDb = getEventsDb(txn, room_id);
-                auto cursor   = lmdb::cursor::open(txn, orderDb);
+                auto orderDb     = getEventOrderDb(txn, room_id);
+                auto evToOrderDb = getEventToOrderDb(txn, room_id);
+                auto o2m         = getOrderToMessageDb(txn, room_id);
+                auto m2o         = getMessageToOrderDb(txn, room_id);
+                auto eventsDb    = getEventsDb(txn, room_id);
+                auto relationsDb = getRelationsDb(txn, room_id);
+                auto cursor      = lmdb::cursor::open(txn, orderDb);
 
                 uint64_t first, last;
                 if (cursor.get(indexVal, val, MDB_LAST)) {
@@ -2678,14 +2771,17 @@ Cache::deleteOldMessages()
 
                 bool start = true;
                 while (cursor.get(indexVal, val, start ? MDB_FIRST : MDB_NEXT) &&
-                       message_count-- < MAX_RESTORED_MESSAGES) {
+                       message_count-- > MAX_RESTORED_MESSAGES) {
                         start    = false;
                         auto obj = json::parse(std::string_view(val.data(), val.size()));
 
                         if (obj.count("event_id") != 0) {
                                 lmdb::val event_id = obj["event_id"].get();
+                                lmdb::dbi_del(txn, evToOrderDb, event_id);
                                 lmdb::dbi_del(txn, eventsDb, event_id);
 
+                                lmdb::dbi_del(txn, relationsDb, event_id);
+
                                 lmdb::val order{};
                                 bool exists = lmdb::dbi_get(txn, m2o, event_id, order);
                                 if (exists) {
diff --git a/src/Cache_p.h b/src/Cache_p.h
index 61d91b0c..d3ec6ee0 100644
--- a/src/Cache_p.h
+++ b/src/Cache_p.h
@@ -208,6 +208,9 @@ public:
           const std::string &room_id);
         void removePendingStatus(const std::string &room_id, const std::string &txn_id);
 
+        //! clear timeline keeping only the latest batch
+        void clearTimeline(const std::string &room_id);
+
         //! Remove old unused data.
         void deleteOldMessages();
         void deleteOldData() noexcept;
diff --git a/src/ChatPage.cpp b/src/ChatPage.cpp
index 518be31c..63d13fb9 100644
--- a/src/ChatPage.cpp
+++ b/src/ChatPage.cpp
@@ -155,6 +155,11 @@ ChatPage::ChatPage(QSharedPointer userSettings, QWidget *parent)
                 trySync();
         });
 
+        connect(text_input_,
+                &TextInputWidget::clearRoomTimeline,
+                view_manager_,
+                &TimelineViewManager::clearCurrentRoomTimeline);
+
         connect(
           new QShortcut(QKeySequence("Ctrl+Down"), this), &QShortcut::activated, this, [this]() {
                   if (isVisible())
diff --git a/src/TextInputWidget.cpp b/src/TextInputWidget.cpp
index 3e3915bb..91846230 100644
--- a/src/TextInputWidget.cpp
+++ b/src/TextInputWidget.cpp
@@ -566,27 +566,29 @@ void
 TextInputWidget::command(QString command, QString args)
 {
         if (command == "me") {
-                sendEmoteMessage(args);
+                emit sendEmoteMessage(args);
         } else if (command == "join") {
-                sendJoinRoomRequest(args);
+                emit sendJoinRoomRequest(args);
         } else if (command == "invite") {
-                sendInviteRoomRequest(args.section(' ', 0, 0), args.section(' ', 1, -1));
+                emit sendInviteRoomRequest(args.section(' ', 0, 0), args.section(' ', 1, -1));
         } else if (command == "kick") {
-                sendKickRoomRequest(args.section(' ', 0, 0), args.section(' ', 1, -1));
+                emit sendKickRoomRequest(args.section(' ', 0, 0), args.section(' ', 1, -1));
         } else if (command == "ban") {
-                sendBanRoomRequest(args.section(' ', 0, 0), args.section(' ', 1, -1));
+                emit sendBanRoomRequest(args.section(' ', 0, 0), args.section(' ', 1, -1));
         } else if (command == "unban") {
-                sendUnbanRoomRequest(args.section(' ', 0, 0), args.section(' ', 1, -1));
+                emit sendUnbanRoomRequest(args.section(' ', 0, 0), args.section(' ', 1, -1));
         } else if (command == "roomnick") {
-                changeRoomNick(args);
+                emit changeRoomNick(args);
         } else if (command == "shrug") {
-                sendTextMessage("¯\\_(ツ)_/¯");
+                emit sendTextMessage("¯\\_(ツ)_/¯");
         } else if (command == "fliptable") {
-                sendTextMessage("(╯°□°)╯︵ ┻━┻");
+                emit sendTextMessage("(╯°□°)╯︵ ┻━┻");
         } else if (command == "unfliptable") {
-                sendTextMessage(" ┯━┯╭( º _ º╭)");
+                emit sendTextMessage(" ┯━┯╭( º _ º╭)");
         } else if (command == "sovietflip") {
-                sendTextMessage("ノ┬─┬ノ ︵ ( \\o°o)\\");
+                emit sendTextMessage("ノ┬─┬ノ ︵ ( \\o°o)\\");
+        } else if (command == "clear-timeline") {
+                emit clearRoomTimeline();
         }
 }
 
diff --git a/src/TextInputWidget.h b/src/TextInputWidget.h
index a0105eb0..cbb6ea95 100644
--- a/src/TextInputWidget.h
+++ b/src/TextInputWidget.h
@@ -156,6 +156,7 @@ private slots:
 signals:
         void sendTextMessage(const QString &msg);
         void sendEmoteMessage(QString msg);
+        void clearRoomTimeline();
         void heightChanged(int height);
 
         void uploadMedia(const QSharedPointer data,
diff --git a/src/timeline/EventStore.cpp b/src/timeline/EventStore.cpp
index a983fe01..fca1d31d 100644
--- a/src/timeline/EventStore.cpp
+++ b/src/timeline/EventStore.cpp
@@ -175,6 +175,26 @@ EventStore::addPending(mtx::events::collections::TimelineEvents event)
         emit processPending();
 }
 
+void
+EventStore::clearTimeline()
+{
+        emit beginResetModel();
+
+        cache::client()->clearTimeline(room_id_);
+        auto range = cache::client()->getTimelineRange(room_id_);
+        if (range) {
+                nhlog::db()->info("Range {} {}", range->last, range->first);
+                this->last  = range->last;
+                this->first = range->first;
+        } else {
+                this->first = std::numeric_limits::max();
+                this->last  = std::numeric_limits::max();
+        }
+        nhlog::ui()->info("Range {} {}", this->last, this->first);
+
+        emit endResetModel();
+}
+
 void
 EventStore::handleSync(const mtx::responses::Timeline &events)
 {
diff --git a/src/timeline/EventStore.h b/src/timeline/EventStore.h
index b5c17d10..d4353a18 100644
--- a/src/timeline/EventStore.h
+++ b/src/timeline/EventStore.h
@@ -101,6 +101,7 @@ signals:
 
 public slots:
         void addPending(mtx::events::collections::TimelineEvents event);
+        void clearTimeline();
 
 private:
         mtx::events::collections::TimelineEvents *decryptEvent(
diff --git a/src/timeline/TimelineModel.h b/src/timeline/TimelineModel.h
index f8a84f17..0bcf42b7 100644
--- a/src/timeline/TimelineModel.h
+++ b/src/timeline/TimelineModel.h
@@ -242,6 +242,7 @@ public slots:
                 }
         }
         void setDecryptDescription(bool decrypt) { decryptDescription = decrypt; }
+        void clearTimeline() { events.clearTimeline(); }
 
 private slots:
         void addPendingMessage(mtx::events::collections::TimelineEvents event);
diff --git a/src/timeline/TimelineViewManager.h b/src/timeline/TimelineViewManager.h
index 63106916..20dbc3bb 100644
--- a/src/timeline/TimelineViewManager.h
+++ b/src/timeline/TimelineViewManager.h
@@ -92,6 +92,12 @@ public slots:
                                uint64_t dsize);
         void updateEncryptedDescriptions();
 
+        void clearCurrentRoomTimeline()
+        {
+                if (timeline_)
+                        timeline_->clearTimeline();
+        }
+
 private:
 #ifdef USE_QUICK_VIEW
         QQuickView *view;

From ed17b0c33b4b1058a7e7077603042df606500867 Mon Sep 17 00:00:00 2001
From: Mihai Fufezan 
Date: Sun, 16 Aug 2020 21:40:51 +0000
Subject: [PATCH 41/50] Added translation using Weblate (Romanian)

---
 resources/langs/nheko_ro.ts | 1815 +++++++++++++++++++++++++++++++++++
 1 file changed, 1815 insertions(+)
 create mode 100644 resources/langs/nheko_ro.ts

diff --git a/resources/langs/nheko_ro.ts b/resources/langs/nheko_ro.ts
new file mode 100644
index 00000000..659c8f6c
--- /dev/null
+++ b/resources/langs/nheko_ro.ts
@@ -0,0 +1,1815 @@
+
+
+
+
+    Cache
+    
+        
+        You joined this room.
+        
+    
+
+
+    ChatPage
+    
+        
+        Failed to invite user: %1
+        
+    
+    
+        
+        
+        Invited user: %1
+        
+    
+    
+        
+        Migrating the cache to the current version failed. This can have different reasons. Please open an issue and try to use an older version in the mean time. Alternatively you can try deleting the cache manually.
+        
+    
+    
+        
+        Room %1 created.
+        
+    
+    
+        
+        Failed to invite %1 to %2: %3
+        
+    
+    
+        
+        Failed to kick %1 to %2: %3
+        
+    
+    
+        
+        Kicked user: %1
+        
+    
+    
+        
+        Failed to ban %1 in %2: %3
+        
+    
+    
+        
+        Banned user: %1
+        
+    
+    
+        
+        Failed to unban %1 in %2: %3
+        
+    
+    
+        
+        Unbanned user: %1
+        
+    
+    
+        
+        Failed to upload media. Please try again.
+        
+    
+    
+        
+        Cache migration failed!
+        
+    
+    
+        
+        Incompatible cache version
+        
+    
+    
+        
+        The cache on your disk is newer than this version of Nheko supports. Please update or clear your cache.
+        
+    
+    
+        
+        Failed to restore OLM account. Please login again.
+        
+    
+    
+        
+        Failed to restore save data. Please login again.
+        
+    
+    
+        
+        Failed to setup encryption keys. Server response: %1 %2. Please try again later.
+        
+    
+    
+        
+        
+        Please try to login again: %1
+        
+    
+    
+        
+        Failed to join room: %1
+        
+    
+    
+        
+        You joined the room
+        
+    
+    
+        
+        Failed to remove invite: %1
+        
+    
+    
+        
+        Room creation failed: %1
+        
+    
+    
+        
+        Failed to leave room: %1
+        
+    
+
+
+    CommunitiesListItem
+    
+        
+        All rooms
+        
+    
+    
+        
+        Favourite rooms
+        
+    
+    
+        
+        Low priority rooms
+        
+    
+    
+        
+        Server Notices
+        Tag translation for m.server_notice
+        
+    
+    
+        
+        
+         (tag)
+        
+    
+    
+        
+         (community)
+        
+    
+
+
+    EditModal
+    
+        
+        Apply
+        
+    
+    
+        
+        Cancel
+        
+    
+    
+        
+        Name
+        
+    
+    
+        
+        Topic
+        
+    
+
+
+    EmojiPicker
+    
+        
+        
+        Search
+        
+    
+    
+        
+        People
+        
+    
+    
+        
+        Nature
+        
+    
+    
+        
+        Food
+        
+    
+    
+        
+        Activity
+        
+    
+    
+        
+        Travel
+        
+    
+    
+        
+        Objects
+        
+    
+    
+        
+        Symbols
+        
+    
+    
+        
+        Flags
+        
+    
+
+
+    EncryptionIndicator
+    
+        
+        Encrypted
+        
+    
+    
+        
+        This message is not encrypted!
+        
+    
+
+
+    InviteeItem
+    
+        
+        Remove
+        
+    
+
+
+    LoginPage
+    
+        
+        Matrix ID
+        
+    
+    
+        
+        e.g @joe:matrix.org
+        
+    
+    
+        
+        Your login name. A mxid should start with @ followed by the user id. After the user id you need to include your server name after a :.
+You can also put your homeserver address there, if your server doesn't support .well-known lookup.
+Example: @user:server.my
+If Nheko fails to discover your homeserver, it will show you a field to enter the server manually.
+        
+    
+    
+        
+        Password
+        
+    
+    
+        
+        Device name
+        
+    
+    
+        
+        A name for this device, which will be shown to others, when verifying your devices. If none is provided a default is used.
+        
+    
+    
+        
+        The address that can be used to contact you homeservers client API.
+Example: https://server.my:8787
+        
+    
+    
+        
+        
+        LOGIN
+        
+    
+    
+        
+        Autodiscovery failed. Received malformed response.
+        
+    
+    
+        
+        Autodiscovery failed. Unknown error when requesting .well-known.
+        
+    
+    
+        
+        The required endpoints were not found. Possibly not a Matrix server.
+        
+    
+    
+        
+        Received malformed response. Make sure the homeserver domain is valid.
+        
+    
+    
+        
+        An unknown error occured. Make sure the homeserver domain is valid.
+        
+    
+    
+        
+        SSO LOGIN
+        
+    
+    
+        
+        Empty password
+        
+    
+    
+        
+        SSO login failed
+        
+    
+
+
+    MemberList
+    
+        
+        Room members
+        
+    
+    
+        
+        OK
+        
+    
+
+
+    MessageDelegate
+    
+        
+        redacted
+        
+    
+    
+        
+        Encryption enabled
+        
+    
+    
+        
+        room name changed to: %1
+        
+    
+    
+        
+        removed room name
+        
+    
+    
+        
+        topic changed to: %1
+        
+    
+    
+        
+        removed topic
+        
+    
+    
+        
+        %1 created and configured room: %2
+        
+    
+
+
+    Placeholder
+    
+        
+        unimplemented event: 
+        
+    
+
+
+    QuickSwitcher
+    
+        
+        Search for a room...
+        
+    
+
+
+    RegisterPage
+    
+        
+        Username
+        
+    
+    
+        
+        The username must not be empty, and must contain only the characters a-z, 0-9, ., _, =, -, and /.
+        
+    
+    
+        
+        Password
+        
+    
+    
+        
+        Please choose a secure password. The exact requirements for password strength may depend on your server.
+        
+    
+    
+        
+        Password confirmation
+        
+    
+    
+        
+        Homeserver
+        
+    
+    
+        
+        A server that allows registration. Since matrix is decentralized, you need to first find a server you can register on or host your own.
+        
+    
+    
+        
+        REGISTER
+        
+    
+    
+        
+        No supported registration flows!
+        
+    
+    
+        
+        Invalid username
+        
+    
+    
+        
+        Password is not long enough (min 8 chars)
+        
+    
+    
+        
+        Passwords don't match
+        
+    
+    
+        
+        Invalid server name
+        
+    
+
+
+    RoomInfo
+    
+        
+        no version stored
+        
+    
+
+
+    RoomInfoListItem
+    
+        
+        Leave room
+        
+    
+    
+        
+        Tag room as:
+        
+    
+    
+        
+        Favourite
+        Standard matrix tag for favourites
+        
+    
+    
+        
+        Low Priority
+        Standard matrix tag for low priority rooms
+        
+    
+    
+        
+        Server Notice
+        Standard matrix tag for server notices
+        
+    
+    
+        
+        Adds or removes the specified tag.
+        WhatsThis hint for tag menu actions
+        
+    
+    
+        
+        New tag...
+        Add a new tag to the room
+        
+    
+    
+        
+        New Tag
+        Tag name prompt title
+        
+    
+    
+        
+        Tag:
+        Tag name prompt
+        
+    
+    
+        
+        Accept
+        
+    
+    
+        
+        Decline
+        
+    
+
+
+    SideBarActions
+    
+        
+        User settings
+        
+    
+    
+        
+        Create new room
+        
+    
+    
+        
+        Join a room
+        
+    
+    
+        
+        Start a new chat
+        
+    
+    
+        
+        Room directory
+        
+    
+
+
+    StatusIndicator
+    
+        
+        Failed
+        
+    
+    
+        
+        Sent
+        
+    
+    
+        
+        Received
+        
+    
+    
+        
+        Read
+        
+    
+
+
+    TextInputWidget
+    
+        
+        Send a file
+        
+    
+    
+        
+        
+        Write a message...
+        
+    
+    
+        
+        Send a message
+        
+    
+    
+        
+        Emoji
+        
+    
+    
+        
+        Select a file
+        
+    
+    
+        
+        All Files (*)
+        
+    
+    
+        
+        Connection lost. Nheko is trying to re-connect...
+        
+    
+
+
+    TimelineModel
+    
+        
+        -- Decryption Error (failed to communicate with DB) --
+        Placeholder, when the message can't be decrypted, because the DB access failed when trying to lookup the session.
+        
+    
+    
+        
+        -- Decryption Error (failed to retrieve megolm keys from db) --
+        Placeholder, when the message can't be decrypted, because the DB access failed.
+        
+    
+    
+        
+        -- Decryption Error (%1) --
+        Placeholder, when the message can't be decrypted. In this case, the Olm decrytion returned an error, which is passed ad %1.
+        
+    
+    
+        
+        Message redaction failed: %1
+        
+    
+    
+        
+        Save image
+        
+    
+    
+        
+        Save video
+        
+    
+    
+        
+        Save audio
+        
+    
+    
+        
+        Save file
+        
+    
+    
+        
+        -- Encrypted Event (No keys found for decryption) --
+        Placeholder, when the message was not decrypted yet or can't be decrypted.
+        
+    
+    
+        
+        -- Encrypted Event (Unknown event type) --
+        Placeholder, when the message was decrypted, but we couldn't parse it, because Nheko/mtxclient don't support that event type yet.
+        
+    
+    
+        
+        %1 and %2 are typing.
+        Multiple users are typing. First argument is a comma separated list of potentially multiple users. Second argument is the last user of that list. (If only one user is typing, %1 is empty. You should still use it in your string though to silence Qt warnings.)
+        
+            
+            
+            
+        
+    
+    
+        
+        %1 opened the room to the public.
+        
+    
+    
+        
+        %1 made this room require and invitation to join.
+        
+    
+    
+        
+        %1 made the room open to guests.
+        
+    
+    
+        
+        %1 has closed the room to guest access.
+        
+    
+    
+        
+        %1 made the room history world readable. Events may be now read by non-joined people.
+        
+    
+    
+        
+        %1 set the room history visible to members from this point on.
+        
+    
+    
+        
+        %1 set the room history visible to members since they were invited.
+        
+    
+    
+        
+        %1 set the room history visible to members since they joined the room.
+        
+    
+    
+        
+        %1 has changed the room's permissions.
+        
+    
+    
+        
+        %1 was invited.
+        
+    
+    
+        
+        %1 changed their display name and avatar.
+        
+    
+    
+        
+        %1 changed their display name.
+        
+    
+    
+        
+        %1 changed their avatar.
+        
+    
+    
+        
+        %1 changed some profile info.
+        
+    
+    
+        
+        %1 joined.
+        
+    
+    
+        
+        %1 rejected their invite.
+        
+    
+    
+        
+        Revoked the invite to %1.
+        
+    
+    
+        
+        %1 left the room.
+        
+    
+    
+        
+        Kicked %1.
+        
+    
+    
+        
+        Unbanned %1.
+        
+    
+    
+        
+        %1 was banned.
+        
+    
+    
+        
+        %1 redacted their knock.
+        
+    
+    
+        
+        You joined this room.
+        
+    
+    
+        
+        Rejected the knock from %1.
+        
+    
+    
+        
+        %1 left after having already left!
+        This is a leave event after the user already left and shouldn't happen apart from state resets
+        
+    
+    
+        
+         Reason: %1
+        
+    
+    
+        
+        %1 knocked.
+        
+    
+
+
+    TimelineRow
+    
+        
+        React
+        
+    
+    
+        
+        Reply
+        
+    
+    
+        
+        Options
+        
+    
+
+
+    TimelineView
+    
+        
+        React
+        
+    
+    
+        
+        Reply
+        
+    
+    
+        
+        Read receipts
+        
+    
+    
+        
+        Mark as read
+        
+    
+    
+        
+        View raw message
+        
+    
+    
+        
+        View decrypted raw message
+        
+    
+    
+        
+        Redact message
+        
+    
+    
+        
+        Save as
+        
+    
+    
+        
+        No room open
+        
+    
+    
+        
+        Close
+        
+    
+
+
+    TopRoomBar
+    
+        
+        Room options
+        
+    
+    
+        
+        Mentions
+        
+    
+    
+        
+        Invite users
+        
+    
+    
+        
+        Members
+        
+    
+    
+        
+        Leave room
+        
+    
+    
+        
+        Settings
+        
+    
+
+
+    TrayIcon
+    
+        
+        Show
+        
+    
+    
+        
+        Quit
+        
+    
+
+
+    UserInfoWidget
+    
+        
+        Logout
+        
+    
+    
+        
+        Set custom status message
+        
+    
+    
+        
+        Custom status message
+        
+    
+    
+        
+        Status:
+        
+    
+    
+        
+        Set presence automatically
+        
+    
+    
+        
+        Online
+        
+    
+    
+        
+        Unavailable
+        
+    
+    
+        
+        Offline
+        
+    
+
+
+    UserSettingsPage
+    
+        
+        Minimize to tray
+        
+    
+    
+        
+        Start in tray
+        
+    
+    
+        
+        Group's sidebar
+        
+    
+    
+        
+        Circular Avatars
+        
+    
+    
+        
+        Keep the application running in the background after closing the client window.
+        
+    
+    
+        
+        Start the application in the background without showing the client window.
+        
+    
+    
+        
+        Change the appearance of user avatars in chats.
+OFF - square, ON - Circle.
+        
+    
+    
+        
+        Show a column containing groups and tags next to the room list.
+        
+    
+    
+        
+        Decrypt messages in sidebar
+        
+    
+    
+        
+        Decrypt the messages shown in the sidebar.
+Only affects messages in encrypted chats.
+        
+    
+    
+        
+        Show buttons in timeline
+        
+    
+    
+        
+        Show buttons to quickly reply, react or access additional options next to each message.
+        
+    
+    
+        
+        Limit width of timeline
+        
+    
+    
+        
+        Set the max width of messages in the timeline (in pixels). This can help readability on wide screen, when Nheko is maximised
+        
+    
+    
+        
+        Typing notifications
+        
+    
+    
+        
+        Show who is typing in a room.
+This will also enable or disable sending typing notifications to others.
+        
+    
+    
+        
+        Sort rooms by unreads
+        
+    
+    
+        
+        Display rooms with new messages first.
+If this is off, the list of rooms will only be sorted by the timestamp of the last message in a room.
+If this is on, rooms which have active notifications (the small circle with a number in it) will be sorted on top. Rooms, that you have muted, will still be sorted by timestamp, since you don't seem to consider them as important as the other rooms.
+        
+    
+    
+        
+        Read receipts
+        
+    
+    
+        
+        Show if your message was read.
+Status is displayed next to timestamps.
+        
+    
+    
+        
+        Send messages as Markdown
+        
+    
+    
+        
+        Allow using markdown in messages.
+When disabled, all messages are sent as a plain text.
+        
+    
+    
+        
+        Desktop notifications
+        
+    
+    
+        
+        Notify about received message when the client is not currently focused.
+        
+    
+    
+        
+        Alert on notification
+        
+    
+    
+        
+        Show an alert when a message is received.
+This usually causes the application icon in the task bar to animate in some fashion.
+        
+    
+    
+        
+        Highlight message on hover
+        
+    
+    
+        
+        Change the background color of messages when you hover over them.
+        
+    
+    
+        
+        Large Emoji in timeline
+        
+    
+    
+        
+        Make font size larger if messages with only a few emojis are displayed.
+        
+    
+    
+        
+        Scale factor
+        
+    
+    
+        
+        Change the scale factor of the whole user interface.
+        
+    
+    
+        
+        Font size
+        
+    
+    
+        
+        Font Family
+        
+    
+    
+        
+        Theme
+        
+    
+    
+        
+        Device ID
+        
+    
+    
+        
+        Device Fingerprint
+        
+    
+    
+        
+        Session Keys
+        
+    
+    
+        
+        IMPORT
+        
+    
+    
+        
+        EXPORT
+        
+    
+    
+        
+        ENCRYPTION
+        
+    
+    
+        
+        GENERAL
+        
+    
+    
+        
+        INTERFACE
+        
+    
+    
+        
+        Emoji Font Family
+        
+    
+    
+        
+        Open Sessions File
+        
+    
+    
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        
+        Error
+        
+    
+    
+        
+        
+        File Password
+        
+    
+    
+        
+        Enter the passphrase to decrypt the file:
+        
+    
+    
+        
+        
+        The password cannot be empty
+        
+    
+    
+        
+        Enter passphrase to encrypt your session keys:
+        
+    
+    
+        
+        File to save the exported session keys
+        
+    
+
+
+    WelcomePage
+    
+        
+        Welcome to nheko! The desktop client for the Matrix protocol.
+        
+    
+    
+        
+        Enjoy your stay!
+        
+    
+    
+        
+        REGISTER
+        
+    
+    
+        
+        LOGIN
+        
+    
+
+
+    descriptiveTime
+    
+        
+        Yesterday
+        
+    
+
+
+    dialogs::CreateRoom
+    
+        
+        Create room
+        
+    
+    
+        
+        Cancel
+        
+    
+    
+        
+        Name
+        
+    
+    
+        
+        Topic
+        
+    
+    
+        
+        Alias
+        
+    
+    
+        
+        Room Visibility
+        
+    
+    
+        
+        Room Preset
+        
+    
+    
+        
+        Direct Chat
+        
+    
+
+
+    dialogs::FallbackAuth
+    
+        
+        Open Fallback in Browser
+        
+    
+    
+        
+        Cancel
+        
+    
+    
+        
+        Confirm
+        
+    
+    
+        
+        Open the fallback, follow the steps and confirm after completing them.
+        
+    
+
+
+    dialogs::InviteUsers
+    
+        
+        Cancel
+        
+    
+    
+        
+        User ID to invite
+        
+    
+
+
+    dialogs::JoinRoom
+    
+        
+        Join
+        
+    
+    
+        
+        Cancel
+        
+    
+    
+        
+        Room ID or alias
+        
+    
+
+
+    dialogs::LeaveRoom
+    
+        
+        Cancel
+        
+    
+    
+        
+        Are you sure you want to leave?
+        
+    
+
+
+    dialogs::Logout
+    
+        
+        Cancel
+        
+    
+    
+        
+        Logout. Are you sure?
+        
+    
+
+
+    dialogs::PreviewUploadOverlay
+    
+        
+        Upload
+        
+    
+    
+        
+        Cancel
+        
+    
+    
+        
+        Media type: %1
+Media size: %2
+
+        
+    
+
+
+    dialogs::ReCaptcha
+    
+        
+        Cancel
+        
+    
+    
+        
+        Confirm
+        
+    
+    
+        
+        Solve the reCAPTCHA and press the confirm button
+        
+    
+
+
+    dialogs::ReadReceipts
+    
+        
+        Read receipts
+        
+    
+    
+        
+        Close
+        
+    
+
+
+    dialogs::ReceiptItem
+    
+        
+        Today %1
+        
+    
+    
+        
+        Yesterday %1
+        
+    
+
+
+    dialogs::RoomSettings
+    
+        
+        Settings
+        
+    
+    
+        
+        Info
+        
+    
+    
+        
+        Internal ID
+        
+    
+    
+        
+        Room Version
+        
+    
+    
+        
+        Notifications
+        
+    
+    
+        
+        Muted
+        
+    
+    
+        
+        Mentions only
+        
+    
+    
+        
+        All messages
+        
+    
+    
+        
+        Room access
+        
+    
+    
+        
+        Anyone and guests
+        
+    
+    
+        
+        Anyone
+        
+    
+    
+        
+        Invited users
+        
+    
+    
+        
+        Encryption
+        
+    
+    
+        
+        End-to-End Encryption
+        
+    
+    
+        
+        Encryption is currently experimental and things might break unexpectedly. <br>Please take note that it can't be disabled afterwards.
+        
+    
+    
+        
+        Respond to key requests
+        
+    
+    
+        
+        Whether or not the client should respond automatically with the session keys
+ upon request. Use with caution, this is a temporary measure to test the
+ E2E implementation until device verification is completed.
+        
+    
+    
+        
+        %n member(s)
+        
+            
+            
+            
+        
+    
+    
+        
+        Failed to enable encryption: %1
+        
+    
+    
+        
+        Select an avatar
+        
+    
+    
+        
+        All Files (*)
+        
+    
+    
+        
+        The selected file is not an image
+        
+    
+    
+        
+        Error while reading file: %1
+        
+    
+    
+        
+        
+        Failed to upload image: %s
+        
+    
+
+
+    dialogs::UserProfile
+    
+        
+        Ban the user from the room
+        
+    
+    
+        
+        Ignore messages from this user
+        
+    
+    
+        
+        Kick the user from the room
+        
+    
+    
+        
+        Start a conversation
+        
+    
+    
+        
+        Devices
+        
+    
+
+
+    emoji::Panel
+    
+        
+        Smileys & People
+        
+    
+    
+        
+        Animals & Nature
+        
+    
+    
+        
+        Food & Drink
+        
+    
+    
+        
+        Activity
+        
+    
+    
+        
+        Travel & Places
+        
+    
+    
+        
+        Objects
+        
+    
+    
+        
+        Symbols
+        
+    
+    
+        
+        Flags
+        
+    
+
+
+    message-description sent:
+    
+        
+        You sent an audio clip
+        
+    
+    
+        
+        %1 sent an audio clip
+        
+    
+    
+        
+        You sent an image
+        
+    
+    
+        
+        %1 sent an image
+        
+    
+    
+        
+        You sent a file
+        
+    
+    
+        
+        %1 sent a file
+        
+    
+    
+        
+        You sent a video
+        
+    
+    
+        
+        %1 sent a video
+        
+    
+    
+        
+        You sent a sticker
+        
+    
+    
+        
+        %1 sent a sticker
+        
+    
+    
+        
+        You sent a notification
+        
+    
+    
+        
+        %1 sent a notification
+        
+    
+    
+        
+        You: %1
+        
+    
+    
+        
+        %1: %2
+        
+    
+    
+        
+        You sent an encrypted message
+        
+    
+    
+        
+        %1 sent an encrypted message
+        
+    
+
+
+    popups::UserMentions
+    
+        
+        This Room
+        
+    
+    
+        
+        All Rooms
+        
+    
+
+
+    utils
+    
+        
+        Unknown Message Type
+        
+    
+
+

From 7c1ca38d98d77f0152321c70c3fc8e9c8123a533 Mon Sep 17 00:00:00 2001
From: Mihai Fufezan 
Date: Mon, 17 Aug 2020 14:19:06 +0000
Subject: [PATCH 42/50] Translated using Weblate (English)

Currently translated at 85.3% (285 of 334 strings)

Translation: Nheko/nheko
Translate-URL: http://weblate.nheko.im/projects/nheko/nheko-master/en/
---
 resources/langs/nheko_en.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/resources/langs/nheko_en.ts b/resources/langs/nheko_en.ts
index f2bb04f9..1e9128d0 100644
--- a/resources/langs/nheko_en.ts
+++ b/resources/langs/nheko_en.ts
@@ -198,7 +198,7 @@
         
         
         Search
-        
+        Search
     
     
         

From 7f7108161e87df272aefb9a14aec708ff427839f Mon Sep 17 00:00:00 2001
From: Nicolas Werner 
Date: Mon, 17 Aug 2020 23:30:36 +0200
Subject: [PATCH 43/50] Hide CallCandidates again in new store

---
 resources/qml/delegates/MessageDelegate.qml |  6 ++++
 src/Cache.cpp                               | 32 +++++++++++++++++++--
 src/timeline/TimelineModel.cpp              |  6 +++-
 src/timeline/TimelineModel.h                |  2 ++
 4 files changed, 43 insertions(+), 3 deletions(-)

diff --git a/resources/qml/delegates/MessageDelegate.qml b/resources/qml/delegates/MessageDelegate.qml
index 7b6e0703..56b8040e 100644
--- a/resources/qml/delegates/MessageDelegate.qml
+++ b/resources/qml/delegates/MessageDelegate.qml
@@ -114,6 +114,12 @@ Item {
 				text: qsTr("%1 ended the call.").arg(model.data.userName)
 			}
 		}
+		DelegateChoice {
+			roleValue: MtxEvent.CallCandidates
+			NoticeMessage {
+				text: qsTr("Negotiating call...")
+			}
+		}
 		DelegateChoice {
 			// TODO: make a more complex formatter for the power levels.
 			roleValue: MtxEvent.PowerLevels
diff --git a/src/Cache.cpp b/src/Cache.cpp
index fd26f63e..e41ad7ca 100644
--- a/src/Cache.cpp
+++ b/src/Cache.cpp
@@ -33,6 +33,7 @@
 #include "Cache_p.h"
 #include "EventAccessors.h"
 #include "Logging.h"
+#include "Olm.h"
 #include "Utils.h"
 
 //! Should be changed when a breaking change occurs in the cache format.
@@ -93,6 +94,33 @@ namespace {
 std::unique_ptr instance_ = nullptr;
 }
 
+static bool
+isHiddenEvent(mtx::events::collections::TimelineEvents e, const std::string &room_id)
+{
+        using namespace mtx::events;
+        if (auto encryptedEvent = std::get_if>(&e)) {
+                MegolmSessionIndex index;
+                index.room_id    = room_id;
+                index.session_id = encryptedEvent->content.session_id;
+                index.sender_key = encryptedEvent->content.sender_key;
+
+                auto result = olm::decryptEvent(index, *encryptedEvent);
+                if (!result.error)
+                        e = result.event.value();
+        }
+
+        static constexpr std::initializer_list hiddenEvents = {
+          EventType::Reaction, EventType::CallCandidates, EventType::Unsupported};
+
+        return std::visit(
+          [](const auto &ev) {
+                  return std::any_of(hiddenEvents.begin(),
+                                     hiddenEvents.end(),
+                                     [ev](EventType type) { return type == ev.type; });
+          },
+          e);
+}
+
 Cache::Cache(const QString &userId, QObject *parent)
   : QObject{parent}
   , env_{nullptr}
@@ -2406,7 +2434,7 @@ Cache::saveTimelineMessages(lmdb::txn &txn,
                         lmdb::dbi_put(txn, evToOrderDb, event_id, lmdb::val(&index, sizeof(index)));
 
                         // TODO(Nico): Allow blacklisting more event types in UI
-                        if (event["type"] != "m.reaction" && event["type"] != "m.dummy") {
+                        if (!isHiddenEvent(e, room_id)) {
                                 ++msgIndex;
                                 lmdb::cursor_put(msgCursor.handle(),
                                                  lmdb::val(&msgIndex, sizeof(msgIndex)),
@@ -2489,7 +2517,7 @@ Cache::saveOldMessages(const std::string &room_id, const mtx::responses::Message
                 lmdb::dbi_put(txn, evToOrderDb, event_id, lmdb::val(&index, sizeof(index)));
 
                 // TODO(Nico): Allow blacklisting more event types in UI
-                if (event["type"] != "m.reaction" && event["type"] != "m.dummy") {
+                if (!isHiddenEvent(e, room_id)) {
                         --msgIndex;
                         lmdb::dbi_put(
                           txn, order2msgDb, lmdb::val(&msgIndex, sizeof(msgIndex)), event_id);
diff --git a/src/timeline/TimelineModel.cpp b/src/timeline/TimelineModel.cpp
index 9695f850..b6c2d4bb 100644
--- a/src/timeline/TimelineModel.cpp
+++ b/src/timeline/TimelineModel.cpp
@@ -136,6 +136,11 @@ struct RoomEventType
         {
                 return qml_mtx_events::EventType::CallHangUp;
         }
+        qml_mtx_events::EventType operator()(
+          const mtx::events::Event &)
+        {
+                return qml_mtx_events::EventType::CallCandidates;
+        }
         // ::EventType::Type operator()(const Event &e) { return
         // ::EventType::LocationMessage; }
 };
@@ -1122,7 +1127,6 @@ struct SendMessageVisitor
                 }
         }
 
-
         // Do-nothing operator for all unhandled events
         template
         void operator()(const mtx::events::Event &)
diff --git a/src/timeline/TimelineModel.h b/src/timeline/TimelineModel.h
index 034ae31a..156606e6 100644
--- a/src/timeline/TimelineModel.h
+++ b/src/timeline/TimelineModel.h
@@ -42,6 +42,8 @@ enum EventType
         CallAnswer,
         /// m.call.hangup
         CallHangUp,
+        /// m.call.candidates
+        CallCandidates,
         /// m.room.canonical_alias
         CanonicalAlias,
         /// m.room.create

From 1402732b5ffec7446edb5c06c9b7004a480b987b Mon Sep 17 00:00:00 2001
From: trilene 
Date: Mon, 17 Aug 2020 17:42:06 -0400
Subject: [PATCH 44/50] Stop SendFile and Call buttons swapping places on file
 upload

---
 src/TextInputWidget.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/TextInputWidget.cpp b/src/TextInputWidget.cpp
index a3392170..633b12ba 100644
--- a/src/TextInputWidget.cpp
+++ b/src/TextInputWidget.cpp
@@ -633,7 +633,7 @@ TextInputWidget::showUploadSpinner()
         topLayout_->removeWidget(sendFileBtn_);
         sendFileBtn_->hide();
 
-        topLayout_->insertWidget(0, spinner_);
+        topLayout_->insertWidget(1, spinner_);
         spinner_->start();
 }
 
@@ -641,7 +641,7 @@ void
 TextInputWidget::hideUploadSpinner()
 {
         topLayout_->removeWidget(spinner_);
-        topLayout_->insertWidget(0, sendFileBtn_);
+        topLayout_->insertWidget(1, sendFileBtn_);
         sendFileBtn_->show();
         spinner_->stop();
 }

From d6bc05fcd6ac0b420460e5e8a456f07615ca505b Mon Sep 17 00:00:00 2001
From: Nicolas Werner 
Date: Mon, 17 Aug 2020 23:59:38 +0200
Subject: [PATCH 45/50] Bump mtxclient

---
 CMakeLists.txt                   | 2 +-
 io.github.NhekoReborn.Nheko.json | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index cf49f21a..7295cc54 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -341,7 +341,7 @@ if(USE_BUNDLED_MTXCLIENT)
 	FetchContent_Declare(
 		MatrixClient
 		GIT_REPOSITORY https://github.com/Nheko-Reborn/mtxclient.git
-		GIT_TAG        fa6e36dbcd922c1920873b3fcdfe0a9d283f082e
+		GIT_TAG        d8666a3f1a5b709b78ccea2b545d540a8cb502ca
 		)
 	FetchContent_MakeAvailable(MatrixClient)
 else()
diff --git a/io.github.NhekoReborn.Nheko.json b/io.github.NhekoReborn.Nheko.json
index 8cdd7b90..b11e587c 100644
--- a/io.github.NhekoReborn.Nheko.json
+++ b/io.github.NhekoReborn.Nheko.json
@@ -146,7 +146,7 @@
       "name": "mtxclient",
       "sources": [
         {
-          "commit": "fa6e36dbcd922c1920873b3fcdfe0a9d283f082e",
+          "commit": "d8666a3f1a5b709b78ccea2b545d540a8cb502ca",
           "type": "git",
           "url": "https://github.com/Nheko-Reborn/mtxclient.git"
         }

From d61d108a4f2cdd296ed86d344572c7074f1921c2 Mon Sep 17 00:00:00 2001
From: Tony O <822863+bqv@users.noreply.github.com>
Date: Sat, 22 Aug 2020 04:55:00 +0100
Subject: [PATCH 46/50] Update README.md

---
 README.md | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/README.md b/README.md
index 20340a46..fb0167c8 100644
--- a/README.md
+++ b/README.md
@@ -75,6 +75,14 @@ sudo eselect repository enable matrix
 sudo emerge -a nheko
 ```
 
+#### Nix(os)
+
+```bash
+nix-env -iA nixpkgs.nheko
+# or
+nix-shell -p nheko --run nheko
+```
+
 #### Alpine Linux (and postmarketOS)
 
 Make sure you have the testing repositories from `edge` enabled. Note that this is not needed on postmarketOS.

From 473293b6a5d06e670065dd4d35af0e456621e9e6 Mon Sep 17 00:00:00 2001
From: trilene 
Date: Sat, 22 Aug 2020 08:18:42 -0400
Subject: [PATCH 47/50] Under GStreamer >= 1.17 gather all candidates before
 sending offer/answer

---
 src/WebRTCSession.cpp | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/WebRTCSession.cpp b/src/WebRTCSession.cpp
index 2248fb1a..b4e7eeb3 100644
--- a/src/WebRTCSession.cpp
+++ b/src/WebRTCSession.cpp
@@ -223,18 +223,19 @@ addLocalICECandidate(GstElement *webrtc G_GNUC_UNUSED,
 {
         nhlog::ui()->debug("WebRTC: local candidate: (m-line:{}):{}", mlineIndex, candidate);
 
+#if GST_CHECK_VERSION(1, 17, 0)
+        localcandidates_.push_back({"audio", (uint16_t)mlineIndex, candidate});
+        return;
+#else
         if (WebRTCSession::instance().state() >= WebRTCSession::State::OFFERSENT) {
                 emit WebRTCSession::instance().newICECandidate(
                   {"audio", (uint16_t)mlineIndex, candidate});
                 return;
         }
 
-        localcandidates_.push_back({"audio", (uint16_t)mlineIndex, candidate});
-
         // GStreamer v1.16: webrtcbin's notify::ice-gathering-state triggers
         // GST_WEBRTC_ICE_GATHERING_STATE_COMPLETE too early. Fixed in v1.17.
         // Use a 100ms timeout in the meantime
-#if !GST_CHECK_VERSION(1, 17, 0)
         static guint timerid = 0;
         if (timerid)
                 g_source_remove(timerid);
@@ -447,6 +448,7 @@ WebRTCSession::startPipeline(int opusPayloadType)
                 g_object_set(webrtc_, "stun-server", stunServer_.c_str(), nullptr);
         }
 
+
         for (const auto &uri : turnServers_) {
                 nhlog::ui()->info("WebRTC: setting TURN server: {}", uri);
                 gboolean udata;

From 9f79b855799e8b11f971d2481621a3de344fac4a Mon Sep 17 00:00:00 2001
From: Nicolas Werner 
Date: Tue, 25 Aug 2020 23:05:20 +0200
Subject: [PATCH 48/50] Speedup db a bit, but loose some crash resiliency

The loss in durability shouldn't matter, if we can just receive the same
events again after a restart
---
 src/Cache.cpp | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/Cache.cpp b/src/Cache.cpp
index e41ad7ca..2231aaac 100644
--- a/src/Cache.cpp
+++ b/src/Cache.cpp
@@ -168,7 +168,10 @@ Cache::setup()
         }
 
         try {
-                env_.open(statePath.toStdString().c_str());
+                // NOTE(Nico): We may want to use (MDB_MAPASYNC | MDB_WRITEMAP) in the future, but
+                // it can really mess up our database, so we shouldn't. For now, hopefully
+                // NOMETASYNC is fast enough.
+                env_.open(statePath.toStdString().c_str(), MDB_NOMETASYNC);
         } catch (const lmdb::error &e) {
                 if (e.code() != MDB_VERSION_MISMATCH && e.code() != MDB_INVALID) {
                         throw std::runtime_error("LMDB initialization failed" +

From 3df4bde0324f9389a59749847cb5b79dff4bea1f Mon Sep 17 00:00:00 2001
From: Nicolas Werner 
Date: Tue, 25 Aug 2020 23:12:01 +0200
Subject: [PATCH 49/50] Add some log messages, that migrations are in progress

---
 src/Cache.cpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/Cache.cpp b/src/Cache.cpp
index 2231aaac..91cde9e7 100644
--- a/src/Cache.cpp
+++ b/src/Cache.cpp
@@ -787,6 +787,7 @@ Cache::runMigrations()
            }},
         };
 
+        nhlog::db()->info("Running migrations, this may take a while!");
         for (const auto &[target_version, migration] : migrations) {
                 if (target_version > stored_version)
                         if (!migration()) {
@@ -794,6 +795,7 @@ Cache::runMigrations()
                                 return false;
                         }
         }
+        nhlog::db()->info("Migrations finished.");
 
         setCurrentFormat();
         return true;

From 67a6ab401b90445169ae35a4107a9973474f8073 Mon Sep 17 00:00:00 2001
From: trilene 
Date: Fri, 28 Aug 2020 10:49:39 -0400
Subject: [PATCH 50/50] Link GStreamer elements before syncing state

---
 src/WebRTCSession.cpp | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/WebRTCSession.cpp b/src/WebRTCSession.cpp
index b4e7eeb3..f5dc49d8 100644
--- a/src/WebRTCSession.cpp
+++ b/src/WebRTCSession.cpp
@@ -283,11 +283,11 @@ linkNewPad(GstElement *decodebin G_GNUC_UNUSED, GstPad *newpad, GstElement *pipe
                 GstElement *resample = gst_element_factory_make("audioresample", nullptr);
                 GstElement *sink     = gst_element_factory_make("autoaudiosink", nullptr);
                 gst_bin_add_many(GST_BIN(pipe), queue, convert, resample, sink, nullptr);
+                gst_element_link_many(queue, convert, resample, sink, nullptr);
                 gst_element_sync_state_with_parent(queue);
                 gst_element_sync_state_with_parent(convert);
                 gst_element_sync_state_with_parent(resample);
                 gst_element_sync_state_with_parent(sink);
-                gst_element_link_many(queue, convert, resample, sink, nullptr);
                 queuepad = gst_element_get_static_pad(queue, "sink");
         }
 
@@ -448,7 +448,6 @@ WebRTCSession::startPipeline(int opusPayloadType)
                 g_object_set(webrtc_, "stun-server", stunServer_.c_str(), nullptr);
         }
 
-
         for (const auto &uri : turnServers_) {
                 nhlog::ui()->info("WebRTC: setting TURN server: {}", uri);
                 gboolean udata;