From aa9b453f8109b70c9e65fd21f2f3fcce0e73b544 Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Tue, 20 Oct 2020 13:46:05 +0200 Subject: [PATCH] Store timestamp with olm sessions --- src/Cache.cpp | 145 +++++++++++++++++++++++++++++++-- src/Cache.h | 6 +- src/CacheCryptoStructs.h | 10 +++ src/Cache_p.h | 8 +- src/Olm.cpp | 26 ++++-- src/Olm.h | 5 ++ src/timeline/EventStore.cpp | 6 ++ src/timeline/EventStore.h | 1 + src/timeline/TimelineModel.cpp | 3 +- 9 files changed, 191 insertions(+), 19 deletions(-) diff --git a/src/Cache.cpp b/src/Cache.cpp index 5a0740f1..d9db99b0 100644 --- a/src/Cache.cpp +++ b/src/Cache.cpp @@ -40,7 +40,7 @@ //! Should be changed when a breaking change occurs in the cache format. //! This will reset client's data. -static const std::string CURRENT_CACHE_FORMAT_VERSION("2020.07.05"); +static const std::string CURRENT_CACHE_FORMAT_VERSION("2020.10.20"); static const std::string SECRET("secret"); static lmdb::val NEXT_BATCH_KEY("next_batch"); @@ -437,7 +437,9 @@ Cache::getOutboundMegolmSession(const std::string &room_id) // void -Cache::saveOlmSession(const std::string &curve25519, mtx::crypto::OlmSessionPtr session) +Cache::saveOlmSession(const std::string &curve25519, + mtx::crypto::OlmSessionPtr session, + uint64_t timestamp) { using namespace mtx::crypto; @@ -447,7 +449,11 @@ Cache::saveOlmSession(const std::string &curve25519, mtx::crypto::OlmSessionPtr const auto pickled = pickle(session.get(), SECRET); const auto session_id = mtx::crypto::session_id(session.get()); - lmdb::dbi_put(txn, db, lmdb::val(session_id), lmdb::val(pickled)); + StoredOlmSession stored_session; + stored_session.pickled_session = pickled; + stored_session.last_message_ts = timestamp; + + lmdb::dbi_put(txn, db, lmdb::val(session_id), lmdb::val(json(stored_session).dump())); txn.commit(); } @@ -466,13 +472,44 @@ Cache::getOlmSession(const std::string &curve25519, const std::string &session_i txn.commit(); if (found) { - auto data = std::string(pickled.data(), pickled.size()); - return unpickle(data, SECRET); + std::string_view raw(pickled.data(), pickled.size()); + auto data = json::parse(raw).get(); + return unpickle(data.pickled_session, SECRET); } return std::nullopt; } +std::optional +Cache::getLatestOlmSession(const std::string &curve25519) +{ + using namespace mtx::crypto; + + auto txn = lmdb::txn::begin(env_); + auto db = getOlmSessionsDb(txn, curve25519); + + std::string session_id, pickled_session; + std::vector res; + + std::optional currentNewest; + + auto cursor = lmdb::cursor::open(txn, db); + while (cursor.get(session_id, pickled_session, MDB_NEXT)) { + auto data = + json::parse(std::string_view(pickled_session.data(), pickled_session.size())) + .get(); + if (!currentNewest || currentNewest->last_message_ts < data.last_message_ts) + currentNewest = data; + } + cursor.close(); + + txn.commit(); + + return currentNewest + ? std::optional(unpickle(currentNewest->pickled_session, SECRET)) + : std::nullopt; +} + std::vector Cache::getOlmSessions(const std::string &curve25519) { @@ -828,6 +865,80 @@ Cache::runMigrations() nhlog::db()->info("Successfully deleted pending receipts database."); return true; }}, + {"2020.10.20", + [this]() { + try { + using namespace mtx::crypto; + + auto txn = lmdb::txn::begin(env_); + + auto mainDb = lmdb::dbi::open(txn, nullptr); + + std::string dbName, ignored; + auto olmDbCursor = lmdb::cursor::open(txn, mainDb); + while (olmDbCursor.get(dbName, ignored, MDB_NEXT)) { + // skip every db but olm session dbs + nhlog::db()->debug("Db {}", dbName); + if (dbName.find("olm_sessions/") != 0) + continue; + + nhlog::db()->debug("Migrating {}", dbName); + + auto olmDb = lmdb::dbi::open(txn, dbName.c_str()); + + std::string session_id, session_value; + + std::vector> sessions; + + auto cursor = lmdb::cursor::open(txn, olmDb); + while (cursor.get(session_id, session_value, MDB_NEXT)) { + nhlog::db()->debug("session_id {}, session_value {}", + session_id, + session_value); + StoredOlmSession session; + bool invalid = false; + for (auto c : session_value) + if (!isprint(c)) { + invalid = true; + break; + } + if (invalid) + continue; + + nhlog::db()->debug("Not skipped"); + + session.pickled_session = session_value; + sessions.emplace_back(session_id, session); + } + cursor.close(); + + olmDb.drop(txn, true); + + auto newDbName = dbName; + newDbName.erase(0, sizeof("olm_sessions") - 1); + newDbName = "olm_sessions.v2" + newDbName; + + auto newDb = lmdb::dbi::open(txn, newDbName.c_str(), MDB_CREATE); + + for (const auto &[key, value] : sessions) { + nhlog::db()->debug("{}\n{}", key, json(value).dump()); + lmdb::dbi_put(txn, + newDb, + lmdb::val(key), + lmdb::val(json(value).dump())); + } + } + olmDbCursor.close(); + + txn.commit(); + } catch (const lmdb::error &) { + nhlog::db()->critical("Failed to migrate olm sessions,"); + return false; + } + + nhlog::db()->info("Successfully migrated olm sessions."); + return true; + }}, }; nhlog::db()->info("Running migrations, this may take a while!"); @@ -3629,6 +3740,19 @@ from_json(const nlohmann::json &obj, MegolmSessionIndex &msg) msg.sender_key = obj.at("sender_key"); } +void +to_json(nlohmann::json &obj, const StoredOlmSession &msg) +{ + obj["ts"] = msg.last_message_ts; + obj["s"] = msg.pickled_session; +} +void +from_json(const nlohmann::json &obj, StoredOlmSession &msg) +{ + msg.last_message_ts = obj.at("ts").get(); + msg.pickled_session = obj.at("s").get(); +} + namespace cache { void init(const QString &user_id) @@ -4114,9 +4238,11 @@ inboundMegolmSessionExists(const MegolmSessionIndex &index) // Olm Sessions // void -saveOlmSession(const std::string &curve25519, mtx::crypto::OlmSessionPtr session) +saveOlmSession(const std::string &curve25519, + mtx::crypto::OlmSessionPtr session, + uint64_t timestamp) { - instance_->saveOlmSession(curve25519, std::move(session)); + instance_->saveOlmSession(curve25519, std::move(session), timestamp); } std::vector getOlmSessions(const std::string &curve25519) @@ -4128,6 +4254,11 @@ getOlmSession(const std::string &curve25519, const std::string &session_id) { return instance_->getOlmSession(curve25519, session_id); } +std::optional +getLatestOlmSession(const std::string &curve25519) +{ + return instance_->getLatestOlmSession(curve25519); +} void saveOlmAccount(const std::string &pickled) diff --git a/src/Cache.h b/src/Cache.h index cd96708e..24b6df9e 100644 --- a/src/Cache.h +++ b/src/Cache.h @@ -292,11 +292,15 @@ inboundMegolmSessionExists(const MegolmSessionIndex &index); // Olm Sessions // void -saveOlmSession(const std::string &curve25519, mtx::crypto::OlmSessionPtr session); +saveOlmSession(const std::string &curve25519, + mtx::crypto::OlmSessionPtr session, + uint64_t timestamp); std::vector getOlmSessions(const std::string &curve25519); std::optional getOlmSession(const std::string &curve25519, const std::string &session_id); +std::optional +getLatestOlmSession(const std::string &curve25519); void saveOlmAccount(const std::string &pickled); diff --git a/src/CacheCryptoStructs.h b/src/CacheCryptoStructs.h index 935d6493..a693e233 100644 --- a/src/CacheCryptoStructs.h +++ b/src/CacheCryptoStructs.h @@ -66,6 +66,16 @@ struct OlmSessionStorage std::mutex group_inbound_mtx; }; +struct StoredOlmSession +{ + std::uint64_t last_message_ts = 0; + std::string pickled_session; +}; +void +to_json(nlohmann::json &obj, const StoredOlmSession &msg); +void +from_json(const nlohmann::json &obj, StoredOlmSession &msg); + //! Verification status of a single user struct VerificationStatus { diff --git a/src/Cache_p.h b/src/Cache_p.h index b3f4c58c..62b1ad37 100644 --- a/src/Cache_p.h +++ b/src/Cache_p.h @@ -266,10 +266,14 @@ public: // // Olm Sessions // - void saveOlmSession(const std::string &curve25519, mtx::crypto::OlmSessionPtr session); + void saveOlmSession(const std::string &curve25519, + mtx::crypto::OlmSessionPtr session, + uint64_t timestamp); std::vector getOlmSessions(const std::string &curve25519); std::optional getOlmSession(const std::string &curve25519, const std::string &session_id); + std::optional getLatestOlmSession( + const std::string &curve25519); void saveOlmAccount(const std::string &pickled); std::string restoreOlmAccount(); @@ -565,7 +569,7 @@ private: lmdb::dbi getOlmSessionsDb(lmdb::txn &txn, const std::string &curve25519_key) { return lmdb::dbi::open( - txn, std::string("olm_sessions/" + curve25519_key).c_str(), MDB_CREATE); + txn, std::string("olm_sessions.v2/" + curve25519_key).c_str(), MDB_CREATE); } QString getDisplayName(const mtx::events::StateEvent &event) diff --git a/src/Olm.cpp b/src/Olm.cpp index fee685a3..e3b0de27 100644 --- a/src/Olm.cpp +++ b/src/Olm.cpp @@ -47,7 +47,7 @@ handle_to_device_messages(const std::vectorwarn( @@ -56,10 +56,6 @@ handle_to_device_messages(const std::vectorwarn("validation error for olm message: {} {}", e.what(), j_msg.dump(2)); - - nhlog::crypto()->warn("validation error for olm message: {} {}", - e.what(), - j_msg.dump(2)); } } else if (msg_type == to_string(mtx::events::EventType::RoomKeyRequest)) { @@ -250,7 +246,8 @@ handle_pre_key_olm_message(const std::string &sender, nhlog::crypto()->debug("decrypted message: \n {}", plaintext.dump(2)); try { - cache::saveOlmSession(sender_key, std::move(inbound_session)); + cache::saveOlmSession( + sender_key, std::move(inbound_session), QDateTime::currentMSecsSinceEpoch()); } catch (const lmdb::error &e) { nhlog::db()->warn( "failed to save inbound olm session from {}: {}", sender, e.what()); @@ -318,7 +315,8 @@ try_olm_decryption(const std::string &sender_key, const mtx::events::msg::OlmCip try { text = olm::client()->decrypt_message(session->get(), msg.type, msg.body); - cache::saveOlmSession(id, std::move(session.value())); + cache::saveOlmSession( + id, std::move(session.value()), QDateTime::currentMSecsSinceEpoch()); } catch (const mtx::crypto::olm_exception &e) { nhlog::crypto()->debug("failed to decrypt olm message ({}, {}) with {}: {}", msg.type, @@ -672,7 +670,9 @@ send_megolm_key_to_device(const std::string &user_id, device_msg = olm::client()->create_olm_encrypted_content( olm_session.get(), json(room_key).dump(), pks.curve25519); - cache::saveOlmSession(pks.curve25519, std::move(olm_session)); + cache::saveOlmSession(pks.curve25519, + std::move(olm_session), + QDateTime::currentMSecsSinceEpoch()); } catch (const json::exception &e) { nhlog::crypto()->warn("creating outbound session: {}", e.what()); @@ -749,4 +749,14 @@ decryptEvent(const MegolmSessionIndex &index, return {std::nullopt, std::nullopt, std::move(te.data)}; } + +//! Send encrypted to device messages, targets is a map from userid to device ids or "*" +void +send_encrypted_to_device_messages(const std::map> targets, + const mtx::events::collections::DeviceEvents &event) +{ + (void)targets; + (void)event; +} + } // namespace olm diff --git a/src/Olm.h b/src/Olm.h index cda9f29a..2556a22d 100644 --- a/src/Olm.h +++ b/src/Olm.h @@ -110,4 +110,9 @@ send_megolm_key_to_device(const std::string &user_id, const std::string &device_id, const mtx::events::msg::ForwardedRoomKey &payload); +//! Send encrypted to device messages, targets is a map from userid to device ids or "*" +void +send_encrypted_to_device_messages(const std::map> targets, + const mtx::events::collections::DeviceEvents &event); + } // namespace olm diff --git a/src/timeline/EventStore.cpp b/src/timeline/EventStore.cpp index d3c5c3fa..3564ffc0 100644 --- a/src/timeline/EventStore.cpp +++ b/src/timeline/EventStore.cpp @@ -54,6 +54,9 @@ EventStore::EventStore(std::string room_id, QObject *) &EventStore::oldMessagesRetrieved, this, [this](const mtx::responses::Messages &res) { + if (cache::client()->previousBatchToken(room_id_) == res.end) + noMoreMessages = true; + uint64_t newFirst = cache::client()->saveOldMessages(room_id_, res); if (newFirst == first) fetchMore(); @@ -687,6 +690,9 @@ EventStore::get(std::string_view id, std::string_view related_to, bool decrypt) void EventStore::fetchMore() { + if (noMoreMessages) + return; + mtx::http::MessagesOpts opts; opts.room_id = room_id_; opts.from = cache::client()->previousBatchToken(room_id_); diff --git a/src/timeline/EventStore.h b/src/timeline/EventStore.h index 954e271c..7f8e2396 100644 --- a/src/timeline/EventStore.h +++ b/src/timeline/EventStore.h @@ -123,4 +123,5 @@ private: std::string current_txn; int current_txn_error_count = 0; + bool noMoreMessages = false; }; diff --git a/src/timeline/TimelineModel.cpp b/src/timeline/TimelineModel.cpp index 359e95bc..5db6aa00 100644 --- a/src/timeline/TimelineModel.cpp +++ b/src/timeline/TimelineModel.cpp @@ -1138,7 +1138,8 @@ TimelineModel::handleClaimedKeys( pks.at(user_id).at(device_id).curve25519); try { - cache::saveOlmSession(id_key, std::move(s)); + cache::saveOlmSession( + id_key, std::move(s), QDateTime::currentMSecsSinceEpoch()); } catch (const lmdb::error &e) { nhlog::db()->critical("failed to save outbound olm session: {}", e.what());