rnhmjoj
/
nheko
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Readd pagination and fix redactions

This commit is contained in:
Nicolas Werner 2020-07-13 00:08:58 +02:00
parent 9479fcde08
commit 9ae7d0dce3
7 changed files with 255 additions and 122 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
resources/qml/delegates/MessageDelegate.qml
View File

@ -66,6 +66,12 @@ Item {
text: qsTr("redacted")
}
}
DelegateChoice {
roleValue: MtxEvent.Redaction
Pill {
text: qsTr("redacted")
}
}
DelegateChoice {
roleValue: MtxEvent.Encryption
Pill {

171
src/Cache.cpp
View File

@ -1241,8 +1241,25 @@ Cache::getTimelineMentions()
return notifs;
}
std::string
Cache::previousBatchToken(const std::string &room_id)
{
auto txn = lmdb::txn::begin(env_, nullptr);
auto orderDb = getEventOrderDb(txn, room_id);
auto cursor = lmdb::cursor::open(txn, orderDb);
lmdb::val indexVal, val;
if (!cursor.get(indexVal, val, MDB_FIRST)) {
return "";
}
auto j = json::parse(std::string_view(val.data(), val.size()));
return j.value("prev_batch", "");
}
Cache::Messages
Cache::getTimelineMessages(lmdb::txn &txn, const std::string &room_id, int64_t index, bool forward)
Cache::getTimelineMessages(lmdb::txn &txn, const std::string &room_id, uint64_t index, bool forward)
{
// TODO(nico): Limit the messages returned by this maybe?
auto orderDb = getOrderToMessageDb(txn, room_id);
@ -1253,16 +1270,16 @@ Cache::getTimelineMessages(lmdb::txn &txn, const std::string &room_id, int64_t i
lmdb::val indexVal, event_id;
auto cursor = lmdb::cursor::open(txn, orderDb);
if (index == std::numeric_limits<int64_t>::max()) {
if (index == std::numeric_limits<uint64_t>::max()) {
if (cursor.get(indexVal, event_id, forward ? MDB_FIRST : MDB_LAST)) {
index = *indexVal.data<int64_t>();
index = *indexVal.data<uint64_t>();
} else {
messages.end_of_cache = true;
return messages;
}
} else {
if (cursor.get(indexVal, event_id, MDB_SET)) {
index = *indexVal.data<int64_t>();
index = *indexVal.data<uint64_t>();
} else {
messages.end_of_cache = true;
return messages;
@ -1296,7 +1313,7 @@ Cache::getTimelineMessages(lmdb::txn &txn, const std::string &room_id, int64_t i
cursor.close();
// std::reverse(timeline.events.begin(), timeline.events.end());
messages.next_index = *indexVal.data<int64_t>();
messages.next_index = *indexVal.data<uint64_t>();
messages.end_of_cache = !ret;
return messages;
@ -1402,16 +1419,16 @@ Cache::getTimelineRange(const std::string &room_id)
}
TimelineRange range{};
range.last = *indexVal.data<int64_t>();
range.last = *indexVal.data<uint64_t>();
if (!cursor.get(indexVal, val, MDB_FIRST)) {
return {};
}
range.first = *indexVal.data<int64_t>();
range.first = *indexVal.data<uint64_t>();
return range;
}
std::optional<int64_t>
std::optional<uint64_t>
Cache::getTimelineIndex(const std::string &room_id, std::string_view event_id)
{
auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY);
@ -1424,11 +1441,11 @@ Cache::getTimelineIndex(const std::string &room_id, std::string_view event_id)
return {};
}
return *val.data<int64_t>();
return *val.data<uint64_t>();
}
std::optional<std::string>
Cache::getTimelineEventId(const std::string &room_id, int64_t index)
Cache::getTimelineEventId(const std::string &room_id, uint64_t index)
{
auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY);
auto orderDb = getOrderToMessageDb(txn, room_id);
@ -2074,6 +2091,9 @@ Cache::saveTimelineMessages(lmdb::txn &txn,
const std::string &room_id,
const mtx::responses::Timeline &res)
{
if (res.events.empty())
return;
auto eventsDb = getEventsDb(txn, room_id);
auto relationsDb = getRelationsDb(txn, room_id);
@ -2090,16 +2110,16 @@ Cache::saveTimelineMessages(lmdb::txn &txn,
using namespace mtx::events::state;
lmdb::val indexVal, val;
int64_t index = 0;
uint64_t index = std::numeric_limits<uint64_t>::max() / 2;
auto cursor = lmdb::cursor::open(txn, orderDb);
if (cursor.get(indexVal, val, MDB_LAST)) {
index = *indexVal.data<int64_t>();
}
int64_t msgIndex = 0;
uint64_t msgIndex = std::numeric_limits<uint64_t>::max() / 2;
auto msgCursor = lmdb::cursor::open(txn, order2msgDb);
if (msgCursor.get(indexVal, val, MDB_LAST)) {
msgIndex = *indexVal.data<int64_t>();
msgIndex = *indexVal.data<uint64_t>();
}
bool first = true;
@ -2111,39 +2131,19 @@ Cache::saveTimelineMessages(lmdb::txn &txn,
continue;
lmdb::val ev{};
bool success =
lmdb::dbi_get(txn, eventsDb, lmdb::val(redaction->redacts), ev);
if (!success)
continue;
lmdb::dbi_put(
txn, eventsDb, lmdb::val(redaction->redacts), lmdb::val(event.dump()));
lmdb::dbi_put(
txn, eventsDb, lmdb::val(redaction->event_id), lmdb::val(event.dump()));
mtx::events::collections::TimelineEvent te;
try {
mtx::events::collections::from_json(
json::parse(std::string_view(ev.data(), ev.size())), te);
} catch (std::exception &e) {
nhlog::db()->error("Failed to parse message from cache {}",
e.what());
continue;
lmdb::val oldIndex{};
if (lmdb::dbi_get(
txn, msg2orderDb, lmdb::val(redaction->redacts), oldIndex)) {
lmdb::dbi_put(
txn, order2msgDb, oldIndex, lmdb::val(redaction->event_id));
lmdb::dbi_put(
txn, msg2orderDb, lmdb::val(redaction->event_id), oldIndex);
}
auto redactedEvent = std::visit(
[](const auto &ev) -> mtx::events::RoomEvent<mtx::events::msg::Redacted> {
mtx::events::RoomEvent<mtx::events::msg::Redacted> replacement =
{};
replacement.event_id = ev.event_id;
replacement.room_id = ev.room_id;
replacement.sender = ev.sender;
replacement.origin_server_ts = ev.origin_server_ts;
replacement.type = ev.type;
return replacement;
},
te.data);
lmdb::dbi_put(txn,
eventsDb,
lmdb::val(redaction->redacts),
lmdb::val(json(redactedEvent).dump()));
} else {
std::string event_id_val = event["event_id"].get<std::string>();
lmdb::val event_id = event_id_val;
@ -2193,6 +2193,83 @@ Cache::saveTimelineMessages(lmdb::txn &txn,
}
}
uint64_t
Cache::saveOldMessages(const std::string &room_id, const mtx::responses::Messages &res)
{
auto txn = lmdb::txn::begin(env_);
auto eventsDb = getEventsDb(txn, room_id);
auto relationsDb = getRelationsDb(txn, room_id);
auto orderDb = getEventOrderDb(txn, room_id);
auto msg2orderDb = getMessageToOrderDb(txn, room_id);
auto order2msgDb = getOrderToMessageDb(txn, room_id);
lmdb::val indexVal, val;
uint64_t index = std::numeric_limits<uint64_t>::max() / 2;
auto cursor = lmdb::cursor::open(txn, orderDb);
if (cursor.get(indexVal, val, MDB_FIRST)) {
index = *indexVal.data<uint64_t>();
}
uint64_t msgIndex = std::numeric_limits<uint64_t>::max() / 2;
auto msgCursor = lmdb::cursor::open(txn, order2msgDb);
if (msgCursor.get(indexVal, val, MDB_FIRST)) {
msgIndex = *indexVal.data<uint64_t>();
}
if (res.chunk.empty())
return index;
std::string event_id_val;
for (const auto &e : res.chunk) {
auto event = mtx::accessors::serialize_event(e);
event_id_val = event["event_id"].get<std::string>();
lmdb::val event_id = event_id_val;
lmdb::dbi_put(txn, eventsDb, event_id, lmdb::val(event.dump()));
--index;
json orderEntry = json::object();
orderEntry["event_id"] = event_id_val;
nhlog::db()->debug("saving '{}'", orderEntry.dump());
lmdb::dbi_put(
txn, orderDb, lmdb::val(&index, sizeof(index)), lmdb::val(orderEntry.dump()));
// TODO(Nico): Allow blacklisting more event types in UI
if (event["type"] != "m.reaction" && event["type"] != "m.dummy") {
--msgIndex;
lmdb::dbi_put(
txn, order2msgDb, lmdb::val(&msgIndex, sizeof(msgIndex)), event_id);
lmdb::dbi_put(
txn, msg2orderDb, event_id, lmdb::val(&msgIndex, sizeof(msgIndex)));
}
if (event.contains("content") && event["content"].contains("m.relates_to")) {
auto temp = event["content"]["m.relates_to"];
std::string relates_to = temp.contains("m.in_reply_to")
? temp["m.in_reply_to"]["event_id"]
: temp["event_id"];
if (!relates_to.empty())
lmdb::dbi_put(txn, relationsDb, lmdb::val(relates_to), event_id);
}
}
json orderEntry = json::object();
orderEntry["event_id"] = event_id_val;
orderEntry["prev_batch"] = res.end;
lmdb::cursor_put(
cursor.handle(), lmdb::val(&index, sizeof(index)), lmdb::val(orderEntry.dump()));
nhlog::db()->debug("saving '{}'", orderEntry.dump());
txn.commit();
return msgIndex;
}
mtx::responses::Notifications
Cache::getTimelineMentionsForRoom(lmdb::txn &txn, const std::string &room_id)
{
@ -2337,14 +2414,14 @@ Cache::deleteOldMessages()
auto eventsDb = getEventsDb(txn, room_id);
auto cursor = lmdb::cursor::open(txn, orderDb);
int64_t first, last;
uint64_t first, last;
if (cursor.get(indexVal, val, MDB_LAST)) {
last = *indexVal.data<int64_t>();
last = *indexVal.data<uint64_t>();
} else {
continue;
}
if (cursor.get(indexVal, val, MDB_FIRST)) {
first = *indexVal.data<int64_t>();
first = *indexVal.data<uint64_t>();
} else {
continue;
}

11
src/Cache_p.h
View File

@ -179,7 +179,7 @@ public:
};
Messages getTimelineMessages(lmdb::txn &txn,
const std::string &room_id,
int64_t index = std::numeric_limits<int64_t>::max(),
uint64_t index = std::numeric_limits<uint64_t>::max(),
bool forward = false);
std::optional<mtx::events::collections::TimelineEvent> getEvent(
@ -190,12 +190,15 @@ public:
const mtx::events::collections::TimelineEvent &event);
struct TimelineRange
{
int64_t first, last;
uint64_t first, last;
};
std::optional<TimelineRange> getTimelineRange(const std::string &room_id);
std::optional<int64_t> getTimelineIndex(const std::string &room_id,
std::optional<uint64_t> getTimelineIndex(const std::string &room_id,
std::string_view event_id);
std::optional<std::string> getTimelineEventId(const std::string &room_id, int64_t index);
std::optional<std::string> getTimelineEventId(const std::string &room_id, uint64_t index);
std::string previousBatchToken(const std::string &room_id);
uint64_t saveOldMessages(const std::string &room_id, const mtx::responses::Messages &res);
//! Remove old unused data.
void deleteOldMessages();

8
src/ChatPage.cpp
View File

@ -795,7 +795,6 @@ ChatPage::loadStateFromCache()
nhlog::db()->info("restoring state from cache");
QtConcurrent::run([this]() {
try {
cache::restoreSessions();
olm::client()->load(cache::restoreOlmAccount(), STORAGE_SECRET_KEY);
@ -811,13 +810,11 @@ ChatPage::loadStateFromCache()
} catch (const mtx::crypto::olm_exception &e) {
nhlog::crypto()->critical("failed to restore olm account: {}", e.what());
emit dropToLoginPageCb(
tr("Failed to restore OLM account. Please login again."));
emit dropToLoginPageCb(tr("Failed to restore OLM account. Please login again."));
return;
} catch (const lmdb::error &e) {
nhlog::db()->critical("failed to restore cache: {}", e.what());
emit dropToLoginPageCb(
tr("Failed to restore save data. Please login again."));
emit dropToLoginPageCb(tr("Failed to restore save data. Please login again."));
return;
} catch (const json::exception &e) {
nhlog::db()->critical("failed to parse cache data: {}", e.what());
@ -831,7 +828,6 @@ ChatPage::loadStateFromCache()
// Start receiving events.
emit trySyncCb();
});
}
void

55
src/timeline/EventStore.cpp
View File

@ -34,12 +34,31 @@ EventStore::EventStore(std::string room_id, QObject *)
cache::client()->storeEvent(room_id_, id, {timeline});
if (!relatedTo.empty()) {
auto idx = idToIndex(id);
auto idx = idToIndex(relatedTo);
if (idx)
emit dataChanged(*idx, *idx);
}
},
Qt::QueuedConnection);
connect(
this,
&EventStore::oldMessagesRetrieved,
this,
[this](const mtx::responses::Messages &res) {
//
uint64_t newFirst = cache::client()->saveOldMessages(room_id_, res);
if (newFirst == first)
fetchMore();
else {
emit beginInsertRows(toExternalIdx(newFirst),
toExternalIdx(this->first - 1));
this->first = newFirst;
emit endInsertRows();
emit fetchedMore();
}
},
Qt::QueuedConnection);
}
void
@ -49,8 +68,16 @@ EventStore::handleSync(const mtx::responses::Timeline &events)
nhlog::db()->warn("{} called from a different thread!", __func__);
auto range = cache::client()->getTimelineRange(room_id_);
if (!range)
return;
if (range && range->last > this->last) {
if (events.limited) {
emit beginResetModel();
this->last = range->last;
this->first = range->first;
emit endResetModel();
} else if (range->last > this->last) {
emit beginInsertRows(toExternalIdx(this->last + 1), toExternalIdx(range->last));
this->last = range->last;
emit endInsertRows();
@ -290,3 +317,27 @@ EventStore::event(std::string_view id, std::string_view related_to, bool decrypt
return event_ptr;
}
void
EventStore::fetchMore()
{
mtx::http::MessagesOpts opts;
opts.room_id = room_id_;
opts.from = cache::client()->previousBatchToken(room_id_);
nhlog::ui()->debug("Paginating room {}, token {}", opts.room_id, opts.from);
http::client()->messages(
opts, [this, opts](const mtx::responses::Messages &res, mtx::http::RequestErr err) {
if (err) {
nhlog::net()->error("failed to call /messages ({}): {} - {} - {}",
opts.room_id,
mtx::errors::to_string(err->matrix_error.errcode),
err->matrix_error.error,
err->parse_error);
return;
}
emit oldMessagesRetrieved(std::move(res));
});
}

17
src/timeline/EventStore.h
View File

@ -8,6 +8,7 @@
#include <qhashfunctions.h>
#include <mtx/events/collections.hpp>
#include <mtx/responses/messages.hpp>
#include <mtx/responses/sync.hpp>
class EventStore : public QObject
@ -20,7 +21,7 @@ public:
struct Index
{
std::string room;
int64_t idx;
uint64_t idx;
friend uint qHash(const Index &i, uint seed = 0) noexcept
{
@ -66,12 +67,12 @@ public:
int size() const
{
return last != std::numeric_limits<int64_t>::max()
return last != std::numeric_limits<uint64_t>::max()
? static_cast<int>(last - first) + 1
: 0;
}
int toExternalIdx(int64_t idx) const { return static_cast<int>(idx - first); }
int64_t toInternalIdx(int idx) const { return first + idx; }
int toExternalIdx(uint64_t idx) const { return static_cast<int>(idx - first); }
uint64_t toInternalIdx(int idx) const { return first + idx; }
std::optional<int> idToIndex(std::string_view id) const;
std::optional<std::string> indexToId(int idx) const;
@ -79,11 +80,15 @@ public:
signals:
void beginInsertRows(int from, int to);
void endInsertRows();
void beginResetModel();
void endResetModel();
void dataChanged(int from, int to);
void newEncryptedImage(mtx::crypto::EncryptedFile encryptionInfo);
void eventFetched(std::string id,
std::string relatedTo,
mtx::events::collections::TimelineEvents timeline);
void oldMessagesRetrieved(const mtx::responses::Messages &);
void fetchedMore();
private:
mtx::events::collections::TimelineEvents *decryptEvent(
@ -92,8 +97,8 @@ private:
std::string room_id_;
int64_t first = std::numeric_limits<int64_t>::max(),
last = std::numeric_limits<int64_t>::max();
uint64_t first = std::numeric_limits<uint64_t>::max(),
last = std::numeric_limits<uint64_t>::max();
static QCache<IdIndex, mtx::events::collections::TimelineEvents> decryptedEvents_;
static QCache<Index, mtx::events::collections::TimelineEvents> events_;

53
src/timeline/TimelineModel.cpp
View File

@ -229,20 +229,33 @@ TimelineModel::TimelineModel(TimelineViewManager *manager, QString room_id, QObj
&EventStore::dataChanged,
this,
[this](int from, int to) {
emit dataChanged(index(events.size() - to, 0), index(events.size() - from, 0));
nhlog::ui()->debug(
"data changed {} to {}", events.size() - to - 1, events.size() - from - 1);
emit dataChanged(index(events.size() - to - 1, 0),
index(events.size() - from - 1, 0));
},
Qt::QueuedConnection);
connect(&events, &EventStore::beginInsertRows, this, [this](int from, int to) {
nhlog::ui()->info("begin insert from {} to {}",
events.size() - to + (to - from),
events.size() - from + (to - from));
beginInsertRows(QModelIndex(),
events.size() - to + (to - from),
events.size() - from + (to - from));
int first = events.size() - to;
int last = events.size() - from;
if (from >= events.size()) {
int batch_size = to - from;
first += batch_size;
last += batch_size;
} else {
first -= 1;
last -= 1;
}
nhlog::ui()->debug("begin insert from {} to {}", first, last);
beginInsertRows(QModelIndex(), first, last);
});
connect(&events, &EventStore::endInsertRows, this, [this]() { endInsertRows(); });
connect(&events, &EventStore::beginResetModel, this, [this]() { beginResetModel(); });
connect(&events, &EventStore::endResetModel, this, [this]() { endResetModel(); });
connect(&events, &EventStore::newEncryptedImage, this, &TimelineModel::newEncryptedImage);
connect(
&events, &EventStore::fetchedMore, this, [this]() { setPaginationInProgress(false); });
}
QHash<int, QByteArray>
@ -512,8 +525,9 @@ TimelineModel::canFetchMore(const QModelIndex &) const
{
if (!events.size())
return true;
if (!std::holds_alternative<mtx::events::StateEvent<mtx::events::state::Create>>(
*events.event(0)))
if (auto first = events.event(0);
first &&
!std::holds_alternative<mtx::events::StateEvent<mtx::events::state::Create>>(*first))
return true;
else
@ -540,27 +554,8 @@ TimelineModel::fetchMore(const QModelIndex &)
}
setPaginationInProgress(true);
mtx::http::MessagesOpts opts;
opts.room_id = room_id_.toStdString();
opts.from = prev_batch_token_.toStdString();
nhlog::ui()->debug("Paginating room {}", opts.room_id);
http::client()->messages(
opts, [this, opts](const mtx::responses::Messages &res, mtx::http::RequestErr err) {
if (err) {
nhlog::net()->error("failed to call /messages ({}): {} - {} - {}",
opts.room_id,
mtx::errors::to_string(err->matrix_error.errcode),
err->matrix_error.error,
err->parse_error);
setPaginationInProgress(false);
return;
}
emit oldMessagesRetrieved(std::move(res));
setPaginationInProgress(false);
});
events.fetchMore();
}
void