diff --git a/resources/qml/TimelineRow.qml b/resources/qml/TimelineRow.qml
index a371f78b..fa427bfd 100644
--- a/resources/qml/TimelineRow.qml
+++ b/resources/qml/TimelineRow.qml
@@ -23,7 +23,7 @@ RowLayout {
 		Reply {
 			visible: model.replyTo
 			modelData: chat.model.getDump(model.replyTo)
-			userColor: chat.model.userColor(modelData.userId, colors.window)
+			userColor: timelineManager.userColor(modelData.userId, colors.window)
 		}
 
 		// actual message content
diff --git a/resources/qml/TimelineView.qml b/resources/qml/TimelineView.qml
index 9730422d..46cf484b 100644
--- a/resources/qml/TimelineView.qml
+++ b/resources/qml/TimelineView.qml
@@ -218,7 +218,7 @@ Item {
 						Text { 
 							id: userName
 							text: chat.model.escapeEmoji(modelData.userName)
-							color: chat.model.userColor(modelData.userId, colors.window)
+							color: timelineManager.userColor(modelData.userId, colors.window)
 							textFormat: Text.RichText
 
 							MouseArea {
@@ -283,7 +283,7 @@ Item {
 						anchors.bottom: parent.bottom
 
 						modelData: chat.model ? chat.model.getDump(timelineManager.replyingEvent) : {}
-						userColor: chat.model ? chat.model.userColor(modelData.userId, colors.window) : colors.window
+						userColor: timelineManager.userColor(modelData.userId, colors.window)
 					}
 
 					ImageButton {
diff --git a/resources/qml/delegates/MessageDelegate.qml b/resources/qml/delegates/MessageDelegate.qml
index f043efec..ab633087 100644
--- a/resources/qml/delegates/MessageDelegate.qml
+++ b/resources/qml/delegates/MessageDelegate.qml
@@ -34,7 +34,7 @@ Item {
 			roleValue: MtxEvent.EmoteMessage
 			NoticeMessage {
 				formatted: chat.model.escapeEmoji(modelData.userName) + " " + model.data.formattedBody
-				color: chat.model.userColor(modelData.userId, colors.window)
+				color: timelineManager.userColor(modelData.userId, colors.window)
 			}
 		}
 		DelegateChoice {
diff --git a/resources/qml/delegates/Reply.qml b/resources/qml/delegates/Reply.qml
index a1ddc8f0..8bb9ed38 100644
--- a/resources/qml/delegates/Reply.qml
+++ b/resources/qml/delegates/Reply.qml
@@ -19,7 +19,7 @@ Rectangle {
 		anchors.bottom: replyContainer.bottom
 		width: 4
 
-		color: chat.model ? chat.model.userColor(reply.modelData.userId, colors.window) : colors.window
+		color: timelineManager.userColor(reply.modelData.userId, colors.window)
 	}
 
 	Column {
diff --git a/src/Cache.cpp b/src/Cache.cpp
index e73f7754..caa3f1ec 100644
--- a/src/Cache.cpp
+++ b/src/Cache.cpp
@@ -2186,7 +2186,6 @@ Cache::roomMembers(const std::string &room_id)
 
 QHash<QString, QString> Cache::DisplayNames;
 QHash<QString, QString> Cache::AvatarUrls;
-QHash<QString, QString> Cache::UserColors;
 
 QString
 Cache::displayName(const QString &room_id, const QString &user_id)
@@ -2218,16 +2217,6 @@ Cache::avatarUrl(const QString &room_id, const QString &user_id)
         return QString();
 }
 
-QString
-Cache::userColor(const QString &user_id)
-{
-        if (UserColors.contains(user_id)) {
-                return UserColors[user_id];
-        }
-
-        return QString();
-}
-
 void
 Cache::insertDisplayName(const QString &room_id,
                          const QString &user_id,
@@ -2258,24 +2247,6 @@ Cache::removeAvatarUrl(const QString &room_id, const QString &user_id)
         AvatarUrls.remove(fmt);
 }
 
-void
-Cache::insertUserColor(const QString &user_id, const QString &color_name)
-{
-        UserColors.insert(user_id, color_name);
-}
-
-void
-Cache::removeUserColor(const QString &user_id)
-{
-        UserColors.remove(user_id);
-}
-
-void
-Cache::clearUserColors()
-{
-        UserColors.clear();
-}
-
 void
 to_json(json &j, const RoomInfo &info)
 {
@@ -2425,12 +2396,6 @@ avatarUrl(const QString &room_id, const QString &user_id)
         return instance_->avatarUrl(room_id, user_id);
 }
 
-QString
-userColor(const QString &user_id)
-{
-        return instance_->userColor(user_id);
-}
-
 void
 removeDisplayName(const QString &room_id, const QString &user_id)
 {
@@ -2441,11 +2406,6 @@ removeAvatarUrl(const QString &room_id, const QString &user_id)
 {
         instance_->removeAvatarUrl(room_id, user_id);
 }
-void
-removeUserColor(const QString &user_id)
-{
-        instance_->removeUserColor(user_id);
-}
 
 void
 insertDisplayName(const QString &room_id, const QString &user_id, const QString &display_name)
@@ -2457,17 +2417,6 @@ insertAvatarUrl(const QString &room_id, const QString &user_id, const QString &a
 {
         instance_->insertAvatarUrl(room_id, user_id, avatar_url);
 }
-void
-insertUserColor(const QString &user_id, const QString &color_name)
-{
-        instance_->insertUserColor(user_id, color_name);
-}
-
-void
-clearUserColors()
-{
-        instance_->clearUserColors();
-}
 
 //! Load saved data for the display names & avatars.
 void
diff --git a/src/Cache.h b/src/Cache.h
index 69b3ef2d..bb042ea9 100644
--- a/src/Cache.h
+++ b/src/Cache.h
@@ -43,25 +43,16 @@ QString
 displayName(const QString &room_id, const QString &user_id);
 QString
 avatarUrl(const QString &room_id, const QString &user_id);
-QString
-userColor(const QString &user_id);
 
 void
 removeDisplayName(const QString &room_id, const QString &user_id);
 void
 removeAvatarUrl(const QString &room_id, const QString &user_id);
-void
-removeUserColor(const QString &user_id);
 
 void
 insertDisplayName(const QString &room_id, const QString &user_id, const QString &display_name);
 void
 insertAvatarUrl(const QString &room_id, const QString &user_id, const QString &avatar_url);
-void
-insertUserColor(const QString &user_id, const QString &color_name);
-
-void
-clearUserColors();
 
 //! Load saved data for the display names & avatars.
 void
diff --git a/src/Cache_p.h b/src/Cache_p.h
index eacb28a7..137408b9 100644
--- a/src/Cache_p.h
+++ b/src/Cache_p.h
@@ -51,11 +51,9 @@ public:
         static std::string displayName(const std::string &room_id, const std::string &user_id);
         static QString displayName(const QString &room_id, const QString &user_id);
         static QString avatarUrl(const QString &room_id, const QString &user_id);
-        static QString userColor(const QString &user_id);
 
         static void removeDisplayName(const QString &room_id, const QString &user_id);
         static void removeAvatarUrl(const QString &room_id, const QString &user_id);
-        static void removeUserColor(const QString &user_id);
 
         static void insertDisplayName(const QString &room_id,
                                       const QString &user_id,
@@ -63,9 +61,6 @@ public:
         static void insertAvatarUrl(const QString &room_id,
                                     const QString &user_id,
                                     const QString &avatar_url);
-        static void insertUserColor(const QString &user_id, const QString &color_name);
-
-        static void clearUserColors();
 
         //! Load saved data for the display names & avatars.
         void populateMembers();
@@ -484,7 +479,6 @@ private:
 
         static QHash<QString, QString> DisplayNames;
         static QHash<QString, QString> AvatarUrls;
-        static QHash<QString, QString> UserColors;
 
         OlmSessionStorage session_storage;
 };
diff --git a/src/MainWindow.cpp b/src/MainWindow.cpp
index 1b378ca9..17a04a41 100644
--- a/src/MainWindow.cpp
+++ b/src/MainWindow.cpp
@@ -115,9 +115,6 @@ MainWindow::MainWindow(QWidget *parent)
 
         connect(
           userSettingsPage_, SIGNAL(trayOptionChanged(bool)), trayIcon_, SLOT(setVisible(bool)));
-        connect(userSettingsPage_, &UserSettingsPage::themeChanged, this, []() {
-                cache::clearUserColors();
-        });
         connect(
           userSettingsPage_, &UserSettingsPage::themeChanged, chat_page_, &ChatPage::themeChanged);
         connect(trayIcon_,
diff --git a/src/timeline/TimelineModel.cpp b/src/timeline/TimelineModel.cpp
index 8de6bec6..cad39bc5 100644
--- a/src/timeline/TimelineModel.cpp
+++ b/src/timeline/TimelineModel.cpp
@@ -196,9 +196,6 @@ TimelineModel::TimelineModel(TimelineViewManager *manager, QString room_id, QObj
                         if (idx >= 0)
                                 emit dataChanged(index(idx, 0), index(idx, 0));
                 });
-
-        connect(
-          ChatPage::instance(), &ChatPage::themeChanged, this, [this]() { userColors.clear(); });
 }
 
 QHash<int, QByteArray>
@@ -650,15 +647,6 @@ TimelineModel::addBackwardsEvents(const mtx::responses::Messages &msgs)
         prev_batch_token_ = QString::fromStdString(msgs.end);
 }
 
-QColor
-TimelineModel::userColor(QString id, QColor background)
-{
-        if (!userColors.contains(id))
-                userColors.insert(
-                  id, QColor(utils::generateContrastingHexColor(id, background.name())));
-        return userColors.value(id);
-}
-
 QString
 TimelineModel::displayName(QString id) const
 {
@@ -1446,7 +1434,8 @@ TimelineModel::formatTypingUsers(const std::vector<QString> &users, QColor bg)
 
         auto formatUser = [this, bg](const QString &user_id) -> QString {
                 auto uncoloredUsername = escapeEmoji(displayName(user_id).toHtmlEscaped());
-                QString prefix = QString("<font color=\"%1\">").arg(userColor(user_id, bg).name());
+                QString prefix =
+                  QString("<font color=\"%1\">").arg(manager_->userColor(user_id, bg).name());
 
                 // color only parts that don't have a font already specified
                 QString coloredUsername;
diff --git a/src/timeline/TimelineModel.h b/src/timeline/TimelineModel.h
index 642e9bbc..f06de5d9 100644
--- a/src/timeline/TimelineModel.h
+++ b/src/timeline/TimelineModel.h
@@ -165,7 +165,6 @@ public:
         bool canFetchMore(const QModelIndex &) const override;
         void fetchMore(const QModelIndex &) override;
 
-        Q_INVOKABLE QColor userColor(QString id, QColor background);
         Q_INVOKABLE QString displayName(QString id) const;
         Q_INVOKABLE QString avatarUrl(QString id) const;
         Q_INVOKABLE QString formatDateSeparator(QDate date) const;
@@ -248,7 +247,6 @@ private:
         bool paginationInProgress = false;
         bool isProcessingPending  = false;
 
-        QHash<QString, QColor> userColors;
         QString currentId;
         std::vector<QString> typingUsers_;
 
diff --git a/src/timeline/TimelineViewManager.cpp b/src/timeline/TimelineViewManager.cpp
index 7cd1432d..14f903a1 100644
--- a/src/timeline/TimelineViewManager.cpp
+++ b/src/timeline/TimelineViewManager.cpp
@@ -18,6 +18,8 @@ Q_DECLARE_METATYPE(mtx::events::collections::TimelineEvents)
 void
 TimelineViewManager::updateColorPalette()
 {
+        userColors.clear();
+
         if (settings->theme() == "light") {
                 QPalette lightActive(/*windowText*/ QColor("#333"),
                                      /*button*/ QColor("#333"),
@@ -53,6 +55,15 @@ TimelineViewManager::updateColorPalette()
         }
 }
 
+QColor
+TimelineViewManager::userColor(QString id, QColor background)
+{
+        if (!userColors.contains(id))
+                userColors.insert(
+                  id, QColor(utils::generateContrastingHexColor(id, background.name())));
+        return userColors.value(id);
+}
+
 TimelineViewManager::TimelineViewManager(QSharedPointer<UserSettings> userSettings, QWidget *parent)
   : imgProvider(new MxcImageProvider())
   , colorImgProvider(new ColorImageProvider())
diff --git a/src/timeline/TimelineViewManager.h b/src/timeline/TimelineViewManager.h
index 625999c9..338101c7 100644
--- a/src/timeline/TimelineViewManager.h
+++ b/src/timeline/TimelineViewManager.h
@@ -40,6 +40,7 @@ public:
         Q_INVOKABLE TimelineModel *activeTimeline() const { return timeline_; }
         Q_INVOKABLE bool isInitialSync() const { return isInitialSync_; }
         Q_INVOKABLE void openImageOverlay(QString mxcUrl, QString eventId) const;
+        Q_INVOKABLE QColor userColor(QString id, QColor background);
 
 signals:
         void clearRoomMessageCount(QString roomid);
@@ -118,4 +119,5 @@ private:
         QString replyingEvent_;
 
         QSharedPointer<UserSettings> settings;
+        QHash<QString, QColor> userColors;
 };