diff --git a/res/css/structures/_GroupFilterPanel.scss b/res/css/structures/_GroupFilterPanel.scss
index c62230edfc..11afe855b8 100644
--- a/res/css/structures/_GroupFilterPanel.scss
+++ b/res/css/structures/_GroupFilterPanel.scss
@@ -20,6 +20,21 @@ limitations under the License.
     }
 }
 
+$groupFilterPanelWidth: 56px; // only applies in this file, used for calculations
+
+.mx_GroupFilterPanelContainer {
+    flex-grow: 0;
+    flex-shrink: 0;
+    width: $groupFilterPanelWidth;
+    height: 100%;
+
+    // Create another flexbox so the GroupFilterPanel fills the container
+    display: flex;
+    flex-direction: column;
+
+    // GroupFilterPanel handles its own CSS
+}
+
 .mx_GroupFilterPanel {
     background-color: $groupFilterPanel-bg-color;
     flex: 1;
diff --git a/res/css/structures/_LeftPanel.scss b/res/css/structures/_LeftPanel.scss
index ba7d9c4523..a172b6c26b 100644
--- a/res/css/structures/_LeftPanel.scss
+++ b/res/css/structures/_LeftPanel.scss
@@ -14,7 +14,6 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-$groupFilterPanelWidth: 56px; // only applies in this file, used for calculations
 $roomListCollapsedWidth: 68px;
 
 .mx_MatrixChat--with-avatar {
@@ -68,19 +67,6 @@ $roomListCollapsedWidth: 68px;
     contain: content;
     position: relative;
 
-    .mx_LeftPanel_GroupFilterPanelContainer {
-        flex-grow: 0;
-        flex-shrink: 0;
-        flex-basis: $groupFilterPanelWidth;
-        height: 100%;
-
-        // Create another flexbox so the GroupFilterPanel fills the container
-        display: flex;
-        flex-direction: column;
-
-        // GroupFilterPanel handles its own CSS
-    }
-
     // Note: The 'room list' in this context is actually everything that isn't the tag
     // panel, such as the menu options, breadcrumbs, filtering, etc
     .mx_LeftPanel_roomListContainer {
diff --git a/src/components/structures/LeftPanel.tsx b/src/components/structures/LeftPanel.tsx
index 6c418e9366..d955271249 100644
--- a/src/components/structures/LeftPanel.tsx
+++ b/src/components/structures/LeftPanel.tsx
@@ -19,8 +19,6 @@ import { createRef } from "react";
 import classNames from "classnames";
 import { Room } from "matrix-js-sdk/src/models/room";
 
-import GroupFilterPanel from "./GroupFilterPanel";
-import CustomRoomTagPanel from "./CustomRoomTagPanel";
 import dis from "../../dispatcher/dispatcher";
 import { _t } from "../../languageHandler";
 import RoomList from "../views/rooms/RoomList";
@@ -33,7 +31,6 @@ import RoomBreadcrumbs from "../views/rooms/RoomBreadcrumbs";
 import { BreadcrumbsStore } from "../../stores/BreadcrumbsStore";
 import { UPDATE_EVENT } from "../../stores/AsyncStore";
 import ResizeNotifier from "../../utils/ResizeNotifier";
-import SettingsStore from "../../settings/SettingsStore";
 import RoomListStore, { LISTS_UPDATE_EVENT } from "../../stores/room-list/RoomListStore";
 import IndicatorScrollbar from "../structures/IndicatorScrollbar";
 import AccessibleTooltipButton from "../views/elements/AccessibleTooltipButton";
@@ -51,7 +48,6 @@ interface IProps {
 
 interface IState {
     showBreadcrumbs: boolean;
-    showGroupFilterPanel: boolean;
     activeSpace?: Room;
 }
 
@@ -68,9 +64,6 @@ const cssClasses = [
 export default class LeftPanel extends React.Component<IProps, IState> {
     private ref: React.RefObject<HTMLDivElement> = createRef();
     private listContainerRef: React.RefObject<HTMLDivElement> = createRef();
-    private groupFilterPanelWatcherRef: string;
-    private groupFilterPanelContainer = createRef<HTMLDivElement>();
-    private bgImageWatcherRef: string;
     private focusedElement = null;
     private isDoingStickyHeaders = false;
 
@@ -79,25 +72,17 @@ export default class LeftPanel extends React.Component<IProps, IState> {
 
         this.state = {
             showBreadcrumbs: BreadcrumbsStore.instance.visible,
-            showGroupFilterPanel: SettingsStore.getValue('TagPanel.enableTagPanel'),
             activeSpace: SpaceStore.instance.activeSpace,
         };
 
         BreadcrumbsStore.instance.on(UPDATE_EVENT, this.onBreadcrumbsUpdate);
         RoomListStore.instance.on(LISTS_UPDATE_EVENT, this.onBreadcrumbsUpdate);
         SpaceStore.instance.on(UPDATE_SELECTED_SPACE, this.updateActiveSpace);
-        this.groupFilterPanelWatcherRef = SettingsStore.watchSetting("TagPanel.enableTagPanel", null, () => {
-            this.setState({ showGroupFilterPanel: SettingsStore.getValue("TagPanel.enableTagPanel") });
-        });
     }
 
     public componentDidMount() {
         UIStore.instance.trackElementDimensions("LeftPanel", this.ref.current);
         UIStore.instance.trackElementDimensions("ListContainer", this.listContainerRef.current);
-        if (this.groupFilterPanelContainer.current) {
-            const componentName = "GroupFilterPanelContainer";
-            UIStore.instance.trackElementDimensions(componentName, this.groupFilterPanelContainer.current);
-        }
         UIStore.instance.on("ListContainer", this.refreshStickyHeaders);
         // Using the passive option to not block the main thread
         // https://developer.mozilla.org/en-US/docs/Web/API/EventTarget/addEventListener#improving_scrolling_performance_with_passive_listeners
@@ -105,7 +90,6 @@ export default class LeftPanel extends React.Component<IProps, IState> {
     }
 
     public componentWillUnmount() {
-        SettingsStore.unwatchSetting(this.groupFilterPanelWatcherRef);
         BreadcrumbsStore.instance.off(UPDATE_EVENT, this.onBreadcrumbsUpdate);
         RoomListStore.instance.off(LISTS_UPDATE_EVENT, this.onBreadcrumbsUpdate);
         SpaceStore.instance.off(UPDATE_SELECTED_SPACE, this.updateActiveSpace);
@@ -422,16 +406,6 @@ export default class LeftPanel extends React.Component<IProps, IState> {
     }
 
     public render(): React.ReactNode {
-        let leftLeftPanel;
-        if (this.state.showGroupFilterPanel) {
-            leftLeftPanel = (
-                <div className="mx_LeftPanel_GroupFilterPanelContainer" ref={this.groupFilterPanelContainer}>
-                    <GroupFilterPanel />
-                    { SettingsStore.getValue("feature_custom_tags") ? <CustomRoomTagPanel /> : null }
-                </div>
-            );
-        }
-
         const roomList = <RoomList
             onKeyDown={this.onKeyDown}
             resizeNotifier={this.props.resizeNotifier}
@@ -455,7 +429,6 @@ export default class LeftPanel extends React.Component<IProps, IState> {
 
         return (
             <div className={containerClasses} ref={this.ref}>
-                { leftLeftPanel }
                 <aside className="mx_LeftPanel_roomListContainer">
                     { this.renderHeader() }
                     { this.renderSearchDialExplore() }
diff --git a/src/components/structures/LoggedInView.tsx b/src/components/structures/LoggedInView.tsx
index 4d0c82e0b8..de35a36dc2 100644
--- a/src/components/structures/LoggedInView.tsx
+++ b/src/components/structures/LoggedInView.tsx
@@ -68,6 +68,8 @@ import GroupView from "./GroupView";
 import BackdropPanel from "./BackdropPanel";
 import SpaceStore from "../../stores/SpaceStore";
 import classNames from 'classnames';
+import GroupFilterPanel from './GroupFilterPanel';
+import CustomRoomTagPanel from './CustomRoomTagPanel';
 
 // We need to fetch each pinned message individually (if we don't already have it)
 // so each pinned message may trigger a request. Limit the number per room for sanity.
@@ -649,6 +651,12 @@ class LoggedInView extends React.Component<IProps, IState> {
                             <BackdropPanel
                                 backgroundImage={this.state.backgroundImage}
                             />
+                            { SettingsStore.getValue('TagPanel.enableTagPanel') &&
+                                (<div className="mx_GroupFilterPanelContainer">
+                                    <GroupFilterPanel />
+                                    { SettingsStore.getValue("feature_custom_tags") ? <CustomRoomTagPanel /> : null }
+                                </div>)
+                            }
                             { SpaceStore.spacesEnabled ? <SpacePanel /> : null }
                             <div
                                 ref={this._resizeContainer}