diff --git a/src/components/views/location/Map.tsx b/src/components/views/location/Map.tsx
index 34b9d4f104..c1dbcab337 100644
--- a/src/components/views/location/Map.tsx
+++ b/src/components/views/location/Map.tsx
@@ -91,26 +91,31 @@ const useMapWithStyle = ({
         }
     }, [map, bounds]);
 
-    const [geolocate] = useState(
-        allowGeolocate
-            ? new maplibregl.GeolocateControl({
-                  positionOptions: {
-                      enableHighAccuracy: true,
-                  },
-                  trackUserLocation: false,
-              })
-            : null,
-    );
+    const [geolocate, setGeolocate] = useState<maplibregl.GeolocateControl | null>(null);
 
     useEffect(() => {
-        if (map && geolocate) {
+        if (!map) {
+            return;
+        }
+        if (allowGeolocate && !geolocate) {
+            const geolocate = new maplibregl.GeolocateControl({
+                positionOptions: {
+                    enableHighAccuracy: true,
+                },
+                trackUserLocation: false,
+            });
+            setGeolocate(geolocate);
             map.addControl(geolocate);
             geolocate.on("error", onGeolocateError);
             return () => {
                 geolocate.off("error", onGeolocateError);
             };
         }
-    }, [map, geolocate]);
+        if (!allowGeolocate && geolocate) {
+            map.removeControl(geolocate);
+            setGeolocate(null);
+        }
+    }, [map, geolocate, allowGeolocate]);
 
     return {
         map,
diff --git a/test/components/views/location/LocationViewDialog-test.tsx b/test/components/views/location/LocationViewDialog-test.tsx
index 41149299ee..c9f6f49099 100644
--- a/test/components/views/location/LocationViewDialog-test.tsx
+++ b/test/components/views/location/LocationViewDialog-test.tsx
@@ -49,7 +49,7 @@ describe("<LocationViewDialog />", () => {
     it("renders marker correctly for self share", () => {
         const selfShareEvent = makeLocationEvent("geo:51.5076,-0.1276", LocationAssetType.Self);
         const member = new RoomMember(roomId, userId);
-        // @ts-ignore cheat assignment to property   
+        // @ts-ignore cheat assignment to property
         selfShareEvent.sender = member;
         const component = getComponent({ mxEvent: selfShareEvent });
         // @ts-ignore fix when moving to rtl