Switching MediaManager to using a Svelte store

This allows cleaner and more expressive code, especially regarding whether the webcam should be on or off.

front/src/Phaser/Game/GameScene.ts

@@ -92,6 +92,7 @@ import {PinchManager} from "../UserInput/PinchManager";
 import {joystickBaseImg, joystickBaseKey, joystickThumbImg, joystickThumbKey} from "../Components/MobileJoystick";
 import {DEPTH_OVERLAY_INDEX} from "./DepthIndexes";
 import {waScaleManager} from "../Services/WaScaleManager";
+import {peerStore} from "../../Stores/PeerStore";
 import {EmoteManager} from "./EmoteManager";
 
 export interface GameSceneInitInterface {
@@ -516,7 +517,7 @@ export class GameScene extends DirtyScene implements CenterListener {
         }
 
         document.addEventListener('visibilitychange', this.onVisibilityChangeCallback);
-        
+
         this.emoteManager = new EmoteManager(this);
     }
 
@@ -622,6 +623,7 @@ export class GameScene extends DirtyScene implements CenterListener {
 
             // When connection is performed, let's connect SimplePeer
             this.simplePeer = new SimplePeer(this.connection, !this.room.isPublic, this.playerName);
+            peerStore.connectToSimplePeer(this.simplePeer);
             this.GlobalMessageManager = new GlobalMessageManager(this.connection);
             userMessageManager.setReceiveBanListener(this.bannedUser.bind(this));
 

front/src/Phaser/Login/EnableCameraScene.ts

@@ -10,6 +10,14 @@ import {PinchManager} from "../UserInput/PinchManager";
 import Zone = Phaser.GameObjects.Zone;
 import { MenuScene } from "../Menu/MenuScene";
 import {ResizableScene} from "./ResizableScene";
+import {
+    audioConstraintStore,
+    enableCameraSceneVisibilityStore,
+    localStreamStore,
+    mediaStreamConstraintsStore,
+    videoConstraintStore
+} from "../../Stores/MediaStore";
+import type {Unsubscriber} from "svelte/store";
 
 export const EnableCameraSceneName = "EnableCameraScene";
 enum LoginTextures {
@@ -40,6 +48,7 @@ export class EnableCameraScene extends ResizableScene {
     private enableCameraSceneElement!: Phaser.GameObjects.DOMElement;
 
     private mobileTapZone!: Zone;
+    private localStreamStoreUnsubscriber!: Unsubscriber;
 
     constructor() {
         super({
@@ -119,9 +128,20 @@ export class EnableCameraScene extends ResizableScene {
 
         HtmlUtils.getElementByIdOrFail<HTMLDivElement>('webRtcSetup').classList.add('active');
 
-        const mediaPromise = mediaManager.getCamera();
+        this.localStreamStoreUnsubscriber = localStreamStore.subscribe((result) => {
+            if (result.type === 'error') {
+                // TODO: proper handling of the error
+                throw result.error;
+            }
+
+            this.getDevices();
+            if (result.stream !== null) {
+                this.setupStream(result.stream);
+            }
+        });
+        /*const mediaPromise = mediaManager.getCamera();
         mediaPromise.then(this.getDevices.bind(this));
-        mediaPromise.then(this.setupStream.bind(this));
+        mediaPromise.then(this.setupStream.bind(this));*/
 
         this.input.keyboard.on('keydown-RIGHT', this.nextCam.bind(this));
         this.input.keyboard.on('keydown-LEFT', this.previousCam.bind(this));
@@ -133,6 +153,8 @@ export class EnableCameraScene extends ResizableScene {
         this.add.existing(this.soundMeterSprite);
 
         this.onResize();
+
+        enableCameraSceneVisibilityStore.showEnableCameraScene();
     }
 
     private previousCam(): void {
@@ -140,7 +162,9 @@ export class EnableCameraScene extends ResizableScene {
             return;
         }
         this.cameraSelected--;
-        mediaManager.setCamera(this.camerasList[this.cameraSelected].deviceId).then(this.setupStream.bind(this));
+        videoConstraintStore.setDeviceId(this.camerasList[this.cameraSelected].deviceId);
+
+        //mediaManager.setCamera(this.camerasList[this.cameraSelected].deviceId).then(this.setupStream.bind(this));
     }
 
     private nextCam(): void {
@@ -148,8 +172,10 @@ export class EnableCameraScene extends ResizableScene {
             return;
         }
         this.cameraSelected++;
+        videoConstraintStore.setDeviceId(this.camerasList[this.cameraSelected].deviceId);
+
         // TODO: the change of camera should be OBSERVED (reactive)
-        mediaManager.setCamera(this.camerasList[this.cameraSelected].deviceId).then(this.setupStream.bind(this));
+        //mediaManager.setCamera(this.camerasList[this.cameraSelected].deviceId).then(this.setupStream.bind(this));
     }
 
     private previousMic(): void {
@@ -157,7 +183,8 @@ export class EnableCameraScene extends ResizableScene {
             return;
         }
         this.microphoneSelected--;
-        mediaManager.setMicrophone(this.microphonesList[this.microphoneSelected].deviceId).then(this.setupStream.bind(this));
+        audioConstraintStore.setDeviceId(this.microphonesList[this.microphoneSelected].deviceId);
+        //mediaManager.setMicrophone(this.microphonesList[this.microphoneSelected].deviceId).then(this.setupStream.bind(this));
     }
 
     private nextMic(): void {
@@ -165,8 +192,9 @@ export class EnableCameraScene extends ResizableScene {
             return;
         }
         this.microphoneSelected++;
+        audioConstraintStore.setDeviceId(this.microphonesList[this.microphoneSelected].deviceId);
         // TODO: the change of camera should be OBSERVED (reactive)
-        mediaManager.setMicrophone(this.microphonesList[this.microphoneSelected].deviceId).then(this.setupStream.bind(this));
+        //mediaManager.setMicrophone(this.microphonesList[this.microphoneSelected].deviceId).then(this.setupStream.bind(this));
     }
 
     /**
@@ -260,15 +288,20 @@ export class EnableCameraScene extends ResizableScene {
         HtmlUtils.getElementByIdOrFail<HTMLDivElement>('webRtcSetup').style.display = 'none';
         this.soundMeter.stop();
 
-        mediaManager.stopCamera();
+        enableCameraSceneVisibilityStore.hideEnableCameraScene();
-        mediaManager.stopMicrophone();
+        this.localStreamStoreUnsubscriber();
+        //mediaManager.stopCamera();
+        //mediaManager.stopMicrophone();
 
-        this.scene.sleep(EnableCameraSceneName)
+        this.scene.sleep(EnableCameraSceneName);
         gameManager.goToStartingMap(this.scene);
     }
 
     private async getDevices() {
+        // TODO: switch this in a store.
         const mediaDeviceInfos = await navigator.mediaDevices.enumerateDevices();
+        this.microphonesList = [];
+        this.camerasList = [];
         for (const mediaDeviceInfo of mediaDeviceInfos) {
             if (mediaDeviceInfo.kind === 'audioinput') {
                 this.microphonesList.push(mediaDeviceInfo);

front/src/Stores/MediaStore.ts

@@ -0,0 +1,360 @@
+import {derived, Readable, readable, writable, Writable} from "svelte/store";
+import {peerStore} from "./PeerStore";
+import {localUserStore} from "../Connexion/LocalUserStore";
+import {ITiledMapGroupLayer, ITiledMapObjectLayer, ITiledMapTileLayer} from "../Phaser/Map/ITiledMap";
+
+/**
+ * A store that contains the camera state requested by the user (on or off).
+ */
+function createRequestedCameraState() {
+    const { subscribe, set, update } = writable(true);
+
+    return {
+        subscribe,
+        enableWebcam: () => set(true),
+        disableWebcam: () => set(false),
+    };
+}
+
+/**
+ * A store that contains the microphone state requested by the user (on or off).
+ */
+function createRequestedMicrophoneState() {
+    const { subscribe, set, update } = writable(true);
+
+    return {
+        subscribe,
+        enableMicrophone: () => set(true),
+        disableMicrophone: () => set(false),
+    };
+}
+
+/**
+ * A store containing whether the current page is visible or not.
+ */
+export const visibilityStore = readable(document.visibilityState === 'visible', function start(set) {
+    const onVisibilityChange = () => {
+        set(document.visibilityState === 'visible');
+    };
+
+    document.addEventListener('visibilitychange', onVisibilityChange);
+
+    return function stop() {
+        document.removeEventListener('visibilitychange', onVisibilityChange);
+    };
+});
+
+/**
+ * A store that contains whether the game overlay is shown or not.
+ * Typically, the overlay is hidden when entering Jitsi meet.
+ */
+function createGameOverlayVisibilityStore() {
+    const { subscribe, set, update } = writable(false);
+
+    return {
+        subscribe,
+        showGameOverlay: () => set(true),
+        hideGameOverlay: () => set(false),
+    };
+}
+
+/**
+ * A store that contains whether the EnableCameraScene is shown or not.
+ */
+function createEnableCameraSceneVisibilityStore() {
+    const { subscribe, set, update } = writable(false);
+
+    return {
+        subscribe,
+        showEnableCameraScene: () => set(true),
+        hideEnableCameraScene: () => set(false),
+    };
+}
+
+export const requestedCameraState = createRequestedCameraState();
+export const requestedMicrophoneState = createRequestedMicrophoneState();
+export const gameOverlayVisibilityStore = createGameOverlayVisibilityStore();
+export const enableCameraSceneVisibilityStore = createEnableCameraSceneVisibilityStore();
+
+/**
+ * A store that contains video constraints.
+ */
+function createVideoConstraintStore() {
+    const { subscribe, set, update } = writable({
+        width: { min: 640, ideal: 1280, max: 1920 },
+        height: { min: 400, ideal: 720 },
+        frameRate: { ideal: localUserStore.getVideoQualityValue() },
+        facingMode: "user",
+        resizeMode: 'crop-and-scale',
+        aspectRatio: 1.777777778
+    } as boolean|MediaTrackConstraints);
+
+    let selectedDeviceId = null;
+
+    return {
+        subscribe,
+        setDeviceId: (deviceId: string) => update((constraints) => {
+            selectedDeviceId = deviceId;
+
+            if (typeof(constraints) === 'boolean') {
+                constraints = {}
+            }
+            constraints.deviceId = {
+                exact: selectedDeviceId
+            };
+
+            return constraints;
+        })
+    };
+}
+
+export const videoConstraintStore = createVideoConstraintStore();
+
+/**
+ * A store that contains video constraints.
+ */
+function createAudioConstraintStore() {
+    const { subscribe, set, update } = writable({
+        //TODO: make these values configurable in the game settings menu and store them in localstorage
+        autoGainControl: false,
+        echoCancellation: true,
+        noiseSuppression: true
+    } as boolean|MediaTrackConstraints);
+
+    let selectedDeviceId = null;
+
+    return {
+        subscribe,
+        setDeviceId: (deviceId: string) => update((constraints) => {
+            selectedDeviceId = deviceId;
+
+            if (typeof(constraints) === 'boolean') {
+                constraints = {}
+            }
+            constraints.deviceId = {
+                exact: selectedDeviceId
+            };
+
+            return constraints;
+        })
+    };
+}
+
+export const audioConstraintStore = createAudioConstraintStore();
+
+
+let timeout: NodeJS.Timeout;
+
+/**
+ * A store containing the media constraints we want to apply.
+ */
+export const mediaStreamConstraintsStore = derived(
+    [
+        requestedCameraState,
+        requestedMicrophoneState,
+        visibilityStore,
+        gameOverlayVisibilityStore,
+        peerStore,
+        enableCameraSceneVisibilityStore,
+        videoConstraintStore,
+        audioConstraintStore,
+    ], (
+        [
+            $requestedCameraState,
+            $requestedMicrophoneState,
+            $visibilityStore,
+            $gameOverlayVisibilityStore,
+            $peerStore,
+            $enableCameraSceneVisibilityStore,
+            $videoConstraintStore,
+            $audioConstraintStore,
+        ], set
+    ) => {
+    let currentVideoConstraint: boolean|MediaTrackConstraints = $videoConstraintStore;
+    let currentAudioConstraint: boolean|MediaTrackConstraints = $audioConstraintStore;
+
+    if ($enableCameraSceneVisibilityStore) {
+        set({
+            video: currentVideoConstraint,
+            audio: currentAudioConstraint,
+        });
+        return;
+    }
+
+    // Disable webcam if the user requested so
+    if ($requestedCameraState === false) {
+        currentVideoConstraint = false;
+    }
+
+    // Disable microphone if the user requested so
+    if ($requestedMicrophoneState === false) {
+        currentAudioConstraint = false;
+    }
+
+    // Disable webcam and microphone when in a Jitsi
+    if ($gameOverlayVisibilityStore === false) {
+        currentVideoConstraint = false;
+        currentAudioConstraint = false;
+    }
+
+    // Disable webcam if the game is not visible and we are talking to noone.
+    if ($visibilityStore === false && $peerStore.size === 0) {
+        currentVideoConstraint = false;
+    }
+
+    if (timeout) {
+        clearTimeout(timeout);
+    }
+
+    // Let's wait a little bit to avoid sending too many constraint changes.
+    timeout = setTimeout(() => {
+        set({
+            video: currentVideoConstraint,
+            audio: currentAudioConstraint,
+        });
+    }, 100)
+}, {
+    video: false,
+    audio: false
+} as MediaStreamConstraints);
+
+export type LocalStreamStoreValue = StreamSuccessValue | StreamErrorValue;
+
+interface StreamSuccessValue {
+    type: "success",
+    stream: MediaStream|null,
+    // The constraints that we got (and not the one that have been requested)
+    constraints: MediaStreamConstraints
+}
+
+interface StreamErrorValue {
+    type: "error",
+    error: Error,
+    constraints: MediaStreamConstraints
+}
+
+let currentStream : MediaStream|null = null;
+
+/**
+ * Stops the camera from filming
+ */
+function stopCamera(): void {
+    if (currentStream) {
+        for (const track of currentStream.getVideoTracks()) {
+            track.stop();
+        }
+    }
+}
+
+/**
+ * Stops the microphone from listening
+ */
+function stopMicrophone(): void {
+    if (currentStream) {
+        for (const track of currentStream.getAudioTracks()) {
+            track.stop();
+        }
+    }
+}
+
+/**
+ * A store containing the MediaStream object (or null if nothing requested, or Error if an error occurred)
+ */
+export const localStreamStore = derived<Readable<MediaStreamConstraints>, LocalStreamStoreValue>(mediaStreamConstraintsStore, ($mediaStreamConstraintsStore, set) => {
+    const constraints = { ...$mediaStreamConstraintsStore };
+
+    if (navigator.mediaDevices === undefined) {
+        if (window.location.protocol === 'http:') {
+            //throw new Error('Unable to access your camera or microphone. You need to use a HTTPS connection.');
+            set({
+                type: 'error',
+                error: new Error('Unable to access your camera or microphone. You need to use a HTTPS connection.'),
+                constraints
+            });
+        } else {
+            //throw new Error('Unable to access your camera or microphone. Your browser is too old.');
+            set({
+                type: 'error',
+                error: new Error('Unable to access your camera or microphone. Your browser is too old.'),
+                constraints
+            });
+        }
+    }
+
+    if (constraints.audio === false) {
+        stopMicrophone();
+    }
+    if (constraints.video === false) {
+        stopCamera();
+    }
+
+    if (constraints.audio === false && constraints.video === false) {
+        set({
+            type: 'success',
+            stream: null,
+            constraints
+        });
+        return;
+    }
+
+    (async () => {
+        try {
+            currentStream = await navigator.mediaDevices.getUserMedia(constraints);
+            set({
+                type: 'success',
+                stream: currentStream,
+                constraints
+            });
+            return;
+        } catch (e) {
+            if (constraints.video !== false) {
+                console.info("Error. Unable to get microphone and/or camera access. Trying audio only.", $mediaStreamConstraintsStore, e);
+                // TODO: does it make sense to pop this error when retrying?
+                set({
+                    type: 'error',
+                    error: e,
+                    constraints
+                });
+                // Let's try without video constraints
+                requestedCameraState.disableWebcam();
+            } else {
+                console.info("Error. Unable to get microphone and/or camera access.", $mediaStreamConstraintsStore, e);
+                set({
+                    type: 'error',
+                    error: e,
+                    constraints
+                });
+            }
+
+            /*constraints.video = false;
+            if (constraints.audio === false) {
+                console.info("Error. Unable to get microphone and/or camera access.", $mediaStreamConstraintsStore, e);
+                set({
+                    type: 'error',
+                    error: e,
+                    constraints
+                });
+                // Let's make as if the user did not ask.
+                requestedCameraState.disableWebcam();
+            } else {
+                console.info("Error. Unable to get microphone and/or camera access. Trying audio only.", $mediaStreamConstraintsStore, e);
+                try {
+                    currentStream = await navigator.mediaDevices.getUserMedia(constraints);
+                    set({
+                        type: 'success',
+                        stream: currentStream,
+                        constraints
+                    });
+                    return;
+                } catch (e2) {
+                    console.info("Error. Unable to get microphone fallback access.", $mediaStreamConstraintsStore, e2);
+                    set({
+                        type: 'error',
+                        error: e,
+                        constraints
+                    });
+                }
+            }*/
+        }
+    })();
+});

front/src/Stores/PeerStore.ts

@@ -0,0 +1,32 @@
+import { derived, writable, Writable } from "svelte/store";
+import type {UserSimplePeerInterface} from "../WebRtc/SimplePeer";
+import type {SimplePeer} from "../WebRtc/SimplePeer";
+
+/**
+ * A store that contains the camera state requested by the user (on or off).
+ */
+function createPeerStore() {
+    let users = new Map<number, UserSimplePeerInterface>();
+
+    const { subscribe, set, update } = writable(users);
+
+    return {
+        subscribe,
+        connectToSimplePeer: (simplePeer: SimplePeer) => {
+            users = new Map<number, UserSimplePeerInterface>();
+            set(users);
+            simplePeer.registerPeerConnectionListener({
+                onConnect(user: UserSimplePeerInterface) {
+                    users.set(user.userId, user);
+                    set(users);
+                },
+                onDisconnect(userId: number) {
+                    users.delete(userId);
+                    set(users);
+                }
+            })
+        }
+    };
+}
+
+export const peerStore = createPeerStore();

front/src/WebRtc/JitsiFactory.ts

@@ -1,6 +1,7 @@
 import {JITSI_URL} from "../Enum/EnvironmentVariable";
 import {mediaManager} from "./MediaManager";
 import {coWebsiteManager} from "./CoWebsiteManager";
+import {requestedCameraState, requestedMicrophoneState} from "../Stores/MediaStore";
 declare const window:any; // eslint-disable-line @typescript-eslint/no-explicit-any
 
 interface jitsiConfigInterface {
@@ -138,14 +139,18 @@ class JitsiFactory {
         //restore previous config
         if(this.previousConfigMeet?.startWithAudioMuted){
             await mediaManager.disableMicrophone();
+            requestedMicrophoneState.disableMicrophone();
         }else{
             await mediaManager.enableMicrophone();
+            requestedMicrophoneState.enableMicrophone();
         }
 
         if(this.previousConfigMeet?.startWithVideoMuted){
             await mediaManager.disableCamera();
+            requestedCameraState.disableWebcam();
         }else{
             await mediaManager.enableCamera();
+            requestedCameraState.enableWebcam();
         }
     }
 

front/src/WebRtc/MediaManager.ts

@@ -6,6 +6,12 @@ import {localUserStore} from "../Connexion/LocalUserStore";
 import type {UserSimplePeerInterface} from "./SimplePeer";
 import {SoundMeter} from "../Phaser/Components/SoundMeter";
 import {DISABLE_NOTIFICATIONS} from "../Enum/EnvironmentVariable";
+import {
+    gameOverlayVisibilityStore,
+    mediaStreamConstraintsStore,
+    requestedCameraState,
+    requestedMicrophoneState
+} from "../Stores/MediaStore";
 
 declare const navigator:any; // eslint-disable-line @typescript-eslint/no-explicit-any
 
@@ -90,12 +96,14 @@ export class MediaManager {
             e.preventDefault();
             this.enableMicrophone();
             //update tracking
+            requestedMicrophoneState.enableMicrophone();
         });
         this.microphone = HtmlUtils.getElementByIdOrFail<HTMLImageElement>('microphone');
         this.microphone.addEventListener('click', (e: MouseEvent) => {
             e.preventDefault();
             this.disableMicrophone();
             //update tracking
+            requestedMicrophoneState.disableMicrophone();
         });
 
         this.cinemaBtn = HtmlUtils.getElementByIdOrFail<HTMLDivElement>('btn-video');
@@ -105,12 +113,14 @@ export class MediaManager {
             e.preventDefault();
             this.enableCamera();
             //update tracking
+            requestedCameraState.enableWebcam();
         });
         this.cinema = HtmlUtils.getElementByIdOrFail<HTMLImageElement>('cinema');
         this.cinema.addEventListener('click', (e: MouseEvent) => {
             e.preventDefault();
             this.disableCamera();
             //update tracking
+            requestedCameraState.disableWebcam();
         });
 
         this.monitorBtn = HtmlUtils.getElementByIdOrFail<HTMLDivElement>('btn-monitor');
@@ -214,6 +224,8 @@ export class MediaManager {
             this.triggerCloseJitsiFrameButton();
         }
         buttonCloseFrame.removeEventListener('click', functionTrigger);
+
+        gameOverlayVisibilityStore.showGameOverlay();
     }
 
     public hideGameOverlay(): void {
@@ -225,6 +237,8 @@ export class MediaManager {
             this.triggerCloseJitsiFrameButton();
         }
         buttonCloseFrame.addEventListener('click', functionTrigger);
+
+        gameOverlayVisibilityStore.hideGameOverlay();
     }
 
     public isGameOverlayVisible(): boolean {

front/style/style.css

@@ -346,6 +346,8 @@ video#myCamVideo{
 #myCamVideoSetup {
     width: 100%;
     height: 100%;
+    -webkit-transform: scaleX(-1);
+    transform: scaleX(-1);
 }
 .webrtcsetup.active{
     display: block;