Kraut.World/server
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'develop' of github.com:thecodingmachine/workadventure into outline

Browse source
This commit is contained in:
David Négrier 2020-08-24 18:15:44 +02:00
commit fca93663b4
13 changed files with 991 additions and 201 deletions
Download patch file Download diff file Expand all files Collapse all files

36
front/src/Connection.ts
View file

@ -6,12 +6,12 @@ import {SetPlayerDetailsMessage} from "./Messages/SetPlayerDetailsMessage";
const SocketIo = require('socket.io-client');
import Socket = SocketIOClient.Socket;
import {PlayerAnimationNames} from "./Phaser/Player/Animation";
import {UserSimplePeer} from "./WebRtc/SimplePeer";
import {UserSimplePeerInterface} from "./WebRtc/SimplePeer";
import {SignalData} from "simple-peer";
enum EventMessage{
WEBRTC_SIGNAL = "webrtc-signal",
WEBRTC_SCREEN_SHARING_SIGNAL = "webrtc-screen-sharing-signal",
WEBRTC_START = "webrtc-start",
JOIN_ROOM = "join-room", // bi-directional
USER_POSITION = "user-position", // bi-directional
@ -73,17 +73,20 @@ export interface GroupCreatedUpdatedMessageInterface {
export interface WebRtcStartMessageInterface {
roomId: string,
clients: UserSimplePeer[]
clients: UserSimplePeerInterface[]
}
export interface WebRtcDisconnectMessageInterface {
userId: string
}
export interface WebRtcSignalMessageInterface {
userId: string,
export interface WebRtcSignalSentMessageInterface {
receiverId: string,
roomId: string,
signal: SignalData
}
export interface WebRtcSignalReceivedMessageInterface {
userId: string,
signal: SignalData
}
@ -201,23 +204,32 @@ export class Connection implements Connection {
this.socket.on(EventMessage.CONNECT_ERROR, callback)
}
public sendWebrtcSignal(signal: unknown, roomId: string, userId? : string|null, receiverId? : string) {
public sendWebrtcSignal(signal: unknown, receiverId : string) {
return this.socket.emit(EventMessage.WEBRTC_SIGNAL, {
userId: userId ? userId : this.userId,
receiverId: receiverId ? receiverId : this.userId,
roomId: roomId,
receiverId: receiverId,
signal: signal
});
} as WebRtcSignalSentMessageInterface);
}
public sendWebrtcScreenSharingSignal(signal: unknown, receiverId : string) {
return this.socket.emit(EventMessage.WEBRTC_SCREEN_SHARING_SIGNAL, {
receiverId: receiverId,
signal: signal
} as WebRtcSignalSentMessageInterface);
}
public receiveWebrtcStart(callback: (message: WebRtcStartMessageInterface) => void) {
this.socket.on(EventMessage.WEBRTC_START, callback);
}
public receiveWebrtcSignal(callback: (message: WebRtcSignalMessageInterface) => void) {
public receiveWebrtcSignal(callback: (message: WebRtcSignalReceivedMessageInterface) => void) {
return this.socket.on(EventMessage.WEBRTC_SIGNAL, callback);
}
public receiveWebrtcScreenSharingSignal(callback: (message: WebRtcSignalReceivedMessageInterface) => void) {
return this.socket.on(EventMessage.WEBRTC_SCREEN_SHARING_SIGNAL, callback);
}
public onServerDisconnected(callback: (reason: string) => void): void {
this.socket.on('disconnect', (reason: string) => {
if (reason === 'io client disconnect') {

40
front/src/Phaser/Game/GameScene.ts
View file

@ -10,7 +10,7 @@ import {
RoomJoinedMessageInterface
} from "../../Connection";
import {CurrentGamerInterface, hasMovedEventName, Player} from "../Player/Player";
import {DEBUG_MODE, POSITION_DELAY, ZOOM_LEVEL} from "../../Enum/EnvironmentVariable";
import {DEBUG_MODE, POSITION_DELAY, RESOLUTION, ZOOM_LEVEL} from "../../Enum/EnvironmentVariable";
import {
ITiledMap,
ITiledMapLayer,
@ -24,10 +24,10 @@ import {PlayerMovement} from "./PlayerMovement";
import {PlayersPositionInterpolator} from "./PlayersPositionInterpolator";
import {RemotePlayer} from "../Entity/RemotePlayer";
import {Queue} from 'queue-typescript';
import {SimplePeer, UserSimplePeer} from "../../WebRtc/SimplePeer";
import {SimplePeer, UserSimplePeerInterface} from "../../WebRtc/SimplePeer";
import {ReconnectingSceneName} from "../Reconnecting/ReconnectingScene";
import {loadAllLayers} from "../Entity/body_character";
import {layoutManager, LayoutMode} from "../../WebRtc/LayoutManager";
import {CenterListener, layoutManager, LayoutMode} from "../../WebRtc/LayoutManager";
import Texture = Phaser.Textures.Texture;
import Sprite = Phaser.GameObjects.Sprite;
import CanvasTexture = Phaser.Textures.CanvasTexture;
@ -78,7 +78,7 @@ interface DeleteGroupEventInterface {
groupId: string
}
export class GameScene extends Phaser.Scene {
export class GameScene extends Phaser.Scene implements CenterListener {
GameManager : GameManager;
Terrains : Array<Phaser.Tilemaps.Tileset>;
CurrentPlayer!: CurrentGamerInterface;
@ -263,7 +263,7 @@ export class GameScene extends Phaser.Scene {
this.simplePeer = new SimplePeer(this.connection);
const self = this;
this.simplePeer.registerPeerConnectionListener({
onConnect(user: UserSimplePeer) {
onConnect(user: UserSimplePeerInterface) {
self.presentationModeSprite.setVisible(true);
self.chatModeSprite.setVisible(true);
},
@ -537,6 +537,9 @@ export class GameScene extends Phaser.Scene {
this.repositionCallback = this.reposition.bind(this);
window.addEventListener('resize', this.repositionCallback);
this.reposition();
// From now, this game scene will be notified of reposition events
layoutManager.setListener(this);
}
private switchLayoutMode(): void {
@ -656,7 +659,7 @@ export class GameScene extends Phaser.Scene {
//todo: in a dedicated class/function?
initCamera() {
this.cameras.main.setBounds(0,0, this.Map.widthInPixels, this.Map.heightInPixels);
this.cameras.main.startFollow(this.CurrentPlayer);
this.updateCameraOffset();
this.cameras.main.setZoom(ZOOM_LEVEL);
}
@ -1058,5 +1061,30 @@ export class GameScene extends Phaser.Scene {
private reposition(): void {
this.presentationModeSprite.setY(this.game.renderer.height - 2);
this.chatModeSprite.setY(this.game.renderer.height - 2);
// Recompute camera offset if needed
this.updateCameraOffset();
}
/**
* Updates the offset of the character compared to the center of the screen according to the layout mananger
* (tries to put the character in the center of the reamining space if there is a discussion going on.
*/
private updateCameraOffset(): void {
const array = layoutManager.findBiggestAvailableArray();
let xCenter = (array.xEnd - array.xStart) / 2 + array.xStart;
let yCenter = (array.yEnd - array.yStart) / 2 + array.yStart;
// Let's put this in Game coordinates by applying the zoom level:
xCenter /= ZOOM_LEVEL * RESOLUTION;
yCenter /= ZOOM_LEVEL * RESOLUTION;
//console.log("updateCameraOffset", array, xCenter, yCenter, this.game.renderer.width, this.game.renderer.height);
this.cameras.main.startFollow(this.CurrentPlayer, true, 1, 1, xCenter - this.game.renderer.width / 2, yCenter - this.game.renderer.height / 2);
}
public onCenterChange(): void {
this.updateCameraOffset();
}
}

138
front/src/WebRtc/LayoutManager.ts
View file

@ -14,6 +14,14 @@ export enum DivImportance {
Normal = "Normal",
}
/**
* Classes implementing this interface can be notified when the center of the screen (the player position) should be
* changed.
*/
export interface CenterListener {
onCenterChange(): void;
}
/**
* This class is in charge of the video-conference layout.
* It receives positioning requests for videos and does its best to place them on the screen depending on the active layout mode.
@ -23,6 +31,11 @@ class LayoutManager {
private importantDivs: Map<string, HTMLDivElement> = new Map<string, HTMLDivElement>();
private normalDivs: Map<string, HTMLDivElement> = new Map<string, HTMLDivElement>();
private listener: CenterListener|null = null;
public setListener(centerListener: CenterListener|null) {
this.listener = centerListener;
}
public add(importance: DivImportance, userId: string, html: string): void {
const div = document.createElement('div');
@ -45,6 +58,7 @@ class LayoutManager {
this.positionDiv(div, importance);
this.adjustVideoChatClass();
this.listener?.onCenterChange();
}
private positionDiv(elem: HTMLDivElement, importance: DivImportance): void {
@ -72,6 +86,7 @@ class LayoutManager {
div.remove();
this.importantDivs.delete(userId);
this.adjustVideoChatClass();
this.listener?.onCenterChange();
return;
}
@ -80,6 +95,7 @@ class LayoutManager {
div.remove();
this.normalDivs.delete(userId);
this.adjustVideoChatClass();
this.listener?.onCenterChange();
return;
}
@ -123,11 +139,133 @@ class LayoutManager {
for (const div of this.normalDivs.values()) {
this.positionDiv(div, DivImportance.Normal);
}
this.listener?.onCenterChange();
}
public getLayoutMode(): LayoutMode {
return this.mode;
}
/*public getGameCenter(): {x: number, y: number} {
}*/
/**
* Tries to find the biggest available box of remaining space (this is a space where we can center the character)
*/
public findBiggestAvailableArray(): {xStart: number, yStart: number, xEnd: number, yEnd: number} {
if (this.mode === LayoutMode.VideoChat) {
const children = document.querySelectorAll<HTMLDivElement>('div.chat-mode > div');
const htmlChildren = Array.from(children.values());
// No chat? Let's go full center
if (htmlChildren.length === 0) {
return {
xStart: 0,
yStart: 0,
xEnd: window.innerWidth,
yEnd: window.innerHeight
}
}
const lastDiv = htmlChildren[htmlChildren.length - 1];
// Compute area between top right of the last div and bottom right of window
const area1 = (window.innerWidth - (lastDiv.offsetLeft + lastDiv.offsetWidth))
* (window.innerHeight - lastDiv.offsetTop);
// Compute area between bottom of last div and bottom of the screen on whole width
const area2 = window.innerWidth
* (window.innerHeight - (lastDiv.offsetTop + lastDiv.offsetHeight));
if (area1 < 0 && area2 < 0) {
// If screen is full, let's not attempt something foolish and simply center character in the middle.
return {
xStart: 0,
yStart: 0,
xEnd: window.innerWidth,
yEnd: window.innerHeight
}
}
if (area1 <= area2) {
console.log('lastDiv', lastDiv.offsetTop, lastDiv.offsetHeight);
return {
xStart: 0,
yStart: lastDiv.offsetTop + lastDiv.offsetHeight,
xEnd: window.innerWidth,
yEnd: window.innerHeight
}
} else {
console.log('lastDiv', lastDiv.offsetTop);
return {
xStart: lastDiv.offsetLeft + lastDiv.offsetWidth,
yStart: lastDiv.offsetTop,
xEnd: window.innerWidth,
yEnd: window.innerHeight
}
}
} else {
// Possible destinations: at the center bottom or at the right bottom.
const mainSectionChildren = Array.from(document.querySelectorAll<HTMLDivElement>('div.main-section > div').values());
const sidebarChildren = Array.from(document.querySelectorAll<HTMLDivElement>('aside.sidebar > div').values());
// Nothing? Let's center
if (mainSectionChildren.length === 0 && sidebarChildren.length === 0) {
return {
xStart: 0,
yStart: 0,
xEnd: window.innerWidth,
yEnd: window.innerHeight
}
}
if (mainSectionChildren.length === 0) {
const lastSidebarDiv = sidebarChildren[sidebarChildren.length-1];
// No presentation? Let's center on the main-section space
return {
xStart: 0,
yStart: 0,
xEnd: lastSidebarDiv.offsetLeft,
yEnd: window.innerHeight
}
}
// At this point, we know we have at least one element in the main section.
const lastPresentationDiv = mainSectionChildren[mainSectionChildren.length-1];
const presentationArea = (window.innerHeight - (lastPresentationDiv.offsetTop + lastPresentationDiv.offsetHeight))
* (lastPresentationDiv.offsetLeft + lastPresentationDiv.offsetWidth);
let leftSideBar: number;
let bottomSideBar: number;
if (sidebarChildren.length === 0) {
leftSideBar = HtmlUtils.getElementByIdOrFail<HTMLDivElement>('sidebar').offsetLeft;
bottomSideBar = 0;
} else {
const lastSideBarChildren = sidebarChildren[sidebarChildren.length - 1];
leftSideBar = lastSideBarChildren.offsetLeft;
bottomSideBar = lastSideBarChildren.offsetTop + lastSideBarChildren.offsetHeight;
}
const sideBarArea = (window.innerWidth - leftSideBar)
* (window.innerHeight - bottomSideBar);
if (presentationArea <= sideBarArea) {
return {
xStart: leftSideBar,
yStart: bottomSideBar,
xEnd: window.innerWidth,
yEnd: window.innerHeight
}
} else {
return {
xStart: 0,
yStart: lastPresentationDiv.offsetTop + lastPresentationDiv.offsetHeight,
xEnd: /*lastPresentationDiv.offsetLeft + lastPresentationDiv.offsetWidth*/ window.innerWidth , // To avoid flickering when a chat start, we center on the center of the screen, not the center of the main content area
yEnd: window.innerHeight
}
}
}
}
}
const layoutManager = new LayoutManager();

187
front/src/WebRtc/MediaManager.ts
View file

@ -1,4 +1,5 @@
import {DivImportance, layoutManager} from "./LayoutManager";
import {HtmlUtils} from "./HtmlUtils";
const videoConstraint: boolean|MediaTrackConstraints = {
width: { ideal: 1280 },
@ -7,15 +8,20 @@ const videoConstraint: boolean|MediaTrackConstraints = {
};
type UpdatedLocalStreamCallback = (media: MediaStream) => void;
type StartScreenSharingCallback = (media: MediaStream) => void;
type StopScreenSharingCallback = (media: MediaStream) => void;
// TODO: Split MediaManager in 2 classes: MediaManagerUI (in charge of HTML) and MediaManager (singleton in charge of the camera only)
// TODO: verify that microphone event listeners are not triggered plenty of time NOW (since MediaManager is created many times!!!!)
export class MediaManager {
localStream: MediaStream|null = null;
localScreenCapture: MediaStream|null = null;
private remoteVideo: Map<string, HTMLVideoElement> = new Map<string, HTMLVideoElement>();
myCamVideo: HTMLVideoElement;
cinemaClose: HTMLImageElement;
cinema: HTMLImageElement;
monitorClose: HTMLImageElement;
monitor: HTMLImageElement;
microphoneClose: HTMLImageElement;
microphone: HTMLImageElement;
webrtcInAudio: HTMLAudioElement;
@ -24,46 +30,81 @@ export class MediaManager {
video: videoConstraint
};
updatedLocalStreamCallBacks : Set<UpdatedLocalStreamCallback> = new Set<UpdatedLocalStreamCallback>();
startScreenSharingCallBacks : Set<StartScreenSharingCallback> = new Set<StartScreenSharingCallback>();
stopScreenSharingCallBacks : Set<StopScreenSharingCallback> = new Set<StopScreenSharingCallback>();
private microphoneBtn: HTMLDivElement;
private cinemaBtn: HTMLDivElement;
private monitorBtn: HTMLDivElement;
constructor() {
this.myCamVideo = this.getElementByIdOrFail<HTMLVideoElement>('myCamVideo');
this.webrtcInAudio = this.getElementByIdOrFail<HTMLAudioElement>('audio-webrtc-in');
this.webrtcInAudio.volume = 0.2;
this.microphoneBtn = this.getElementByIdOrFail<HTMLDivElement>('btn-micro');
this.microphoneClose = this.getElementByIdOrFail<HTMLImageElement>('microphone-close');
this.microphoneClose.style.display = "none";
this.microphoneClose.addEventListener('click', (e: MouseEvent) => {
e.preventDefault();
this.enabledMicrophone();
this.enableMicrophone();
//update tracking
});
this.microphone = this.getElementByIdOrFail<HTMLImageElement>('microphone');
this.microphone.addEventListener('click', (e: MouseEvent) => {
e.preventDefault();
this.disabledMicrophone();
this.disableMicrophone();
//update tracking
});
this.cinemaBtn = this.getElementByIdOrFail<HTMLDivElement>('btn-video');
this.cinemaClose = this.getElementByIdOrFail<HTMLImageElement>('cinema-close');
this.cinemaClose.style.display = "none";
this.cinemaClose.addEventListener('click', (e: MouseEvent) => {
e.preventDefault();
this.enabledCamera();
this.enableCamera();
//update tracking
});
this.cinema = this.getElementByIdOrFail<HTMLImageElement>('cinema');
this.cinema.addEventListener('click', (e: MouseEvent) => {
e.preventDefault();
this.disabledCamera();
this.disableCamera();
//update tracking
});
this.monitorBtn = this.getElementByIdOrFail<HTMLDivElement>('btn-monitor');
this.monitorClose = this.getElementByIdOrFail<HTMLImageElement>('monitor-close');
this.monitorClose.style.display = "block";
this.monitorClose.addEventListener('click', (e: MouseEvent) => {
e.preventDefault();
this.enableScreenSharing();
//update tracking
});
this.monitor = this.getElementByIdOrFail<HTMLImageElement>('monitor');
this.monitor.style.display = "none";
this.monitor.addEventListener('click', (e: MouseEvent) => {
e.preventDefault();
this.disableScreenSharing();
//update tracking
});
}
onUpdateLocalStream(callback: UpdatedLocalStreamCallback): void {
public onUpdateLocalStream(callback: UpdatedLocalStreamCallback): void {
this.updatedLocalStreamCallBacks.add(callback);
}
public onStartScreenSharing(callback: StartScreenSharingCallback): void {
this.startScreenSharingCallBacks.add(callback);
}
public onStopScreenSharing(callback: StopScreenSharingCallback): void {
this.stopScreenSharingCallBacks.add(callback);
}
removeUpdateLocalStreamEventListener(callback: UpdatedLocalStreamCallback): void {
this.updatedLocalStreamCallBacks.delete(callback);
}
@ -74,13 +115,26 @@ export class MediaManager {
}
}
activeVisio(){
private triggerStartedScreenSharingCallbacks(stream: MediaStream): void {
for (const callback of this.startScreenSharingCallBacks) {
callback(stream);
}
}
private triggerStoppedScreenSharingCallbacks(stream: MediaStream): void {
for (const callback of this.stopScreenSharingCallBacks) {
callback(stream);
}
}
showGameOverlay(){
const gameOverlay = this.getElementByIdOrFail('game-overlay');
gameOverlay.classList.add('active');
}
enabledCamera() {
private enableCamera() {
this.cinemaClose.style.display = "none";
this.cinemaBtn.classList.remove("disabled");
this.cinema.style.display = "block";
this.constraintsMedia.video = videoConstraint;
this.getCamera().then((stream: MediaStream) => {
@ -88,9 +142,10 @@ export class MediaManager {
});
}
disabledCamera() {
private disableCamera() {
this.cinemaClose.style.display = "block";
this.cinema.style.display = "none";
this.cinemaBtn.classList.add("disabled");
this.constraintsMedia.video = false;
this.myCamVideo.srcObject = null;
if (this.localStream) {
@ -103,18 +158,20 @@ export class MediaManager {
});
}
enabledMicrophone() {
private enableMicrophone() {
this.microphoneClose.style.display = "none";
this.microphone.style.display = "block";
this.microphoneBtn.classList.remove("disabled");
this.constraintsMedia.audio = true;
this.getCamera().then((stream) => {
this.triggerUpdatedLocalStreamCallbacks(stream);
});
}
disabledMicrophone() {
private disableMicrophone() {
this.microphoneClose.style.display = "block";
this.microphone.style.display = "none";
this.microphoneBtn.classList.add("disabled");
this.constraintsMedia.audio = false;
if(this.localStream) {
this.localStream.getAudioTracks().forEach((MediaStreamTrack: MediaStreamTrack) => {
@ -126,6 +183,80 @@ export class MediaManager {
});
}
private enableScreenSharing() {
this.monitorClose.style.display = "none";
this.monitor.style.display = "block";
this.monitorBtn.classList.add("enabled");
this.getScreenMedia().then((stream) => {
this.triggerStartedScreenSharingCallbacks(stream);
});
}
private disableScreenSharing() {
this.monitorClose.style.display = "block";
this.monitor.style.display = "none";
this.monitorBtn.classList.remove("enabled");
this.removeActiveScreenSharingVideo('me');
this.localScreenCapture?.getTracks().forEach((track: MediaStreamTrack) => {
track.stop();
});
if (this.localScreenCapture === null) {
console.warn('Weird: trying to remove a screen sharing that is not enabled');
return;
}
const localScreenCapture = this.localScreenCapture;
this.getCamera().then((stream) => {
this.triggerStoppedScreenSharingCallbacks(localScreenCapture);
});
this.localScreenCapture = null;
}
//get screen
getScreenMedia() : Promise<MediaStream>{
try {
return this._startScreenCapture()
.then((stream: MediaStream) => {
this.localScreenCapture = stream;
// If stream ends (for instance if user clicks the stop screen sharing button in the browser), let's close the view
for (const track of stream.getTracks()) {
track.onended = () => {
this.disableScreenSharing();
};
}
this.addScreenSharingActiveVideo('me', DivImportance.Normal);
HtmlUtils.getElementByIdOrFail<HTMLVideoElement>('screen-sharing-me').srcObject = stream;
return stream;
})
.catch((err: unknown) => {
console.error("Error => getScreenMedia => ", err);
throw err;
});
}catch (err) {
return new Promise((resolve, reject) => { // eslint-disable-line no-unused-vars
reject(err);
});
}
}
private _startScreenCapture() {
// getDisplayMedia was moved to mediaDevices in 2018. Typescript definitions are not up to date yet.
// See: https://github.com/w3c/mediacapture-screen-share/pull/86
// https://github.com/microsoft/TypeScript/issues/31821
if ((navigator as any).getDisplayMedia) { // eslint-disable-line @typescript-eslint/no-explicit-any
return (navigator as any).getDisplayMedia({video: true}); // eslint-disable-line @typescript-eslint/no-explicit-any
} else if ((navigator.mediaDevices as any).getDisplayMedia) { // eslint-disable-line @typescript-eslint/no-explicit-any
return (navigator.mediaDevices as any).getDisplayMedia({video: true}); // eslint-disable-line @typescript-eslint/no-explicit-any
} else {
//return navigator.mediaDevices.getUserMedia(({video: {mediaSource: 'screen'}} as any));
return new Promise((resolve, reject) => { // eslint-disable-line no-unused-vars
reject("error sharing screen");
});
}
}
//get camera
async getCamera(): Promise<MediaStream> {
if (navigator.mediaDevices === undefined) {
@ -205,6 +336,25 @@ export class MediaManager {
this.remoteVideo.set(userId, this.getElementByIdOrFail<HTMLVideoElement>(userId));
}
/**
*
* @param userId
*/
addScreenSharingActiveVideo(userId : string, divImportance: DivImportance = DivImportance.Important){
//this.webrtcInAudio.play();
userId = `screen-sharing-${userId}`;
const html = `
<div id="div-${userId}" class="video-container">
<video id="${userId}" autoplay></video>
</div>
`;
layoutManager.add(divImportance, userId, html);
this.remoteVideo.set(userId, this.getElementByIdOrFail<HTMLVideoElement>(userId));
}
/**
*
* @param userId
@ -272,6 +422,15 @@ export class MediaManager {
}
remoteVideo.srcObject = stream;
}
addStreamRemoteScreenSharing(userId : string, stream : MediaStream){
// In the case of screen sharing (going both ways), we may need to create the HTML element if it does not exist yet
const remoteVideo = this.remoteVideo.get(`screen-sharing-${userId}`);
if (remoteVideo === undefined) {
this.addScreenSharingActiveVideo(userId);
}
this.addStreamRemoteVideo(`screen-sharing-${userId}`, stream);
}
/**
*
@ -281,6 +440,9 @@ export class MediaManager {
layoutManager.remove(userId);
this.remoteVideo.delete(userId);
}
removeActiveScreenSharingVideo(userId : string) {
this.removeActiveVideo(`screen-sharing-${userId}`)
}
isConnecting(userId : string): void {
const connectingSpinnerDiv = this.getSpinner(userId);
@ -299,6 +461,7 @@ export class MediaManager {
}
isError(userId : string): void {
console.log("isError", `div-${userId}`);
const element = document.getElementById(`div-${userId}`);
if(!element){
return;
@ -309,6 +472,10 @@ export class MediaManager {
}
errorDiv.style.display = 'block';
}
isErrorScreenSharing(userId : string): void {
this.isError(`screen-sharing-${userId}`);
}
private getSpinner(userId : string): HTMLDivElement|null {
const element = document.getElementById(`div-${userId}`);

127
front/src/WebRtc/ScreenSharingPeer.ts Normal file
View file

@ -0,0 +1,127 @@
import * as SimplePeerNamespace from "simple-peer";
import {mediaManager} from "./MediaManager";
import {Connection} from "../Connection";
const Peer: SimplePeerNamespace.SimplePeer = require('simple-peer');
/**
* A peer connection used to transmit video / audio signals between 2 peers.
*/
export class ScreenSharingPeer extends Peer {
/**
* Whether this connection is currently receiving a video stream from a remote user.
*/
private isReceivingStream:boolean = false;
constructor(private userId: string, initiator: boolean, private connection: Connection) {
super({
initiator: initiator ? initiator : false,
reconnectTimer: 10000,
config: {
iceServers: [
{
urls: 'stun:stun.l.google.com:19302'
},
{
urls: 'turn:numb.viagenie.ca',
username: 'g.parant@thecodingmachine.com',
credential: 'itcugcOHxle9Acqi$'
},
]
}
});
//start listen signal for the peer connection
this.on('signal', (data: unknown) => {
this.sendWebrtcScreenSharingSignal(data);
});
this.on('stream', (stream: MediaStream) => {
this.stream(stream);
});
this.on('close', () => {
this.destroy();
});
this.on('data', (chunk: Buffer) => {
// We unfortunately need to rely on an event to let the other party know a stream has stopped.
// It seems there is no native way to detect that.
const message = JSON.parse(chunk.toString('utf8'));
if (message.streamEnded !== true) {
console.error('Unexpected message on screen sharing peer connection');
}
mediaManager.removeActiveScreenSharingVideo(this.userId);
});
// eslint-disable-next-line @typescript-eslint/no-explicit-any
this.on('error', (err: any) => {
console.error(`screen sharing error => ${this.userId} => ${err.code}`, err);
//mediaManager.isErrorScreenSharing(this.userId);
});
this.on('connect', () => {
// FIXME: we need to put the loader on the screen sharing connection
mediaManager.isConnected(this.userId);
console.info(`connect => ${this.userId}`);
});
this.pushScreenSharingToRemoteUser();
}
private sendWebrtcScreenSharingSignal(data: unknown) {
console.log("sendWebrtcScreenSharingSignal", data);
try {
this.connection.sendWebrtcScreenSharingSignal(data, this.userId);
}catch (e) {
console.error(`sendWebrtcScreenSharingSignal => ${this.userId}`, e);
}
}
/**
* Sends received stream to screen.
*/
private stream(stream?: MediaStream) {
console.log(`ScreenSharingPeer::stream => ${this.userId}`, stream);
console.log(`stream => ${this.userId} => `, stream);
if(!stream){
mediaManager.removeActiveScreenSharingVideo(this.userId);
this.isReceivingStream = false;
} else {
mediaManager.addStreamRemoteScreenSharing(this.userId, stream);
this.isReceivingStream = true;
}
}
public isReceivingScreenSharingStream(): boolean {
return this.isReceivingStream;
}
public destroy(error?: Error): void {
try {
mediaManager.removeActiveScreenSharingVideo(this.userId);
// FIXME: I don't understand why "Closing connection with" message is displayed TWICE before "Nb users in peerConnectionArray"
// I do understand the method closeConnection is called twice, but I don't understand how they manage to run in parallel.
//console.log('Closing connection with '+userId);
super.destroy(error);
//console.log('Nb users in peerConnectionArray '+this.PeerConnectionArray.size);
} catch (err) {
console.error("ScreenSharingPeer::destroy", err)
}
}
private pushScreenSharingToRemoteUser() {
const localScreenCapture: MediaStream | null = mediaManager.localScreenCapture;
if(!localScreenCapture){
return;
}
this.addStream(localScreenCapture);
return;
}
public stopPushingScreenSharingToRemoteUser(stream: MediaStream) {
this.removeStream(stream);
this.write(new Buffer(JSON.stringify({streamEnded: true})));
}
}

345
front/src/WebRtc/SimplePeer.ts
View file

@ -1,21 +1,23 @@
import {
Connection,
WebRtcDisconnectMessageInterface,
WebRtcSignalMessageInterface,
WebRtcSignalReceivedMessageInterface,
WebRtcStartMessageInterface
} from "../Connection";
import { mediaManager } from "./MediaManager";
import * as SimplePeerNamespace from "simple-peer";
import {ScreenSharingPeer} from "./ScreenSharingPeer";
import {VideoPeer} from "./VideoPeer";
const Peer: SimplePeerNamespace.SimplePeer = require('simple-peer');
export interface UserSimplePeer{
export interface UserSimplePeerInterface{
userId: string;
name?: string;
initiator?: boolean;
}
export interface PeerConnectionListener {
onConnect(user: UserSimplePeer): void;
onConnect(user: UserSimplePeerInterface): void;
onDisconnect(userId: string): void;
}
@ -26,18 +28,25 @@ export interface PeerConnectionListener {
export class SimplePeer {
private Connection: Connection;
private WebRtcRoomId: string;
private Users: Array<UserSimplePeer> = new Array<UserSimplePeer>();
private Users: Array<UserSimplePeerInterface> = new Array<UserSimplePeerInterface>();
private PeerConnectionArray: Map<string, SimplePeerNamespace.Instance> = new Map<string, SimplePeerNamespace.Instance>();
private readonly updateLocalStreamCallback: (media: MediaStream) => void;
private PeerScreenSharingConnectionArray: Map<string, ScreenSharingPeer> = new Map<string, ScreenSharingPeer>();
private PeerConnectionArray: Map<string, VideoPeer> = new Map<string, VideoPeer>();
private readonly sendLocalVideoStreamCallback: (media: MediaStream) => void;
private readonly sendLocalScreenSharingStreamCallback: (media: MediaStream) => void;
private readonly stopLocalScreenSharingStreamCallback: (media: MediaStream) => void;
private readonly peerConnectionListeners: Array<PeerConnectionListener> = new Array<PeerConnectionListener>();
constructor(Connection: Connection, WebRtcRoomId: string = "test-webrtc") {
this.Connection = Connection;
this.WebRtcRoomId = WebRtcRoomId;
// We need to go through this weird bound function pointer in order to be able to "free" this reference later.
this.updateLocalStreamCallback = this.updatedLocalStream.bind(this);
mediaManager.onUpdateLocalStream(this.updateLocalStreamCallback);
this.sendLocalVideoStreamCallback = this.sendLocalVideoStream.bind(this);
this.sendLocalScreenSharingStreamCallback = this.sendLocalScreenSharingStream.bind(this);
this.stopLocalScreenSharingStreamCallback = this.stopLocalScreenSharingStream.bind(this);
mediaManager.onUpdateLocalStream(this.sendLocalVideoStreamCallback);
mediaManager.onStartScreenSharing(this.sendLocalScreenSharingStreamCallback);
mediaManager.onStopScreenSharing(this.stopLocalScreenSharingStreamCallback);
this.initialise();
}
@ -55,11 +64,16 @@ export class SimplePeer {
private initialise() {
//receive signal by gemer
this.Connection.receiveWebrtcSignal((message: WebRtcSignalMessageInterface) => {
this.Connection.receiveWebrtcSignal((message: WebRtcSignalReceivedMessageInterface) => {
this.receiveWebrtcSignal(message);
});
mediaManager.activeVisio();
//receive signal by gemer
this.Connection.receiveWebrtcScreenSharingSignal((message: WebRtcSignalReceivedMessageInterface) => {
this.receiveWebrtcScreenSharingSignal(message);
});
mediaManager.showGameOverlay();
mediaManager.getCamera().then(() => {
//receive message start
@ -79,7 +93,7 @@ export class SimplePeer {
private receiveWebrtcStart(data: WebRtcStartMessageInterface) {
this.WebRtcRoomId = data.roomId;
this.Users = data.clients;
// Note: the clients array contain the list of all clients (event the ones we are already connected to in case a user joints a group)
// Note: the clients array contain the list of all clients (even the ones we are already connected to in case a user joints a group)
// So we can receive a request we already had before. (which will abort at the first line of createPeerConnection)
// TODO: refactor this to only send a message to connect to one user (rather than several users).
// This would be symmetrical to the way we handle disconnection.
@ -93,7 +107,8 @@ export class SimplePeer {
* server has two people connected, start the meet
*/
private startWebRtc() {
this.Users.forEach((user: UserSimplePeer) => {
console.warn('startWebRtc startWebRtc');
this.Users.forEach((user: UserSimplePeerInterface) => {
//if it's not an initiator, peer connection will be created when gamer will receive offer signal
if(!user.initiator){
return;
@ -105,102 +120,63 @@ export class SimplePeer {
/**
* create peer connection to bind users
*/
private createPeerConnection(user : UserSimplePeer) {
if(this.PeerConnectionArray.has(user.userId)) {
return;
private createPeerConnection(user : UserSimplePeerInterface) : VideoPeer | null{
if(
this.PeerConnectionArray.has(user.userId)
){
return null;
}
//console.log("Creating connection with peer "+user.userId);
let name = user.name;
if(!name){
const userSearch = this.Users.find((userSearch: UserSimplePeer) => userSearch.userId === user.userId);
const userSearch = this.Users.find((userSearch: UserSimplePeerInterface) => userSearch.userId === user.userId);
if(userSearch) {
name = userSearch.name;
}
}
mediaManager.removeActiveVideo(user.userId);
mediaManager.addActiveVideo(user.userId, name);
const peer : SimplePeerNamespace.Instance = new Peer({
initiator: user.initiator ? user.initiator : false,
reconnectTimer: 10000,
config: {
iceServers: [
{
urls: 'stun:stun.l.google.com:19302'
},
{
urls: 'turn:numb.viagenie.ca',
username: 'g.parant@thecodingmachine.com',
credential: 'itcugcOHxle9Acqi$'
},
]
},
const peer = new VideoPeer(user.userId, user.initiator ? user.initiator : false, this.Connection);
// When a connection is established to a video stream, and if a screen sharing is taking place,
// the user sharing screen should also initiate a connection to the remote user!
peer.on('connect', () => {
if (mediaManager.localScreenCapture) {
this.sendLocalScreenSharingStreamToUser(user.userId);
}
});
this.PeerConnectionArray.set(user.userId, peer);
//start listen signal for the peer connection
peer.on('signal', (data: unknown) => {
this.sendWebrtcSignal(data, user.userId);
});
peer.on('stream', (stream: MediaStream) => {
let videoActive = false;
let microphoneActive = false;
stream.getTracks().forEach((track : MediaStreamTrack) => {
if(track.kind === "audio"){
microphoneActive = true;
}
if(track.kind === "video"){
videoActive = true;
}
});
if(microphoneActive){
mediaManager.enabledMicrophoneByUserId(user.userId);
}else{
mediaManager.disabledMicrophoneByUserId(user.userId);
}
if(videoActive){
mediaManager.enabledVideoByUserId(user.userId);
}else{
mediaManager.disabledVideoByUserId(user.userId);
}
this.stream(user.userId, stream);
});
/*peer.on('track', (track: MediaStreamTrack, stream: MediaStream) => {
this.stream(user.userId, stream);
});*/
peer.on('close', () => {
this.closeConnection(user.userId);
});
// eslint-disable-next-line @typescript-eslint/no-explicit-any
peer.on('error', (err: any) => {
console.error(`error => ${user.userId} => ${err.code}`, err);
mediaManager.isError(user.userId);
});
peer.on('connect', () => {
mediaManager.isConnected(user.userId);
console.info(`connect => ${user.userId}`);
});
peer.on('data', (chunk: Buffer) => {
const data = JSON.parse(chunk.toString('utf8'));
if(data.type === "stream"){
this.stream(user.userId, data.stream);
}
});
this.addMedia(user.userId);
for (const peerConnectionListener of this.peerConnectionListeners) {
peerConnectionListener.onConnect(user);
}
return peer;
}
/**
* create peer connection to bind users
*/
private createPeerScreenSharingConnection(user : UserSimplePeerInterface) : ScreenSharingPeer | null{
if(
this.PeerScreenSharingConnectionArray.has(user.userId)
){
return null;
}
// We should display the screen sharing ONLY if we are not initiator
if (!user.initiator) {
mediaManager.removeActiveScreenSharingVideo(user.userId);
mediaManager.addScreenSharingActiveVideo(user.userId);
}
const peer = new ScreenSharingPeer(user.userId, user.initiator ? user.initiator : false, this.Connection);
this.PeerScreenSharingConnectionArray.set(user.userId, peer);
for (const peerConnectionListener of this.peerConnectionListeners) {
peerConnectionListener.onConnect(user);
}
return peer;
}
/**
@ -210,17 +186,18 @@ export class SimplePeer {
*/
private closeConnection(userId : string) {
try {
mediaManager.removeActiveVideo(userId);
//mediaManager.removeActiveVideo(userId);
const peer = this.PeerConnectionArray.get(userId);
if (peer === undefined) {
console.warn("Tried to close connection for user "+userId+" but could not find user")
return;
}
peer.destroy();
// FIXME: I don't understand why "Closing connection with" message is displayed TWICE before "Nb users in peerConnectionArray"
// I do understand the method closeConnection is called twice, but I don't understand how they manage to run in parallel.
//console.log('Closing connection with '+userId);
peer.destroy();
this.PeerConnectionArray.delete(userId)
this.PeerConnectionArray.delete(userId);
this.closeScreenSharingConnection(userId);
//console.log('Nb users in peerConnectionArray '+this.PeerConnectionArray.size);
for (const peerConnectionListener of this.peerConnectionListeners) {
peerConnectionListener.onDisconnect(userId);
@ -230,34 +207,49 @@ export class SimplePeer {
}
}
/**
* This is triggered twice. Once by the server, and once by a remote client disconnecting
*
* @param userId
*/
private closeScreenSharingConnection(userId : string) {
try {
mediaManager.removeActiveScreenSharingVideo(userId);
const peer = this.PeerScreenSharingConnectionArray.get(userId);
if (peer === undefined) {
console.warn("Tried to close connection for user "+userId+" but could not find user")
return;
}
// FIXME: I don't understand why "Closing connection with" message is displayed TWICE before "Nb users in peerConnectionArray"
// I do understand the method closeConnection is called twice, but I don't understand how they manage to run in parallel.
//console.log('Closing connection with '+userId);
peer.destroy();
this.PeerScreenSharingConnectionArray.delete(userId)
//console.log('Nb users in peerConnectionArray '+this.PeerConnectionArray.size);
} catch (err) {
console.error("closeConnection", err)
}
}
public closeAllConnections() {
for (const userId of this.PeerConnectionArray.keys()) {
this.closeConnection(userId);
}
for (const userId of this.PeerScreenSharingConnectionArray.keys()) {
this.closeScreenSharingConnection(userId);
}
}
/**
* Unregisters any held event handler.
*/
public unregister() {
mediaManager.removeUpdateLocalStreamEventListener(this.updateLocalStreamCallback);
}
/**
*
* @param userId
* @param data
*/
private sendWebrtcSignal(data: unknown, userId : string) {
try {
this.Connection.sendWebrtcSignal(data, this.WebRtcRoomId, null, userId);
}catch (e) {
console.error(`sendWebrtcSignal => ${userId}`, e);
}
mediaManager.removeUpdateLocalStreamEventListener(this.sendLocalVideoStreamCallback);
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
private receiveWebrtcSignal(data: WebRtcSignalMessageInterface) {
private receiveWebrtcSignal(data: WebRtcSignalReceivedMessageInterface) {
try {
//if offer type, create peer connection
if(data.signal.type === "offer"){
@ -274,53 +266,126 @@ export class SimplePeer {
}
}
/**
*
* @param userId
* @param stream
*/
private stream(userId : string, stream: MediaStream) {
if(!stream){
mediaManager.disabledVideoByUserId(userId);
mediaManager.disabledMicrophoneByUserId(userId);
return;
private receiveWebrtcScreenSharingSignal(data: WebRtcSignalReceivedMessageInterface) {
console.log("receiveWebrtcScreenSharingSignal", data);
try {
//if offer type, create peer connection
if(data.signal.type === "offer"){
this.createPeerScreenSharingConnection(data);
}
const peer = this.PeerScreenSharingConnectionArray.get(data.userId);
if (peer !== undefined) {
peer.signal(data.signal);
} else {
console.error('Could not find peer whose ID is "'+data.userId+'" in receiveWebrtcScreenSharingSignal');
}
} catch (e) {
console.error(`receiveWebrtcSignal => ${data.userId}`, e);
}
mediaManager.addStreamRemoteVideo(userId, stream);
}
/**
*
* @param userId
*/
private addMedia (userId : string) {
private pushVideoToRemoteUser(userId : string) {
try {
const localStream: MediaStream|null = mediaManager.localStream;
const peer = this.PeerConnectionArray.get(userId);
if(localStream === null) {
//send fake signal
if(peer === undefined){
return;
}
peer.write(new Buffer(JSON.stringify({
type: "stream",
stream: null
})));
const PeerConnection = this.PeerConnectionArray.get(userId);
if (!PeerConnection) {
throw new Error('While adding media, cannot find user with ID ' + userId);
}
const localStream: MediaStream | null = mediaManager.localStream;
PeerConnection.write(new Buffer(JSON.stringify(mediaManager.constraintsMedia)));
if(!localStream){
return;
}
if (peer === undefined) {
throw new Error('While adding media, cannot find user with ID '+userId);
}
for (const track of localStream.getTracks()) {
peer.addTrack(track, localStream);
PeerConnection.addTrack(track, localStream);
}
}catch (e) {
console.error(`addMedia => addMedia => ${userId}`, e);
console.error(`pushVideoToRemoteUser => ${userId}`, e);
}
}
updatedLocalStream(){
this.Users.forEach((user: UserSimplePeer) => {
this.addMedia(user.userId);
private pushScreenSharingToRemoteUser(userId : string) {
const PeerConnection = this.PeerScreenSharingConnectionArray.get(userId);
if (!PeerConnection) {
throw new Error('While pushing screen sharing, cannot find user with ID ' + userId);
}
const localScreenCapture: MediaStream | null = mediaManager.localScreenCapture;
if(!localScreenCapture){
return;
}
for (const track of localScreenCapture.getTracks()) {
PeerConnection.addTrack(track, localScreenCapture);
}
return;
}
public sendLocalVideoStream(){
this.Users.forEach((user: UserSimplePeerInterface) => {
this.pushVideoToRemoteUser(user.userId);
})
}
/**
* Triggered locally when clicking on the screen sharing button
*/
public sendLocalScreenSharingStream() {
if (!mediaManager.localScreenCapture) {
console.error('Could not find localScreenCapture to share')
return;
}
for (const user of this.Users) {
this.sendLocalScreenSharingStreamToUser(user.userId);
}
}
/**
* Triggered locally when clicking on the screen sharing button
*/
public stopLocalScreenSharingStream(stream: MediaStream) {
for (const user of this.Users) {
this.stopLocalScreenSharingStreamToUser(user.userId, stream);
}
}
private sendLocalScreenSharingStreamToUser(userId: string): void {
// If a connection already exists with user (because it is already sharing a screen with us... let's use this connection)
if (this.PeerScreenSharingConnectionArray.has(userId)) {
this.pushScreenSharingToRemoteUser(userId);
return;
}
const screenSharingUser: UserSimplePeerInterface = {
userId,
initiator: true
};
const PeerConnectionScreenSharing = this.createPeerScreenSharingConnection(screenSharingUser);
if (!PeerConnectionScreenSharing) {
return;
}
}
private stopLocalScreenSharingStreamToUser(userId: string, stream: MediaStream): void {
const PeerConnectionScreenSharing = this.PeerScreenSharingConnectionArray.get(userId);
if (!PeerConnectionScreenSharing) {
throw new Error('Weird, screen sharing connection to user ' + userId + 'not found')
}
console.log("updatedScreenSharing => destroy", PeerConnectionScreenSharing);
// Stop sending stream and close peer connection if peer is not sending stream too
PeerConnectionScreenSharing.stopPushingScreenSharingToRemoteUser(stream);
if (!PeerConnectionScreenSharing.isReceivingScreenSharingStream()) {
PeerConnectionScreenSharing.destroy();
this.PeerScreenSharingConnectionArray.delete(userId);
}
}
}

128
front/src/WebRtc/VideoPeer.ts Normal file
View file

@ -0,0 +1,128 @@
import * as SimplePeerNamespace from "simple-peer";
import {mediaManager} from "./MediaManager";
import {Connection} from "../Connection";
const Peer: SimplePeerNamespace.SimplePeer = require('simple-peer');
/**
* A peer connection used to transmit video / audio signals between 2 peers.
*/
export class VideoPeer extends Peer {
constructor(private userId: string, initiator: boolean, private connection: Connection) {
super({
initiator: initiator ? initiator : false,
reconnectTimer: 10000,
config: {
iceServers: [
{
urls: 'stun:stun.l.google.com:19302'
},
{
urls: 'turn:numb.viagenie.ca',
username: 'g.parant@thecodingmachine.com',
credential: 'itcugcOHxle9Acqi$'
},
]
}
});
//start listen signal for the peer connection
this.on('signal', (data: unknown) => {
this.sendWebrtcSignal(data);
});
this.on('stream', (stream: MediaStream) => {
this.stream(stream);
});
/*peer.on('track', (track: MediaStreamTrack, stream: MediaStream) => {
});*/
this.on('close', () => {
this.destroy();
});
// eslint-disable-next-line @typescript-eslint/no-explicit-any
this.on('error', (err: any) => {
console.error(`error => ${this.userId} => ${err.code}`, err);
mediaManager.isError(userId);
});
this.on('connect', () => {
mediaManager.isConnected(this.userId);
console.info(`connect => ${this.userId}`);
});
this.on('data', (chunk: Buffer) => {
const constraint = JSON.parse(chunk.toString('utf8'));
console.log("data", constraint);
if (constraint.audio) {
mediaManager.enabledMicrophoneByUserId(this.userId);
} else {
mediaManager.disabledMicrophoneByUserId(this.userId);
}
if (constraint.video || constraint.screen) {
mediaManager.enabledVideoByUserId(this.userId);
} else {
this.stream(undefined);
mediaManager.disabledVideoByUserId(this.userId);
}
});
this.pushVideoToRemoteUser();
}
private sendWebrtcSignal(data: unknown) {
try {
this.connection.sendWebrtcSignal(data, this.userId);
}catch (e) {
console.error(`sendWebrtcSignal => ${this.userId}`, e);
}
}
/**
* Sends received stream to screen.
*/
private stream(stream?: MediaStream) {
console.log(`VideoPeer::stream => ${this.userId}`, stream);
if(!stream){
mediaManager.disabledVideoByUserId(this.userId);
mediaManager.disabledMicrophoneByUserId(this.userId);
} else {
mediaManager.addStreamRemoteVideo(this.userId, stream);
}
}
/**
* This is triggered twice. Once by the server, and once by a remote client disconnecting
*/
public destroy(error?: Error): void {
try {
mediaManager.removeActiveVideo(this.userId);
// FIXME: I don't understand why "Closing connection with" message is displayed TWICE before "Nb users in peerConnectionArray"
// I do understand the method closeConnection is called twice, but I don't understand how they manage to run in parallel.
//console.log('Closing connection with '+userId);
super.destroy(error);
} catch (err) {
console.error("VideoPeer::destroy", err)
}
}
private pushVideoToRemoteUser() {
try {
const localStream: MediaStream | null = mediaManager.localStream;
this.write(new Buffer(JSON.stringify(mediaManager.constraintsMedia)));
if(!localStream){
return;
}
for (const track of localStream.getTracks()) {
this.addTrack(track, localStream);
}
}catch (e) {
console.error(`pushVideoToRemoteUser => ${this.userId}`, e);
}
}
}