Kraut.World/server
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Migrating the video overlay in Svelte (WIP)

Browse source
This commit is contained in:
David Négrier 2021-06-11 11:29:36 +02:00
parent e6264948b1
commit e7b0f859a5
20 changed files with 630 additions and 97 deletions
Download patch file Download diff file Expand all files Collapse all files

37
front/src/WebRtc/MediaManager.ts
View file

@ -7,7 +7,7 @@ import type {UserSimplePeerInterface} from "./SimplePeer";
import {SoundMeter} from "../Phaser/Components/SoundMeter";
import {DISABLE_NOTIFICATIONS} from "../Enum/EnvironmentVariable";
import {
gameOverlayVisibilityStore, localStreamStore,
localStreamStore,
} from "../Stores/MediaStore";
import {
screenSharingLocalStreamStore
@ -22,6 +22,7 @@ export type ShowReportCallBack = (userId: string, userName: string|undefined) =>
export type HelpCameraSettingsCallBack = () => void;
import {cowebsiteCloseButtonId} from "./CoWebsiteManager";
import {gameOverlayVisibilityStore} from "../Stores/GameOverlayStoreVisibility";
export class MediaManager {
private remoteVideo: Map<string, HTMLVideoElement> = new Map<string, HTMLVideoElement>();
@ -65,7 +66,7 @@ export class MediaManager {
}
});
let isScreenSharing = false;
//let isScreenSharing = false;
screenSharingLocalStreamStore.subscribe((result) => {
if (result.type === 'error') {
console.error(result.error);
@ -75,7 +76,7 @@ export class MediaManager {
return;
}
if (result.stream !== null) {
/*if (result.stream !== null) {
isScreenSharing = true;
this.addScreenSharingActiveVideo('me', DivImportance.Normal);
HtmlUtils.getElementByIdOrFail<HTMLVideoElement>('screen-sharing-me').srcObject = result.stream;
@ -84,7 +85,7 @@ export class MediaManager {
isScreenSharing = false;
this.removeActiveScreenSharingVideo('me');
}
}
}*/
});
@ -134,7 +135,7 @@ export class MediaManager {
gameOverlayVisibilityStore.hideGameOverlay();
}
addActiveVideo(user: UserSimplePeerInterface, userName: string = ""){
/*addActiveVideo(user: UserSimplePeerInterface, userName: string = ""){
const userId = ''+user.userId
userName = userName.toUpperCase();
@ -194,7 +195,7 @@ export class MediaManager {
layoutManager.add(divImportance, userId, html);
this.remoteVideo.set(userId, HtmlUtils.getElementByIdOrFail<HTMLVideoElement>(userId));
}
}*/
private getScreenSharingId(userId: string): string {
return `screen-sharing-${userId}`;
@ -242,19 +243,12 @@ export class MediaManager {
const blockLogoElement = HtmlUtils.getElementByIdOrFail<HTMLImageElement>('blocking-'+userId);
show ? blockLogoElement.classList.add('active') : blockLogoElement.classList.remove('active');
}
addStreamRemoteVideo(userId: string, stream : MediaStream): void {
/*addStreamRemoteVideo(userId: string, stream : MediaStream): void {
const remoteVideo = this.remoteVideo.get(userId);
if (remoteVideo === undefined) {
throw `Unable to find video for ${userId}`;
}
remoteVideo.srcObject = stream;
//FIX ME SOUNDMETER: check stalability of sound meter calculation
//sound metter
/*const soundMeter = new SoundMeter();
soundMeter.connectToSource(stream, new AudioContext());
this.soundMeters.set(userId, soundMeter);
this.soundMeterElements.set(userId, HtmlUtils.getElementByIdOrFail<HTMLImageElement>('soundMeter-'+userId));*/
}
addStreamRemoteScreenSharing(userId: string, stream : MediaStream){
// In the case of screen sharing (going both ways), we may need to create the HTML element if it does not exist yet
@ -264,23 +258,18 @@ export class MediaManager {
}
this.addStreamRemoteVideo(this.getScreenSharingId(userId), stream);
}
}*/
removeActiveVideo(userId: string){
layoutManager.remove(userId);
this.remoteVideo.delete(userId);
//FIX ME SOUNDMETER: check stalability of sound meter calculation
/*this.soundMeters.get(userId)?.stop();
this.soundMeters.delete(userId);
this.soundMeterElements.delete(userId);*/
//layoutManager.remove(userId);
//this.remoteVideo.delete(userId);
//permit to remove user in discussion part
this.removeParticipant(userId);
}
removeActiveScreenSharingVideo(userId: string) {
/*removeActiveScreenSharingVideo(userId: string) {
this.removeActiveVideo(this.getScreenSharingId(userId))
}
}*/
isConnecting(userId: string): void {
const connectingSpinnerDiv = this.getSpinner(userId);

73
front/src/WebRtc/ScreenSharingPeer.ts
View file

@ -1,9 +1,11 @@
import type * as SimplePeerNamespace from "simple-peer";
import {mediaManager} from "./MediaManager";
import {STUN_SERVER, TURN_SERVER, TURN_USER, TURN_PASSWORD} from "../Enum/EnvironmentVariable";
import {STUN_SERVER, TURN_PASSWORD, TURN_SERVER, TURN_USER} from "../Enum/EnvironmentVariable";
import type {RoomConnection} from "../Connexion/RoomConnection";
import {MESSAGE_TYPE_CONSTRAINT} from "./VideoPeer";
import type {UserSimplePeerInterface} from "./SimplePeer";
import {Readable, readable, writable, Writable} from "svelte/store";
import {DivImportance} from "./LayoutManager";
const Peer: SimplePeerNamespace.SimplePeer = require('simple-peer');
@ -17,9 +19,13 @@ export class ScreenSharingPeer extends Peer {
private isReceivingStream:boolean = false;
public toClose: boolean = false;
public _connected: boolean = false;
private userId: number;
public readonly userId: number;
public readonly uniqueId: string;
public readonly streamStore: Readable<MediaStream | null>;
public readonly importanceStore: Writable<DivImportance>;
public readonly statusStore: Readable<"connecting" | "connected" | "error" | "closed">;
constructor(user: UserSimplePeerInterface, initiator: boolean, private connection: RoomConnection, stream: MediaStream | null) {
constructor(user: UserSimplePeerInterface, initiator: boolean, public readonly userName: string, private connection: RoomConnection, stream: MediaStream | null) {
super({
initiator: initiator ? initiator : false,
//reconnectTimer: 10000,
@ -38,6 +44,56 @@ export class ScreenSharingPeer extends Peer {
});
this.userId = user.userId;
this.uniqueId = 'screensharing_'+this.userId;
this.streamStore = readable<MediaStream|null>(null, (set) => {
const onStream = (stream: MediaStream|null) => {
set(stream);
};
const onData = (chunk: Buffer) => {
// We unfortunately need to rely on an event to let the other party know a stream has stopped.
// It seems there is no native way to detect that.
// TODO: we might rely on the "ended" event: https://developer.mozilla.org/en-US/docs/Web/API/HTMLMediaElement/ended_event
const message = JSON.parse(chunk.toString('utf8'));
if (message.streamEnded !== true) {
console.error('Unexpected message on screen sharing peer connection');
return;
}
set(null);
}
this.on('stream', onStream);
this.on('data', onData);
return () => {
this.off('stream', onStream);
this.off('data', onData);
};
});
this.importanceStore = writable(DivImportance.Important);
this.statusStore = readable<"connecting" | "connected" | "error" | "closed">("connecting", (set) => {
const onConnect = () => {
set('connected');
};
const onError = () => {
set('error');
};
const onClose = () => {
set('closed');
};
this.on('connect', onConnect);
this.on('error', onError);
this.on('close', onClose);
return () => {
this.off('connect', onConnect);
this.off('error', onError);
this.off('close', onClose);
};
});
//start listen signal for the peer connection
this.on('signal', (data: unknown) => {
@ -54,16 +110,17 @@ export class ScreenSharingPeer extends Peer {
this.destroy();
});
this.on('data', (chunk: Buffer) => {
/*this.on('data', (chunk: Buffer) => {
// We unfortunately need to rely on an event to let the other party know a stream has stopped.
// It seems there is no native way to detect that.
// TODO: we might rely on the "ended" event: https://developer.mozilla.org/en-US/docs/Web/API/HTMLMediaElement/ended_event
const message = JSON.parse(chunk.toString('utf8'));
if (message.streamEnded !== true) {
console.error('Unexpected message on screen sharing peer connection');
return;
}
mediaManager.removeActiveScreenSharingVideo("" + this.userId);
});
});*/
// eslint-disable-next-line @typescript-eslint/no-explicit-any
this.on('error', (err: any) => {
@ -103,10 +160,10 @@ export class ScreenSharingPeer extends Peer {
//console.log(`ScreenSharingPeer::stream => ${this.userId}`, stream);
//console.log(`stream => ${this.userId} => `, stream);
if(!stream){
mediaManager.removeActiveScreenSharingVideo("" + this.userId);
//mediaManager.removeActiveScreenSharingVideo("" + this.userId);
this.isReceivingStream = false;
} else {
mediaManager.addStreamRemoteScreenSharing("" + this.userId, stream);
//mediaManager.addStreamRemoteScreenSharing("" + this.userId, stream);
this.isReceivingStream = true;
}
}
@ -121,7 +178,7 @@ export class ScreenSharingPeer extends Peer {
if(!this.toClose){
return;
}
mediaManager.removeActiveScreenSharingVideo("" + this.userId);
//mediaManager.removeActiveScreenSharingVideo("" + this.userId);
// FIXME: I don't understand why "Closing connection with" message is displayed TWICE before "Nb users in peerConnectionArray"
// I do understand the method closeConnection is called twice, but I don't understand how they manage to run in parallel.
//console.log('Closing connection with '+userId);

36
front/src/WebRtc/SimplePeer.ts
View file

@ -28,8 +28,10 @@ export interface UserSimplePeerInterface{
webRtcPassword?: string|undefined;
}
export type RemotePeer = VideoPeer | ScreenSharingPeer;
export interface PeerConnectionListener {
onConnect(user: UserSimplePeerInterface): void;
onConnect(user: RemotePeer): void;
onDisconnect(userId: number): void;
}
@ -159,20 +161,17 @@ export class SimplePeer {
let name = user.name;
if (!name) {
const userSearch = this.Users.find((userSearch: UserSimplePeerInterface) => userSearch.userId === user.userId);
if (userSearch) {
name = userSearch.name;
}
name = this.getName(user.userId);
}
mediaManager.removeActiveVideo("" + user.userId);
mediaManager.addActiveVideo(user, name);
//mediaManager.addActiveVideo(user, name);
this.lastWebrtcUserName = user.webRtcUser;
this.lastWebrtcPassword = user.webRtcPassword;
const peer = new VideoPeer(user, user.initiator ? user.initiator : false, this.Connection, localStream);
const peer = new VideoPeer(user, user.initiator ? user.initiator : false, name, this.Connection, localStream);
//permit to send message
mediaManager.addSendMessageCallback(user.userId,(message: string) => {
@ -196,11 +195,20 @@ export class SimplePeer {
this.PeerConnectionArray.set(user.userId, peer);
for (const peerConnectionListener of this.peerConnectionListeners) {
peerConnectionListener.onConnect(user);
peerConnectionListener.onConnect(peer);
}
return peer;
}
private getName(userId: number): string {
const userSearch = this.Users.find((userSearch: UserSimplePeerInterface) => userSearch.userId === userId);
if (userSearch) {
return userSearch.name || '';
} else {
return '';
}
}
/**
* create peer connection to bind users
*/
@ -222,10 +230,10 @@ export class SimplePeer {
}
// We should display the screen sharing ONLY if we are not initiator
if (!user.initiator) {
/* if (!user.initiator) {
mediaManager.removeActiveScreenSharingVideo("" + user.userId);
mediaManager.addScreenSharingActiveVideo("" + user.userId);
}
}*/
// Enrich the user with last known credentials (if they are not set in the user object, which happens when a user triggers the screen sharing)
if (user.webRtcUser === undefined) {
@ -233,11 +241,13 @@ export class SimplePeer {
user.webRtcPassword = this.lastWebrtcPassword;
}
const peer = new ScreenSharingPeer(user, user.initiator ? user.initiator : false, this.Connection, stream);
const name = this.getName(user.userId);
const peer = new ScreenSharingPeer(user, user.initiator ? user.initiator : false, name, this.Connection, stream);
this.PeerScreenSharingConnectionArray.set(user.userId, peer);
for (const peerConnectionListener of this.peerConnectionListeners) {
peerConnectionListener.onConnect(user);
peerConnectionListener.onConnect(peer);
}
return peer;
}
@ -288,7 +298,7 @@ export class SimplePeer {
*/
private closeScreenSharingConnection(userId : number) {
try {
mediaManager.removeActiveScreenSharingVideo("" + userId);
//mediaManager.removeActiveScreenSharingVideo("" + userId);
const peer = this.PeerScreenSharingConnectionArray.get(userId);
if (peer === undefined) {
console.warn("closeScreenSharingConnection => Tried to close connection for user "+userId+" but could not find user")

80
front/src/WebRtc/VideoPeer.ts
View file

@ -5,8 +5,9 @@ import type {RoomConnection} from "../Connexion/RoomConnection";
import {blackListManager} from "./BlackListManager";
import type {Subscription} from "rxjs";
import type {UserSimplePeerInterface} from "./SimplePeer";
import {get} from "svelte/store";
import {get, readable, Readable, writable, Writable} from "svelte/store";
import {obtainedMediaConstraintStore} from "../Stores/MediaStore";
import {DivImportance} from "./LayoutManager";
const Peer: SimplePeerNamespace.SimplePeer = require('simple-peer');
@ -22,12 +23,16 @@ export class VideoPeer extends Peer {
public _connected: boolean = false;
private remoteStream!: MediaStream;
private blocked: boolean = false;
private userId: number;
private userName: string;
public readonly userId: number;
public readonly uniqueId: string;
private onBlockSubscribe: Subscription;
private onUnBlockSubscribe: Subscription;
public readonly streamStore: Readable<MediaStream | null>;
public readonly importanceStore: Writable<DivImportance>;
public readonly statusStore: Readable<"connecting" | "connected" | "error" | "closed">;
public readonly constraintsStore: Readable<MediaStreamConstraints|null>;
constructor(public user: UserSimplePeerInterface, initiator: boolean, private connection: RoomConnection, localStream: MediaStream | null) {
constructor(public user: UserSimplePeerInterface, initiator: boolean, public readonly userName: string, private connection: RoomConnection, localStream: MediaStream | null) {
super({
initiator: initiator ? initiator : false,
//reconnectTimer: 10000,
@ -46,7 +51,70 @@ export class VideoPeer extends Peer {
});
this.userId = user.userId;
this.userName = user.name || '';
this.uniqueId = 'video_'+this.userId;
this.streamStore = readable<MediaStream|null>(null, (set) => {
const onStream = (stream: MediaStream|null) => {
set(stream);
};
const onData = (chunk: Buffer) => {
this.on('data', (chunk: Buffer) => {
const message = JSON.parse(chunk.toString('utf8'));
if (message.type === MESSAGE_TYPE_CONSTRAINT) {
if (!message.video) {
set(null);
}
}
});
}
this.on('stream', onStream);
this.on('data', onData);
return () => {
this.off('stream', onStream);
this.off('data', onData);
};
});
this.constraintsStore = readable<MediaStreamConstraints|null>(null, (set) => {
const onData = (chunk: Buffer) => {
const message = JSON.parse(chunk.toString('utf8'));
if(message.type === MESSAGE_TYPE_CONSTRAINT) {
set(message);
}
}
this.on('data', onData);
return () => {
this.off('data', onData);
};
});
this.importanceStore = writable(DivImportance.Normal);
this.statusStore = readable<"connecting" | "connected" | "error" | "closed">("connecting", (set) => {
const onConnect = () => {
set('connected');
};
const onError = () => {
set('error');
};
const onClose = () => {
set('closed');
};
this.on('connect', onConnect);
this.on('error', onError);
this.on('close', onClose);
return () => {
this.off('connect', onConnect);
this.off('error', onError);
this.off('close', onClose);
};
});
//start listen signal for the peer connection
this.on('signal', (data: unknown) => {
@ -152,7 +220,7 @@ export class VideoPeer extends Peer {
if (blackListManager.isBlackListed(this.userId) || this.blocked) {
this.toggleRemoteStream(false);
}
mediaManager.addStreamRemoteVideo("" + this.userId, stream);
//mediaManager.addStreamRemoteVideo("" + this.userId, stream);
}catch (err){
console.error(err);
}