Kraut.World/server
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update screen sharing feature

Browse source
This commit is contained in:
Gregoire Parant 2020-06-11 23:18:06 +02:00
parent 4a843fea77
commit 3f579914dc
4 changed files with 161 additions and 41 deletions
Download patch file Download diff file Expand all files Collapse all files

33
back/src/Controller/IoSocketController.ts
View file

@ -28,6 +28,7 @@ enum SockerIoEvent {
USER_MOVED = "user-moved", // From server to client
USER_LEFT = "user-left", // From server to client
WEBRTC_SIGNAL = "webrtc-signal",
WEBRTC_SCREEN_SHARING_SIGNAL = "webrtc-screen-sharing-signal",
WEBRTC_START = "webrtc-start",
WEBRTC_DISCONNECT = "webrtc-disconect",
MESSAGE_ERROR = "message-error",
@ -222,18 +223,11 @@ export class IoSocketController {
});
socket.on(SockerIoEvent.WEBRTC_SIGNAL, (data: unknown) => {
if (!isWebRtcSignalMessageInterface(data)) {
socket.emit(SockerIoEvent.MESSAGE_ERROR, {message: 'Invalid WEBRTC_SIGNAL message.'});
console.warn('Invalid WEBRTC_SIGNAL message received: ', data);
return;
}
//send only at user
const client = this.sockets.get(data.receiverId);
if (client === undefined) {
console.warn("While exchanging a WebRTC signal: client with id ", data.receiverId, " does not exist. This might be a race condition.");
return;
}
return client.emit(SockerIoEvent.WEBRTC_SIGNAL, data);
this.emit((socket as ExSocketInterface), data, SockerIoEvent.WEBRTC_SIGNAL);
});
socket.on(SockerIoEvent.WEBRTC_SCREEN_SHARING_SIGNAL, (data: unknown) => {
this.emit((socket as ExSocketInterface), data, SockerIoEvent.WEBRTC_SCREEN_SHARING_SIGNAL);
});
socket.on(SockerIoEvent.DISCONNECT, () => {
@ -280,6 +274,21 @@ export class IoSocketController {
});
}
emit(socket: ExSocketInterface, data: unknown, event: SockerIoEvent){
if (!isWebRtcSignalMessageInterface(data)) {
socket.emit(SockerIoEvent.MESSAGE_ERROR, {message: 'Invalid WEBRTC_SIGNAL message.'});
console.warn('Invalid WEBRTC_SIGNAL message received: ', data);
return;
}
//send only at user
const client = this.sockets.get(data.receiverId);
if (client === undefined) {
console.warn("While exchanging a WebRTC signal: client with id ", data.receiverId, " does not exist. This might be a race condition.");
return;
}
return client.emit(event, data);
}
searchClientByIdOrFail(userId: string): ExSocketInterface {
const client: ExSocketInterface|undefined = this.sockets.get(userId);
if (client === undefined) {

19
front/src/Connection.ts
View file

@ -9,9 +9,9 @@ import Socket = SocketIOClient.Socket;
import {PlayerAnimationNames} from "./Phaser/Player/Animation";
import {UserSimplePeerInterface} from "./WebRtc/SimplePeer";
enum EventMessage{
WEBRTC_SIGNAL = "webrtc-signal",
WEBRTC_SCREEN_SHARING_SIGNAL = "webrtc-screen-sharing-signal",
WEBRTC_START = "webrtc-start",
WEBRTC_JOIN_ROOM = "webrtc-join-room",
JOIN_ROOM = "join-room", // bi-directional
@ -124,8 +124,12 @@ export interface ConnectionInterface {
/*webrtc*/
sendWebrtcSignal(signal: any, roomId: string, userId?: string|null, receiverId?: string): void;
sendWebrtcScreenSharingSignal(signal: any, roomId: string, userId?: string|null, receiverId?: string): void;
receiveWebrtcSignal(callBack: Function): void;
receiveWebrtcScreenSharingSignal(callBack: Function): void;
receiveWebrtcStart(callBack: (message: WebRtcStartMessageInterface) => void): void;
disconnectMessage(callBack: (message: WebRtcDisconnectMessageInterface) => void): void;
@ -290,6 +294,15 @@ export class Connection implements ConnectionInterface {
});
}
sendWebrtcScreenSharingSignal(signal: any, roomId: string, userId? : string|null, receiverId? : string) {
return this.getSocket().emit(EventMessage.WEBRTC_SCREEN_SHARING_SIGNAL, {
userId: userId ? userId : this.userId,
receiverId: receiverId ? receiverId : this.userId,
roomId: roomId,
signal: signal
});
}
receiveWebrtcStart(callback: (message: WebRtcStartMessageInterface) => void) {
this.getSocket().on(EventMessage.WEBRTC_START, callback);
}
@ -298,6 +311,10 @@ export class Connection implements ConnectionInterface {
return this.getSocket().on(EventMessage.WEBRTC_SIGNAL, callback);
}
receiveWebrtcScreenSharingSignal(callback: Function) {
return this.getSocket().on(EventMessage.WEBRTC_SCREEN_SHARING_SIGNAL, callback);
}
private errorMessage(): void {
this.getSocket().on(EventMessage.MESSAGE_ERROR, (message: string) => {
console.error(EventMessage.MESSAGE_ERROR, message);

15
front/src/WebRtc/MediaManager.ts
View file

@ -241,13 +241,12 @@ export class MediaManager {
*
* @param userId
*/
addScreenSharingActiveVideo(userId : string, userName: string = ""){
addScreenSharingActiveVideo(userId : string){
userId = `screen-sharing-${userId}`;
this.webrtcInAudio.play();
let elementRemoteVideo = this.getElementByIdOrFail("activeScreenSharing");
userName = userName.toUpperCase();
let color = this.getColorByString(userName);
elementRemoteVideo.insertAdjacentHTML('beforeend', `
<div id="div-${userId}" class="screen-sharing-video-container" style="border-color: ${color};">
<div id="div-${userId}" class="screen-sharing-video-container">
<video id="${userId}" autoplay></video>
</div>
`);
@ -255,6 +254,7 @@ export class MediaManager {
if(!activeHTMLVideoElement){
return;
}
console.log(userId, (activeHTMLVideoElement as HTMLVideoElement));
this.remoteVideo.set(userId, (activeHTMLVideoElement as HTMLVideoElement));
}
@ -320,8 +320,12 @@ export class MediaManager {
* @param stream
*/
addStreamRemoteVideo(userId : string, stream : MediaStream){
console.log("this.remoteVideo.get(userId)"+" => "+userId, this.remoteVideo.get(userId));
this.remoteVideo.get(userId).srcObject = stream;
}
addStreamRemoteScreenSharing(userId : string, stream : MediaStream){
this.addStreamRemoteVideo(`screen-sharing-${userId}`, stream);
}
/**
*
@ -334,6 +338,9 @@ export class MediaManager {
}
element.remove();
}
removeActiveScreenSharingVideo(userId : string) {
this.removeActiveVideo(`screen-sharing-${userId}`)
}
isConnecting(userId : string): void {
let connectingSpinnerDiv = this.getSpinner(userId);

135
front/src/WebRtc/SimplePeer.ts
View file

@ -19,6 +19,7 @@ export class SimplePeer {
private MediaManager: MediaManager;
private PeerScreenSharingConnectionArray: Map<string, SimplePeerNamespace.Instance> = new Map<string, SimplePeerNamespace.Instance>();
private PeerConnectionArray: Map<string, SimplePeerNamespace.Instance> = new Map<string, SimplePeerNamespace.Instance>();
constructor(Connection: ConnectionInterface, WebRtcRoomId: string = "test-webrtc") {
@ -45,6 +46,11 @@ export class SimplePeer {
this.receiveWebrtcSignal(message);
});
//receive signal by gemer
this.Connection.receiveWebrtcScreenSharingSignal((message: any) => {
this.receiveWebrtcScreenSharingSignal(message);
});
this.MediaManager.activeVisio();
this.MediaManager.getCamera().then(() => {
@ -91,7 +97,8 @@ export class SimplePeer {
/**
* create peer connection to bind users
*/
private createPeerConnection(user : UserSimplePeerInterface) : SimplePeerNamespace.Instance | null{
private createPeerConnection(user : UserSimplePeerInterface, screenSharing: boolean = false) : SimplePeerNamespace.Instance | null{
console.log("createPeerConnection => screenSharing", screenSharing)
if(this.PeerConnectionArray.has(user.userId)) {
return null;
}
@ -104,12 +111,11 @@ export class SimplePeer {
}
}
let screenSharing : boolean = name !== undefined && name.indexOf("screenSharing") > -1;
this.MediaManager.removeActiveVideo(user.userId);
if(!screenSharing) {
this.MediaManager.addActiveVideo(user.userId, name);
if(screenSharing) {
this.MediaManager.addScreenSharingActiveVideo(user.userId);
}else{
this.MediaManager.addScreenSharingActiveVideo(user.userId, name);
this.MediaManager.addActiveVideo(user.userId, name);
}
let peer : SimplePeerNamespace.Instance = new Peer({
@ -128,10 +134,19 @@ export class SimplePeer {
]
},
});
this.PeerConnectionArray.set(user.userId, peer);
if(screenSharing){
this.PeerScreenSharingConnectionArray.set(user.userId, peer);
}else {
this.PeerConnectionArray.set(user.userId, peer);
}
//start listen signal for the peer connection
peer.on('signal', (data: any) => {
console.log("screenSharing", screenSharing);
if(screenSharing){
this.sendWebrtcScreenSharingSignal(data, user.userId);
return;
}
this.sendWebrtcSignal(data, user.userId);
});
@ -143,6 +158,9 @@ export class SimplePeer {
});*/
peer.on('close', () => {
if(screenSharing){
this.closeScreenSharingConnection(user.userId);
}
this.closeConnection(user.userId);
});
@ -172,7 +190,11 @@ export class SimplePeer {
}
});
this.addMedia(user.userId);
if(screenSharing){
this.addMediaScreenSharing(user.userId);
}else {
this.addMedia(user.userId);
}
return peer;
}
@ -200,12 +222,37 @@ export class SimplePeer {
}
}
/**
* This is triggered twice. Once by the server, and once by a remote client disconnecting
*
* @param userId
*/
private closeScreenSharingConnection(userId : string) {
try {
this.MediaManager.removeActiveScreenSharingVideo(userId);
let peer = this.PeerScreenSharingConnectionArray.get(userId);
if (peer === undefined) {
console.warn("Tried to close connection for user "+userId+" but could not find user")
return;
}
// FIXME: I don't understand why "Closing connection with" message is displayed TWICE before "Nb users in peerConnectionArray"
// I do understand the method closeConnection is called twice, but I don't understand how they manage to run in parallel.
//console.log('Closing connection with '+userId);
peer.destroy();
this.PeerScreenSharingConnectionArray.delete(userId)
//console.log('Nb users in peerConnectionArray '+this.PeerConnectionArray.size);
} catch (err) {
console.error("closeConnection", err)
}
}
/**
*
* @param userId
* @param data
*/
private sendWebrtcSignal(data: any, userId : string) {
console.log("sendWebrtcSignal", data);
try {
this.Connection.sendWebrtcSignal(data, this.WebRtcRoomId, null, userId);
}catch (e) {
@ -213,7 +260,22 @@ export class SimplePeer {
}
}
/**
*
* @param userId
* @param data
*/
private sendWebrtcScreenSharingSignal(data: any, userId : string) {
console.log("sendWebrtcScreenSharingSignal", data);
try {
this.Connection.sendWebrtcScreenSharingSignal(data, this.WebRtcRoomId, null, userId);
}catch (e) {
console.error(`sendWebrtcSignal => ${userId}`, e);
}
}
private receiveWebrtcSignal(data: any) {
console.log("receiveWebrtcSignal", data);
try {
//if offer type, create peer connection
if(data.signal.type === "offer"){
@ -230,6 +292,24 @@ export class SimplePeer {
}
}
private receiveWebrtcScreenSharingSignal(data: any) {
console.log("receiveWebrtcScreenSharingSignal", data);
try {
//if offer type, create peer connection
if(data.signal.type === "offer"){
this.createPeerConnection(data, true);
}
let peer = this.PeerConnectionArray.get(data.userId);
if (peer !== undefined) {
peer.signal(data.signal);
} else {
console.error('Could not find peer whose ID is "'+data.userId+'" in PeerConnectionArray');
}
} catch (e) {
console.error(`receiveWebrtcSignal => ${data.userId}`, e);
}
}
/**
*
* @param userId
@ -254,18 +334,8 @@ export class SimplePeer {
if (!PeerConnection) {
throw new Error('While adding media, cannot find user with ID ' + userId);
}
if(userId.indexOf("screenSharing") > -1 && this.MediaManager.localScreenCapture){
for (const track of this.MediaManager.localScreenCapture.getTracks()) {
PeerConnection.addTrack(track, this.MediaManager.localScreenCapture);
}
return;
}
let localStream: MediaStream | null = this.MediaManager.localStream;
let localScreenCapture: MediaStream | null = this.MediaManager.localScreenCapture;
PeerConnection.write(new Buffer(JSON.stringify(Object.assign(this.MediaManager.constraintsMedia, {screen: localScreenCapture !== null}))));
PeerConnection.write(new Buffer(JSON.stringify(this.MediaManager.constraintsMedia)));
if(!localStream){
return;
@ -280,6 +350,21 @@ export class SimplePeer {
}
}
private addMediaScreenSharing (userId : any = null) {
let PeerConnection = this.PeerScreenSharingConnectionArray.get(userId);
if (!PeerConnection) {
throw new Error('While adding media, cannot find user with ID ' + userId);
}
let localScreenCapture: MediaStream | null = this.MediaManager.localScreenCapture;
if(!localScreenCapture){
return;
}
/*for (const track of localScreenCapture.getTracks()) {
PeerConnection.addTrack(track, localScreenCapture);
}*/
return;
}
updatedLocalStream(){
this.Users.forEach((user: UserSimplePeerInterface) => {
this.addMedia(user.userId);
@ -288,12 +373,14 @@ export class SimplePeer {
updatedScreenSharing() {
if (this.MediaManager.localScreenCapture) {
if(!this.Connection.userId){
return;
}
let screenSharingUser: UserSimplePeerInterface = {
userId: `screenSharing-${this.Connection.userId}`,
name: 'screenSharing',
userId: this.Connection.userId,
initiator: true
};
let PeerConnectionScreenSharing = this.createPeerConnection(screenSharingUser);
let PeerConnectionScreenSharing = this.createPeerConnection(screenSharingUser, true);
if (!PeerConnectionScreenSharing) {
return;
}
@ -304,12 +391,12 @@ export class SimplePeer {
}catch (e) {
console.error("updatedScreenSharing => ", e);
}
this.MediaManager.addStreamRemoteVideo(screenSharingUser.userId, this.MediaManager.localScreenCapture);
this.MediaManager.addStreamRemoteScreenSharing(screenSharingUser.userId, this.MediaManager.localScreenCapture);
} else {
if (!this.PeerConnectionArray.has(`screenSharing-${this.Connection.userId}`)) {
if (!this.Connection.userId || !this.PeerScreenSharingConnectionArray.has(this.Connection.userId)) {
return;
}
let PeerConnectionScreenSharing = this.PeerConnectionArray.get(`screenSharing-${this.Connection.userId}`);
let PeerConnectionScreenSharing = this.PeerScreenSharingConnectionArray.get(this.Connection.userId);
if (!PeerConnectionScreenSharing) {
return;
}