How can i get remote streams on react-native-webrtc from a Kurento Media Server - react-native

Question

I am trying to connect a Kurento Media Server to React Native on iOS for a group video call.
The server is running on Nodejs and Docker, and they already have a TURN server set up.
The app is already running in the web browser via kurento-utils, but I can't stream the web video to RN, instead, the RN video looks just fine on the web.
I'm not usig kurento-utils on client side of React Native.
This is my code:
Video Screen (emmiter and receptor)
import React, { useState, useEffect } from 'react';
import {
View,
StyleSheet,
Text,
} from 'react-native';
import Display from 'react-native-display';
import InCallManager from 'react-native-incall-manager';
import { TouchableOpacity } from 'react-native-gesture-handler';
import { socket } from '../../screens/User';
import ReceiveScreen from './ReceiveScreen';
import {
RTCView,
RTCIceCandidate,
} from 'react-native-webrtc';
import {
startCommunication,
receiveVideo,
addIceCandidate,
ProcessAnswer,
ViewPCArray
} from '../../utils/webrtc-utils';
const participants = {};
function sendMessage(message) {
if (socket) {
socket.emit('message', message);
}
}
const VideoScreen = () => {
const [videoURL, setVideoURL] = useState(null)
const [remoteURL, setRemoteURL] = useState(null)
const [userName, setUserName] = useState(socket.id)
const [roomName, setRoomName] = useState('9b33737f-737f-4a3d-a323-e1cd3f4b68b2')
useEffect(() => {
if(socket) {
var message = {
event: 'joinRoom',
userName: userName,
roomName: roomName
}
sendMessage(message);
}
socket.on('message', message => {
messageProcessHandler(message);
});
return () => {
socket.close();
}
}, [])
/**
*
* #param {*} msg
*/
const messageProcessHandler = (msg) => {
// console.log(`MessageProcessHandler: ${msg.event}`);
switch (msg.event) {
case 'existingParticipants':
console.log('LOG:176 onExistingParticipants= ' + JSON.stringify(msg));
startCommunication(sendMessage, userName, (stream) => {
setVideoURL(stream.toURL())
msg.existingUsers.forEach((object) => {
participants[object.name] = object.id;
console.log("participants:" + JSON.stringify(participants));
receiveVideo(sendMessage, object.name, (pc) => {
console.log('getRemoteStreams ', pc);
pc.getRemoteStreams().forEach(track => {
// console.log('TRACK: ', track);
console.log('STREAM', track);
// pc.addTrack(track, stream);
setRemoteURL(track.toURL())
})
});
});
});
break;
case 'newParticipantArrived':
participants[msg.name] = msg.name;
if (remoteURL == null || remoteURL === '') {
receiveVideo(sendMessage, msg.userName, (pc) => {
pc.getRemoteStreams().forEach(track => {
console.log('STREAM', track);
setRemoteURL(track.toURL())
})
});
}
break;
case 'participantLeft':
participantLeft(msg.name);
break;
case 'receiveVideoAnswer':
ProcessAnswer(msg.senderid, msg.sdpAnswer, (err) => {
if (err) {
console.log('the error: ' + err);
}
});
break;
case 'candidate':
addIceCandidate(msg.userid, new RTCIceCandidate(msg.candidate));
break;
default:
console.error('Unrecognized message', msg.message);
}
}
/**
*
* #param {*} name
*/
const participantLeft = (name) => {
if (participants[name]) {
delete participants[name];
}
if (Object.keys(participants).length == 0) {
setRemoteURL(null)
}
}
return (
<View style={styles.container}>
<View style={{position: "absolute", top: 40, right: 20, zIndex: 200}}>
<Text style={{fontSize: 20, fontWeight: "bold", color: "#fff"}}>{userName}</Text>
</View>
<RTCView zOrder={0} objectFit='cover' style={styles.videoContainer} streamURL={videoURL} />
<Display enable={remoteURL != null}>
<View style={styles.floatView}>
<ReceiveScreen videoURL={remoteURL} />
</View>
</Display>
</View>
);
}
const styles = StyleSheet.create({
container: {
flex: 1
},
videoContainer: {
flex: 1
},
floatView: {
position: 'absolute',
width: 250,
height: 210,
bottom: 15,
right: 20,
backgroundColor: 'rgba(255, 255, 255, 0.1)',
borderRadius: 15
}
});
export default VideoScreen
Display (where the browser video is supposed to be)
import React, { Component } from 'react';
import config from "../../config/app.js";
import {
View,
Text,
ScrollView,
StyleSheet
} from 'react-native';
import { RTCView } from 'react-native-webrtc';
const styles = StyleSheet.create({
root: {
flex: 1
},
titleContainer: {
flex: 0.1,
paddingVertical: 2,
paddingLeft: 10
},
title: {
fontSize: 15,
color: 'rgba(255, 255, 255, 0.5)'
},
contentContainer: {
flex: 1,
backgroundColor: 'transparent'
}
});
const ReceiveScreen = (props) => {
return (
<View style={styles.root}>
<View style={styles.titleContainer}>
<Text style={styles.title}>Nuevo miembro</Text>
</View>
<RTCView objectFit='cover' zOrder={1} style={styles.contentContainer} streamURL={props.videoURL} mirror={true} />
</View>
)
}
export default ReceiveScreen;
WebRtc Utils
import {
getUserMedia,
MediaStreamTrack,
mediaDevices,
RTCPeerConnection,
RTCSessionDescription,
} from 'react-native-webrtc';
import { PERMISSIONS, request, checkMultiple } from 'react-native-permissions';
import { socket } from '../screens/User';
let pcArray = {};
let isFront = true;
let isEnableAudio = true;
let isEnableVideo = true;
let localstream = null;
const ICE_CONFIG = {
'iceServers': [
{
'urls': 'turn:xxx.xxx.xxx.xxx:3478',
'username': 'xxxxxxxx',
'credential': 'xxxxxxxxxxxxxxxxxxxxxxx',
'credentialType': 'password'
}
]
};
/**
* Obtenga elementos multimedia locales (transmisiones de video)
*
* #param {*} _sendMessage
* #param {*} _name
* #param {*} callback
*/
export function startCommunication(_sendMessage, _name, callback) {
getStream(true, stream => {
localstream = stream;
let options = {
audio: true,
video : {
mandatory : {
maxWidth : 320,
maxFrameRate : 15,
minFrameRate : 1
}
}
};
let pc = createPC(_sendMessage, _name, true, options);
pcArray[_name] = pc;
callback(stream);
});
}
/**
* Obtener transmisión de video remota
*
* #param {*} _sendMessae
* #param {*} _name
* #param {*} callback
*/
export function receiveVideo(_sendMessae, _name, callback) {
console.log("receiveVideo:", _sendMessae, _name, callback);
let options = {
audio: true,
video : {
mandatory : {
// maxWidth : 320,
// maxFrameRate : 15,
// minFrameRate : 15
maxWidth : 120,
maxHeight: 80,
minWidth : 80,
minHeight: 60,
maxFrameRate : 10,
minFrameRate : 10
}
}
};
let pc = createPC(_sendMessae, _name, true, options);
// console.log(`PC CREATED FOR ${_name}: ${JSON.stringify(pc)}`);
pcArray[_name] = pc;
// callback(pc);
callback(pcArray);
}
/**
* Encender/apagar el micrófono
*/
export function toggleAudio() {
if (localstream) {
isEnableAudio = !isEnableAudio;
localstream.getAudioTracks().forEach((track) => {
track.enabled = isEnableAudio;
});
} else {
console.log('in toggleAudio(), localstream is empty');
}
return isEnableAudio;
}
/**
* Activar/desactivar vídeo
*/
export function toggleVideo() {
if (localstream) {
isEnableVideo = !isEnableVideo;
localstream.getVideoTracks().forEach((track) => {
track.enabled = isEnableVideo;
});
} else {
console.log('in toggleVideo(), localstream is empty');
}
return isEnableVideo;
}
/**
* cambiar de cámara
*
*/
export function switchVideoType() {
if (localstream) {
localstream.getVideoTracks().forEach(track => {
track._switchCamera();
});
} else {
console.log('error');
}
}
/**
* Crear una transmisión de video local
*
* #param {*} isFront
* #param {*} callback
*/
export function getStream(isFront, callback) {
mediaDevices.enumerateDevices().then(sourceInfos => {
//console.log('Log: '+ JSON.stringify(sourceInfos)); // -> Es un objecto de 3
let videoSourceId;
for (let i = 0; i < sourceInfos.length; i++) {
const sourceInfo = sourceInfos[i];
if (sourceInfo.kind === 'videoinput' && sourceInfo.facing === (isFront ? 'front' : 'back')) {
// if (sourceInfo.kind === 'videoinput' && sourceInfo.facing === (isFront ? 'front' : 'back')) { Was video only, not videoinput
videoSourceId = sourceInfo.id;
}
}
request(PERMISSIONS.IOS.CAMERA).then((response) => {
if (response === 'granted') {
request(PERMISSIONS.IOS.CAMERA).then((response) => {
if (response === 'granted') {
mediaDevices.getUserMedia({
audio: true,
video: {
width: 640,
height: 480,
frameRate: 30,
facingMode: (isFront ? "user" : "environment"),
deviceId: videoSourceId
}
})
.then(stream => {
callback(stream)
})
.catch(error => {
// Log error
console.log(error)
});
}
})
}
})
});
}
/**
*
* Crear una conexión WebRTC
*
* #param {*} sendMessage
* #param {*} name
* #param {*} isOffer
* #param {*} options
*/
export function createPC(sendMessage, name, isOffer, options) {
let pc = new RTCPeerConnection(ICE_CONFIG);
pc.onnegotiationneeded = () => {
console.log('onnegotiationneeded');
if (isOffer) {
isOffer = false;
createOffer();
}
};
pc.onicecandidate = (event) => {
console.log('onicecandidate');
if (event.candidate) {
let msg = {
event: 'candidate',
// event: 'onIceCandidate',
userid: socket.id,
roomName: '9b33737f-737f-4a3d-a323-e1cd3f4b68b2',
candidate: event.candidate
};
sendMessage(msg);
}
};
pc.oniceconnectionstatechange = (event) => {
// console.log('oniceconnectionstatechange');
if (event.target.iceConnectionState === 'disconnected') {
//localstream.release();
// localstream = null;
if (pc !== null) {
pc.close();
pc = null;
}
}
};
pc.onsignalingstatechange = (event) => {
console.log('onsignalingstatechange: ', event.target.signalingState);
};
// send local stream
pc.addStream(localstream);
function createOffer() {
// console.log('createOffer');
pc.createOffer()
.then(function (desc) {
// console.log('...createOffer...');
pc.setLocalDescription(desc);
// something to do
let msg = {
event: 'receiveVideoFrom',
userid: socket.id,
roomName: '9b33737f-737f-4a3d-a323-e1cd3f4b68b2',
// sdpOffer: desc.sdp,
sdpOffer: desc.sdp,
};
sendMessage(msg);
})
.catch(err => console.error(err));
}
return pc;
}
/**
* Adición incremental de iceCandidate
*
* #param {*} name
* #param {*} candidate
*/
export function addIceCandidate(name, candidate) {
let pc = pcArray[name];
if (pc) {
pc.addIceCandidate(candidate);
} else {
console.log('pc.addIceCandidate failed : pc not exists');
}
}
/**
* Proceso SdpAnswer
*
* #param {*} name
* #param {*} sdp
* #param {*} callback
*/
export async function ProcessAnswer(name, sdp, callback) {
let pc = pcArray[name];
if (pc) {
let answer = {
type: 'answer',
sdp: sdp,
};
if (pc) {
// const remoteDesc = new RTCSessionDescription(answer);
// await pc.setRemoteDescription(remoteDesc);
await pc.setRemoteDescription(new RTCSessionDescription(answer)).then(function () {
console.log('LOG:252 Answer = ' + answer);
callback();
})
.catch(function (reason) {
callback(reason);
console.log('ERROR REASON:', reason);
});
}
} else {
console.log('ProcessAnswer failed : pc not exists');
}
}
/**
*
* Cierra la conexión y libera el streamer local.
*
*/
export function ViewPCArray (username) {
// console.log(pcArray[username].getRemoteStreams()[0]);
return pcArray
}
export function ReleaseMeidaSource() {
if (localstream) {
localstream.release();
localstream = null;
}
if (pcArray !== null) {
for (let mem in pcArray) {
pcArray[mem].close();
delete pcArray[mem];
}
}
}
function logError(error) {
console.log('logError', error);
}
NodeJs Kurento
module.exports = function(io, socket) {
// variables
const kurento = require('kurento-client');
const minimist = require('minimist');
var kurentoClient = null;
var iceCandidateQueues = {};
// constants
var argv = minimist(process.argv.slice(2), {
default: {
// as_uri: 'https://localhost:3000/',
// ws_uri: 'ws://localhost:8888/kurento'
as_uri: 'https://localhost:4000/',
ws_uri: 'ws://localhost:8888/kurento'
}
});
socket.on('message', function (message) {
console.log('Message received: ', message.event);
switch (message.event) {
case 'joinRoom':
joinRoom(socket, message.userName, message.roomName, err => {
if (err) {
console.log(err);
}
});
break;
case 'receiveVideoFrom':
receiveVideoFrom(socket, message.userid, message.roomName, message.sdpOffer, err => {
if (err) {
console.log(err);
}
});
break;
case 'candidate':
addIceCandidate(socket, message.userid, message.roomName, message.candidate, err => {
if (err) {
console.log(err);
}
});
break;
}
socket.on('disconnect', () => {
leaveRoom(socket, message.roomName, err => {
if (err) {
console.log(err);
};
});
})
});
// signaling functions
function joinRoom(socket, username, roomname, callback) {
console.log(('Smeone in the Room').green + socket + " " + username + " " + roomname)
getRoom(socket, roomname, (err, myRoom) => {
if (err) {
return callback(err);
}
myRoom.pipeline.create('WebRtcEndpoint', (err, outgoingMedia) => {
outgoingMedia.setMaxVideoSendBandwidth(100);
if (err) {
return callback(err);
}
var user = {
id: socket.id,
name: username,
outgoingMedia: outgoingMedia,
incomingMedia: {}
}
let iceCandidateQueue = iceCandidateQueues[user.id];
if (iceCandidateQueue) {
while (iceCandidateQueue.length) {
let ice = iceCandidateQueue.shift();
console.error(`user: ${user.name} collect candidate for outgoing media`);
user.outgoingMedia.addIceCandidate(ice.candidate);
}
}
user.outgoingMedia.on('OnIceCandidate', event => {
let candidate = kurento.register.complexTypes.IceCandidate(event.candidate);
socket.emit('message', {
event: 'candidate',
userid: user.id,
candidate: candidate
});
});
socket.to(roomname).emit('message', {
event: 'newParticipantArrived',
userid: user.id,
username: user.name
});
let existingUsers = [];
for (let i in myRoom.participants) {
if (myRoom.participants[i].id != user.id) {
existingUsers.push({
id: myRoom.participants[i].id,
name: myRoom.participants[i].name
});
}
}
socket.emit('message', {
event: 'existingParticipants',
existingUsers: existingUsers,
userid: user.id
});
myRoom.participants[user.id] = user;
});
});
}
// Disconnect from room
function leaveRoom(socket, roomname, callback) {
if (io.sockets.adapter.rooms[roomname] == null) {
// ROOMS
// console.log((io.sockets.adapter.rooms))
return;
}
var userSession = io.sockets.adapter.rooms[roomname].participants[socket.id];
var myRoom = io.sockets.adapter.rooms[roomname] || { length: 0 };
// MY ROOM
// console.log(myRoom);
if (!userSession) {
return;
}
var room = io.sockets.adapter.rooms[roomname];
if(!room){
return;
}
console.log('notify all user that ' + userSession.name + ' is leaving the room ' + roomname);
var usersInRoom = room.participants;
delete usersInRoom[userSession.id];
userSession.outgoingMedia.release();
// release incoming media for the leaving user
for (var i in userSession.incomingMedia) {
userSession.incomingMedia[i].release();
delete userSession.incomingMedia[i];
}
for (var i in usersInRoom) {
var user = usersInRoom[i];
// release viewer from this
user.incomingMedia[userSession.id].release();
delete user.incomingMedia[userSession.id];
// notify all user in the room
io.emit('message', {
event: 'participantLeft',
name: userSession.id
});
console.log('Mensaje emitido')
}
// Release pipeline and delete room when room is empty
if (Object.keys(room.participants).length == 0) {
room.pipeline.release();
delete rooms[userSession.roomName];
}
delete userSession.roomName;
// console.log(myRoom);
callback(null);
}
function receiveVideoFrom(socket, userid, roomname, sdpOffer, callback) {
getEndpointForUser(socket, roomname, userid, (err, endpoint) => {
if (err) {
return callback(err);
}
endpoint.processOffer(sdpOffer, (err, sdpAnswer) => {
if (err) {
return callback(err);
}
socket.emit('message', {
event: 'receiveVideoAnswer',
senderid: userid,
sdpAnswer: sdpAnswer
});
endpoint.gatherCandidates(err => {
if (err) {
return callback(err);
}
});
});
});
}
function addIceCandidate(socket, senderid, roomname, iceCandidate, callback) {
// console.log(io.sockets.adapter.rooms);
let user = io.sockets.adapter.rooms[roomname].participants[socket.id];
if (user != null) {
let candidate = kurento.register.complexTypes.IceCandidate(iceCandidate);
if (senderid == user.id) {
if (user.outgoingMedia) {
user.outgoingMedia.addIceCandidate(candidate);
} else {
iceCandidateQueues[user.id].push({ candidate: candidate });
}
} else {
if (user.incomingMedia[senderid]) {
user.incomingMedia[senderid].addIceCandidate(candidate);
} else {
if (!iceCandidateQueues[senderid]) {
iceCandidateQueues[senderid] = [];
}
iceCandidateQueues[senderid].push({ candidate: candidate });
}
}
callback(null);
} else {
callback(new Error("addIceCandidate failed"));
}
}
// useful functions
function getRoom(socket, roomname, callback) {
var myRoom = io.sockets.adapter.rooms[roomname] || { length: 0 };
var numClients = myRoom.length;
console.log(roomname, ' has ', numClients, ' clients');
if (numClients == 0) {
socket.join(roomname, () => {
myRoom = io.sockets.adapter.rooms[roomname];
getKurentoClient((error, kurento) => {
kurento.create('MediaPipeline', (err, pipeline) => {
if (error) {
return callback(err);
}
myRoom.pipeline = pipeline;
myRoom.participants = {};
callback(null, myRoom);
});
});
});
} else {
socket.join(roomname);
callback(null, myRoom);
}
}
function getEndpointForUser(socket, roomname, senderid, callback) {
var myRoom = io.sockets.adapter.rooms[roomname];
var asker = myRoom.participants[socket.id];
var sender = myRoom.participants[senderid];
if (asker.id === sender.id) {
return callback(null, asker.outgoingMedia);
}
if (asker.incomingMedia[sender.id]) {
sender.outgoingMedia.connect(asker.incomingMedia[sender.id], err => {
if (err) {
return callback(err);
}
callback(null, asker.incomingMedia[sender.id]);
});
} else {
myRoom.pipeline.create('WebRtcEndpoint', (err, incoming) => {
incoming.setMaxVideoSendBandwidth(100);
if (err) {
return callback(err);
}
asker.incomingMedia[sender.id] = incoming;
let iceCandidateQueue = iceCandidateQueues[sender.id];
if (iceCandidateQueue) {
while (iceCandidateQueue.length) {
let ice = iceCandidateQueue.shift();
console.error(`user: ${sender.name} collect candidate for outgoing media`);
incoming.addIceCandidate(ice.candidate);
}
}
incoming.on('OnIceCandidate', event => {
let candidate = kurento.register.complexTypes.IceCandidate(event.candidate);
console.log("CANDIDATE: ", event.candidate);
socket.emit('message', {
event: 'candidate',
userid: sender.id,
candidate: candidate
});
});
sender.outgoingMedia.connect(incoming, err => {
if (err) {
return callback(err);
}
callback(null, incoming);
});
});
}
}
function getKurentoClient(callback) {
if (kurentoClient !== null) {
return callback(null, kurentoClient);
}
kurento(argv.ws_uri, function (error, _kurentoClient) {
if (error) {
console.log("Could not find media server at address " + argv.ws_uri);
return callback("Could not find media server at address" + argv.ws_uri
+ ". Exiting with error " + error);
}
kurentoClient = _kurentoClient;
callback(null, kurentoClient);
});
}
}