Dynamically addTrack to offerer from answerer onnegotiationneeded in webrtc - webrtc

Is there anyway to notify offerer that non-existing track before just added to get the new stream from the answerer from the code below?
For my current issue now here is that the offerer can add new non-existing track and onnegotiationneeded will be fired and will also be able to createOffer and update media successfully, but when answerer do same process onnegotiationneeded fired normally also from the answerer but no media will be exchanged just because offerer do not have any new track on his end!
I use replaceOrAddTrack(remotePartiID, track, TrackKind) in adding and replacing of tracks
Only the replace works with either ends if it has same track kind from initial connection
_cfg = {
sdpConstraints: {
mandatory: {
OfferToReceiveAudio: true,
OfferToReceiveVideo: true,
VoiceActivityDetection: true,
IceRestart: true
},
optional: []
}
...
};
var channels_wrap = (function() {
return {
...
init: function() {
_cfg.defaultChannel.on('message', (message) => {
if (_cfg.enableLog) {
console.log('Client received message:', message);
}
if (message.type === 'newparticipant') {
var partID = message.from;
var partData = message.fromData;
// Open a new communication channel to the new participant
_cfg.offerChannels[partID] = this.openSignalingChannel(partID);
// Wait for answers (to offers) from the new participant
_cfg.offerChannels[partID].on('message', (msg) => {
if (msg.dest === _cfg.myID) {
if (msg.type === 'reoffer') {
if (_cfg.opc.hasOwnProperty(msg.from)) {
console.log('reoffering')
_cfg.opc[msg.from].negotiationNeeded();
}
} else
if (msg.type === 'answer') {
_cfg.opc[msg.from].peer.setRemoteDescription(new RTCSessionDescription(msg.snDescription),
handlers_wrap.setRemoteDescriptionSuccess,
handlers_wrap.setRemoteDescriptionError);
} else if (msg.type === 'candidate') {
var candidate = new RTCIceCandidate({
sdpMLineIndex: msg.label,
candidate: msg.candidate
});
if (_cfg.enableLog) {
console.log('got ice candidate from ' + msg.from);
}
_cfg.opc[msg.from].peer.addIceCandidate(candidate, handlers_wrap.addIceCandidateSuccess, handlers_wrap.addIceCandidateError);
}
}
});
// Send an offer to the new participant
dialogs_wrap.createOffer(partID, partData);
} else if (message.type === 'bye') {
handlers_wrap.hangup(message.from, message.fromData);
}
});
},
initPrivateChannel: function() {
// Open a private channel (namespace = _cfg.myID) to receive offers
_cfg.privateAnswerChannel = this.openSignalingChannel(_cfg.myID);
// Wait for offers or ice candidates
_cfg.privateAnswerChannel.on('message', (message) => {
if (message.dest === _cfg.myID) {
if (message.type === 'offer') {
var to = message.from;
dialogs_wrap.createAnswer(message.snDescription, _cfg.privateAnswerChannel, to, message.fromData);
} else if (message.type === 'candidate') {
var candidate = new RTCIceCandidate({
sdpMLineIndex: message.label,
candidate: message.candidate
});
_cfg.apc[message.from].peer.addIceCandidate(candidate, handlers_wrap.addIceCandidateSuccess, handlers_wrap.addIceCandidateError);
}
}
});
}
};
})();
var tracks_wrap = (function() {
return {
getParticipants: function(partID = null) {
var participants = {};
if (partID) {
if (_cfg.opc.hasOwnProperty(partID)) {
participants[partID] = {
ID: partID,
type: 'opc'
};
} else
if (_cfg.apc.hasOwnProperty(partID)) {
participants[partID] = {
ID: partID,
type: 'apc'
};
}
} else {
for (let key in _cfg.opc) {
participants[key] = {
ID: key,
type: 'opc'
};
}
for (let key in _cfg.apc) {
participants[key] = {
ID: key,
type: 'apc'
};
}
}
return participants;
},
replaceOrAddTrack: function(remotePartiID, track, TrackKind) {
if (!TrackKind) {
return;
}
var participants = this.getParticipants(remotePartiID);
for (var partiID in participants) {
var peer = null;
if (participants[partiID].type === 'apc' && _cfg.apc.hasOwnProperty(partiID)) {
peer = _cfg.apc[partiID].peer;
} else if (participants[partiID].type === 'opc' && _cfg.opc.hasOwnProperty(partiID)) {
peer = _cfg.opc[partiID].peer;
} else {
continue;
}
var foundTrack = null;
peer.getSenders().forEach(function(rtpSender) {
if (rtpSender.track && TrackKind === rtpSender.track.kind) {
foundTrack = true;
rtpSender.replaceTrack(track);
}
});
if (!foundTrack) {
peer.addTrack(track, _cfg.localStream); //This work only if it is offerrer that add track but not working with answerer even if i tell the offerer to send offer again
}
}
}
};
})();
var dialogs_wrap = (function() {
return {
/**
*
* Send an offer to peer with id partID and metadata as partData
*
*/
createOffer: function(partID, partData) {
if (_cfg.enableLog) {
console.log('Creating offer for peer ' + partID, partData);
}
var opcPeer = new RTCPeerConnection(_cfg.pcConfig, _cfg.peerSetup);
_cfg.opc[partID] = {};
_cfg.opc[partID].peer = opcPeer;
_cfg.opc[partID].peer.onicecandidate = handlers_wrap.handleIceCandidateAnswer(_cfg.offerChannels[partID], partID, partData);
_cfg.opc[partID].peer.ontrack = handlers_wrap.handleRemoteStreamAdded(partID, partData);
_cfg.opc[partID].peer.onremovetrack = handlers_wrap.handleRemoteStreamRemoved(partID, partData);
_cfg.localStream.getTracks().forEach(track => _cfg.opc[partID].peer.addTrack(track, _cfg.localStream));
try {
_cfg.sendChannel[partID] = _cfg.opc[partID].peer.createDataChannel("sendDataChannel", {
reliable: false
});
_cfg.sendChannel[partID].onmessage = handlers_wrap.handleMessage;
if (_cfg.enableLog) {
console.log('Created send data channel');
}
} catch (e) {
alert('Failed to create data channel. \n You need supported RtpDataChannel enabled browser');
console.log('createDataChannel() failed with exception: ', e.message);
}
_cfg.sendChannel[partID].onopen = handlers_wrap.handleSendChannelStateChange(partID);
_cfg.sendChannel[partID].onclose = handlers_wrap.handleSendChannelStateChange(partID);
var onSuccess = (partID, partData) => {
var channel = _cfg.offerChannels[partID];
if (_cfg.enableLog) {
console.log('Sending offering');
}
channel.emit('message', {
snDescription: _cfg.opc[partID].peer.localDescription,
from: _cfg.myID,
fromData: _cfg.myData,
type: 'offer',
dest: partID,
destData: partData
});
}
_cfg.opc[partID].negotiationNeeded = () => {
_cfg.opc[partID].peer.createOffer(_cfg.sdpConstraints).then(offer => {
offer.sdp = sdp_wrap.SDPController(offer.sdp);
return _cfg.opc[partID].peer.setLocalDescription(offer)
})
.then(() => onSuccess(partID, partData)).catch(handlers_wrap.handleCreateOfferError);
}
_cfg.opc[partID].peer.onnegotiationneeded = () => {
_cfg.opc[partID].negotiationNeeded();
}
},
createAnswer: function(snDescription, cnl, to, toData) {
if (_cfg.enableLog) {
console.log('Creating answer for peer ' + to);
}
if (!_cfg.apc.hasOwnProperty(to)) {
var apcPeer = new RTCPeerConnection(_cfg.pcConfig, _cfg.peerSetup);
//apcPeer.setConfiguration(_cfg.pcConfig);
_cfg.apc[to] = {};
_cfg.apc[to].peer = apcPeer;
_cfg.apc[to].peer.onicecandidate = handlers_wrap.handleIceCandidateAnswer(cnl, to, toData);
_cfg.apc[to].peer.ontrack = handlers_wrap.handleRemoteStreamAdded(to, toData);
_cfg.apc[to].peer.onremovetrack = handlers_wrap.handleRemoteStreamRemoved(to, toData);
_cfg.localStream.getTracks().forEach(track => _cfg.apc[to].peer.addTrack(track, _cfg.localStream));
_cfg.apc[to].peer.ondatachannel = handlers_wrap.gotReceiveChannel(to);
}
_cfg.apc[to].peer.setRemoteDescription(new RTCSessionDescription(snDescription), handlers_wrap.setRemoteDescriptionSuccess, handlers_wrap.setRemoteDescriptionError);
var onSuccess = (channel) => {
if (_cfg.enableLog) {
console.log('Sending answering');
}
channel.emit('message', {
snDescription: _cfg.apc[to].peer.localDescription,
from: _cfg.myID,
fromData: _cfg.myData,
type: 'answer',
dest: to,
destData: toData
});
}
_cfg.apc[to].peer.createAnswer().then(function(answer) {
answer.sdp = sdp_wrap.SDPController(answer.sdp);
return _cfg.apc[to].peer.setLocalDescription(answer);
})
.then(() => onSuccess(cnl))
.catch(handlers_wrap.handleCreateAnswerError);
var negotiationNeeded = false;
_cfg.apc[to].peer.onnegotiationneeded = (ev) => {
if (!negotiationNeeded) {
negotiationNeeded = true;
return;
}
//So i tried to create this to tell the offerer to do offer again, offerer do resend offer but nothing seem to happen
cnl.emit('message', {
from: _cfg.myID,
fromData: _cfg.myData,
type: 'reoffer',
dest: to,
destData: toData
});
}
}
};
})();

Related

Error integrating Agora.io with nuxt.js Error in created hook: "ReferenceError: AgoraRTC is not defined"

I am integrating Agora Web SDK with nuxt.js.
I have included all the methods I need and my page has the following methods and lifecycle hooks:
methods: {
streamInit(uid, attendeeMode, videoProfile, config) {
let defaultConfig = {
streamID: uid,
audio: true,
video: true,
screen: false
};
switch (attendeeMode) {
case "audio-only":
defaultConfig.video = false;
break;
case "audience":
defaultConfig.video = false;
defaultConfig.audio = false;
break;
default:
case "video":
break;
}
let stream = AgoraRTC.createStream(merge(defaultConfig, config));
stream.setVideoProfile(videoProfile);
return stream;
},
subscribeStreamEvents() {
let rt = this;
rt.client.on("stream-added", function(evt) {
let stream = evt.stream;
console.log("New stream added: " + stream.getId());
console.log("At " + new Date().toLocaleTimeString());
console.log("Subscribe ", stream);
rt.client.subscribe(stream, function(err) {
console.log("Subscribe stream failed", err);
});
});
rt.client.on("peer-leave", function(evt) {
console.log("Peer has left: " + evt.uid);
console.log(new Date().toLocaleTimeString());
console.log(evt);
rt.removeStream(evt.uid);
});
rt.client.on("stream-subscribed", function(evt) {
let stream = evt.stream;
console.log("Got stream-subscribed event");
console.log(new Date().toLocaleTimeString());
console.log("Subscribe remote stream successfully: " + stream.getId());
console.log(evt);
rt.addStream(stream);
});
rt.client.on("stream-removed", function(evt) {
let stream = evt.stream;
console.log("Stream removed: " + stream.getId());
console.log(new Date().toLocaleTimeString());
console.log(evt);
rt.removeStream(stream.getId());
});
},
removeStream(uid) {
this.streamList.map((item, index) => {
if (item.getId() === uid) {
item.close();
let element = document.querySelector("#ag-item-" + uid);
if (element) {
element.parentNode.removeChild(element);
}
let tempList = [...this.streamList];
tempList.splice(index, 1);
this.streamList = tempList;
}
});
},
addStream(stream, push = false) {
let repeatition = this.streamList.some(item => {
return item.getId() === stream.getId();
});
if (repeatition) {
return;
}
if (push) {
this.streamList = this.streamList.concat([stream]);
} else {
this.streamList = [stream].concat(this.streamList);
}
},
handleCamera(e) {
e.currentTarget.classList.toggle("off");
this.localStream.isVideoOn()
? this.localStream.disableVideo()
: this.localStream.enableVideo();
},
handleMic(e) {
e.currentTarget.classList.toggle("off");
this.localStream.isAudioOn()
? this.localStream.disableAudio()
: this.localStream.enableAudio();
},
switchDisplay(e) {
if (
e.currentTarget.classList.contains("disabled") ||
this.streamList.length <= 1
) {
return;
}
if (this.displayMode === "pip") {
this.displayMode = "tile";
} else if (this.displayMode === "tile") {
this.displayMode = "pip";
} else if (this.displayMode === "share") {
// do nothing or alert, tbd
} else {
console.error("Display Mode can only be tile/pip/share");
}
},
hideRemote(e) {
if (
e.currentTarget.classList.contains("disabled") ||
this.streamList.length <= 1
) {
return;
}
let list;
let id = this.streamList[this.streamList.length - 1].getId();
list = Array.from(
document.querySelectorAll(`.ag-item:not(#ag-item-${id})`)
);
list.map(item => {
if (item.style.display !== "none") {
item.style.display = "none";
} else {
item.style.display = "block";
}
});
},
handleExit(e) {
if (e.currentTarget.classList.contains("disabled")) {
return;
}
try {
this.client && this.client.unpublish(this.localStream);
this.localStream && this.localStream.close();
this.client &&
this.client.leave(
() => {
console.log("Client succeed to leave.");
},
() => {
console.log("Client failed to leave.");
}
);
} finally {
this.readyState = false;
this.client = null;
this.localStream = null;
// redirect to index
this.$router.push("/");
}
}
},
created() {
let $ = this;
// init AgoraRTC local client
$.client = AgoraRTC.createClient({ mode: $.transcode });
$.client.init($.appId, () => {
console.log("AgoraRTC client initialized");
$.subscribeStreamEvents();
$.client.join($.appId, $.channel, $.uid, uid => {
console.log("User " + uid + " join channel successfully");
console.log("At " + new Date().toLocaleTimeString());
// create local stream
// It is not recommended to setState in function addStream
$.localStream = this.streamInit(uid, $.attendeeMode, $.videoProfile);
$.localStream.init(
() => {
if ($.attendeeMode !== "audience") {
$.addStream($.localStream, true);
$.client.publish($.localStream, err => {
console.log("Publish local stream error: " + err);
});
}
$.readyState = true;
},
err => {
console.log("getUserMedia failed", err);
$.readyState = true;
}
);
});
});
},
mounted() {
this.$nextTick(() => {
// add listener to control btn group
let canvas = document.querySelector("#ag-canvas");
let btnGroup = document.querySelector(".ag-btn-group");
canvas.addEventListener("mousemove", () => {
if (global._toolbarToggle) {
clearTimeout(global._toolbarToggle);
}
btnGroup.classList.add("active");
global._toolbarToggle = setTimeout(function() {
btnGroup.classList.remove("active");
}, 2000);
});
});
},
beforeUpdate() {
let $ = this;
// rerendering
let canvas = document.querySelector("#ag-canvas");
// pip mode (can only use when less than 4 people in channel)
if ($.displayMode === "pip") {
let no = $.streamList.length;
if (no > 4) {
$.displayMode = "tile";
return;
}
$.streamList.map((item, index) => {
let id = item.getId();
let dom = document.querySelector("#ag-item-" + id);
if (!dom) {
dom = document.createElement("section");
dom.setAttribute("id", "ag-item-" + id);
dom.setAttribute("class", "ag-item");
canvas.appendChild(dom);
item.play("ag-item-" + id);
}
if (index === no - 1) {
dom.setAttribute("style", `grid-area: span 12/span 24/13/25`);
} else {
dom.setAttribute(
"style",
`grid-area: span 3/span 4/${4 + 3 * index}/25;
z-index:1;width:calc(100% - 20px);height:calc(100% - 20px)`
);
}
item.player.resize && item.player.resize();
});
} else if ($.displayMode === "tile") {
// tile mode
let no = $.streamList.length;
$.streamList.map((item, index) => {
let id = item.getId();
let dom = document.querySelector("#ag-item-" + id);
if (!dom) {
dom = document.createElement("section");
dom.setAttribute("id", "ag-item-" + id);
dom.setAttribute("class", "ag-item");
canvas.appendChild(dom);
item.play("ag-item-" + id);
}
dom.setAttribute("style", `grid-area: ${tile_canvas[no][index]}`);
item.player.resize && item.player.resize();
});
} else if ($.displayMode === "share") {
// screen share mode (tbd)
}
},
beforeDestroy () {
this.client && this.client.unpublish(this.localStream);
this.localStream && this.localStream.close();
this.client &&
this.client.leave(
() => {
console.log("Client succeed to leave.");
},
() => {
console.log("Client failed to leave.");
}
);
}
I have installed agora-rtc-sdk from npm.
My plugins/agora.js file looks like this
import Vue from "vue";
import AgoraRTC from 'agora-rtc-sdk';
Vue.use(AgoraRTC);
My nuxt.config.js has plugins declared as:
{
src: "~/plugins/agora.js",
ssr: false
}
The application on loading the page gives AgoraRTC is not defined. How do I add this AgoraRTC to my nuxt.js application?
Agora works only on the client side, fully independent of a server and hence you need to define the mode as client in the nuxt.config.js like this:
{ src: '~/plugins/agora.js', mode: 'client' },

WebRTC ontrack on caller side not firing after answer received

In my multi peer webrtc client (testing on chrome) a stable connection is successfully established but after I receive the answer from the callee, the ontrack event is not firing and thus the stream sourceObj is not attached to my DOM. Why? The callee shows both videos (local and remote) but on the caller side the remote video is not added and it seems that this is due to the ontrack event not firing.
I create all the RTCPeerConnection before sending the Offer/Answer and add the local Tracks at creation time to it after I had bound the ontrack events to it. Then I send the offer/answer and follow the signaling proccess.
class WebRTC_Client {
private serversCfg = {
iceServers: [{
urls: ["stun:stun.l.google.com:19302"]
}]
};
...
private gotStream(stream) {
window.localStream = stream;
...
}
private stopLocalTracks(){
if (window.localStream) {
window.localStream.getTracks().forEach(function (track) {
track.stop();
});
var videoTracks = window.localStream.getVideoTracks();
for (var i = 0; i !== videoTracks.length; ++i) {
videoTracks[i].stop();
}
}
}
private start() {
var self = this;
...
this.stopLocalTracks();
...
navigator.mediaDevices.getUserMedia(this.getConstrains())
.then((stream) => {
self.gotStream(stream);
trace('Send signal to peers that I am ready to be called: onReadyForTeamspeak');
self.SignalingChannel.send(JSON.stringify({type: 'onReadyForTeamspeak'}));
})
.catch( self.errorHandler );
}
public addPeerId(peerId){
this.availablePeerIds[peerId] = peerId;
this.preparePeerConnection(peerId);
}
private preparePeerConnection(peerId){
var self = this;
if( this.peerConns[peerId] ){
return;
}
this.peerConns[peerId] = new RTCPeerConnection(this.serversCfg);
this.peerConns[peerId].ontrack = function (evt) { self.gotRemoteStream(evt, peerId); };
this.peerConns[peerId].onicecandidate = function (evt) { self.iceCallback(evt, peerId); };
this.addTracks(peerId);
}
private addTracks(peerId){
var self = this;
var localTracksCount = 0;
window.localStream.getTracks().forEach(
function (track) {
self.peerConns[peerId].addTrack(
track,
window.localStream
);
}
);
}
private call() {
var self = this;
for( let peerId in this.availablePeerIds ){
if( !this.availablePeerIds.hasOwnProperty(peerId) ){
continue;
}
if( this.isCaller(peerId) ) {
this.preparePeerConnection(peerId);
this.createOffer(peerId);
}
}
}
private createOffer(peerId){
var self = this;
this.peerConns[peerId].createOffer( this.offerOptions )
.then( function (offer) { return self.peerConns[peerId].setLocalDescription(offer); } )
.then( function () {
self.SignalingChannel.send(JSON.stringify({ "sdp": self.peerConns[peerId].localDescription, "remotePeerId": peerId, "type": "onWebRTCPeerConn" }));
})
.catch( this.errorHandler );
}
private answerCall(peerId){
var self = this;
this.peerConns[peerId].createAnswer()
.then( function (answer) { return self.peerConns[peerId].setLocalDescription(answer); } )
.then( function () {
self.SignalingChannel.send(JSON.stringify({ "sdp": self.peerConns[peerId].localDescription, "remotePeerId": peerId, "type": "onWebRTCPeerConn" }));
})
.catch( this.errorHandler );
}
...
private gotRemoteStream(e, peerId) {
if (!this.videoBillboards[peerId]) {
this.createMediaElements(peerId);
}
if (this.videoAssets[peerId].srcObject !== e.streams[0]) {
this.videoAssets[peerId].srcObject = e.streams[0];
}
}
private iceCallback(event, peerId) {
this.SignalingChannel.send(JSON.stringify({ "candidate": event.candidate, "remotePeerId": peerId, "type": "onWebRTCPeerConn" }));
}
private handleCandidate(candidate, peerId) {
this.peerConns[peerId].addIceCandidate(candidate)
.then(
this.onAddIceCandidateSuccess,
this.onAddIceCandidateError
);
}
...
private handleSignals(signal){
var self = this,
peerId = signal.connectionId;
this.addPeerId(peerId);
if( signal.sdp ) {
var desc = new RTCSessionDescription(signal.sdp);
this.peerConns[peerId].setRemoteDescription(desc)
.then(function () {
if (desc.type === 'offer') {
self.answerCall(peerId);
}
})
.catch( this.errorHandler );
} else if( signal.candidate ){
this.handleCandidate(new RTCIceCandidate(signal.candidate), peerId);
} else if( signal.closeConn ){
this.endCall(peerId,true);
}
}
}
Found a solution! Something was wrong with the options I put in to this.peerConns[peerId].createOffer( this.offerOptions ).
Actually in the code I provided one could not see it but the method that was dynamically creating my class member variable this.offerOptions had a bug. This was obviously telling the callee to NOT send any streams back to the caller.

Webrtc video chat not working in different networks

I am trying to implement video chat with webrtc using code from google code labs . It works fine on same network but does not work on different network. i have installed coturn on my server. I am not sure if it is working correctly.
Any one used google codelabs code and made it working?
This is the modified main.js file
'use strict';
var isChannelReady = false;
var isInitiator = false;
var isStarted = false;
var localStream;
var pc;
var remoteStream;
var turnReady;
var pcConfig = {
"iceServers":[
{'urls': 'stun:stun.l.google.com:19302'},
{"urls":["turn:78.129.167.90"],"username":"wyzturner1","credential":"wyzturnp2ss"}]
};
// Set up audio and video regardless of what devices are present.
var sdpConstraints = {
offerToReceiveAudio: true,
offerToReceiveVideo: true
};
/////////////////////////////////////////////
var room = 'foo';
// Could prompt for room name:
// room = prompt('Enter room name:');
var socket = io.connect('https://www.samplesite.com:8080');
if (room !== '') {
socket.emit('create or join', room);
console.log('Attempted to create or join room', room);
}
socket.on('created', function(room) {
console.log('Created room ' + room);
isInitiator = true;
});
socket.on('full', function(room) {
console.log('Room ' + room + ' is full');
});
socket.on('join', function (room){
console.log('Another peer made a request to join room ' + room);
console.log('This peer is the initiator of room ' + room + '!');
isChannelReady = true;
});
socket.on('joined', function(room) {
console.log('joined: ' + room);
isChannelReady = true;
});
socket.on('log', function(array) {
console.log.apply(console, array);
});
////////////////////////////////////////////////
function sendMessage(message) {
console.log('Client sending message: ', message);
socket.emit('message', message);
}
// This client receives a message
socket.on('message', function(message) {
console.log('Client received message:', message);
if (message === 'got user media') {
maybeStart();
} else if (message.type === 'offer') {
if (!isInitiator && !isStarted) {
maybeStart();
}
pc.setRemoteDescription(new RTCSessionDescription(message));
doAnswer();
} else if (message.type === 'answer' && isStarted) {
pc.setRemoteDescription(new RTCSessionDescription(message));
} else if (message.type === 'candidate' && isStarted) {
var candidate = new RTCIceCandidate({
sdpMLineIndex: message.label,
candidate: message.candidate
});
pc.addIceCandidate(candidate);
} else if (message === 'bye' && isStarted) {
handleRemoteHangup();
}
});
////////////////////////////////////////////////////
var localVideo = document.querySelector('#localVideo');
var remoteVideo = document.querySelector('#remoteVideo');
navigator.mediaDevices.getUserMedia({
audio: false,
video: true
})
.then(gotStream)
.catch(function(e) {
alert('getUserMedia() error: ' + e.name);
});
function gotStream(stream) {
console.log('Adding local stream.');
localVideo.src = window.URL.createObjectURL(stream);
localStream = stream;
sendMessage('got user media');
if (isInitiator) {
maybeStart();
}
}
var constraints = {
video: true
};
console.log('Getting user media with constraints', constraints);
//if (location.hostname !== 'localhost') {
// requestTurn(
// 'https://computeengineondemand.appspot.com/turn?username=41784574&key=4080218913'
// );
//}
function maybeStart() {
console.log('>>>>>>> maybeStart() ', isStarted, localStream, isChannelReady);
if (!isStarted && typeof localStream !== 'undefined' && isChannelReady) {
console.log('>>>>>> creating peer connection');
createPeerConnection();
isStarted = true;
console.log('isInitiator', isInitiator);
if (isInitiator) {
doCall();
}
}
}
window.onbeforeunload = function() {
sendMessage('bye');
};
/////////////////////////////////////////////////////////
function createPeerConnection() {
try {
pc = new RTCPeerConnection(pcConfig);
pc.addStream(localStream);
pc.onicecandidate = handleIceCandidate;
pc.onaddstream = handleRemoteStreamAdded;
pc.oniceconnectionstatechange = function(){
console.log('ICE state: ',pc.iceConnectionState);
}
pc.onremovestream = handleRemoteStreamRemoved;
console.log('Created RTCPeerConnnection');
} catch (e) {
console.log('Failed to create PeerConnection, exception: ' + e.message);
alert('Cannot create RTCPeerConnection object.');
return;
}
}
function handleIceCandidate(event) {
console.log('icecandidate event: ', event);
if (event.candidate) {
sendMessage({
type: 'candidate',
label: event.candidate.sdpMLineIndex,
id: event.candidate.sdpMid,
candidate: event.candidate.candidate
});
} else {
console.log('End of candidates.');
}
}
function handleRemoteStreamAdded(event) {
console.log('Remote stream added.');
remoteVideo.src = window.URL.createObjectURL(event.stream);
remoteStream = event.stream;
}
function handleCreateOfferError(event) {
console.log('createOffer() error: ', event);
}
function doCall() {
console.log('Sending offer to peer');
pc.createOffer(setLocalAndSendMessage, handleCreateOfferError);
}
function doAnswer() {
console.log('Sending answer to peer.');
pc.createAnswer().then(
setLocalAndSendMessage,
onCreateSessionDescriptionError
);
}
function setLocalAndSendMessage(sessionDescription) {
// Set Opus as the preferred codec in SDP if Opus is present.
// sessionDescription.sdp = preferOpus(sessionDescription.sdp);
pc.setLocalDescription(sessionDescription);
console.log('setLocalAndSendMessage sending message', sessionDescription);
sendMessage(sessionDescription);
}
function onCreateSessionDescriptionError(error) {
trace('Failed to create session description: ' + error.toString());
}
function requestTurn(turnURL) {
var turnExists = false;
for (var i in pcConfig.iceServers) {
if (pcConfig.iceServers[i].urls.substr(0, 5) === 'turn:') {
turnExists = true;
turnReady = true;
break;
}
}
// if (!turnExists) {
// console.log('Getting TURN server from ', turnURL);
// // No TURN server. Get one from computeengineondemand.appspot.com:
// var xhr = new XMLHttpRequest();
// xhr.onreadystatechange = function() {
// if (xhr.readyState === 4 && xhr.status === 200) {
// var turnServer = JSON.parse(xhr.responseText);
// console.log('Got TURN server: ', turnServer);
// pcConfig.iceServers.push({
// 'url': 'turn:' + turnServer.username + '#' + turnServer.turn,
// 'credential': turnServer.password
// });
// turnReady = true;
// }
// };
// xhr.open('GET', turnURL, true);
// xhr.send();
// }
}
function handleRemoteStreamAdded(event) {
console.log('Remote stream added.');
remoteVideo.src = window.URL.createObjectURL(event.stream);
remoteStream = event.stream;
}
function handleRemoteStreamRemoved(event) {
console.log('Remote stream removed. Event: ', event);
}
function hangup() {
console.log('Hanging up.');
stop();
sendMessage('bye');
}
function handleRemoteHangup() {
console.log('Session terminated.');
stop();
isInitiator = false;
}
function stop() {
isStarted = false;
// isAudioMuted = false;
// isVideoMuted = false;
pc.close();
pc = null;
}
///////////////////////////////////////////
// Set Opus as the default audio codec if it's present.
function preferOpus(sdp) {
var sdpLines = sdp.split('\r\n');
var mLineIndex;
// Search for m line.
for (var i = 0; i < sdpLines.length; i++) {
if (sdpLines[i].search('m=audio') !== -1) {
mLineIndex = i;
break;
}
}
if (mLineIndex === null) {
return sdp;
}
// If Opus is available, set it as the default in m line.
for (i = 0; i < sdpLines.length; i++) {
if (sdpLines[i].search('opus/48000') !== -1) {
var opusPayload = extractSdp(sdpLines[i], /:(\d+) opus\/48000/i);
if (opusPayload) {
sdpLines[mLineIndex] = setDefaultCodec(sdpLines[mLineIndex],
opusPayload);
}
break;
}
}
// Remove CN in m line and sdp.
sdpLines = removeCN(sdpLines, mLineIndex);
sdp = sdpLines.join('\r\n');
return sdp;
}
function extractSdp(sdpLine, pattern) {
var result = sdpLine.match(pattern);
return result && result.length === 2 ? result[1] : null;
}
// Set the selected codec to the first in m line.
function setDefaultCodec(mLine, payload) {
var elements = mLine.split(' ');
var newLine = [];
var index = 0;
for (var i = 0; i < elements.length; i++) {
if (index === 3) { // Format of media starts from the fourth.
newLine[index++] = payload; // Put target payload to the first.
}
if (elements[i] !== payload) {
newLine[index++] = elements[i];
}
}
return newLine.join(' ');
}
// Strip CN from sdp before CN constraints is ready.
function removeCN(sdpLines, mLineIndex) {
var mLineElements = sdpLines[mLineIndex].split(' ');
// Scan from end for the convenience of removing an item.
for (var i = sdpLines.length - 1; i >= 0; i--) {
var payload = extractSdp(sdpLines[i], /a=rtpmap:(\d+) CN\/\d+/i);
if (payload) {
var cnPos = mLineElements.indexOf(payload);
if (cnPos !== -1) {
// Remove CN payload from m line.
mLineElements.splice(cnPos, 1);
}
// Remove CN line in sdp
sdpLines.splice(i, 1);
}
}
sdpLines[mLineIndex] = mLineElements.join(' ');
return sdpLines;
}

WebRTC DataChannel Errors

I'm trying to connect 2 peers with webrtc and datachannel without camera and microphone.
try {
socket = new WebSocket("ws://localhost:1337/");
var servers = {iceServers:[{url:"stun:stun.l.google.com:19302"}]};
peerConn = new webkitRTCPeerConnection(servers, {optional:[{RtpDataChannels: true}]});
channel = peerConn.createDataChannel("abcd1234", {reliable: false});
peerConn.onicecandidate = function(evt) {
if(evt.candidate) {
socket.send(JSON.stringify({"candidate": evt.candidate}));
}
};
channel.onopen = function () {
console.log("channel is open");
channel.send('first text message over RTP data ports');
};
channel.onmessage = function (event) {
console.log('received a message:', event.data);
};
peerConn.createOffer(function(desc) {
peerConn.setLocalDescription(desc);
socket.send(JSON.stringify({"sdp": desc}));
});
socket.onmessage = function(evt) {
var signal = JSON.parse(evt.data);
if(signal.sdp) {
peerConn.setRemoteDescription(new RTCSessionDescription(signal.sdp));
alert("desc");
} else {
peerConn.addIceCandidate(new RTCIceCandidate(signal.candidate));
alert("ice");
}
}
} catch(e) {
console.log(e.message);
}
In Chrome this errors out with:
Uncaught Error: InvalidStateError: DOM Exception 11
Open two tabs; click "Create Offer" button from 1st tab; and watch console logs:
<script>
// webkitRTCPeerConnection && RTCDataChannel specific code goes here
var iceServers = {
iceServers: [{
url: 'stun:stun.l.google.com:19302'
}]
};
var optionalRtpDataChannels = {
optional: [{
RtpDataChannels: true
}]
};
var mediaConstraints = {
optional: [],
mandatory: {
OfferToReceiveAudio: false, // Hmm!!
OfferToReceiveVideo: false // Hmm!!
}
};
var offerer, answerer, answererDataChannel, offererDataChannel;
function createOffer() {
offerer = new webkitRTCPeerConnection(iceServers, optionalRtpDataChannels);
offererDataChannel = offerer.createDataChannel('RTCDataChannel', {
reliable: false
});
setChannelEvents(offererDataChannel, 'offerer');
offerer.onicecandidate = function (event) {
if (!event.candidate) returnSDP();
};
offerer.ongatheringchange = function (event) {
if (event.currentTarget && event.currentTarget.iceGatheringState === 'complete') returnSDP();
};
function returnSDP() {
socket.send({
sender: 'offerer',
sdp: offerer.localDescription
});
}
offerer.createOffer(function (sessionDescription) {
offerer.setLocalDescription(sessionDescription);
}, null, mediaConstraints);
}
function createAnswer(offerSDP) {
answerer = new webkitRTCPeerConnection(iceServers, optionalRtpDataChannels);
answererDataChannel = answerer.createDataChannel('RTCDataChannel', {
reliable: false
});
setChannelEvents(answererDataChannel, 'answerer');
answerer.onicecandidate = function (event) {
if (!event.candidate) returnSDP();
};
answerer.ongatheringchange = function (event) {
if (event.currentTarget && event.currentTarget.iceGatheringState === 'complete') returnSDP();
};
function returnSDP() {
socket.send({
sender: 'answerer',
sdp: answerer.localDescription
});
}
answerer.setRemoteDescription(new RTCSessionDescription(offerSDP));
answerer.createAnswer(function (sessionDescription) {
answerer.setLocalDescription(sessionDescription);
}, null, mediaConstraints);
}
function setChannelEvents(channel, channelNameForConsoleOutput) {
channel.onmessage = function (event) {
console.debug(channelNameForConsoleOutput, 'received a message:', event.data);
};
channel.onopen = function () {
channel.send('first text message over RTP data ports');
};
}
// WebSocket specific code goes here
var socket = new WebSocket('ws://localhost:1337');
socket.onmessage = function (e) {
var data = JSON.parse(e.data);
console.log(data);
if (data.sdp) {
if (data.sender == 'offerer') createAnswer(data.sdp);
else offerer.setRemoteDescription(new RTCSessionDescription(data.sdp));
}
};
socket.push = socket.send;
socket.send = function (data) {
socket.push(JSON.stringify(data));
};
</script>
<button id="create-offer">Create Offer</button>
<script>
document.getElementById('create-offer').onclick = function () {
this.disabled = true;
createOffer();
};
</script>
now i'm trying the following:
First i create an offer on client #1 and send the description:
try {
peerConn = new webkitRTCPeerConnection(stunServers, {optional:[{RtpDataChannels: true}]});
peerConn.createOffer(function(desc) {
peerConn.setLocalDescription(desc);
socket.send("createpeer|" + JSON.stringify(desc));
}, null, mediaConstraints);
peerConn.onconnection = function () {
console.log("[webrtc] connected with peer");
peerChannel = peerConn.createDataChannel("test", {reliable: false});
peerChannel.onmessage = function (event) {
alert("Server: " + event.data);
};
peerChannel.onopen = function () {
peerChannel.send("Hello Server!");
};
};
} catch(error) {
console.log(error);
}
Client #2 receives that and sends his description:
case "createpeer":
console.log("[websocket] received create peer request from " + cmd[1] + " on " + cmd[2]);
try {
peerConn = new webkitRTCPeerConnection(stunServers, {optional:[{RtpDataChannels: true}]});
peerConn.setRemoteDescription(new RTCSessionDescription(JSON.parse(cmd[3])));
peerConn.createAnswer(function(desc) {
peerConn.setLocalDescription(desc);
socket.send("openpeer|" + cmd[1] + "|" + cmd[2] + "|" + JSON.stringify(desc));
}, null, mediaConstraints);
peerConn.ondatachannel = function (channel) {
channel.onmessage = function (event) {
alert("Client: " + event.data);
};
channel.onopen = function () {
channel.send("Hello Client!");
};
};
} catch(error) {
console.log(error);
}
break;
Finally client #1 recives the description from client #2
case "openpeer":
console.log("[websocket] received open peer");
peerConn.setRemoteDescription(new RTCSessionDescription(JSON.parse(cmd[1])));
break;
Everything works fine without errors, but the connection is not established and the onconnection method is not called.
Greetings

Failed to set local answer sdp: Called in wrong state: kStable

for a couple of days I'm now stuck with trying to get my webRTC client to work and I can't figure out what I'm doing wrong.
I'm trying to create multi peer webrtc client and am testing both sides with Chrome.
When the callee receives the call and create the Answer, I get the following error:
Failed to set local answer sdp: Called in wrong state: kStable
The receiving side correctly establishes both video connnections and is showing the local and remote streams. But the caller seems not to receive the callees answer. Can someone hint me what I am doing wrong here?
Here is the code I am using (it's a stripped version to just show the relevant parts and make it better readable)
class WebRTC_Client
{
private peerConns = {};
private sendAudioByDefault = true;
private sendVideoByDefault = true;
private offerOptions = {
offerToReceiveAudio: true,
offerToReceiveVideo: true
};
private constraints = {
"audio": true,
"video": {
frameRate: 5,
width: 256,
height: 194
}
};
private serversCfg = {
iceServers: [{
urls: ["stun:stun.l.google.com:19302"]
}]
};
private SignalingChannel;
public constructor(SignalingChannel){
this.SignalingChannel = SignalingChannel;
this.bindSignalingHandlers();
}
/*...*/
private gotStream(stream) {
(<any>window).localStream = stream;
this.videoAssets[0].srcObject = stream;
}
private stopLocalTracks(){}
private start() {
var self = this;
if( !this.isReady() ){
console.error('Could not start WebRTC because no WebSocket user connectionId had been assigned yet');
}
this.buttonStart.disabled = true;
this.stopLocalTracks();
navigator.mediaDevices.getUserMedia(this.getConstrains())
.then((stream) => {
self.gotStream(stream);
self.SignalingChannel.send(JSON.stringify({type: 'onReadyForTeamspeak'}));
})
.catch(function(error) { trace('getUserMedia error: ', error); });
}
public addPeerId(peerId){
this.availablePeerIds[peerId] = peerId;
this.preparePeerConnection(peerId);
}
private preparePeerConnection(peerId){
var self = this;
if( this.peerConns[peerId] ){
return;
}
this.peerConns[peerId] = new RTCPeerConnection(this.serversCfg);
this.peerConns[peerId].ontrack = function (evt) { self.gotRemoteStream(evt, peerId); };
this.peerConns[peerId].onicecandidate = function (evt) { self.iceCallback(evt, peerId); };
this.peerConns[peerId].onnegotiationneeded = function (evt) { if( self.isCallingTo(peerId) ) { self.createOffer(peerId); } };
this.addLocalTracks(peerId);
}
private addLocalTracks(peerId){
var self = this;
var localTracksCount = 0;
(<any>window).localStream.getTracks().forEach(
function (track) {
self.peerConns[peerId].addTrack(
track,
(<any>window).localStream
);
localTracksCount++;
}
);
trace('Added ' + localTracksCount + ' local tracks to remote peer #' + peerId);
}
private call() {
var self = this;
trace('Start calling all available new peers if any available');
// only call if there is anyone to call
if( !Object.keys(this.availablePeerIds).length ){
trace('There are no callable peers available that I know of');
return;
}
for( let peerId in this.availablePeerIds ){
if( !this.availablePeerIds.hasOwnProperty(peerId) ){
continue;
}
this.preparePeerConnection(peerId);
}
}
private createOffer(peerId){
var self = this;
this.peerConns[peerId].createOffer( this.offerOptions )
.then( function (offer) { return self.peerConns[peerId].setLocalDescription(offer); } )
.then( function () {
trace('Send offer to peer #' + peerId);
self.SignalingChannel.send(JSON.stringify({ "sdp": self.peerConns[peerId].localDescription, "remotePeerId": peerId, "type": "onWebRTCPeerConn" }));
})
.catch(function(error) { self.onCreateSessionDescriptionError(error); });
}
private answerCall(peerId){
var self = this;
trace('Answering call from peer #' + peerId);
this.peerConns[peerId].createAnswer()
.then( function (answer) { return self.peerConns[peerId].setLocalDescription(answer); } )
.then( function () {
trace('Send answer to peer #' + peerId);
self.SignalingChannel.send(JSON.stringify({ "sdp": self.peerConns[peerId].localDescription, "remotePeerId": peerId, "type": "onWebRTCPeerConn" }));
})
.catch(function(error) { self.onCreateSessionDescriptionError(error); });
}
private onCreateSessionDescriptionError(error) {
console.warn('Failed to create session description: ' + error.toString());
}
private gotRemoteStream(e, peerId) {
if (this.audioAssets[peerId].srcObject !== e.streams[0]) {
this.videoAssets[peerId].srcObject = e.streams[0];
trace('Added stream source of remote peer #' + peerId + ' to DOM');
}
}
private iceCallback(event, peerId) {
this.SignalingChannel.send(JSON.stringify({ "candidate": event.candidate, "remotePeerId": peerId, "type": "onWebRTCPeerConn" }));
}
private handleCandidate(candidate, peerId) {
this.peerConns[peerId].addIceCandidate(candidate)
.then(
this.onAddIceCandidateSuccess,
this.onAddIceCandidateError
);
trace('Peer #' + peerId + ': New ICE candidate: ' + (candidate ? candidate.candidate : '(null)'));
}
private onAddIceCandidateSuccess() {
trace('AddIceCandidate success.');
}
private onAddIceCandidateError(error) {
console.warn('Failed to add ICE candidate: ' + error.toString());
}
private hangup() {}
private bindSignalingHandlers(){
this.SignalingChannel.registerHandler('onWebRTCPeerConn', (signal) => this.handleSignals(signal));
}
private handleSignals(signal){
var self = this,
peerId = signal.connectionId;
if( signal.sdp ) {
trace('Received sdp from peer #' + peerId);
this.peerConns[peerId].setRemoteDescription(new RTCSessionDescription(signal.sdp))
.then( function () {
if( self.peerConns[peerId].remoteDescription.type === 'answer' ){
trace('Received sdp answer from peer #' + peerId);
} else if( self.peerConns[peerId].remoteDescription.type === 'offer' ){
trace('Received sdp offer from peer #' + peerId);
self.answerCall(peerId);
} else {
trace('Received sdp ' + self.peerConns[peerId].remoteDescription.type + ' from peer #' + peerId);
}
})
.catch(function(error) { trace('Unable to set remote description for peer #' + peerId + ': ' + error); });
} else if( signal.candidate ){
this.handleCandidate(new RTCIceCandidate(signal.candidate), peerId);
} else if( signal.closeConn ){
trace('Closing signal received from peer #' + peerId);
this.endCall(peerId,true);
}
}
}
I have been using a similar construct to build up the WebRTC connections between sender and receiver peers, by calling the method RTCPeerConnection.addTrack twice (one for the audio track, and one for the video track).
I used the same structure as shown in the Stage 2 example shown in The evolution of WebRTC 1.0:
let pc1 = new RTCPeerConnection(), pc2 = new RTCPeerConnection(), stream, videoTrack, videoSender;
(async () => {
try {
stream = await navigator.mediaDevices.getUserMedia({video: true, audio: true});
videoTrack = stream.getVideoTracks()[0];
pc1.addTrack(stream.getAudioTracks()[0], stream);
} catch (e) {
console.log(e);
}
})();
checkbox.onclick = () => {
if (checkbox.checked) {
videoSender = pc1.addTrack(videoTrack, stream);
} else {
pc1.removeTrack(videoSender);
}
}
pc2.ontrack = e => {
video.srcObject = e.streams[0];
e.track.onended = e => video.srcObject = video.srcObject; // Chrome/Firefox bug
}
pc1.onicecandidate = e => pc2.addIceCandidate(e.candidate);
pc2.onicecandidate = e => pc1.addIceCandidate(e.candidate);
pc1.onnegotiationneeded = async e => {
try {
await pc1.setLocalDescription(await pc1.createOffer());
await pc2.setRemoteDescription(pc1.localDescription);
await pc2.setLocalDescription(await pc2.createAnswer());
await pc1.setRemoteDescription(pc2.localDescription);
} catch (e) {
console.log(e);
}
}
Test it here: https://jsfiddle.net/q8Lw39fd/
As you'll notice, in this example the method createOffer is never called directly; instead, it is indirectly called via addTrack triggering an RTCPeerConnection.onnegotiationneeded event.
However, just as in your case, Chrome triggers this event twice, once for each track, and this causes the error message you mentioned:
DOMException: Failed to set local answer sdp: Called in wrong state: kStable
This doesn't happen in Firefox, by the way: it triggers the event only once.
The solution to this issue is to write a workaround for the Chrome behavior: a guard that prevents nested calls to the (re)negotiation mechanism.
The relevant part of the fixed example would be like this:
pc1.onicecandidate = e => pc2.addIceCandidate(e.candidate);
pc2.onicecandidate = e => pc1.addIceCandidate(e.candidate);
var isNegotiating = false; // Workaround for Chrome: skip nested negotiations
pc1.onnegotiationneeded = async e => {
if (isNegotiating) {
console.log("SKIP nested negotiations");
return;
}
isNegotiating = true;
try {
await pc1.setLocalDescription(await pc1.createOffer());
await pc2.setRemoteDescription(pc1.localDescription);
await pc2.setLocalDescription(await pc2.createAnswer());
await pc1.setRemoteDescription(pc2.localDescription);
} catch (e) {
console.log(e);
}
}
pc1.onsignalingstatechange = (e) => { // Workaround for Chrome: skip nested negotiations
isNegotiating = (pc1.signalingState != "stable");
}
Test it here: https://jsfiddle.net/q8Lw39fd/8/
You should be able to easily implement this guard mechanism into your own code.
You're sending the answer here:
.then( function (answer) { return self.peerConns[peerId].setLocalDescription(answer); } )
Look at mine:
var callback = function (answer) {
createdDescription(answer, fromId);
};
peerConnection[fromId].createAnswer().then(callback).catch(errorHandler);
function createdDescription(description, fromId) {
console.log('Got description');
peerConnection[fromId].setLocalDescription(description).then(function() {
console.log("Sending SDP:", fromId, description);
serverConnection.emit('signal', fromId, {'sdp': description});
}).catch(errorHandler);
}