I am developing a voice call app for android using peer.js. It is working but there is a lag in audio (around 1-5 seconds). So I was searching for a fix and found out that I can enable DTX Audio to reduce the number of packets sent. But I have no idea How to access and change SDP object in peer.js. Here is my code,
let audioConstraints = {
channelCount: 1,
sampleRate: 16000,
sampleSize: 8,
volume: 1,
latency: 0.003,
echoCancellation: true,
noiseSuppression: true,
autoGainControl: true,
};
constraints = { audio: audioConstraints, video: false };
let localStream;
//listen for calls
function listen() {
peer.on("call", (call) => {
navigator.mediaDevices
.getUserMedia(constraints)
.then(function (stream) {
localVideo.srcObject = stream;
localStream = stream;
call.answer(stream);
call.on("stream", (remoteStream) => {
Android.onPeerStream();
remoteVideo.srcObject = remoteStream;
});
})
.catch(function (err) {
Android.logEvent("getUserMedia error: " + err);
});
});
}
//start call
function startCall(otherUserId) {
navigator.mediaDevices
.getUserMedia(constraints)
.then(function (stream) {
localVideo.srcObject = stream;
localStream = stream;
const call = peer.call(otherUserId, stream);
call.on("stream", (remoteStream) => {
Android.onPeerStream();
remoteVideo.srcObject = remoteStream;
});
})
.catch(function (err) {
Android.logEvent("getUserMedia error: " + err);
});
}
could someone please share a working example of how to access and modify SDP in peer.js? Also, how can I tune the audioConstraints to achieve low network lag while keeping decent voice quality?
UPDATE
Found the way to access SDP in peer.js but still can't figure it out how to enable DTX audio.
Related
I am using simple-peer and it really worked well on server before I added APIs for my project. I already did secure with https in the beginning and the only thing that has changed is releasing the server with APIs... Here is my code and now I only can check console.log(1), (3) for initiator peer and console.log(2), (8) for requesting peer. On requestinng peer internet tab, "Uncaught ReferenceError: process is not defined" this error occurs and I don't know why this error occurs on client side. Also it worked well on both of Chrome and Edge but now I can't even get my own stream on Chrome.
const myVideo = useRef();
const userVideo = useRef();
const connectionRef = useRef();
const roomName = "123";
let userStream = null;
let creator = false;
useEffect(() => {
const socket = io("https://www.jg-jg.shop");
socket.emit("joinRoom", roomName);
socket.on("created", () => {
creator = true;
navigator.mediaDevices
.getUserMedia({ video: true, audio: true })
.then((stream) => {
userStream = stream;
myVideo.current.srcObject = stream;
console.log(1);
});
});
socket.on("joined", () => {
navigator.mediaDevices
.getUserMedia({ video: true, audio: true })
.then((stream) => {
userStream = stream;
myVideo.current.srcObject = stream;
console.log(2);
});
socket.emit("ready", roomName);
});
socket.on("ready", () => {
if (creator) {
const peer = new Peer({
initiator: true,
trickle: false,
stream: userStream,
});
peer.on("signal", (signal) => {
socket.emit("sendingSignal", { signal, roomName });
console.log(3);
});
peer.on("stream", (stream) => {
userVideo.current.srcObject = stream;
console.log(4);
});
socket.on("receivingSignal", (signal) => {
peer.signal(signal);
console.log(5);
});
connectionRef.current = peer;
}
});
socket.on("offer", (incomingSignal) => {
if (!creator) {
const peer = new Peer({
initiator: false,
trickle: false,
stream: userStream,
});
peer.on("signal", (signal) => {
socket.emit("returningSignal", { signal, roomName });
console.log(6);
});
peer.on("stream", (stream) => {
userVideo.current.srcObject = stream;
console.log(7);
});
peer.signal(incomingSignal);
console.log(8);
connectionRef.current = peer;
}
});
}, []);
we are using peer js webrtc for video call. Everything is working fine just the problem is i am not able to switch camera during call. I have done some work where i can switch camera in local during call but its doesnt effect on remote area.
here is my code
$('select').on('change', function (e) {
navigator.mediaDevices.enumerateDevices().then(function (devices) {
var valueSelected = $("#myselect option:selected").val();
alert(valueSelected);
//var myselect = 0;
if (valueSelected == "0") {
var cameras = [];
devices.forEach(function (device) {
'videoinput' === device.kind && cameras.push(device.deviceId);
});
var constraints = { video: { deviceId: { exact: cameras[0] } } };
navigator.mediaDevices.getUserMedia(constraints).then(function (stream) {
window.localStream = stream;
myapp.setMyVideo(window.localStream)
//if (callback)
// callback();
}, function (err) {
console.log("The following error occurred: " + err.name);
alert('Unable to call ' + err.name)
});
}
else {
var cameras = [];
devices.forEach(function (device) {
'videoinput' === device.kind && cameras.push(device.deviceId);
});
var constraints = { video: { deviceId: { exact: cameras[1] } } };
navigator.mediaDevices.getUserMedia(constraints).then(function (stream) {
window.localStream = stream;
myapp.setMyVideo(window.localStream)
//if (callback)
// callback();
}, function (err) {
console.log("The following error occurred: " + err.name);
alert('Unable to call ' + err.name)
});
}
//var myselect = $("#myselect option:selected").val();
});
});
The recommended way to change stream when a peer-to-peer connection is established is to use replaceTrack function that does not require ICE renegotiation:
RTCRtpSender.replaceTrack
The documentation says:
Among the use cases for replaceTrack() is the common need to switch between the rear- and front-facing cameras on a phone. With replaceTrack(), you can simply have a track object for each camera and switch between the two as needed. See the example...
I can't receive any data from the remote stream and the div with the id of the remote streams that I've created with Javascript doesn't show I don't understand why ! The Javascript code doesn't work on the browser it's a little bit strange it's my first time coming across with this kind of problem please help me solve it.
This is the code :
// Local stream
// rtc object
var rtc = {
client: null,
joined: false,
published: false,
localStream: null,
remoteStreams: [],
params: {}
};
// Options for joining a channel
var option = {
mode: "rtc",
codec: "h264",
appID: "22a27d03d0edf54749a03597d72ad82aaa78",
channel: "qiossa",
uid: null,
token: "006a27d03d0edf54749a03597d72ad82aaaIADHJF46Q3g4Jn+mjfRgh5Le76OO2BpUfEuvw1Qv+35XKFwgy+4AAAAAEACfOV6k44bGXgEAAQCIh8Ze"
};
// Create a client
rtc.client = AgoraRTC.createClient({mode: option.mode, codec: option.codec});
// Initialize the client
rtc.client.init(option.appID, function () {
console.log("init success");
}, (err) => {
console.error(err);
});
// Join a channel
rtc.client.join(option.token, option.channel, option.uid, function (uid) {
console.log("join channel: " + option.channel + " success, uid: " + uid);
rtc.params.uid = uid;
}, function(err) {
console.error("client join failed", err);
});
// Create a local stream
rtc.localStream = AgoraRTC.createStream({
streamID: rtc.params.uid,
audio: true,
video: true,
screen: false,
});
// Initialize the local stream
rtc.localStream.init(function () {
console.log("init local stream success");
// play stream with html element id "local_stream"
rtc.localStream.play("local_stream");
}, function (err) {
console.error("init local stream failed ", err);
});
// Publish the local stream
rtc.client.publish(rtc.localStream, function (err) {
console.log("publish failed");
console.error(err);
});
//*************************************************************************************************************//
// Remote stream
rtc.client.on("stream-added", function (evt) {
var remoteStream = evt.stream;
var id = remoteStream.getId();
if (id !== rtc.params.uid) {
rtc.client.subscribe(remoteStream, function (err) {
console.log("stream subscribe failed", err);
});
}
console.log("stream-added remote-uid: ", id);
});
rtc.client.on("stream-subscribed", function (evt) {
var remoteStream = evt.stream;
var id = remoteStream.getId();
// Add a view for the remote stream.
let streamDiv=document.createElement("div"); // Create a new div for every stream
streamDiv.id= id; // Assigning id to div
streamDiv.style.transform="rotateY(180deg)"; // Takes care of lateral inversion (mirror image)
remoteContainer.appendChild(streamDiv);
// Play the remote stream.
remoteStream.play("remote_video_" + id);
console.log("stream-subscribed remote-uid: ", id);
});
Photo of the problem
The local stream needs to be created, initialised, played and published within the join function.
This is the corrected code for the rtc.client.join() function:
rtc.client.join(option.token, option.channel, option.uid, (uid)=>{
console.log("join channel: " + option.channel + " success, uid: " + uid);
rtc.params.uid = uid;
// Create a local stream
rtc.localStream = AgoraRTC.createStream({
streamID: rtc.params.uid,
audio: true,
video: true,
screen: false,
});
// Initialize the local stream
rtc.localStream.init(function () {
console.log("init local stream success");
// play stream with html element id "local_stream"
rtc.localStream.play("local_stream");
}, function (err) {
console.error("init local stream failed ", err);
});
// Publish the local stream
rtc.client.publish(rtc.localStream, function (err) {
console.log("publish failed");
console.error(err);
});
}, function(err) {
console.error("client join failed", err);
});
Get back to us for any further queries.
console problems
// Local stream
// rtc object
var rtc = {
client: null,
joined: false,
published: false,
localStream: null,
remoteStreams: [],
params: {}
};
// Options for joining a channel
var option = {
mode: "rtc",
codec: "h264",
appID: "",
channel: "qiossa",
uid: null,
token: "006a27d03d0edf54749a03597d72ad82aaaIADkVIvop7lo0OEkm/7Tuz/Tp4M+TXhFd9DkLAAwu9fNllwgy+4AAAAAEAD4aAmV2FzKXgEAAQBjT8pe"
};
// Create a client
rtc.client = AgoraRTC.createClient({mode: option.mode, codec: option.codec});
// Initialize the client
rtc.client.init(option.appID, function () {
console.log("init success");
}, (err) => {
console.error(err);
});
// Join a channel
rtc.client.join(option.token, option.channel, option.uid, function (uid) {
console.log("join channel: " + option.channel + " success, uid: " + uid);
rtc.params.uid = uid;
// Create a local stream
rtc.localStream = AgoraRTC.createStream({
streamID: rtc.params.uid,
audio: true,
video: true,
screen: false,
});
// Initialize the local stream
rtc.localStream.init(function () {
console.log("init local stream success");
// play stream with html element id "local_stream"
rtc.localStream.play("local_stream");
}, function (err) {
console.error("init local stream failed ", err);
});
// Publish the local stream
rtc.client.publish(rtc.localStream, function (err) {
console.log("publish failed");
console.error(err);
});
}, function(err) {
console.error("client join failed", err);
});
//*************************************************************************************************************//
// Remote stream
rtc.client.on("stream-added", function (evt) {
var remoteStream = evt.stream;
var id = remoteStream.getId();
if (id !== rtc.params.uid) {
rtc.client.subscribe(remoteStream, function (err) {
console.log("stream subscribe failed", err);
});
}
console.log("stream-added remote-uid: ", id);
});
rtc.client.on("stream-subscribed", function (evt) {
var remoteStream = evt.stream;
var id = remoteStream.getId();
// Add a view for the remote stream.
addView(id);
// Play the remote stream.
remoteStream.play("remote_video_" + id);
console.log("stream-subscribed remote-uid: ", id);
});
rtc.client.on("stream-removed", function (evt) {
var remoteStream = evt.stream;
var id = remoteStream.getId();
// Stop playing the remote stream.
remoteStream.stop("remote_video_" + id);
// Remove the view of the remote stream.
removeView(id);
console.log("stream-removed remote-uid: ", id);
});
// Leave the channel
rtc.client.leave(function () {
// Stop playing the local stream
rtc.localStream.stop();
// Close the local stream
rtc.localStream.close();
// Stop playing the remote streams and remove the views
while (rtc.remoteStreams.length > 0) {
var stream = rtc.remoteStreams.shift();
var id = stream.getId();
stream.stop();
removeView(id);
}
console.log("client leaves channel success");
}, function (err) {
console.log("channel leave failed");
console.error(err);
});
function addView (id, show) {
if (!$("#" + id)[0]) {
$("<div/>", {
id: "remote_video_panel_" + id,
class: "video-view",
}).appendTo("#video")
$("<div/>", {
id: "remote_video_" + id,
class: "video-placeholder",
}).appendTo("#remote_video_panel_" + id)
$("<div/>", {
id: "remote_video_info_" + id,
class: "video-profile " + (show ? "" : "hide"),
}).appendTo("#remote_video_panel_" + id)
$("<div/>", {
id: "video_autoplay_"+ id,
class: "autoplay-fallback hide",
}).appendTo("#remote_video_panel_" + id)
}
}
function removeView (id) {
if ($("#remote_video_panel_" + id)[0]) {
$("#remote_video_panel_"+id).remove()
}
}
chrome version: 62.0.3202.94;
firefox version: 57.0.1;
I write a simple demo use webrtc and socket.io.
It works with pages. For example, I open one page to connect socket, and waiting for PeerConnection info from the main page(which get the local media). When I open the main, I create ice and sdp and then exchange them by socket.io to create connection.
Here is the code.
// The server side:
const express = require('express')
const app = express()
const path = require('path')
app.use(express.static(path.join(__dirname, 'public')))
app.get('/phone', function(req, res) {
res.sendfile(__dirname + '/phone.html')
})
app.get('/', function(req, res) {
res.sendfile(__dirname + '/index.html')
})
const server = require('http').createServer(app)
const io = require('socket.io')(server)
let clients = []
io.on('connection', function(socket) {
clients.push(socket)
const referer = socket.handshake.headers.referer
// socket connect from '/phone'
if (referer.match('/phone')) {
// send the ice from phone to others
socket.on('phone_ice_candidate', function(res) {
socket.broadcast.emit('pc_add_ice', {
ice: res.ice
})
})
// send the sdp from phone to others
socket.on('send_phone_sdp', function(data) {
socket.broadcast.emit('set_pc_remote_sdp', {
desc: data.desc
})
})
}
// phone add ice from web
socket.on('remote_ice_candidate', function(ice) {
socket.to(getId(clients, '/phone')).emit('send_ice_to_pc', {
ice: ice
})
})
// phone add sdp from web
socket.on('send_pc_sdp', function(data) {
// send to phone
socket.to(getId(clients, '/phone')).emit('set_phone_remote_sdp', {
desc: data
})
})
// socket disconnect and remove it from clients
socket.on('disconnect', () => {
let id = socket.id
clients.forEach((client, index) => {
if (client.id === id) {
clients.splice(index, 1)
}
})
})
})
// get the socket id to emit
function getId(sockets, exp) {
let id
sockets.forEach(socket => {
if (socket.handshake.headers.referer.match(exp)) {
id = socket.id
}
})
return id
}
server.listen(3000, function() {
console.log('port listening at 3000')
})
// --------------------------------------------- //
// web.js
var socket = io();
var server = {
// "iceServers": [{
// "url": "stun:stun.l.google.com:19302"
// }]
},
pc = new RTCPeerConnection(null),
v = document.querySelector('#video2')
// web onicecandidate
pc.onicecandidate = function(event) {
if (event.candidate) {
socket.emit('remote_ice_candidate', {
ice: event.candidate
})
}
}
// web addIceCandidate
socket.on('pc_add_ice', function(event) {
pc.addIceCandidate(new RTCIceCandidate(event.ice))
})
// didn't trigger
pc.ontrack = function(e) {
// v.srcObject = e.streams[0];
console.log(e, 'pc.ontrack')
}
// web setRemoteDescription and createAnswer
socket.on('set_pc_remote_sdp', function(e) {
pc.setRemoteDescription(e.desc).then(
function() {
console.log('remote setRemoteDescription success')
pc.createAnswer().then(function(desc) {
pc.setLocalDescription(desc).then(
function() {
socket.emit('send_pc_sdp', {
desc: desc
})
},
function(err) {
console.log(err)
}
);
})
},
function() {
console.log('pc setLocalDescription error')
}
)
})
// web iceConnectionState
pc.oniceconnectionstatechange = function() {
console.log('web oniceconnectionstatechange', pc.iceConnectionState)
// log checking -> connected
};
//---------------------------------------------//
// phone.js
var socket = io();
var server = {
// "iceServers": [{
// "url": "stun:stun.l.google.com:19302"
// }]
},
pc = new RTCPeerConnection(null),
v = document.querySelector('#video1')
// phone onicecandidate
pc.onicecandidate = function(event) {
if (event.candidate) {
socket.emit('phone_ice_candidate', {
ice: event.candidate
})
}
}
// phone addIceCandidate
socket.on('send_ice_to_pc', function(event) {
pc.addIceCandidate(new RTCIceCandidate(event.ice.ice))
})
// getUserMedia
navigator.mediaDevices.getUserMedia({
video: {
width: 400,
height: 300
},
audio: false
})
.then(function(stream) {
v.src = window.URL.createObjectURL(stream);
pc.addStream(stream);
})
.then(function() {
// create offer
pc.createOffer({
offerToReceiveVideo: 1
}).then(function(e) {
// pc setLocalDescription
pc.setLocalDescription(e).then(
function() {
socket.emit('send_phone_sdp', {
desc: e
})
},
function() {
console.log('pc setLocalDescription error')
}
)
});
})
.catch(function(err) {
console.log(err.name + ": " + err.message);
})
// phone setRemoteDescription
socket.on('set_phone_remote_sdp', function(e) {
pc.setRemoteDescription(e.desc.desc).then(
function() {
console.log('pc setRemoteDescription success')
},
function(err) {
console.log(err)
})
})
// phone iceConnectionState
pc.oniceconnectionstatechange = function() {
console.log('phone oniceconnectionstatechange', pc.iceConnectionState)
// log checking -> connected -> completed
};
When i use firefox to open it, there is an error ICE failed, add a STUN server and see about:webrtc for more details in console.
In chrome the 'phone iceConnectionState' changed checking -> connected -> completed, the 'web iceConnectionState' changed checking -> connected.
have you set autoplay in your html? I have the same issue, and it turns out I should have set autoplay in my html tag. Namely:
<video autoplay></video>
Hope this helps!
getUserMedia is an async function. You are calling createOffer before you call pc.addStream which means there is nothing to negotiate.
Make the promise callback return your pc.createOffer() after pc.addStream(stream);
PTAL at https://www.html5rocks.com/en/tutorials/webrtc/basics/#toc-signaling and compare your code to the example and see if you can figure it out.
I had earlier posted some questions on this problem. At that time I had two separate programs for caller and receiver. I was also using old-fashioned callback API. Thanks to help from #jib on that post, I was able to understand the need for some fundamental changes. I rewrote the program to make it an integrated one for both caller and receiver and have used the WebRTC promises API. My problem is that I am not getting remote video from either end. One part I understand but do not know the solution: The receiver does not create SDPs for Video in the first place, only for audio. The caller part does create SDPS for Video and audio but on the receiver end there is no event generated for remote stream.
I have checked, through console logs, that the core functions work. Offer SDP is created, sent out, received, answer SDP created, sent out, received, etc. Candidates get exchanged and added too. But the .onaddstream event handler is never triggered. Local video is shown but that is trivial.
I have spent a LOT of time on this. I simply need to get that exciting feeling of seeing remote video on both ends which has kept me going. ANY HELP WILL BE SINCERELY APPRECIATED.
<script>
$(document).ready(function () {
var iceCandidates = [], countIceCandidates=0;
var socket = io.connect();
socket.on('connect',function() { console.log("Socket connected"); });
var pc = new RTCPeerConnection({"iceServers":[{"url":"stun:stun.l.google.com:19302"}]});
//If remote video stream comes in, display it in DIV vid2
pc.onaddStream = function (event) {
stream = event.stream;
var video = $('#vid2');
video.attr('src', URL.createObjectURL(stream));
video.onloadedmetadata = function(e) { video.play(); }
}
//Display media in both Caller and Receiver
navigator.mediaDevices.getUserMedia({ audio: true, video: true })
.then(function(stream) {
var video = $('#vid1');
video.attr('src', URL.createObjectURL(stream));
video.onloadedmetadata = function(e) { video.play(); };
pc.addStream(stream);
})
.catch(function(err) { console.log(err);});
//INITIATE CALL
$('#call').click(function() {
pc.createOffer({ offerToReceiveVideo: true, offerToReceiveAudio: true })
.then(function(offer) {
localSessionDescription = new RTCSessionDescription(offer);
pc.setLocalDescription(localSessionDescription)
.then (function() { socket.emit('sdpOffer',localSessionDescription); })
.catch(function(err) { console.log("Error in setLocalDescription"); console.log(err); })
.catch(function(err) { console.log("Error in createOffer"); console.log(err); })
});
})
pc.onicecandidate = function (event) {
socket.emit('candidate',event.candidate);
};
socket.on('candidate',function (data) {
if (data != null) {
pc.addIceCandidate(new RTCIceCandidate(data))
.then(function() { console.log("peer candidate added");})
.catch(function(err) {console.log(err); console.log("Error during peer candidate addition");});
}
});
socket.on('disconnect',function() { alert("Disconnected"); });
function error(err) {
console.log("The following error occurred: " + err.name);
}
socket.on('sdpAnswer',function(data) {
sdpAnswer = new RTCSessionDescription(data.sdpAnswer);
pc.setRemoteDescription(sdpAnswer)
.then(function() { console.log("Answer SDP Set:"); console.log(sdpAnswer); })
.catch(function(err) { console.log("Error enountered when setting remote SDP Answer"); console.log(err)});
});
socket.on('sdpOffer', function(data) {
sdpOffer = new RTCSessionDescription(data.sdpOffer);
pc.setRemoteDescription(sdpOffer)
.then(function() { console.log("Remote SDP set in receiver");
pc.createAnswer()
.then(function(sdpAnswer) {
localSessionDescription = new RTCSessionDescription(sdpAnswer);
socket.emit('sdpAnswer',localSessionDescription);
pc.setLocalDescription(localSessionDescription)
.then(function(){
console.log("Local SDP Description set in receiver:");
})
.catch(function(err) { console.log("Error enountered when setting local SDP in receiver"); console.log(err)});
})
.catch(function(err) { console.log("Error enountered when creating answer SDP in receiver"); console.log(err)});
});
});
}); //End of document.ready function
</script>
ON THE SERVER SIDE (RELEVANT CODE ONLY). I have included here just in case there are any datatype related issues - object types, etc. getting changed when sent thru the server.
io.sockets.on('connection', function(socket) {
socket.on('sdpOffer', function(data) {
sdpOffer = data.sdp;
socket.broadcast.emit('sdpOffer',{"sdpOffer":data});
});
socket.on('sdpAnswer', function(data) {
sdpAnswer = data.sdp;
socket.broadcast.emit('sdpAnswer',{"sdpAnswer":data});
});
socket.on('candidate', function(data) {
socket.broadcast.emit('candidate',data);
});
});
Rename pc.onaddStream to pc.onaddstream.