Can't get WebRTC to work in different networks - vue.js

I am trying to transfer a video streaming from one browser to another with WebRTC and socket.io. It works just fine in the same network. No image is getting through across different ones.
I use socket-io as a signal server. I register two browsers in a "room" and then start sending signals.
The code which is executed in the browser from which the streaming is sent:
function joinRoom(room) {
if (room === '') {
alert('Please type a room ID')
} else {
data = { room: room};
socket.emit('join', data);
}
}
// SOCKET EVENT CALLBACKS =====================================================
socket.on('room_created', async () => {
console.log('Socket event callback: room_created')
await setLocalStream(mediaConstraints)
socket.emit('startc', {room: roomId, clientip: clientip})
isRoomCreator = true
})
socket.on('full_room', () => {
console.log('Socket event callback: full_room')
alert('The room is full, please try another one')
})
socket.on('startc', async () => {
console.log('Socket event callback: start_call')
if (isRoomCreator) {
rtcPeerConnection = new RTCPeerConnection(iceServers)
addLocalTracks(rtcPeerConnection)
rtcPeerConnection.ontrack = setRemoteStream
rtcPeerConnection.onicecandidate = sendIceCandidate
await createOffer(rtcPeerConnection)
}
})
socket.on('offer', async (event) => {
console.log('Socket event callback: offer')
if (!isRoomCreator) {
rtcPeerConnection = new RTCPeerConnection(iceServers)
addLocalTracks(rtcPeerConnection)
rtcPeerConnection.ontrack = setRemoteStream
rtcPeerConnection.onicecandidate = sendIceCandidate
rtcPeerConnection.setRemoteDescription(new RTCSessionDescription(event))
await createAnswer(rtcPeerConnection)
}
})
socket.on('answer', (event) => {
console.log('answer');
console.log('Socket event callback: webrtc_answer')
rtcPeerConnection.setRemoteDescription(new RTCSessionDescription(event))
})
socket.on('webrtc_ice_candidate', (event) => {
console.log('Socket event callback: webrtc_ice_candidate')
// ICE candidate configuration.
var candidate = new RTCIceCandidate({
sdpMLineIndex: event.label,
candidate: event.candidate,
})
rtcPeerConnection.addIceCandidate(candidate)
})
The code that accepts the streamed media (vuejs):
socket.on("offer", (data) => {
this.$emit("closeWaitingToConnect");
this.createAnswer(data);
});
joinMeToRoom() {
console.log("joinToRoom: ", this.room);
this.$socket.emit("join", this.room);
}, //joinMeToRoom ()
createAnswer: function(event) {
var roomId = this.room.room;
let sessionDescription
this.peer.ontrack = this.setRemoteStream
this.peer.onicecandidate = this.sendIceCandidate
this.peer.setRemoteDescription(new RTCSessionDescription(event))
try {
sessionDescription = this.peer.createAnswer().then((answer) => {
var anwer =
console.log('sessionDescription');
console.log(answer);
this.$socket.emit('answer', {
type: 'webrtc_answer',
sdp: answer,
sessionDescription: JSON.stringify(answer),
roomId,
})
return this.peer.setLocalDescription(answer)
});
} catch (error) {
console.error('cae: '+error)
}
},
getScreenPosition() {
const right = this.$refs.screen.getBoundingClientRect().right;
const bottom = this.$refs.screen.getBoundingClientRect().bottom;
return { bottom: bottom, right: right };
},
setRemoteStream(event) {
console.log('event setRemoteStream');
console.log(event);
var stream_screen = document.querySelector("video");
stream_screen.srcObject = event.streams[0];
stream_screen.play();
var remoteStream = event.stream
},
I have setup my own TURN server and tried paid versions. Still can't get the stream across different networks.
What am I missing?

Related

Getting pc.iceConnectionState Checking, failed in pc.oniceconnectionstatechange event in webRTC

I'm using webrtc for video calling in react native app.
If I call to someone else and receivers receives call then I get stream of receiver but there is a problem at receiver side.
Receiver gets remotestream but it shows blank view.
import AsyncStorage from '#react-native-async-storage/async-storage';
import {
RTCIceCandidate,
RTCPeerConnection,
RTCSessionDescription,
} from 'react-native-webrtc';
import io from '../scripts/socket.io';
const PC_CONFIG = {
iceServers: [
{ url: 'stun:motac85002'},
],
};
export const pc = new RTCPeerConnection(PC_CONFIG);
// Signaling methods
export const onData = data => {
handleSignalingData(data.data);
};
const ENDPOINT = 'http://52.52.75.250:3000/';
const socket = io(ENDPOINT);
// const PeerConnection = () => {
const sendData = async data => {
const roomId = await AsyncStorage.getItem('roomId');
const userId = parseInt(await AsyncStorage.getItem('user_id'));
socket.emit('data', roomId, userId, data);
};
export const createPeerConnection = async(stream, setUsers) => {
try {
pc.onicecandidate = onIceCandidate;
const userId = parseInt(await AsyncStorage.getItem('user_id'));
pc.onaddstream = e => {
setUsers(e.stream);
};
pc.addStream(stream)
pc.oniceconnectionstatechange = function () {
// console.log('ICE state: ', pc);
console.log('iceConnectionState', pc.iceConnectionState);
if (pc.iceConnectionState === "failed" ||
pc.iceConnectionState === "disconnected" ||
pc.iceConnectionState === "closed") {
console.log('iceConnectionState restart', userId);
// console.log('ICE state: ', pc);
// Handle the failure
pc.restartIce();
}
};
console.log('PeerConnection created', userId);
// sendOffer();
} catch (error) {
console.error('PeerConnection failed: ', error);
}
};
export const callSomeone = () => {
pc.createOffer({}).then(setAndSendLocalDescription, error => {
console.error('Send offer failed: ', error);
});
};
const setAndSendLocalDescription = sessionDescription => {
pc.setLocalDescription(sessionDescription);
sendData(sessionDescription);
};
const onIceCandidate = event => {
if (event.candidate) {
sendData({
type: 'candidate',
candidate: event.candidate,
});
}
};
export const disconnectPeer = () =>{
pc.close();
}
const sendAnswer = () => {
pc.createAnswer().then(setAndSendLocalDescription, error => {
console.error('Send answer failed: ', error);
});
};
export const handleSignalingData = data => {
switch (data.type) {
case 'offer':
pc.setRemoteDescription(new RTCSessionDescription(data));
sendAnswer();
break;
case 'answer':
pc.setRemoteDescription(new RTCSessionDescription(data));
break;
case 'candidate':
pc.addIceCandidate(new RTCIceCandidate(data.candidate));
break;
}
};
// }
// export default PeerConnection
Can anyone please tell me why at receiver side video stream is not displaying?
Also there is a remoteStream issue in motorola device. Why is this happening?
This statement:
const PC_CONFIG = {
iceServers: [
{ url: 'stun:motac85002'},
],
};
Has two potential issues:
First, the parameter for the iceServers object should be urls, not url. (Although it wouldn't surprise me if the browsers accepted either).
Second, as I mentioned in comments to your question, the STUN address itself looks to be a local address instead of an Internet address. That might explain why you aren't seeing any srflx or UDP candidates in the SDP. And as such, that might explain connectivity issues.
So instead of the above, could you try this:
const PC_CONFIG= {iceServers: [{urls: "stun:stun.stunprotocol.org"}]};

WebRTC connectiong only send one candidate

I am new coding in Javascript.
I am creating a WebRTC connection between my iPhone and my browser.
The connection works but my code only send one candidate and I don't know if I am doing anything wrong. I would appreciate any comment or support.
Thanks
const createPeerConnection = (signaling) => {
const peerConnection = new RTCPeerConnection({
iceServers: [],
});
const offerOptions = {
offerToReceiveVideo: true, offerToReceiveAudio: true
};
peerConnection.createOffer(offerOptions);
createAndSendOffer(signaling, peerConnection);
peerConnection.onicecandidate = (iceEvent) => {
if (iceEvent && iceEvent.candidate) {
signaling.send(JSON.stringify({
type: MESSAGE_TYPE.IceCandidate,
payload: iceEvent.candidate,
}));
}
};
peerConnection.onconnectionstatechange = (state ) => {
console.log(peerConnection.connectionState);
};
return peerConnection;
};
const createAndSendOffer = async (signaling, peerConnection) => {
const offer = await peerConnection.createOffer();
await peerConnection.setLocalDescription(offer);
signaling.send(JSON.stringify({ type: MESSAGE_TYPE.SessionDescription, payload: offer }));
};

WebSocket onmessage is not triggered when onsend is called

I am developing a stateless typescript backend with WebSocket. I created a SocketMiddleware as a middleware to my redux state based on dev.io tutorial. The first socket.send() message from onopen works fine. However, I can't trigger the subsequent onmessage using SEND_MSG dispatch.
The backend shows that it receives a log but it is not received by the clients. I am sure that the connection_id is already set correctly
const socketMiddleware = () => {
let socket = null;
const onOpen = (store) => (event) => {
store.dispatch({ type: "WS_CONNECTED" });
};
const onClose = (store) => () => {
store.dispatch({ type: "WS_DISCONNECTED" });
};
const onMessage = (store) => (message) => {
console.log("message received #middleware, ", message.data);
const payload = JSON.parse(message.data);
switch (payload.action) {
case "get_connection_id":
const { connectionId } = payload;
store.dispatch({
type: "UPDATE_MY_CONNECTION_ID",
payload: { myConnectionId: connectionId },
});
break;
case "join_room_socket":
const { match_id, players, connectionIdArr } = payload;
if (match_id) {
store.dispatch({
type: "UPDATE_ROOM",
payload: {
players: players,
match_id: match_id,
connectionIdArr: connectionIdArr,
},
});
}
break;
case "broadcast_action":
const { move } = body;
store.dispatch({
type: "UPDATE_GAME_STATE",
payload: { move: move },
});
}
};
return (store) => (next) => (action) => {
switch (action.type) {
case "WS_CONNECT":
if (socket !== null) socket.close();
socket = new WebSocket(process.env.WSS_ENDPOINT);
socket.onmessage = onMessage(store);
socket.onclose = onClose(store);
socket.onopen = onOpen(store);
break;
case "WS_CONNECTED":
console.log("WebSocket client is connected");
socket.send(JSON.stringify({ action: "get_connection_id" }));
case "WS_DISCONNECTED":
console.log("WebSocket client is disconnected");
case "SEND_MSG":
console.log("sending a message", action);
socket.send(JSON.stringify({ ...action.payload }));
default:
console.log("the next action:", action);
return next(action);
}
};
};
export default socketMiddleware();
my redux store
...
const persistConfig = {
key: "root",
storage: AsyncStorage,
};
const persistedReducer = persistReducer(persistConfig, reducer);
const store = createStore(
persistedReducer,
compose(applyMiddleware(reduxThunk, wsMiddleware))
);
my backend side:
joinRoomSocket: (data) => {
const body = JSON.parse(data.body);
body.connectionIdArr.map((connectionId) => {
const endpoint = `${data.requestContext.domainName}/${data.requestContext.stage}`;
const apigwManagementApi = new AWS.ApiGatewayManagementApi({
apiVersion: "2018-11-29",
endpoint,
});
const params = {
ConnectionId: connectionId,
Data: JSON.stringify(body),
};
return apigwManagementApi.postToConnection(params).promise();
});
},
Which onmessage are you referring to server or client? If you are referring to onMessage callback of client. For this you need to send something from the server using websocket.send('text message').
For client to receive message on onmessage event, server needs to send data.
Here is the flow:
Client ws.onsend('abc') ----------------> Server ws.onmessage(data)
Server ws.onsend('abc') ----------------> Client ws.onmessage(data)```

WebRTC state checking

I would like connect 2 devices with WebRTC on localhost. All devices have no internet access. They are connected to a same local wifi.
I try this on React Native App.
In this context offline, do I need to trickle ICE candidates and addIceCandidate ? If I understund correctly, ICE candidates is for iceServer. But my case, iceServer is null (because i'm offline only, connected on same localhost wifi) :
const configuration = { iceServers: [{ urls: [] }] };
So actualty i exchange offer and answer, but after setRemoteDescription the answer, the connectionState stay on checking.
You can see my React Component :
constructor(props) {
super(props);
this.pc = new RTCPeerConnection(configuration);
}
state = initialState;
componentDidMount() {
const { pc } = this;
if (pc) {
this.setState({
peerCreated: true
});
}
this.setConnectionState();
pc.oniceconnectionstatechange = () => this.setConnectionState();
pc.onaddstream = ({ stream }) => {
if (stream) {
this.setState({
receiverVideoURL: stream.toURL()
});
}
};
pc.onnegotiationneeded = () => {
if (this.state.initiator) {
this.createOffer();
}
};
pc.onicecandidate = ({ candidate }) => {
if (candidate === null) {
const { offer } = this.state;
const field = !offer ? 'offer' : 'data';
setTimeout(() => {
alert('setTimeout started');
this.setState({
[field]: JSON.stringify(pc.localDescription)
});
}, 2000);
}
};
}
#autobind
setConnectionState() {
this.setState({
connectionState: this.pc.iceConnectionState
});
}
getUserMedia() {
MediaStreamTrack.getSources(() => {
getUserMedia(
{
audio: false,
video: true
},
this.getUserMediaSuccess,
this.getUserMediaError
);
});
}
#autobind
async getUserMediaSuccess(stream) {
const { pc } = this;
pc.addStream(stream);
await this.setState({ videoURL: stream.toURL() });
if (this.state.initiator) {
return this.createOffer();
}
return this.createAnswer();
}
getUserMediaError(error) {
console.log(error);
}
#autobind
logError(error) {
const errorArray = [...this.state.error, error];
return this.setState({
error: errorArray
});
}
/**
* Create offer
*
* #memberof HomeScreen
*/
#autobind
createOffer() {
const { pc } = this;
pc.createOffer()
.then(offer => pc.setLocalDescription(offer))
.then(() => {
this.setState({
offerCreated: true
});
})
.catch(this.logError);
}
/**
* Create anwser
*
* #memberof HomeScreen
*/
#autobind
async createAnswer() {
const { pc } = this;
const { data } = this.state;
if (data) {
const sd = new RTCSessionDescription(JSON.parse(data));
await this.setState({
offerImported: true
});
pc.setRemoteDescription(sd)
.then(() => pc.createAnswer())
.then(answer => pc.setLocalDescription(answer))
.then(() => {
this.setState({
answerCreated: true
});
})
.catch(this.logError);
}
}
#autobind
receiveAnswer() {
const { pc } = this;
const { data } = this.state;
const sd = new RTCSessionDescription(JSON.parse(data));
return pc
.setRemoteDescription(sd)
.then(() => {
this.setState({
answerImported: true
});
})
.catch(this.logError);
}
/**
* Start communication
*
* #param {boolean} [initiator=true]
* #returns
* #memberof HomeScreen
*/
#autobind
async start(initiator = true) {
if (!initiator) {
await this.setState({
initiator: false
});
}
return this.getUserMedia();
}
Anyone can help me?
No iceServers is fine on a LAN, but peers must still exchange at least one candidate: their host candidate (based on their machine's LAN IP address).
Either:
Trickle candidates using onicecandidate -> signaling -> addIceCandidate as usual, or
Out-wait the ICE process (a few seconds) before exchanging pc.localDescription.
It looks like you're attempting the latter. This approach works because...
Trickle ICE is an optimization.
The signaling (trickling) of individual ice candidates using onicecandidate, is an optimization meant to speed up negotiation. Once setLocalDescription succeeds, the browser's internal ICE Agent starts, inserting ICE candidates, as they're discovered, into the localDescription itself. Wait a few seconds to negotiate, and trickling isn't necessary at all: all ICE candidates will be in the offer and answer transmitted.
Your code
From your onicecandidate code it looks like you're already attempting to gather the localDescription after ICE completion (and you've written it to work from both ends):
pc.onicecandidate = ({ candidate }) => {
if (!candidate) {
const { offer } = this.state;
const field = !offer ? 'offer' : 'data';
this.setState({
[field]: JSON.stringify(pc.localDescription)
});
}
};
In the offerer side you've correctly commented out the equivalent code in createOffer:
pc.createOffer()
.then(offer => pc.setLocalDescription(offer))
.catch(this.logError);
// .then(() => {
// this.setState({
// offer: JSON.stringify(pc.localDescription)
// });
// });
But on the answerer side, you have not, and that's likely the problem:
createAnswer() {
const { pc } = this;
const { data } = this.state;
if (data) {
const sd = new RTCSessionDescription(JSON.parse(data));
pc.setRemoteDescription(sd)
.then(() => pc.createAnswer())
.then(answer => pc.setLocalDescription(answer))
.then(() => {
this.setState({
offer: JSON.stringify(pc.localDescription)
});
})
.catch(this.logError);
}
}
This means it sends an answer back immediately, before the answerer's ICE agent has had time to insert any candidates into the answer. This is probably why it fails.
On a side-note: Nothing appears to wait for getUserMedia to finish either, so answers likely won't contain any video either, depending on the timing of your getUserMediaSuccess function, which fails to add any tracks or streams to the connection. But assuming you're just doing data channels, this should work with my recommended fixes.

webrtc each step success but no video share

chrome version: 62.0.3202.94;
firefox version: 57.0.1;
I write a simple demo use webrtc and socket.io.
It works with pages. For example, I open one page to connect socket, and waiting for PeerConnection info from the main page(which get the local media). When I open the main, I create ice and sdp and then exchange them by socket.io to create connection.
Here is the code.
// The server side:
const express = require('express')
const app = express()
const path = require('path')
app.use(express.static(path.join(__dirname, 'public')))
app.get('/phone', function(req, res) {
res.sendfile(__dirname + '/phone.html')
})
app.get('/', function(req, res) {
res.sendfile(__dirname + '/index.html')
})
const server = require('http').createServer(app)
const io = require('socket.io')(server)
let clients = []
io.on('connection', function(socket) {
clients.push(socket)
const referer = socket.handshake.headers.referer
// socket connect from '/phone'
if (referer.match('/phone')) {
// send the ice from phone to others
socket.on('phone_ice_candidate', function(res) {
socket.broadcast.emit('pc_add_ice', {
ice: res.ice
})
})
// send the sdp from phone to others
socket.on('send_phone_sdp', function(data) {
socket.broadcast.emit('set_pc_remote_sdp', {
desc: data.desc
})
})
}
// phone add ice from web
socket.on('remote_ice_candidate', function(ice) {
socket.to(getId(clients, '/phone')).emit('send_ice_to_pc', {
ice: ice
})
})
// phone add sdp from web
socket.on('send_pc_sdp', function(data) {
// send to phone
socket.to(getId(clients, '/phone')).emit('set_phone_remote_sdp', {
desc: data
})
})
// socket disconnect and remove it from clients
socket.on('disconnect', () => {
let id = socket.id
clients.forEach((client, index) => {
if (client.id === id) {
clients.splice(index, 1)
}
})
})
})
// get the socket id to emit
function getId(sockets, exp) {
let id
sockets.forEach(socket => {
if (socket.handshake.headers.referer.match(exp)) {
id = socket.id
}
})
return id
}
server.listen(3000, function() {
console.log('port listening at 3000')
})
// --------------------------------------------- //
// web.js
var socket = io();
var server = {
// "iceServers": [{
// "url": "stun:stun.l.google.com:19302"
// }]
},
pc = new RTCPeerConnection(null),
v = document.querySelector('#video2')
// web onicecandidate
pc.onicecandidate = function(event) {
if (event.candidate) {
socket.emit('remote_ice_candidate', {
ice: event.candidate
})
}
}
// web addIceCandidate
socket.on('pc_add_ice', function(event) {
pc.addIceCandidate(new RTCIceCandidate(event.ice))
})
// didn't trigger
pc.ontrack = function(e) {
// v.srcObject = e.streams[0];
console.log(e, 'pc.ontrack')
}
// web setRemoteDescription and createAnswer
socket.on('set_pc_remote_sdp', function(e) {
pc.setRemoteDescription(e.desc).then(
function() {
console.log('remote setRemoteDescription success')
pc.createAnswer().then(function(desc) {
pc.setLocalDescription(desc).then(
function() {
socket.emit('send_pc_sdp', {
desc: desc
})
},
function(err) {
console.log(err)
}
);
})
},
function() {
console.log('pc setLocalDescription error')
}
)
})
// web iceConnectionState
pc.oniceconnectionstatechange = function() {
console.log('web oniceconnectionstatechange', pc.iceConnectionState)
// log checking -> connected
};
//---------------------------------------------//
// phone.js
var socket = io();
var server = {
// "iceServers": [{
// "url": "stun:stun.l.google.com:19302"
// }]
},
pc = new RTCPeerConnection(null),
v = document.querySelector('#video1')
// phone onicecandidate
pc.onicecandidate = function(event) {
if (event.candidate) {
socket.emit('phone_ice_candidate', {
ice: event.candidate
})
}
}
// phone addIceCandidate
socket.on('send_ice_to_pc', function(event) {
pc.addIceCandidate(new RTCIceCandidate(event.ice.ice))
})
// getUserMedia
navigator.mediaDevices.getUserMedia({
video: {
width: 400,
height: 300
},
audio: false
})
.then(function(stream) {
v.src = window.URL.createObjectURL(stream);
pc.addStream(stream);
})
.then(function() {
// create offer
pc.createOffer({
offerToReceiveVideo: 1
}).then(function(e) {
// pc setLocalDescription
pc.setLocalDescription(e).then(
function() {
socket.emit('send_phone_sdp', {
desc: e
})
},
function() {
console.log('pc setLocalDescription error')
}
)
});
})
.catch(function(err) {
console.log(err.name + ": " + err.message);
})
// phone setRemoteDescription
socket.on('set_phone_remote_sdp', function(e) {
pc.setRemoteDescription(e.desc.desc).then(
function() {
console.log('pc setRemoteDescription success')
},
function(err) {
console.log(err)
})
})
// phone iceConnectionState
pc.oniceconnectionstatechange = function() {
console.log('phone oniceconnectionstatechange', pc.iceConnectionState)
// log checking -> connected -> completed
};
When i use firefox to open it, there is an error ICE failed, add a STUN server and see about:webrtc for more details in console.
In chrome the 'phone iceConnectionState' changed checking -> connected -> completed, the 'web iceConnectionState' changed checking -> connected.
have you set autoplay in your html? I have the same issue, and it turns out I should have set autoplay in my html tag. Namely:
<video autoplay></video>
Hope this helps!
getUserMedia is an async function. You are calling createOffer before you call pc.addStream which means there is nothing to negotiate.
Make the promise callback return your pc.createOffer() after pc.addStream(stream);
PTAL at https://www.html5rocks.com/en/tutorials/webrtc/basics/#toc-signaling and compare your code to the example and see if you can figure it out.