CodenameOne WebRTC CN1LIB Serverless, Manual Signaling - webrtc

Question

I am trying to breakdown the process of the webrtc signaling using the PeerConnectionStates Demo by copying the signaling strings manually like as in here Youtube:https://www.youtube.com/watch?v=YLPRBYTeoF4&t=1594s .
Github: https://github.com/chrisuehlinger/serverless-webrtc (from 25:00)
With success I would be able to troubleshoot any signaling problems from Pubnub, Firebase or any other signaling solution I may choose.
I have buttons Start, Call, Exchange and Hang.
The 'Exchange' Button shows an Interactive Dialog which has buttons
Copy Offer , Paste Offer, Set Offer, Copy Answer, Paste Answer, Set Answer. And also 'Copy Offer 2 from text file', and 'Copy Answer 2 from text file'.
If you click on the 1st set of the Dialog buttons in that order (excluding the last 2 file selection buttons), you complete the negotiation as in the original demo.
But I want it to be between 2 devices so I put the offer string from device A into a text file and paste into the text field of device B using Copy Offer 2 from text file, then click 'set offer' to generate an answer which I copy into a text file and send to device A using Copy Answer 2 from text file.
I was not able to place a finger on what I was missing. Any help is appreciated.
public class NewClass extends Form implements AutoCloseable {
private RTCVideoElement video1, video2;
private Button startButton = new Button("Start"),
callButton = new Button("Call"),
hangupButton = new Button("Hang up"),
sendOffer = new Button("Exchange"),
sendAnswer = new Button("Send Answer"),
setOffer = new Button("Set Offer"),
setAnswer = new Button("Set Answer"),
copyOffer = new Button("copy Offer"),
pasteOffer = new Button("paste Offer"),
copyAnswer = new Button("copy Answer"),
pasteAnswer = new Button("paste Answer"),
copyOffer2 = new Button("copy Offer 2- from text file"),
setOffer2 = new Button("set Offer 2"),
copyAnswer2 = new Button("copy Answer 2- from text file"),
setAnswer2 = new Button("set Answer 2");
private TextArea toffer1 = new TextArea("", 5, 7, TextArea.ANY),
toffer2 = new TextArea("", 5, 7, TextArea.ANY),
tAnswer1 = new TextArea("", 5, 7, TextArea.ANY),
tAnswer2 = new TextArea("", 5, 7, TextArea.ANY);
{
startButton.setEnabled(true);
callButton.setEnabled(false);
hangupButton.setEnabled(false);
startButton.addActionListener(evt->start());
callButton.addActionListener(evt->call());
hangupButton.addActionListener(evt->hangup());
sendOffer.addActionListener(evt->sendOffer());
// sendAnswer.addActionListener(evt->sendAnswer());
setOffer.addActionListener(evt->setOffer());
setAnswer.addActionListener(evt->setAnswer());
copyOffer.addActionListener(evt->{ Display.getInstance().copyToClipboard(toffer1.getText()); });
pasteOffer.addActionListener(evt->{ toffer2.setText((String)Display.getInstance().getPasteDataFromClipboard()); });
copyAnswer.addActionListener(evt->{ Display.getInstance().copyToClipboard(tAnswer1.getText()); });
pasteAnswer.addActionListener(evt->{ tAnswer2.setText((String)Display.getInstance().getPasteDataFromClipboard()); });
copyOffer2.addActionListener(evt->{
if (FileChooser.isAvailable()) {
FileChooser.showOpenDialog(".xls, .csv, text/plain", e2-> {
String file = (String)e2.getSource();
if (file == null) {
// hi.add("No file was selected");
// hi.revalidate();
} else {
String extension = null;
if (file.lastIndexOf(".") > 0) {
extension = file.substring(file.lastIndexOf(".")+1);
}
if ("txt".equals(extension)) {
FileSystemStorage fs = FileSystemStorage.getInstance();
try {
InputStream fis = fs.openInputStream(file);
// hi.addComponent(new SpanLabel(Util.readToString(fis)));
toffer1.setText(Util.readToString(fis));
} catch (Exception ex) {
Log.e(ex);
}
} else {
// hi.add("Selected file "+file);
}
}
//hi.revalidate();
});
}
});
copyAnswer2.addActionListener(evt->{
if (FileChooser.isAvailable()) {
FileChooser.showOpenDialog("text/plain", e2-> {
String file = (String)e2.getSource();
if (file == null) {
// hi.add("No file was selected");
// hi.revalidate();
} else {
String extension = null;
if (file.lastIndexOf(".") > 0) {
extension = file.substring(file.lastIndexOf(".")+1);
}
if ("txt".equals(extension)) {
FileSystemStorage fs = FileSystemStorage.getInstance();
try {
InputStream fis = fs.openInputStream(file);
// hi.addComponent(new SpanLabel(Util.readToString(fis)));
tAnswer1.setText(Util.readToString(fis));
} catch (Exception ex) {
Log.e(ex);
}
} else {
// hi.add("Selected file "+file);
}
}
//hi.revalidate();
});
}
});
}
InteractionDialog dlg = new InteractionDialog(" - - -");
{dlg.setLayout(new BoxLayout(BoxLayout.Y_AXIS));
dlg.setScrollable(true);
dlg.setDisposeWhenPointerOutOfBounds(true);
dlg.add(" the offer - copy this offer ");dlg.add(copyOffer);dlg.add(copyOffer2);
dlg.addComponent(toffer1); dlg.add("===============================");
dlg.add(" paste offer here ");dlg.add(pasteOffer);
dlg.addComponent(toffer2);
dlg.addComponent(setOffer);
dlg.add("===============================");
dlg.add(" copy this answer ");dlg.add(copyAnswer);dlg.add(copyAnswer2);
dlg.addComponent(tAnswer1);
dlg.add("===============================");
dlg.add("paste answer here");dlg.add(pasteAnswer);
dlg.addComponent(tAnswer2);
dlg.addComponent(setAnswer);
}
private void sendOffer() {
//= call
Dimension pre = dlg.getContentPane().getPreferredSize();
dlg.show(Display.getInstance().getDisplayHeight()/16,0, Display.getInstance().getDisplayWidth()/16, 0);
} /*
private void sendAnswer() {
}*/
private Date startTime;
private MediaStream localStream;
private RTCPeerConnection pc1, pc2;
private SpanLabel pc1StateDiv = new SpanLabel();
private SpanLabel pc2StateDiv = new SpanLabel();
private SpanLabel pc1IceStateDiv = new SpanLabel();
private SpanLabel pc2IceStateDiv = new SpanLabel();
private SpanLabel pc1ConnStateDiv = new SpanLabel();
private SpanLabel pc2ConnStateDiv = new SpanLabel();
private static final RTCOfferOptions offerOptions = new RTCOfferOptions()
.offerToReceiveAudio(true)
.offerToReceiveVideo(true);
private RTC rtc;
public NewClass() {
super("Peer Connection Demo", new BorderLayout());
Container center = new Container(BoxLayout.y());
Container videoCnt = new Container(new BorderLayout());
String intro = "This sample was adapted from the \"PeerConnection: States Demo\" on the WebRTC web site.";
Button viewSource = new Button("View Source");
FontImage.setMaterialIcon(viewSource, FontImage.MATERIAL_LINK);
viewSource.addActionListener(evt->CN.execute("https://github.com/shannah/CN1WebRTC/blob/master/src/com/codename1/webrtc/demos/PeerConnectionStatesDemo.java"));
add(BorderLayout.NORTH, BoxLayout.encloseY(new SpanLabel(intro), viewSource));
//,toffer1,tAnswer1,setAnswer,toffer2,setOffer,tAnswer2
videoCnt.add(BorderLayout.SOUTH, FlowLayout.encloseCenter(startButton, callButton,sendOffer, hangupButton));
videoCnt.setPreferredH(CN.getDisplayHeight()/2);
center.add(videoCnt);
center.add("PC1 state:").
add(pc1StateDiv).
add("PC1 ICE state:").
add(pc1IceStateDiv).
add("PC1 connection state:").
add(pc1ConnStateDiv).
add("PC2 state:").
add(pc2StateDiv).
add("PC2 ICE state:").
add(pc2IceStateDiv).
add("PC2 connection state:").
add(pc2ConnStateDiv).
add(new SpanLabel("View the console to see logging. The MediaStream object localStream, and the RTCPeerConnection objects localPeerConnection and remotePeerConnection are in global scope, so you can inspect them in the console as well."));
center.setScrollableY(true);
add(BorderLayout.CENTER, center);
RTC.createRTC().onSuccess(r->{
rtc = r;
video1 = rtc.createVideo();
video1.setAutoplay(true);
video1.setMuted(true);
video1.applyStyle("position:fixed;width:50%;height:100%;top:0;left:0;bottom:0;");
video2 = rtc.createVideo();
video2.setAutoplay(true);
video2.applyStyle("position:fixed;width:50%;height:100%;top:0;right:0;bottom:0;");
rtc.append(video1);
rtc.append(video2);
video1.onloadedmetadata(evt->{
System.out.println("Local video videoWidth: "+video1.getVideoWidth()+"px, videoHeight: "+video1.getVideoHeight()+"px");
});
video2.onloadedmetadata(evt->{
System.out.println("Remote video size changed to "+video2.getVideoWidth()+"x"+video2.getVideoHeight());
if (startTime != null) {
long elapsedTime = System.currentTimeMillis() - startTime.getTime();
System.out.println("Setup time: "+elapsedTime+"ms");
startTime = null;
}
});
videoCnt.add(BorderLayout.CENTER, rtc.getVideoComponent());
revalidateWithAnimationSafety();
});
}
private void gotStream(MediaStream stream) {
Log.p("Received local stream");
video1.setSrcObject(stream);
localStream = stream;
stream.retain();
callButton.setEnabled(true);
}
private void start() {
Log.p("Requesting local stream");
startButton.setEnabled(false);
rtc.getUserMedia(new MediaStreamConstraints().audio(true).video(true)).onSuccess(stream->{
gotStream(stream);
}).onFail(t->{
Log.e((Throwable)t);
Dialog.show("Error", "getUserMedia() error: "+((Throwable)t).getMessage(), "OK", null);
});
}
private void call() {
callButton.setEnabled(false);
hangupButton.setEnabled(true);
Log.p("Starting call");
startTime = new Date();
MediaStreamTracks videoTracks = localStream.getVideoTracks();
MediaStreamTracks audioTracks = localStream.getAudioTracks();
if (videoTracks.size() > 0) {
Log.p("Using video device "+videoTracks.get(0).getLabel());
}
if (audioTracks.size() > 0) {
Log.p("Using audio device "+audioTracks.get(0).getLabel());
}
RTCConfiguration servers = new RTCConfiguration();
pc1 = rtc.newRTCPeerConnection(servers);
Log.p("Created local peer connection object pc1");
pc1StateDiv.setText(pc1.getSignalingState()+"");
pc1.onsignalingstatechange(evt->stateCallback1());
pc1IceStateDiv.setText(pc1.getIceConnectionState()+"");
pc1.oniceconnectionstatechange(evt->iceStateCallback1())
.onconnectionstatechange(evt->connStateCallback1())
.onicecandidate(e->onIceCandidate(pc1, e));
/* */
pc2 = rtc.newRTCPeerConnection(servers);
Log.p("Created remote peer connection object pc2");
pc2StateDiv.setText(pc2.getSignalingState()+"");
pc2.onsignalingstatechange(evt->stateCallback2());
pc2IceStateDiv.setText(pc2.getIceConnectionState()+"");
pc2.oniceconnectionstatechange(evt->iceStateCallback2())
.onconnectionstatechange(evt->connStateCallback2())
.onicecandidate(evt->{
onIceCandidate(pc2, evt);
});
pc2.ontrack(evt->gotRemoteStream(evt));
for (MediaStreamTrack track : localStream.getTracks()) {
pc1.addTrack(track, localStream);
}
Log.p("Adding local stream to peer connection");
/* */
pc1.createOffer(offerOptions).onSuccess(offer->{
gotDescription1(offer);
})
.onFail(e-> {
onCreateSessionDescriptionError((Throwable)e);
});
}
private void onCreateSessionDescriptionError(Throwable e) {
Log.p("Failed to create session description: "+e.getMessage(), Log.ERROR);
}
private void gotDescription1(RTCSessionDescription description) {
pc1.setLocalDescription(description);
Log.p("Offer from pc1:\n"+description.getSdp());
sendOffer(description.getSdp());
}
private void sendOffer(String jsDesc) {
toffer1.setText(jsDesc);
// getOffer(jsDesc);
}
private void setOffer() {
String desc = toffer2.getText();
if(desc.endsWith("\n")){getOffer(desc);}else{getOffer(desc+"\n");}
}
private void getOffer(String jsDesc) {
RTCSessionDescription description = rtc.createSessionDescription(RTCSessionDescription.RTCSdpType.Offer, jsDesc);
pc2.setRemoteDescription(description).onSuccess(ef->{
})
.onFail(e->onCreateSessionDescriptionError((Throwable)e)); ;
pc2.createAnswer()
.onSuccess(desc->{ sendAnswer(desc.getSdp()); } )
.onFail(e->onCreateSessionDescriptionError((Throwable)e));
}
private void sendAnswer(String jsDesc) {
tAnswer1.setText(jsDesc);
// getAnswer(jsDesc);
}
private void setAnswer() {
String desc = tAnswer2.getText();
//getAnswer(desc+"\n");
//getAnswer(desc);
if(desc.endsWith("\n")){getAnswer(desc);}else{getAnswer(desc+"\n");}
}
private void getAnswer(String jsDesc) {
RTCSessionDescription description = rtc.createSessionDescription(RTCSessionDescription.RTCSdpType.Answer, jsDesc);
gotDescription2(description);
}
private void gotDescription2(RTCSessionDescription description) {
pc2.setLocalDescription(description);
Log.p("Answer from pc2\n"+description.getSdp());
pc1.setRemoteDescription(description);
}
private void hangup() {
Log.p("Ending call");
pc1.close();
pc2.close();
pc1StateDiv.setText(pc1StateDiv.getText() + pc1.getSignalingState());
pc2StateDiv.setText(pc2StateDiv.getText() + pc2.getSignalingState());
pc1.release();
pc1 = null;
pc2.release();
pc2 = null;
hangupButton.setEnabled(false);
callButton.setEnabled(true);
}
private void gotRemoteStream(RTCTrackEvent e) {
if (video2.getSrcObject() != e.getStreams().get(0)) {
video2.setSrcObject(e.getStreams().get(0));
Log.p("Got remote stream");
}
}
private void stateCallback1() {
if (pc1 != null) {
Log.p("pc1 state change callback, state: "+pc1.getSignalingState());
pc1StateDiv.setText(pc1StateDiv.getText() + pc1.getSignalingState());
}
}
private void stateCallback2() {
if (pc2 != null) {
Log.p("pc2 state change callback, state: "+pc2.getSignalingState());
pc2StateDiv.setText(pc2StateDiv.getText() + pc2.getSignalingState());
}
}
private void iceStateCallback1() {
if (pc1 != null) {
Log.p("pc1 ICE connection state change callback, state: "+pc1.getIceConnectionState());
pc1IceStateDiv.setText(pc1IceStateDiv.getText() + pc1.getIceConnectionState());
}
}
private void iceStateCallback2() {
if (pc2 != null) {
Log.p("pc2 ICE connection state change callback, state: "+pc2.getIceConnectionState());
pc2IceStateDiv.setText(pc2IceStateDiv.getText() + pc2.getIceConnectionState());
}
}
private void connStateCallback1() {
if (pc1 != null) {
Log.p("pc1 connection state change callback, state: "+pc1.getConnectionState());
pc1ConnStateDiv.setText(pc1ConnStateDiv.getText() + pc1.getConnectionState());
}
}
private void connStateCallback2() {
if (pc2 != null) {
Log.p("pc2 connection state change callback, state: "+pc2.getConnectionState());
pc2ConnStateDiv.setText(pc2ConnStateDiv.getText() + pc2.getConnectionState());
}
}
private RTCPeerConnection getOtherPc(RTCPeerConnection pc) {
return pc == pc1 ? pc2 : pc1;
}
private String getName(RTCPeerConnection pc) {
return pc == pc1 ? "pc1" : "pc2";
}
private void onIceCandidate(RTCPeerConnection pc, RTCPeerConnectionIceEvent event) {
getOtherPc(pc).addIceCandidate(event.getCandidate())
.onSuccess(res->onAddIceCandidateSuccess(pc))
.onFail(e->{
Log.e((Throwable)e);
onAddIceCandidateError(pc, (Throwable)e);
}).onComplete(e->{
Log.p(getName(pc)+" ICE candidate:\n"+(event.getCandidate() != null ? event.getCandidate().getCandidate() : "(null"));
});
}
private void onAddIceCandidateSuccess(RTCPeerConnection pc) {
Log.p(getName(pc)+" addIceCandidate success");
}
private void onAddIceCandidateError(RTCPeerConnection pc, Throwable error) {
Log.p(getName(pc)+" failed to add ICE Candidate: "+error.getMessage(), Log.ERROR);
}
#Override
public void close() throws Exception {
if (rtc != null) {
rtc.close();
rtc = null;
}
if (pc1 != null) {
pc1.release();
pc1 = null;
}
if (pc2 != null) {
pc2.release();
pc2 = null;
}
}
}
Call is initiated from Phone - as device A to Simulator
On device A - My Phone
PC1 state:
nullHaveLocalOfferStable
PC1 ICE state:
New
PC1 connection state:
PC2 state:
nullHaveRemoteOffer
PC2 ICE state:
New
PC2 connection state:
On Device B - PC Simulator
PC1 state:
nullHaveLocalOfferStable
PC1 ICE state:
NewCheckingConnected
PC1 connection state:
Connecting
PC2 state:
nullHaveRemoteOfferStable
PC2 ICE state:
NewChecking
PC2 connection state:
Connecting
There is an error message:
java.lang.RuntimeException: Failed to execute 'addIceCandidate' on 'RTCPeerConnection': Error processing ICE candidate
But the final 3 lines on the console are
[EDT] 0:20:48,103 - pc2 addIceCandidate success
[EDT] 0:20:48,103 - pc2 ICE candidate:
candidate:1503035259 1 udp 7935 154.127.57.220 50066 typ relay raddr 129.205.113.2 rport 6591 generation 0 ufrag lXG+ network-cost 999
My Steps are:
from A
Start Button
Call Button - Creates Offer
Copy Offer from toffer1 (using Exchange Button)
Paste into toffer2
Set Offer Button
on B
6. Start Button
7. Call Button
8. Paste Offer of A into TextArea toffer1 and toffer2 (using Exchange Button)
9. Set Offer Button
10. Copy Answer from tAnswer1
11. Paste Answer into tAnswer2
on A
Paste Answer on tAnswer1 and tAnswer2
Set Answer Button
on B
14. set Answer Button