Related
I have a NextJS (Typescript) setup with Express . I'm using Moralis, Wagmi and Web3Auth for authentication. The Metamask authentication works, but when I try to setup Web3 Authentication by importing Web3AuthConnector it throws me an error.
My code is like here in the documentation but looks to be an error about CommonJS and ESM which I can't handle. This is the error:
error - Error [ERR_REQUIRE_ESM]: require() of ES Module
C:...\node_modules#web3auth\web3auth-wagmi-connector\node_modules#wagmi\core\dist\index.js
from
C:...\node_modules#web3auth\web3auth-wagmi-connector\dist\web3authWagmiConnector.cjs.js
not supported. Instead change the require of index.js in
C:...\node_modules#web3auth\web3auth-wagmi-connector\dist\web3authWagmiConnector.cjs.js
to a dynamic import() which is available in all CommonJS modules.
I tried to set in package.json "type:module", and in tsconfig: module:"ES6" or "ESNext" but got the same error.
I got the same problem than you and I resolved it by create my own Connector.
I didn't found any other way to do it.
import {
Chain,
Connector,
ConnectorData,
normalizeChainId,
UserRejectedRequestError,
} from "#wagmi/core";
import {
ADAPTER_EVENTS,
ADAPTER_STATUS,
CHAIN_NAMESPACES,
CustomChainConfig,
getChainConfig,
SafeEventEmitterProvider,
WALLET_ADAPTER_TYPE,
WALLET_ADAPTERS,
BaseAdapterConfig,
CONNECTED_EVENT_DATA,
IAdapter,
storageAvailable,
ADAPTER_CATEGORY,
} from "#web3auth/base";
import { Web3AuthCore } from "#web3auth/core";
import { MetamaskAdapter } from "#web3auth/metamask-adapter";
import { OpenloginAdapter } from "#web3auth/openlogin-adapter";
import { TorusWalletAdapter } from "#web3auth/torus-evm-adapter";
import LoginModal, {
getAdapterSocialLogins,
LOGIN_MODAL_EVENTS,
OPENLOGIN_PROVIDERS,
} from "#web3auth/ui";
import {
AdaptersModalConfig,
defaultEvmDappModalConfig,
ModalConfig,
} from "#web3auth/web3auth";
import { Options } from "#web3auth/web3auth-wagmi-connector/dist/types/lib/interfaces";
import { ethers, Signer } from "ethers";
import { getAddress } from "ethers/lib/utils";
import { WalletConnectV1Adapter } from "#web3auth/wallet-connect-v1-adapter";
import QRCodeModal from "#walletconnect/qrcode-modal";
import { ConnectorEvents, defaultChains } from "wagmi";
import EventEmitter from "events";
import { Provider } from "react";
const IS_SERVER = typeof window === "undefined";
const ADAPTER_CACHE_KEY = "Web3Auth-cachedAdapter";
export class Web3AuthConnectorLocal extends Connector {
ready = !IS_SERVER;
readonly id = "web3Auth";
readonly name = "web3Auth";
provider: SafeEventEmitterProvider;
web3AuthInstance?: Web3AuthCore;
isModalOpen = false;
web3AuthOptions: Options;
private loginModal: LoginModal;
private socialLoginAdapter: OpenloginAdapter;
private torusWalletAdapter: TorusWalletAdapter;
private metamaskAdapter: MetamaskAdapter;
private walletConnectV1Adapter: WalletConnectV1Adapter;
private adapters: Record<string, IAdapter<unknown>> = {};
private modalConfig: AdaptersModalConfig = defaultEvmDappModalConfig;
private storage: "sessionStorage" | "localStorage" = "localStorage";
constructor(config: { chains?: Chain[]; options: Options }) {
super(config);
this.web3AuthOptions = config.options;
const chainId = config.options.chainId
? parseInt(config.options.chainId, 16)
: 1;
const chainConfig = this.chains.filter((x) => x.id === chainId);
const defaultChainConfig = getChainConfig(
CHAIN_NAMESPACES.EIP155,
config.options.chainId || "0x1"
);
let finalChainConfig: CustomChainConfig = {
chainNamespace: CHAIN_NAMESPACES.EIP155,
...defaultChainConfig,
};
if (chainConfig.length > 0) {
finalChainConfig = {
...finalChainConfig,
chainNamespace: CHAIN_NAMESPACES.EIP155,
chainId: config.options.chainId || "0x1",
rpcTarget: chainConfig[0].rpcUrls.default,
displayName: chainConfig[0].name,
tickerName: chainConfig[0].nativeCurrency?.name,
ticker: chainConfig[0].nativeCurrency?.symbol,
blockExplorer: chainConfig[0]?.blockExplorers.default?.url,
};
}
this.web3AuthInstance = new Web3AuthCore({
clientId: config.options.clientId,
chainConfig: {
chainNamespace: CHAIN_NAMESPACES.EIP155,
chainId: "0x1",
rpcTarget: "https://rpc.ankr.com/eth", // This is the public RPC we have added, please pass on your own endpoint while creating an app
},
});
this.socialLoginAdapter = new OpenloginAdapter({
adapterSettings: {
...config.options,
},
chainConfig: {
chainId: "0x1",
chainNamespace: CHAIN_NAMESPACES.EIP155,
rpcTarget: "https://rpc.ankr.com/eth",
displayName: "mainnet",
blockExplorer: "https://etherscan.io/",
ticker: "ETH",
tickerName: "Ethereum",
},
});
this.torusWalletAdapter = new TorusWalletAdapter({
adapterSettings: {
buttonPosition: "bottom-left",
},
loginSettings: {
verifier: "google",
},
initParams: {
buildEnv: "testing",
},
chainConfig: {
chainNamespace: CHAIN_NAMESPACES.EIP155,
chainId: "0x1",
rpcTarget: "https://rpc.ankr.com/eth", // This is the mainnet RPC we have added, please pass on your own endpoint while creating an app
displayName: "Ethereum Mainnet",
blockExplorer: "https://etherscan.io/",
ticker: "ETH",
tickerName: "Ethereum",
},
});
this.metamaskAdapter = new MetamaskAdapter({
clientId: config.options.clientId,
});
this.walletConnectV1Adapter = new WalletConnectV1Adapter({
adapterSettings: {
bridge: "https://bridge.walletconnect.org",
qrcodeModal: QRCodeModal,
},
clientId: config.options.clientId,
});
this.web3AuthInstance.configureAdapter(this.socialLoginAdapter);
this.web3AuthInstance.configureAdapter(this.torusWalletAdapter);
this.web3AuthInstance.configureAdapter(this.metamaskAdapter);
this.web3AuthInstance.configureAdapter(this.walletConnectV1Adapter);
this.adapters[this.socialLoginAdapter.name] = this.socialLoginAdapter;
this.adapters[this.torusWalletAdapter.name] = this.torusWalletAdapter;
this.adapters[this.metamaskAdapter.name] = this.metamaskAdapter;
this.adapters[this.walletConnectV1Adapter.name] =
this.walletConnectV1Adapter;
this.loginModal = new LoginModal({
theme: this.options.uiConfig?.theme,
appLogo: this.options.uiConfig?.appLogo || "",
version: "",
adapterListener: this.web3AuthInstance,
displayErrorsOnModal: this.options.displayErrorsOnModal,
});
// this.loginModal.initExternalWalletContainer();
this.subscribeToLoginModalEvents();
}
async connect(): Promise<Required<ConnectorData>> {
this.web3AuthInstance.init();
const adapterEventsPromise = this.subscribeToAdpaterConnectionEvents()
await this.init();
this.loginModal.open();
const elem = document.getElementById("w3a-container");
elem.style.zIndex = "10000000000";
return (await adapterEventsPromise) as Required<ConnectorData>;
}
async subscribeToAdpaterConnectionEvents(): Promise<Required<ConnectorData>> {
return new Promise((resolve, reject) => {
this.web3AuthInstance.once(ADAPTER_EVENTS.CONNECTED, async () => {
console.log(
"Received event connected: ",
this.web3AuthInstance.connectedAdapterName
);
console.log("Requesting Signer");
const signer = await this.getSigner();
const account = await signer.getAddress();
const provider = await this.getProvider();
if (provider.on) {
provider.on("accountsChanged", this.onAccountsChanged.bind(this));
provider.on("chainChanged", this.onChainChanged.bind(this));
provider.on("disconnect", this.onDisconnect.bind(this));
}
return resolve({
account,
chain: {
id: 0,
unsupported: false,
},
provider,
});
});
this.web3AuthInstance.once(ADAPTER_EVENTS.ERRORED, (err: unknown) => {
console.log("error while connecting", err);
return reject(err);
});
});
}
async init(): Promise<void> {
console.log("What is this type: ", typeof this)
console.log("What is this instance: ", this instanceof Web3AuthConnectorLocal)
try {
await this.loginModal.initModal();
const allAdapters = [
...new Set([
...Object.keys(this.modalConfig.adapters || {}),
...Object.keys(this.adapters),
]),
];
const adapterNames = [
"torus-evm",
"metamask",
"openlogin",
"wallet-connect-v1",
];
const hasInAppWallets = true;
// Now, initialize the adapters.
const initPromises = adapterNames.map(async (adapterName) => {
if (!adapterName) return;
try {
const adapter = this.adapters[adapterName];
console.log("Adapter Found: ", adapterName);
console.log("Cached Adapter: ", this.web3AuthInstance.cachedAdapter);
// only initialize a external adapter here if it is a cached adapter.
if (
this.web3AuthInstance.cachedAdapter !== adapterName &&
adapter.type === ADAPTER_CATEGORY.EXTERNAL
) {
console.log(adapterName, " Adapter is not External");
return;
}
// in-app wallets or cached wallet (being connected or already connected) are initialized first.
// if adapter is configured thn only initialize in app or cached adapter.
// external wallets are initialized on INIT_EXTERNAL_WALLET event.
this.subscribeToAdapterEvents(adapter);
if (adapter.status === ADAPTER_STATUS.NOT_READY) {
await adapter.init({
autoConnect: this.web3AuthInstance.cachedAdapter === adapterName,
});
console.log(
"Initializing In Wallet: COMPLETED",
adapter,
adapter.status
);
}
// note: not adding cachedWallet to modal if it is external wallet.
// adding it later if no in-app wallets are available.
if (adapter.type === ADAPTER_CATEGORY.IN_APP) {
this.loginModal.addSocialLogins(
WALLET_ADAPTERS.OPENLOGIN,
getAdapterSocialLogins(
WALLET_ADAPTERS.OPENLOGIN,
this.socialLoginAdapter,
this.options.uiConfig?.loginMethodConfig
),
this.options.uiConfig?.loginMethodsOrder || OPENLOGIN_PROVIDERS
);
}
} catch (error) {
console.log(error, "error while initializing adapter");
}
});
this.web3AuthInstance.status = ADAPTER_STATUS.READY;
await Promise.all(initPromises);
const hasExternalWallets = allAdapters.some((adapterName) => {
return (
this.adapters[adapterName]?.type === ADAPTER_CATEGORY.EXTERNAL &&
this.modalConfig.adapters?.[adapterName].showOnModal
);
});
console.log("Has External Wallets: ", hasExternalWallets);
if (hasExternalWallets) {
this.loginModal.initExternalWalletContainer();
}
if (!hasInAppWallets && hasExternalWallets) {
await this.initExternalWalletAdapters(false, {
showExternalWalletsOnly: true,
});
}
} catch (error) {
console.log("error while connecting", error);
throw new UserRejectedRequestError("Something went wrong");
}
}
async getAccount(): Promise<string> {
const provider = new ethers.providers.Web3Provider(
await this.getProvider()
);
const signer = provider.getSigner();
const account = await signer.getAddress();
return account;
}
async getProvider() {
if (this.provider) {
return this.provider;
}
this.provider = this.web3AuthInstance.provider;
return this.provider;
}
async getSigner(): Promise<Signer> {
console.log("Getting Signer");
const provider = new ethers.providers.Web3Provider(
await this.getProvider()
);
const signer = provider.getSigner();
return signer;
}
async isAuthorized() {
try {
const account = await this.getAccount();
return !!(account && this.provider);
} catch {
return false;
}
}
async getChainId(): Promise<number> {
try {
const networkOptions = this.socialLoginAdapter.chainConfigProxy;
if (typeof networkOptions === "object") {
const chainID = networkOptions.chainId;
if (chainID) {
return normalizeChainId(chainID);
}
}
throw new Error("Chain ID is not defined");
} catch (error) {
console.log("error", error);
throw error;
}
}
async disconnect(): Promise<void> {
await this.web3AuthInstance.logout();
this.provider = null;
}
protected onAccountsChanged(accounts: string[]): void {
if (accounts.length === 0) this.emit("disconnect");
else this.emit("change", { account: getAddress(accounts[0]) });
}
protected onChainChanged(chainId: string | number): void {
const id = normalizeChainId(chainId);
const unsupported = this.isChainUnsupported(id);
this.emit("change", { chain: { id, unsupported } });
}
protected onDisconnect(): void {
this.emit("disconnect");
}
private subscribeToLoginModalEvents(): void {
this.loginModal.on(
LOGIN_MODAL_EVENTS.LOGIN,
async (params: {
adapter: WALLET_ADAPTER_TYPE;
loginParams: unknown;
}) => {
try {
console.log("Wallet Adapters: ", +params.adapter);
await this.web3AuthInstance.connectTo<unknown>(
params.adapter,
params.loginParams
);
} catch (error) {
console.log(
`Error while connecting to adapter: ${params.adapter}`,
error
);
}
}
);
this.loginModal.on(
LOGIN_MODAL_EVENTS.INIT_EXTERNAL_WALLETS,
async (params: { externalWalletsInitialized: boolean }) => {
await this.initExternalWalletAdapters(
params.externalWalletsInitialized
);
}
);
this.loginModal.on(LOGIN_MODAL_EVENTS.DISCONNECT, async () => {
try {
await this.disconnect();
} catch (error) {
console.log(`Error while disconnecting`, error);
}
});
}
private async initExternalWalletAdapters(
externalWalletsInitialized: boolean,
options?: { showExternalWalletsOnly: boolean }
): Promise<void> {
if (externalWalletsInitialized) return;
const adaptersConfig: Record<string, BaseAdapterConfig> = {};
const adaptersData: Record<string, unknown> = {};
const adapterPromises = Object.keys(this.adapters).map(
async (adapterName) => {
try {
const adapter = this.adapters[adapterName];
if (adapter?.type === ADAPTER_CATEGORY.EXTERNAL) {
console.log("init external wallet", adapterName);
this.subscribeToAdapterEvents(adapter);
// we are not initializing cached adapter here as it is already being initialized in initModal before.
if (this.web3AuthInstance.cachedAdapter === adapterName) {
return;
}
if (adapter.status === ADAPTER_STATUS.NOT_READY) {
console.log("Adapter not Ready: " + adapterName);
return await Promise.race([
adapter
.init({
autoConnect:
this.web3AuthInstance.cachedAdapter === adapterName,
})
.then(() => {
adaptersConfig[adapterName] = (
defaultEvmDappModalConfig.adapters as Record<
WALLET_ADAPTER_TYPE,
ModalConfig
>
)[adapterName];
adaptersData[adapterName] = adapter.adapterData || {};
console.log("Adapter Init: ", adapterName);
return adapterName;
}),
new Promise((resolve) => {
setTimeout(() => {
return resolve(null);
}, 5000);
}),
]);
} else {
console.log("Adapter Ready: " + adapterName);
return adapterName;
}
}
} catch (error) {
console.log(error, "error while initializing adapter");
}
}
);
const adapterInitResults = await Promise.all(adapterPromises);
console.log("Adapter Init Results: ", adapterInitResults);
const finalAdaptersConfig: Record<WALLET_ADAPTER_TYPE, BaseAdapterConfig> =
{};
adapterInitResults.forEach((result: string | undefined) => {
if (result) {
finalAdaptersConfig[result] = adaptersConfig[result];
}
});
this.loginModal.addWalletLogins(finalAdaptersConfig, {
showExternalWalletsOnly: !!options?.showExternalWalletsOnly,
});
}
private subscribeToAdapterEvents(walletAdapter: IAdapter<unknown>): void {
console.log("Running adapter events");
walletAdapter.on(ADAPTER_EVENTS.CONNECTED, (data: CONNECTED_EVENT_DATA) => {
let status = ADAPTER_STATUS.CONNECTED;
this.web3AuthInstance.connectedAdapterName = data.adapter;
this.cacheWallet(data.adapter);
console.log(
"connected",
status,
this.web3AuthInstance.connectedAdapterName
);
this.web3AuthInstance.emit(ADAPTER_EVENTS.CONNECTED, {
...data,
} as CONNECTED_EVENT_DATA);
});
walletAdapter.on(ADAPTER_EVENTS.DISCONNECTED, async (data) => {
// get back to ready state for rehydrating.
let status = ADAPTER_STATUS.READY;
if (storageAvailable(this.storage)) {
const cachedAdapter = window[this.storage].getItem(ADAPTER_CACHE_KEY);
if (this.web3AuthInstance.connectedAdapterName === cachedAdapter) {
this.web3AuthInstance.clearCache();
}
}
console.log(
"disconnected",
status,
this.web3AuthInstance.connectedAdapterName
);
this.web3AuthInstance.connectedAdapterName = null;
this.web3AuthInstance.emit(ADAPTER_EVENTS.DISCONNECTED, data);
});
walletAdapter.on(ADAPTER_EVENTS.CONNECTING, (data) => {
let status = ADAPTER_STATUS.CONNECTING;
this.web3AuthInstance.emit(ADAPTER_EVENTS.CONNECTING, data);
console.log(
"connecting",
status,
this.web3AuthInstance.connectedAdapterName
);
});
walletAdapter.on(ADAPTER_EVENTS.ERRORED, (data) => {
let status = ADAPTER_STATUS.ERRORED;
this.web3AuthInstance.clearCache();
this.web3AuthInstance.emit(ADAPTER_EVENTS.ERRORED, data);
console.log(
"errored",
status,
this.web3AuthInstance.connectedAdapterName
);
});
walletAdapter.on(ADAPTER_EVENTS.ADAPTER_DATA_UPDATED, (data) => {
console.log("adapter data updated", data);
this.web3AuthInstance.emit(ADAPTER_EVENTS.ADAPTER_DATA_UPDATED, data);
});
}
private cacheWallet(walletName: string) {
if (!storageAvailable(this.storage)) return;
window[this.storage].setItem(ADAPTER_CACHE_KEY, walletName);
this.web3AuthInstance.cachedAdapter = walletName;
}
}
I got the code from https://forum.moralis.io/t/solved-web3auth-wagmi-connector-doesnt-show-web3-wallets-as-a-login-option/20110/4.
Let me know if it's working for you ?
( You should add all the missing packages with yarn add )
i am trying to create a room video call something like Whatsapp with firebase as signalisation server for two users it works perfect as expected but when a third user join everything stop working
if the user A the first who join and the user B the second until the user C join everything work fine but when C join user A freeze and lost user B connection and B share screen with C but also sometimes freezing
here is my code
onst joinRoom = async () => {
const roomRef = await firestore().collection('meets').doc(params.callId);
roomRef
.collection('users')
.add({name, type: 'join'})
.then(() => {
roomRef
.collection('users')
.get()
.then(async querySnapshot => {
if (querySnapshot.empty) {
console.log('EMPTY');
} else {
querySnapshot.forEach(async snap => {
if (snap.data().name !== name && snap.data().type === 'join') {
// data.push(snap.data().name);
await creatOffer(snap.data().name);
}
// if (data.length > 0) {
// Promise.all(data).then(async user => {
// return await creatOffer(user);
// });
// }
});
}
});
});
// // listen on any new offers
roomRef.collection('offers').onSnapshot(data => {
data.docChanges().forEach(async change => {
if (change.type === 'added') {
// console.log('changes', change.doc.data());
if (change.doc.data().to === name) {
await createAnswer(change.doc.data().from, change.doc.data().offer);
}
}
});
});
//listen to answers
roomRef.collection('answers').onSnapshot(async snapshot => {
snapshot.docChanges().forEach(async change => {
if (change.type === 'added') {
const pc = pcPeers[change.doc.data().from];
if (change.doc.data().to === name) {
const rtcSessionDescription = new RTCSessionDescription(
change.doc.data().answer,
);
if (pc && rtcSessionDescription) {
await pc.setRemoteDescription(rtcSessionDescription);
}
}
}
});
});
//listen to candidate change
roomRef.collection('candidates').onSnapshot(async snapshot => {
snapshot.docChanges().forEach(async change => {
// console.log('answers', change.doc.data());
if (change.type === 'added') {
console.log('added', Platform.OS);
if (change.doc.data().to === name) {
const pc = pcPeers[change.doc.data().from];
// console.log(pc);
if (pc) {
await pc.addIceCandidate(
new RTCIceCandidate(change.doc.data().candidate),
);
}
}
}
});
});
};
and function to create offer
const creatOffer = async to => {
try {
const {roomRef, localPC} = await initializePeer(to);
const offer = await localPC.createOffer();
// console.log('offer', offer);
if (offer) {
await localPC.setLocalDescription(offer);
await roomRef.collection('offers').add({from: name, to, offer});
}
pcPeers = {...pcPeers, [to]: localPC};
} catch (e) {
console.log(e);
}
};
function to create answer
const createAnswer = async (from, offer) => {
try {
const {localPC, roomRef} = await initializePeer(from);
await localPC.setRemoteDescription(new RTCSessionDescription(offer));
const answer = await localPC.createAnswer();
await localPC.setLocalDescription(answer);
// await checkIfAnswerAlreadyCreated(from, name);
await roomRef.collection('answers').add({from: name, to: from, answer});
pcPeers = {...pcPeers, [from]: localPC};
} catch (e) {
console.log(e);
}
};
and here is how i initialise a peer
setLocalStream(initialStream);
const roomRef = await firestore().collection('meets').doc(params.callId);
const collection = roomRef.collection('candidates');
localPC.onicecandidate = async e => {
if (!e.candidate) {
return;
}
// console.log('canditates', Platform.OS);
const state = localPC.iceGatheringState;
if (state !== 'complete') {
await collection.add({
from: name,
candidate: e.candidate.toJSON(),
to,
date: new Date(),
});
} else {
Alert.alert('tes');
}
// InCallManager.setForceSpeakerphoneOn(true);
// InCallManager.setSpeakerphoneOn(true);
};
localPC.onsignalingstatechange = async event => {
// when the signal state become stable record the data and stop ringback
if (event.target.signalingState === 'stable') {
if (Platform.OS === 'ios') {
localStream?.getVideoTracks().forEach(track => {
//For ios to trigger the camera on
track._switchCamera();
track._switchCamera();
});
}
}
};
localPC.onaddstream = e => {
if (e?.stream) {
// console.log(
// `RemotePC received the stream call ${Platform.OS}_${Platform.Version}`,
// e?.stream,
// );
console.log(Platform.OS, ' ', Platform.Version);
if (remoteStream === null) {
// Alert.alert('stream 1');
setRemoteStream(e?.stream);
} else {
// Alert.alert('stream 2');
setRemoteStream_(e?.stream);
}
}
};
return {localPC, roomRef};
};
can anyone help me with some hints where is tthe problem i’ve created for my self here and thank you
What I'm trying to do is get real time transcription for video recorded in the browser with webRTC. Use case is basically subtitles in real time like google hangouts has.
So I have a WebRTC program running in the browser. It sends webm objects back to the server. They are linear32 audio encodings. Google speech to text only accepts linear16 or Flac files.
Is there a way to convert linear32 to linear16 in real time?
Otherwise has anyone been able to hook up webRTC with Google speech to get real time transcriptions working?
Any advice on where to look to solve this problem would be great
Check out this repository it might help you - https://github.com/muaz-khan/Translator
Translator.js is a JavaScript library built top on Google Speech-Recognition & Translation API to transcript and translate voice and text. It supports many locales and brings globalization in WebRTC!
I had the same problem and failed with webRTC. I recommend you use the Web Audio Api instead if you are just interested in transcribing the audio from the video.
Here is how I did it with a nodejs sever and react client app. It is uploaded to github here
You need an audio worklet script. (Put it in the public folder because that is where the API expects to find it)
recorderWorkletProcessor.js (saved in public/src/worklets/recorderWorkletProcessor.js)
/**
An in-place replacement for ScriptProcessorNode using AudioWorklet
*/
class RecorderProcessor extends AudioWorkletProcessor {
// 0. Determine the buffer size (this is the same as the 1st argument of ScriptProcessor)
bufferSize = 2048;
// 1. Track the current buffer fill level
_bytesWritten = 0;
// 2. Create a buffer of fixed size
_buffer = new Float32Array(this.bufferSize);
constructor() {
super();
this.initBuffer();
}
initBuffer() {
this._bytesWritten = 0;
}
isBufferEmpty() {
return this._bytesWritten === 0;
}
isBufferFull() {
return this._bytesWritten === this.bufferSize;
}
/**
* #param {Float32Array[][]} inputs
* #returns {boolean}
*/
process(inputs) {
// Grabbing the 1st channel similar to ScriptProcessorNode
this.append(inputs[0][0]);
return true;
}
/**
*
* #param {Float32Array} channelData
*/
append(channelData) {
if (this.isBufferFull()) {
this.flush();
}
if (!channelData) return;
for (let i = 0; i < channelData.length; i++) {
this._buffer[this._bytesWritten++] = channelData[i];
}
}
flush() {
// trim the buffer if ended prematurely
const buffer = this._bytesWritten < this.bufferSize ? this._buffer.slice(0, this._bytesWritten) : this._buffer;
const result = this.downsampleBuffer(buffer, 44100, 16000);
this.port.postMessage(result);
this.initBuffer();
}
downsampleBuffer(buffer, sampleRate, outSampleRate) {
if (outSampleRate == sampleRate) {
return buffer;
}
if (outSampleRate > sampleRate) {
throw new Error("downsampling rate show be smaller than original sample rate");
}
var sampleRateRatio = sampleRate / outSampleRate;
var newLength = Math.round(buffer.length / sampleRateRatio);
var result = new Int16Array(newLength);
var offsetResult = 0;
var offsetBuffer = 0;
while (offsetResult < result.length) {
var nextOffsetBuffer = Math.round((offsetResult + 1) * sampleRateRatio);
var accum = 0,
count = 0;
for (var i = offsetBuffer; i < nextOffsetBuffer && i < buffer.length; i++) {
accum += buffer[i];
count++;
}
result[offsetResult] = Math.min(1, accum / count) * 0x7fff;
offsetResult++;
offsetBuffer = nextOffsetBuffer;
}
return result.buffer;
}
}
registerProcessor("recorder.worklet", RecorderProcessor);
Install Socket.io-client on front end
npm i socket.io-client
React component code
/* eslint-disable react-hooks/exhaustive-deps */
import { default as React, useEffect, useState, useRef } from "react";
import { Button } from "react-bootstrap";
import Container from "react-bootstrap/Container";
import * as io from "socket.io-client";
const sampleRate = 16000;
const getMediaStream = () =>
navigator.mediaDevices.getUserMedia({
audio: {
deviceId: "default",
sampleRate: sampleRate,
sampleSize: 16,
channelCount: 1,
},
video: false,
});
interface WordRecognized {
final: boolean;
text: string;
}
const AudioToText: React.FC = () => {
const [connection, setConnection] = useState<io.Socket>();
const [currentRecognition, setCurrentRecognition] = useState<string>();
const [recognitionHistory, setRecognitionHistory] = useState<string[]>([]);
const [isRecording, setIsRecording] = useState<boolean>(false);
const [recorder, setRecorder] = useState<any>();
const processorRef = useRef<any>();
const audioContextRef = useRef<any>();
const audioInputRef = useRef<any>();
const speechRecognized = (data: WordRecognized) => {
if (data.final) {
setCurrentRecognition("...");
setRecognitionHistory((old) => [data.text, ...old]);
} else setCurrentRecognition(data.text + "...");
};
const connect = () => {
connection?.disconnect();
const socket = io.connect("http://localhost:8081");
socket.on("connect", () => {
console.log("connected", socket.id);
setConnection(socket);
});
socket.emit("send_message", "hello world");
socket.emit("startGoogleCloudStream");
socket.on("receive_message", (data) => {
console.log("received message", data);
});
socket.on("receive_audio_text", (data) => {
speechRecognized(data);
console.log("received audio text", data);
});
socket.on("disconnect", () => {
console.log("disconnected", socket.id);
});
};
const disconnect = () => {
if (!connection) return;
connection?.emit("endGoogleCloudStream");
connection?.disconnect();
processorRef.current?.disconnect();
audioInputRef.current?.disconnect();
audioContextRef.current?.close();
setConnection(undefined);
setRecorder(undefined);
setIsRecording(false);
};
useEffect(() => {
(async () => {
if (connection) {
if (isRecording) {
return;
}
const stream = await getMediaStream();
audioContextRef.current = new window.AudioContext();
await audioContextRef.current.audioWorklet.addModule(
"/src/worklets/recorderWorkletProcessor.js"
);
audioContextRef.current.resume();
audioInputRef.current =
audioContextRef.current.createMediaStreamSource(stream);
processorRef.current = new AudioWorkletNode(
audioContextRef.current,
"recorder.worklet"
);
processorRef.current.connect(audioContextRef.current.destination);
audioContextRef.current.resume();
audioInputRef.current.connect(processorRef.current);
processorRef.current.port.onmessage = (event: any) => {
const audioData = event.data;
connection.emit("send_audio_data", { audio: audioData });
};
setIsRecording(true);
} else {
console.error("No connection");
}
})();
return () => {
if (isRecording) {
processorRef.current?.disconnect();
audioInputRef.current?.disconnect();
if (audioContextRef.current?.state !== "closed") {
audioContextRef.current?.close();
}
}
};
}, [connection, isRecording, recorder]);
return (
<React.Fragment>
<Container className="py-5 text-center">
<Container fluid className="py-5 bg-primary text-light text-center ">
<Container>
<Button
className={isRecording ? "btn-danger" : "btn-outline-light"}
onClick={connect}
disabled={isRecording}
>
Start
</Button>
<Button
className="btn-outline-light"
onClick={disconnect}
disabled={!isRecording}
>
Stop
</Button>
</Container>
</Container>
<Container className="py-5 text-center">
{recognitionHistory.map((tx, idx) => (
<p key={idx}>{tx}</p>
))}
<p>{currentRecognition}</p>
</Container>
</Container>
</React.Fragment>
);
};
export default AudioToText;
server.js
const express = require("express");
const speech = require("#google-cloud/speech");
//use logger
const logger = require("morgan");
//use body parser
const bodyParser = require("body-parser");
//use corrs
const cors = require("cors");
const http = require("http");
const { Server } = require("socket.io");
const app = express();
app.use(cors());
app.use(logger("dev"));
app.use(bodyParser.json());
const server = http.createServer(app);
const io = new Server(server, {
cors: {
origin: "http://localhost:3000",
methods: ["GET", "POST"],
},
});
//TODO: run in terminal first to setup credentials export GOOGLE_APPLICATION_CREDENTIALS="./speech-to-text-key.json"
const speechClient = new speech.SpeechClient();
io.on("connection", (socket) => {
let recognizeStream = null;
console.log("** a user connected - " + socket.id + " **\n");
socket.on("disconnect", () => {
console.log("** user disconnected ** \n");
});
socket.on("send_message", (message) => {
console.log("message: " + message);
setTimeout(() => {
io.emit("receive_message", "got this message" + message);
}, 1000);
});
socket.on("startGoogleCloudStream", function (data) {
startRecognitionStream(this, data);
});
socket.on("endGoogleCloudStream", function () {
console.log("** ending google cloud stream **\n");
stopRecognitionStream();
});
socket.on("send_audio_data", async (audioData) => {
io.emit("receive_message", "Got audio data");
if (recognizeStream !== null) {
try {
recognizeStream.write(audioData.audio);
} catch (err) {
console.log("Error calling google api " + err);
}
} else {
console.log("RecognizeStream is null");
}
});
function startRecognitionStream(client) {
console.log("* StartRecognitionStream\n");
try {
recognizeStream = speechClient
.streamingRecognize(request)
.on("error", console.error)
.on("data", (data) => {
const result = data.results[0];
const isFinal = result.isFinal;
const transcription = data.results
.map((result) => result.alternatives[0].transcript)
.join("\n");
console.log(`Transcription: `, transcription);
client.emit("receive_audio_text", {
text: transcription,
final: isFinal,
});
});
} catch (err) {
console.error("Error streaming google api " + err);
}
}
function stopRecognitionStream() {
if (recognizeStream) {
console.log("* StopRecognitionStream \n");
recognizeStream.end();
}
recognizeStream = null;
}
});
server.listen(8081, () => {
console.log("WebSocket server listening on port 8081.");
});
// =========================== GOOGLE CLOUD SETTINGS ================================ //
// The encoding of the audio file, e.g. 'LINEAR16'
// The sample rate of the audio file in hertz, e.g. 16000
// The BCP-47 language code to use, e.g. 'en-US'
const encoding = "LINEAR16";
const sampleRateHertz = 16000;
const languageCode = "en-US"; //en-US
const alternativeLanguageCodes = ["en-US", "ko-KR"];
const request = {
config: {
encoding: encoding,
sampleRateHertz: sampleRateHertz,
languageCode: languageCode,
//alternativeLanguageCodes: alternativeLanguageCodes,
enableWordTimeOffsets: true,
enableAutomaticPunctuation: true,
enableWordConfidence: true,
enableSpeakerDiarization: true,
diarizationSpeakerCount: 2,
model: "video",
//model: "command_and_search",
useEnhanced: true,
speechContexts: [
{
phrases: ["hello", "안녕하세요"],
},
],
},
interimResults: true,
};
I am developing a mobile app, which uses React-Native, Redux and React-Native-Contacts.
In an action creator, I am trying to call a function (from React-Native-Contacts), which has callbacks (rather than promises). Following is my code.
I want the action "acContactImport" to wait for the helper function "fContactImport" to finish before proceeding to the "dispatch" statement. It does not wait for it. How can I make it wait for it?
// Action Creator:
import Contacts from 'react-native-contacts';
import { PermissionsAndroid } from 'react-native';
export const acContactImport = (userID) => {
return async (dispatch) => {
let contactList = [];
try {
// The following statement should be executed asynchronously.
// However, it is not. How can I make it asynchronous?
contactList = await fContactImport();
dispatch({
type: CONTACT_IMPORT,
payload: { contactList: contactList },
});
}
catch (err) {
console.log("acContactImport - catch: ", err);
}
};
};
// Helper Function 1
export const fContactImport = async () => {
let contactList = [];
if (Platform.OS === "android") {
PermissionsAndroid.request(PermissionsAndroid.PERMISSIONS.READ_CONTACTS, {
title: "Contacts",
message: "This app would like to view your contacts."
})
.then(() => {
contactList = _loadContacts();
});
} else {
contactList = _loadContacts();
}
}
// Helper Function 2
export const _loadContacts = () => {
Contacts.getAll((err, data2) => {
if (err) {
return [];
}
else {
let candidates = [];
for (let i = 0; i < data2.length; i++) {
let candidateObject = { name: candidateRecord.givenName + " " + candidateRecord.middleName + " " + candidateRecord.familyName };
candidates.push(candidateObject);
}
return contactList;
}
});
}
Apparently, the problem is NOT in the action creator, but in the helper functions, which do not support the new style of promises (async/wait). So, here is the solution...
// Action Creator:
// No changes are required. Original code is good.
// Helper Function 1
export const fContactImport = async () => {
let contactList = [];
if (Platform.OS === "android") {
try {
const granted = await PermissionsAndroid.request(
PermissionsAndroid.PERMISSIONS.READ_CONTACTS,
{
title: "Contacts",
message: "This app would like to view your contacts."
},
);
if (granted === PermissionsAndroid.RESULTS.GRANTED) {
contactList = await _loadContacts();
} else {
console.log('Contacts access denied');
}
} catch (err) {
console.warn(err);
}
} else {
contactList = await _loadContacts();
}
return contactList;
}
// Helper Function 2
export const _loadContacts = () => {
return new Promise((resolve, reject) => {
Contacts.getAll((err, data2) => {
if (err) {
reject([]);
}
else {
let candidates = [];
for (let i = 0; i < data2.length; i++) {
let candidateObject = { name: candidateRecord.givenName + " " + candidateRecord.middleName + " " + candidateRecord.familyName };
candidates.push(candidateObject);
}
resolve(candidates);
}
});
});
}
I try to update a now playing state in Redux but everytime i call the update and it passes the check it get's added twice to the state.
result is that my filter when mapping (hide the first item) is not working due to the 2 additions.
Sample reducer:
import {
UPDATE_NOW_PLAYING,
UPDATE_ARTIST_AND_TITLE,
UPDATE_CURRENT_ARTIST
} from '../actions/actionTypes'
import { nowPlayingUrl, showNumberOfTracks, lastFmApiKey } from '../constants'
const currentState = {
playHistory: [],
currentArtist: '',
currentTitle: '',
currentShowName: '',
currentAlbumUrl: 'https://https://url.to.logo/app/app/logo-app.png'
}
const NowPlayingReducer = (state = currentState, action) => {
switch (action.type) {
case UPDATE_ARTIST_AND_TITLE: {
if (state.currentArtist !== action.array[0].artist) {
return {
...state,
playHistory: state.playHistory.concat(action.array[0]),
currentArtist: action.array[0].artist,
currentTitle: action.array[0].title,
currentShowName: action.array[0].showName,
currentAlbumUrl: action.array[0].albumUrl
}
}
return state
}
case UPDATE_NOW_PLAYING: {
return {
...state,
playHistory: action.payload,
currentArtist: action.payload[0].artist,
currentTitle: action.payload[0].title,
currentShowName: action.payload[0].showName,
currentAlbumUrl: action.payload[0].albumUrl
}
}
default: return state
}
}
export default NowPlayingReducer
So it works, but the only part is the second addition somehow :(
//Edit:
componentWillMount() {
this.getNowPlaying()
}
componentDidMount() {
setInterval(async () => {
await this.updateCurrentInformation()
}, 10000);
}
updateCurrentInformation = async () => {
try {
let currentPlayingResponse = await fetch(nowPlayingUrl + 1 )
let currentPlayingJson = await currentPlayingResponse.json()
let newArray = []
Promise.all(
currentPlayingJson.nowplaying.map(async (album) => {
const albumObj = await this.getAlbumInformation(album);
newArray.push(albumObj);
this.props.updateArtistAndTitle(newArray)
})
);
} catch(error) { console.log(error)};
}
togglePlayerType() {
}
getNowPlaying = async () => {
try {
let nowPlayingResponse = await fetch(nowPlayingUrl + showNumberOfTracks);
let nowPlayingJson = await nowPlayingResponse.json();
let newArray = [];
Promise.all(
nowPlayingJson.nowplaying.map(async (album) => {
const albumObj = await this.getAlbumInformation(album);
newArray.push(albumObj);
this.props.updateNowPlaying(newArray);
})
);
} catch (err) {
console.log(err);
}
}
getAlbumInformation = async (album) => {
return new Promise(async (resolve, reject) => {
let id = JSON.stringify(album.id).replace(/"/g, '');
let title = JSON.stringify(album.title)
.replace(/"/g, '')
.replace(/([&][^&; ]+[;])/g, '');
let artist = JSON.stringify(album.artist)
.replace(/"/g, '')
.replace(/([&][^&; ]+[;])/g, '');
let dateTime = JSON.stringify(album.datetime).replace(/"/g, '');
let showName = JSON.stringify(album.showName).replace(/"/g, '');
let albumUrl = 'https://url.to.logo/app/logo-app.png';
let obj = { id, title, artist, dateTime, showName, albumUrl };
try {
let albumResponse = await fetch(
`https://ws.audioscrobbler.com/2.0/?method=track.getInfo&format=json&api_key=${lastFmApiKey}&artist=${artist}&track=${title}`
);
let albumJson = await albumResponse.json();
if (albumJson) {
url = albumJson.track.album.image[3]['#text'];
if (url > '') {
obj.albumUrl = albumJson.track.album.image[3]['#text'];
} else {
obj.albumUrl = 'https://url.to.logo/app/logo-app.png';
}
}
resolve(obj);
} catch (err) {
resolve(obj);
}
});
}