Related
I have a NextJS (Typescript) setup with Express . I'm using Moralis, Wagmi and Web3Auth for authentication. The Metamask authentication works, but when I try to setup Web3 Authentication by importing Web3AuthConnector it throws me an error.
My code is like here in the documentation but looks to be an error about CommonJS and ESM which I can't handle. This is the error:
error - Error [ERR_REQUIRE_ESM]: require() of ES Module
C:...\node_modules#web3auth\web3auth-wagmi-connector\node_modules#wagmi\core\dist\index.js
from
C:...\node_modules#web3auth\web3auth-wagmi-connector\dist\web3authWagmiConnector.cjs.js
not supported. Instead change the require of index.js in
C:...\node_modules#web3auth\web3auth-wagmi-connector\dist\web3authWagmiConnector.cjs.js
to a dynamic import() which is available in all CommonJS modules.
I tried to set in package.json "type:module", and in tsconfig: module:"ES6" or "ESNext" but got the same error.
I got the same problem than you and I resolved it by create my own Connector.
I didn't found any other way to do it.
import {
Chain,
Connector,
ConnectorData,
normalizeChainId,
UserRejectedRequestError,
} from "#wagmi/core";
import {
ADAPTER_EVENTS,
ADAPTER_STATUS,
CHAIN_NAMESPACES,
CustomChainConfig,
getChainConfig,
SafeEventEmitterProvider,
WALLET_ADAPTER_TYPE,
WALLET_ADAPTERS,
BaseAdapterConfig,
CONNECTED_EVENT_DATA,
IAdapter,
storageAvailable,
ADAPTER_CATEGORY,
} from "#web3auth/base";
import { Web3AuthCore } from "#web3auth/core";
import { MetamaskAdapter } from "#web3auth/metamask-adapter";
import { OpenloginAdapter } from "#web3auth/openlogin-adapter";
import { TorusWalletAdapter } from "#web3auth/torus-evm-adapter";
import LoginModal, {
getAdapterSocialLogins,
LOGIN_MODAL_EVENTS,
OPENLOGIN_PROVIDERS,
} from "#web3auth/ui";
import {
AdaptersModalConfig,
defaultEvmDappModalConfig,
ModalConfig,
} from "#web3auth/web3auth";
import { Options } from "#web3auth/web3auth-wagmi-connector/dist/types/lib/interfaces";
import { ethers, Signer } from "ethers";
import { getAddress } from "ethers/lib/utils";
import { WalletConnectV1Adapter } from "#web3auth/wallet-connect-v1-adapter";
import QRCodeModal from "#walletconnect/qrcode-modal";
import { ConnectorEvents, defaultChains } from "wagmi";
import EventEmitter from "events";
import { Provider } from "react";
const IS_SERVER = typeof window === "undefined";
const ADAPTER_CACHE_KEY = "Web3Auth-cachedAdapter";
export class Web3AuthConnectorLocal extends Connector {
ready = !IS_SERVER;
readonly id = "web3Auth";
readonly name = "web3Auth";
provider: SafeEventEmitterProvider;
web3AuthInstance?: Web3AuthCore;
isModalOpen = false;
web3AuthOptions: Options;
private loginModal: LoginModal;
private socialLoginAdapter: OpenloginAdapter;
private torusWalletAdapter: TorusWalletAdapter;
private metamaskAdapter: MetamaskAdapter;
private walletConnectV1Adapter: WalletConnectV1Adapter;
private adapters: Record<string, IAdapter<unknown>> = {};
private modalConfig: AdaptersModalConfig = defaultEvmDappModalConfig;
private storage: "sessionStorage" | "localStorage" = "localStorage";
constructor(config: { chains?: Chain[]; options: Options }) {
super(config);
this.web3AuthOptions = config.options;
const chainId = config.options.chainId
? parseInt(config.options.chainId, 16)
: 1;
const chainConfig = this.chains.filter((x) => x.id === chainId);
const defaultChainConfig = getChainConfig(
CHAIN_NAMESPACES.EIP155,
config.options.chainId || "0x1"
);
let finalChainConfig: CustomChainConfig = {
chainNamespace: CHAIN_NAMESPACES.EIP155,
...defaultChainConfig,
};
if (chainConfig.length > 0) {
finalChainConfig = {
...finalChainConfig,
chainNamespace: CHAIN_NAMESPACES.EIP155,
chainId: config.options.chainId || "0x1",
rpcTarget: chainConfig[0].rpcUrls.default,
displayName: chainConfig[0].name,
tickerName: chainConfig[0].nativeCurrency?.name,
ticker: chainConfig[0].nativeCurrency?.symbol,
blockExplorer: chainConfig[0]?.blockExplorers.default?.url,
};
}
this.web3AuthInstance = new Web3AuthCore({
clientId: config.options.clientId,
chainConfig: {
chainNamespace: CHAIN_NAMESPACES.EIP155,
chainId: "0x1",
rpcTarget: "https://rpc.ankr.com/eth", // This is the public RPC we have added, please pass on your own endpoint while creating an app
},
});
this.socialLoginAdapter = new OpenloginAdapter({
adapterSettings: {
...config.options,
},
chainConfig: {
chainId: "0x1",
chainNamespace: CHAIN_NAMESPACES.EIP155,
rpcTarget: "https://rpc.ankr.com/eth",
displayName: "mainnet",
blockExplorer: "https://etherscan.io/",
ticker: "ETH",
tickerName: "Ethereum",
},
});
this.torusWalletAdapter = new TorusWalletAdapter({
adapterSettings: {
buttonPosition: "bottom-left",
},
loginSettings: {
verifier: "google",
},
initParams: {
buildEnv: "testing",
},
chainConfig: {
chainNamespace: CHAIN_NAMESPACES.EIP155,
chainId: "0x1",
rpcTarget: "https://rpc.ankr.com/eth", // This is the mainnet RPC we have added, please pass on your own endpoint while creating an app
displayName: "Ethereum Mainnet",
blockExplorer: "https://etherscan.io/",
ticker: "ETH",
tickerName: "Ethereum",
},
});
this.metamaskAdapter = new MetamaskAdapter({
clientId: config.options.clientId,
});
this.walletConnectV1Adapter = new WalletConnectV1Adapter({
adapterSettings: {
bridge: "https://bridge.walletconnect.org",
qrcodeModal: QRCodeModal,
},
clientId: config.options.clientId,
});
this.web3AuthInstance.configureAdapter(this.socialLoginAdapter);
this.web3AuthInstance.configureAdapter(this.torusWalletAdapter);
this.web3AuthInstance.configureAdapter(this.metamaskAdapter);
this.web3AuthInstance.configureAdapter(this.walletConnectV1Adapter);
this.adapters[this.socialLoginAdapter.name] = this.socialLoginAdapter;
this.adapters[this.torusWalletAdapter.name] = this.torusWalletAdapter;
this.adapters[this.metamaskAdapter.name] = this.metamaskAdapter;
this.adapters[this.walletConnectV1Adapter.name] =
this.walletConnectV1Adapter;
this.loginModal = new LoginModal({
theme: this.options.uiConfig?.theme,
appLogo: this.options.uiConfig?.appLogo || "",
version: "",
adapterListener: this.web3AuthInstance,
displayErrorsOnModal: this.options.displayErrorsOnModal,
});
// this.loginModal.initExternalWalletContainer();
this.subscribeToLoginModalEvents();
}
async connect(): Promise<Required<ConnectorData>> {
this.web3AuthInstance.init();
const adapterEventsPromise = this.subscribeToAdpaterConnectionEvents()
await this.init();
this.loginModal.open();
const elem = document.getElementById("w3a-container");
elem.style.zIndex = "10000000000";
return (await adapterEventsPromise) as Required<ConnectorData>;
}
async subscribeToAdpaterConnectionEvents(): Promise<Required<ConnectorData>> {
return new Promise((resolve, reject) => {
this.web3AuthInstance.once(ADAPTER_EVENTS.CONNECTED, async () => {
console.log(
"Received event connected: ",
this.web3AuthInstance.connectedAdapterName
);
console.log("Requesting Signer");
const signer = await this.getSigner();
const account = await signer.getAddress();
const provider = await this.getProvider();
if (provider.on) {
provider.on("accountsChanged", this.onAccountsChanged.bind(this));
provider.on("chainChanged", this.onChainChanged.bind(this));
provider.on("disconnect", this.onDisconnect.bind(this));
}
return resolve({
account,
chain: {
id: 0,
unsupported: false,
},
provider,
});
});
this.web3AuthInstance.once(ADAPTER_EVENTS.ERRORED, (err: unknown) => {
console.log("error while connecting", err);
return reject(err);
});
});
}
async init(): Promise<void> {
console.log("What is this type: ", typeof this)
console.log("What is this instance: ", this instanceof Web3AuthConnectorLocal)
try {
await this.loginModal.initModal();
const allAdapters = [
...new Set([
...Object.keys(this.modalConfig.adapters || {}),
...Object.keys(this.adapters),
]),
];
const adapterNames = [
"torus-evm",
"metamask",
"openlogin",
"wallet-connect-v1",
];
const hasInAppWallets = true;
// Now, initialize the adapters.
const initPromises = adapterNames.map(async (adapterName) => {
if (!adapterName) return;
try {
const adapter = this.adapters[adapterName];
console.log("Adapter Found: ", adapterName);
console.log("Cached Adapter: ", this.web3AuthInstance.cachedAdapter);
// only initialize a external adapter here if it is a cached adapter.
if (
this.web3AuthInstance.cachedAdapter !== adapterName &&
adapter.type === ADAPTER_CATEGORY.EXTERNAL
) {
console.log(adapterName, " Adapter is not External");
return;
}
// in-app wallets or cached wallet (being connected or already connected) are initialized first.
// if adapter is configured thn only initialize in app or cached adapter.
// external wallets are initialized on INIT_EXTERNAL_WALLET event.
this.subscribeToAdapterEvents(adapter);
if (adapter.status === ADAPTER_STATUS.NOT_READY) {
await adapter.init({
autoConnect: this.web3AuthInstance.cachedAdapter === adapterName,
});
console.log(
"Initializing In Wallet: COMPLETED",
adapter,
adapter.status
);
}
// note: not adding cachedWallet to modal if it is external wallet.
// adding it later if no in-app wallets are available.
if (adapter.type === ADAPTER_CATEGORY.IN_APP) {
this.loginModal.addSocialLogins(
WALLET_ADAPTERS.OPENLOGIN,
getAdapterSocialLogins(
WALLET_ADAPTERS.OPENLOGIN,
this.socialLoginAdapter,
this.options.uiConfig?.loginMethodConfig
),
this.options.uiConfig?.loginMethodsOrder || OPENLOGIN_PROVIDERS
);
}
} catch (error) {
console.log(error, "error while initializing adapter");
}
});
this.web3AuthInstance.status = ADAPTER_STATUS.READY;
await Promise.all(initPromises);
const hasExternalWallets = allAdapters.some((adapterName) => {
return (
this.adapters[adapterName]?.type === ADAPTER_CATEGORY.EXTERNAL &&
this.modalConfig.adapters?.[adapterName].showOnModal
);
});
console.log("Has External Wallets: ", hasExternalWallets);
if (hasExternalWallets) {
this.loginModal.initExternalWalletContainer();
}
if (!hasInAppWallets && hasExternalWallets) {
await this.initExternalWalletAdapters(false, {
showExternalWalletsOnly: true,
});
}
} catch (error) {
console.log("error while connecting", error);
throw new UserRejectedRequestError("Something went wrong");
}
}
async getAccount(): Promise<string> {
const provider = new ethers.providers.Web3Provider(
await this.getProvider()
);
const signer = provider.getSigner();
const account = await signer.getAddress();
return account;
}
async getProvider() {
if (this.provider) {
return this.provider;
}
this.provider = this.web3AuthInstance.provider;
return this.provider;
}
async getSigner(): Promise<Signer> {
console.log("Getting Signer");
const provider = new ethers.providers.Web3Provider(
await this.getProvider()
);
const signer = provider.getSigner();
return signer;
}
async isAuthorized() {
try {
const account = await this.getAccount();
return !!(account && this.provider);
} catch {
return false;
}
}
async getChainId(): Promise<number> {
try {
const networkOptions = this.socialLoginAdapter.chainConfigProxy;
if (typeof networkOptions === "object") {
const chainID = networkOptions.chainId;
if (chainID) {
return normalizeChainId(chainID);
}
}
throw new Error("Chain ID is not defined");
} catch (error) {
console.log("error", error);
throw error;
}
}
async disconnect(): Promise<void> {
await this.web3AuthInstance.logout();
this.provider = null;
}
protected onAccountsChanged(accounts: string[]): void {
if (accounts.length === 0) this.emit("disconnect");
else this.emit("change", { account: getAddress(accounts[0]) });
}
protected onChainChanged(chainId: string | number): void {
const id = normalizeChainId(chainId);
const unsupported = this.isChainUnsupported(id);
this.emit("change", { chain: { id, unsupported } });
}
protected onDisconnect(): void {
this.emit("disconnect");
}
private subscribeToLoginModalEvents(): void {
this.loginModal.on(
LOGIN_MODAL_EVENTS.LOGIN,
async (params: {
adapter: WALLET_ADAPTER_TYPE;
loginParams: unknown;
}) => {
try {
console.log("Wallet Adapters: ", +params.adapter);
await this.web3AuthInstance.connectTo<unknown>(
params.adapter,
params.loginParams
);
} catch (error) {
console.log(
`Error while connecting to adapter: ${params.adapter}`,
error
);
}
}
);
this.loginModal.on(
LOGIN_MODAL_EVENTS.INIT_EXTERNAL_WALLETS,
async (params: { externalWalletsInitialized: boolean }) => {
await this.initExternalWalletAdapters(
params.externalWalletsInitialized
);
}
);
this.loginModal.on(LOGIN_MODAL_EVENTS.DISCONNECT, async () => {
try {
await this.disconnect();
} catch (error) {
console.log(`Error while disconnecting`, error);
}
});
}
private async initExternalWalletAdapters(
externalWalletsInitialized: boolean,
options?: { showExternalWalletsOnly: boolean }
): Promise<void> {
if (externalWalletsInitialized) return;
const adaptersConfig: Record<string, BaseAdapterConfig> = {};
const adaptersData: Record<string, unknown> = {};
const adapterPromises = Object.keys(this.adapters).map(
async (adapterName) => {
try {
const adapter = this.adapters[adapterName];
if (adapter?.type === ADAPTER_CATEGORY.EXTERNAL) {
console.log("init external wallet", adapterName);
this.subscribeToAdapterEvents(adapter);
// we are not initializing cached adapter here as it is already being initialized in initModal before.
if (this.web3AuthInstance.cachedAdapter === adapterName) {
return;
}
if (adapter.status === ADAPTER_STATUS.NOT_READY) {
console.log("Adapter not Ready: " + adapterName);
return await Promise.race([
adapter
.init({
autoConnect:
this.web3AuthInstance.cachedAdapter === adapterName,
})
.then(() => {
adaptersConfig[adapterName] = (
defaultEvmDappModalConfig.adapters as Record<
WALLET_ADAPTER_TYPE,
ModalConfig
>
)[adapterName];
adaptersData[adapterName] = adapter.adapterData || {};
console.log("Adapter Init: ", adapterName);
return adapterName;
}),
new Promise((resolve) => {
setTimeout(() => {
return resolve(null);
}, 5000);
}),
]);
} else {
console.log("Adapter Ready: " + adapterName);
return adapterName;
}
}
} catch (error) {
console.log(error, "error while initializing adapter");
}
}
);
const adapterInitResults = await Promise.all(adapterPromises);
console.log("Adapter Init Results: ", adapterInitResults);
const finalAdaptersConfig: Record<WALLET_ADAPTER_TYPE, BaseAdapterConfig> =
{};
adapterInitResults.forEach((result: string | undefined) => {
if (result) {
finalAdaptersConfig[result] = adaptersConfig[result];
}
});
this.loginModal.addWalletLogins(finalAdaptersConfig, {
showExternalWalletsOnly: !!options?.showExternalWalletsOnly,
});
}
private subscribeToAdapterEvents(walletAdapter: IAdapter<unknown>): void {
console.log("Running adapter events");
walletAdapter.on(ADAPTER_EVENTS.CONNECTED, (data: CONNECTED_EVENT_DATA) => {
let status = ADAPTER_STATUS.CONNECTED;
this.web3AuthInstance.connectedAdapterName = data.adapter;
this.cacheWallet(data.adapter);
console.log(
"connected",
status,
this.web3AuthInstance.connectedAdapterName
);
this.web3AuthInstance.emit(ADAPTER_EVENTS.CONNECTED, {
...data,
} as CONNECTED_EVENT_DATA);
});
walletAdapter.on(ADAPTER_EVENTS.DISCONNECTED, async (data) => {
// get back to ready state for rehydrating.
let status = ADAPTER_STATUS.READY;
if (storageAvailable(this.storage)) {
const cachedAdapter = window[this.storage].getItem(ADAPTER_CACHE_KEY);
if (this.web3AuthInstance.connectedAdapterName === cachedAdapter) {
this.web3AuthInstance.clearCache();
}
}
console.log(
"disconnected",
status,
this.web3AuthInstance.connectedAdapterName
);
this.web3AuthInstance.connectedAdapterName = null;
this.web3AuthInstance.emit(ADAPTER_EVENTS.DISCONNECTED, data);
});
walletAdapter.on(ADAPTER_EVENTS.CONNECTING, (data) => {
let status = ADAPTER_STATUS.CONNECTING;
this.web3AuthInstance.emit(ADAPTER_EVENTS.CONNECTING, data);
console.log(
"connecting",
status,
this.web3AuthInstance.connectedAdapterName
);
});
walletAdapter.on(ADAPTER_EVENTS.ERRORED, (data) => {
let status = ADAPTER_STATUS.ERRORED;
this.web3AuthInstance.clearCache();
this.web3AuthInstance.emit(ADAPTER_EVENTS.ERRORED, data);
console.log(
"errored",
status,
this.web3AuthInstance.connectedAdapterName
);
});
walletAdapter.on(ADAPTER_EVENTS.ADAPTER_DATA_UPDATED, (data) => {
console.log("adapter data updated", data);
this.web3AuthInstance.emit(ADAPTER_EVENTS.ADAPTER_DATA_UPDATED, data);
});
}
private cacheWallet(walletName: string) {
if (!storageAvailable(this.storage)) return;
window[this.storage].setItem(ADAPTER_CACHE_KEY, walletName);
this.web3AuthInstance.cachedAdapter = walletName;
}
}
I got the code from https://forum.moralis.io/t/solved-web3auth-wagmi-connector-doesnt-show-web3-wallets-as-a-login-option/20110/4.
Let me know if it's working for you ?
( You should add all the missing packages with yarn add )
infinity loading when try to scrap title article
i was trying to scrap title articles with cheerio in node js but i could not make it happen
infinity loading in google chrome and no content . here is my controller.js file that i trying to use cheerio to scrap title articles with cheerio and show handlesbars view engine
but view engie is working and everything is fine . but for scrap infinity loading in chrome with no error
classname that i use for this website is .c-entry-box--compact__title
maybe i am wrong . but i can not figure out
var express = require("express");
var router = express.Router();
var path = require("path");
var request = require("request");
var cheerio = require("cheerio");
var Comment = require("../models/Comment.js");
var Article = require("../models/Article.js");
router.get("/", function(req, res) {
res.redirect("/articles");
});
router.get("/scrape", function(req, res) {
request("http://www.theverge.com", function(error, response, html) {
var $ = cheerio.load(html);
var titlesArray = [];
$(".c-entry-box--compact__title").each(function(i, element) {
var result = {};
result.title = $(this)
.children("a")
.text();
result.link = $(this)
.children("a")
.attr("href");
if (result.title !== "" && result.link !== "") {
if (titlesArray.indexOf(result.title) == -1) {
titlesArray.push(result.title);
Article.count({ title: result.title }, function(err, test) {
if (test === 0) {
var entry = new Article(result);
entry.save(function(err, doc) {
if (err) {
console.log(err);
} else {
console.log(doc);
}
});
}
});
} else {
console.log("Article already exists.");
}
} else {
console.log("Not saved to DB, missing data");
}
});
res.redirect("/");
});
});
router.get("/articles", function(req, res) {
Article.find()
.sort({ _id: -1 })
.exec(function(err, doc) {
if (err) {
console.log(err);
} else {
var artcl = { article: doc };
res.render("index", artcl);
}
});
});
router.get("/articles-json", function(req, res) {
Article.find({}, function(err, doc) {
if (err) {
console.log(err);
} else {
res.json(doc);
}
});
});
router.get("/clearAll", function(req, res) {
Article.remove({}, function(err, doc) {
if (err) {
console.log(err);
} else {
console.log("removed all articles");
}
});
res.redirect("/articles-json");
});
router.get("/readArticle/:id", function(req, res) {
var articleId = req.params.id;
var hbsObj = {
article: [],
body: []
};
Article.findOne({ _id: articleId })
.populate("comment")
.exec(function(err, doc) {
if (err) {
console.log("Error: " + err);
} else {
hbsObj.article = doc;
var link = doc.link;
request(link, function(error, response, html) {
var $ = cheerio.load(html);
$(".l-col__main").each(function(i, element) {
hbsObj.body = $(this)
.children(".c-entry-content")
.children("p")
.text();
res.render("article", hbsObj);
return false;
});
});
}
});
});
router.post("/comment/:id", function(req, res) {
var user = req.body.name;
var content = req.body.comment;
var articleId = req.params.id;
var commentObj = {
name: user,
body: content
};
var newComment = new Comment(commentObj);
newComment.save(function(err, doc) {
if (err) {
console.log(err);
} else {
console.log(doc._id);
console.log(articleId);
Article.findOneAndUpdate(
{ _id: req.params.id },
{ $push: { comment: doc._id } },
{ new: true }
).exec(function(err, doc) {
if (err) {
console.log(err);
} else {
res.redirect("/readArticle/" + articleId);
}
});
}
});
});
module.exports = router;
The http://www.theverge.com Add content dynamically by scrolling.
This is example how to get the title by puppeteer
const puppeteer = require("puppeteer");
const getTitle = async () => {
try {
const browser = await puppeteer.launch({
headless: false,
});
const page = await browser.newPage();
await page.setDefaultNavigationTimeout(0);
await page.goto('https://www.theverge.com');
await page.setViewport({
width:1920,
height:1080
});
// scroll down end of page
await page.evaluate(() => {
window.scrollTo(0, window.document.body.scrollHeight);
});
await page.waitForNavigation({ waitUntil: 'networkidle0' }), // (0 network connections for 500ms)
// get the title
titles = await page.evaluate(() => {
const textSelector = 'div.inline.pr-4.font-bold'
texts = Array.from(document.querySelectorAll(textSelector), row => row.innerText.trim() );
return texts;
});
await browser.close();
return Promise.resolve(titles);
} catch (error) {
return Promise.reject(error);
}
}
getTitle()
.then((titles) => {
console.log(titles); // first news search
})
This is result
[
'Is an upgraded M2 Ultra enough for a new Mac Pro and the Mac Studio?',
'Here’s the official trailer for Christopher Nolan’s next IMAX-filmed epic, Oppenheimer.',
'Sam Bankman-Fried’s ready to surrender himself to the US for extradition.',
'Who knew the thumb drive had such a contentious origin story?',
'But how many pebbles do you have in a jar?',
'This way for The Way of Water.',
'Netflix is taking Blockbuster behind the woodshed again.',
'I can’t escape the year-end wrap-ups.',
'The clock’s ticking if you want to get your gifts on time.',
'Want solar panels on your California home? Now might be the time.',
'Twitter Spaces has returned.',
'Apple’s facing another accusation of breaking labor laws.',
'Every game should have this feature.',
'Google’s working on simplifying smart home control on the wrist.',
'Apple could open up iOS, and the feds finally make a case against SBF.',
'I’m not the first, and I won’t be the last... but I do feel early.',
'For what it’s worth, Avatar: The Way of Water’s a good looking movie.',
'You may not want to upgrade to Apple’s new Home architecture.'
]
I have a test to test my cloudflare worker that looks like this:
const workerScript = fs.readFileSync(
path.resolve(__dirname, '../pkg-prd/worker.js'),
'utf8'
);
describe('worker unit test', function () {
// this.timeout(60000);
let worker;
beforeEach(() => {
worker = new Cloudworker(workerScript, {
bindings: {
HTMLRewriter
},
});
});
it('tests requests and responses', async () => {
const request = new Cloudworker.Request('https://www.example.com/pathname')
const response = await worker.dispatch(request);
console.log(response);
// const body = await response.json();
expect(response.status).to.eql(200);
// expect(body).to.eql({message: 'Hello mocha!'});
});
});
In my worker I do something like this:
const response = await fetch(BASE_URL, request);
const modifiedResponse = new Response(response.body, response);
// Remove the webflow badge
class ElementHandler {
element(element) {
element.append('<style type="text/css">body .w-webflow-badge {display: none!important}</style>', {html: true})
}
}
console.log(3);
return new HTMLRewriter()
.on('head', new ElementHandler()).transform(modifiedResponse);
Now when i run my test I get this error message:
● worker unit test › tests requests and responses
TypeError: Cannot read property 'transform' of undefined
at evalmachine.<anonymous>:1:1364
at FetchEvent.respondWith (node_modules/#dollarshaveclub/cloudworker/lib/cloudworker.js:39:17)
What seems to be wrong?
HTMLRewriter i created looks like this:
function HTMLRewriter() {
const elementHandler = {};
const on = (selector, handler) => {
if (handler && handler.element) {
if (!elementHandler[selector]) {
elementHandler[selector] = [];
}
elementHandler[selector].push(handler.element.bind(handler));
}
};
const transform = async response => {
const tempResponse = response.clone();
const doc = HTMLParser.parse(await tempResponse.text());
Object.keys(elementHandler).forEach(selector => {
const el = doc.querySelector(selector);
if (el) {
elementHandler[selector].map(callback => {
callback(new _Element(el));
});
}
});
return new Response(doc.toString(), response);
};
return {
on,
transform
};
}
Since HTMLRewriter() is called with new, the function needs to be a constructor. In JavaScript, a constructor function should set properties on this and should not return a value. But, your function is written to return a value.
So, try changing this:
return {
on,
transform
};
To this:
this.on = on;
this.transform = transform;
What I'm trying to do is get real time transcription for video recorded in the browser with webRTC. Use case is basically subtitles in real time like google hangouts has.
So I have a WebRTC program running in the browser. It sends webm objects back to the server. They are linear32 audio encodings. Google speech to text only accepts linear16 or Flac files.
Is there a way to convert linear32 to linear16 in real time?
Otherwise has anyone been able to hook up webRTC with Google speech to get real time transcriptions working?
Any advice on where to look to solve this problem would be great
Check out this repository it might help you - https://github.com/muaz-khan/Translator
Translator.js is a JavaScript library built top on Google Speech-Recognition & Translation API to transcript and translate voice and text. It supports many locales and brings globalization in WebRTC!
I had the same problem and failed with webRTC. I recommend you use the Web Audio Api instead if you are just interested in transcribing the audio from the video.
Here is how I did it with a nodejs sever and react client app. It is uploaded to github here
You need an audio worklet script. (Put it in the public folder because that is where the API expects to find it)
recorderWorkletProcessor.js (saved in public/src/worklets/recorderWorkletProcessor.js)
/**
An in-place replacement for ScriptProcessorNode using AudioWorklet
*/
class RecorderProcessor extends AudioWorkletProcessor {
// 0. Determine the buffer size (this is the same as the 1st argument of ScriptProcessor)
bufferSize = 2048;
// 1. Track the current buffer fill level
_bytesWritten = 0;
// 2. Create a buffer of fixed size
_buffer = new Float32Array(this.bufferSize);
constructor() {
super();
this.initBuffer();
}
initBuffer() {
this._bytesWritten = 0;
}
isBufferEmpty() {
return this._bytesWritten === 0;
}
isBufferFull() {
return this._bytesWritten === this.bufferSize;
}
/**
* #param {Float32Array[][]} inputs
* #returns {boolean}
*/
process(inputs) {
// Grabbing the 1st channel similar to ScriptProcessorNode
this.append(inputs[0][0]);
return true;
}
/**
*
* #param {Float32Array} channelData
*/
append(channelData) {
if (this.isBufferFull()) {
this.flush();
}
if (!channelData) return;
for (let i = 0; i < channelData.length; i++) {
this._buffer[this._bytesWritten++] = channelData[i];
}
}
flush() {
// trim the buffer if ended prematurely
const buffer = this._bytesWritten < this.bufferSize ? this._buffer.slice(0, this._bytesWritten) : this._buffer;
const result = this.downsampleBuffer(buffer, 44100, 16000);
this.port.postMessage(result);
this.initBuffer();
}
downsampleBuffer(buffer, sampleRate, outSampleRate) {
if (outSampleRate == sampleRate) {
return buffer;
}
if (outSampleRate > sampleRate) {
throw new Error("downsampling rate show be smaller than original sample rate");
}
var sampleRateRatio = sampleRate / outSampleRate;
var newLength = Math.round(buffer.length / sampleRateRatio);
var result = new Int16Array(newLength);
var offsetResult = 0;
var offsetBuffer = 0;
while (offsetResult < result.length) {
var nextOffsetBuffer = Math.round((offsetResult + 1) * sampleRateRatio);
var accum = 0,
count = 0;
for (var i = offsetBuffer; i < nextOffsetBuffer && i < buffer.length; i++) {
accum += buffer[i];
count++;
}
result[offsetResult] = Math.min(1, accum / count) * 0x7fff;
offsetResult++;
offsetBuffer = nextOffsetBuffer;
}
return result.buffer;
}
}
registerProcessor("recorder.worklet", RecorderProcessor);
Install Socket.io-client on front end
npm i socket.io-client
React component code
/* eslint-disable react-hooks/exhaustive-deps */
import { default as React, useEffect, useState, useRef } from "react";
import { Button } from "react-bootstrap";
import Container from "react-bootstrap/Container";
import * as io from "socket.io-client";
const sampleRate = 16000;
const getMediaStream = () =>
navigator.mediaDevices.getUserMedia({
audio: {
deviceId: "default",
sampleRate: sampleRate,
sampleSize: 16,
channelCount: 1,
},
video: false,
});
interface WordRecognized {
final: boolean;
text: string;
}
const AudioToText: React.FC = () => {
const [connection, setConnection] = useState<io.Socket>();
const [currentRecognition, setCurrentRecognition] = useState<string>();
const [recognitionHistory, setRecognitionHistory] = useState<string[]>([]);
const [isRecording, setIsRecording] = useState<boolean>(false);
const [recorder, setRecorder] = useState<any>();
const processorRef = useRef<any>();
const audioContextRef = useRef<any>();
const audioInputRef = useRef<any>();
const speechRecognized = (data: WordRecognized) => {
if (data.final) {
setCurrentRecognition("...");
setRecognitionHistory((old) => [data.text, ...old]);
} else setCurrentRecognition(data.text + "...");
};
const connect = () => {
connection?.disconnect();
const socket = io.connect("http://localhost:8081");
socket.on("connect", () => {
console.log("connected", socket.id);
setConnection(socket);
});
socket.emit("send_message", "hello world");
socket.emit("startGoogleCloudStream");
socket.on("receive_message", (data) => {
console.log("received message", data);
});
socket.on("receive_audio_text", (data) => {
speechRecognized(data);
console.log("received audio text", data);
});
socket.on("disconnect", () => {
console.log("disconnected", socket.id);
});
};
const disconnect = () => {
if (!connection) return;
connection?.emit("endGoogleCloudStream");
connection?.disconnect();
processorRef.current?.disconnect();
audioInputRef.current?.disconnect();
audioContextRef.current?.close();
setConnection(undefined);
setRecorder(undefined);
setIsRecording(false);
};
useEffect(() => {
(async () => {
if (connection) {
if (isRecording) {
return;
}
const stream = await getMediaStream();
audioContextRef.current = new window.AudioContext();
await audioContextRef.current.audioWorklet.addModule(
"/src/worklets/recorderWorkletProcessor.js"
);
audioContextRef.current.resume();
audioInputRef.current =
audioContextRef.current.createMediaStreamSource(stream);
processorRef.current = new AudioWorkletNode(
audioContextRef.current,
"recorder.worklet"
);
processorRef.current.connect(audioContextRef.current.destination);
audioContextRef.current.resume();
audioInputRef.current.connect(processorRef.current);
processorRef.current.port.onmessage = (event: any) => {
const audioData = event.data;
connection.emit("send_audio_data", { audio: audioData });
};
setIsRecording(true);
} else {
console.error("No connection");
}
})();
return () => {
if (isRecording) {
processorRef.current?.disconnect();
audioInputRef.current?.disconnect();
if (audioContextRef.current?.state !== "closed") {
audioContextRef.current?.close();
}
}
};
}, [connection, isRecording, recorder]);
return (
<React.Fragment>
<Container className="py-5 text-center">
<Container fluid className="py-5 bg-primary text-light text-center ">
<Container>
<Button
className={isRecording ? "btn-danger" : "btn-outline-light"}
onClick={connect}
disabled={isRecording}
>
Start
</Button>
<Button
className="btn-outline-light"
onClick={disconnect}
disabled={!isRecording}
>
Stop
</Button>
</Container>
</Container>
<Container className="py-5 text-center">
{recognitionHistory.map((tx, idx) => (
<p key={idx}>{tx}</p>
))}
<p>{currentRecognition}</p>
</Container>
</Container>
</React.Fragment>
);
};
export default AudioToText;
server.js
const express = require("express");
const speech = require("#google-cloud/speech");
//use logger
const logger = require("morgan");
//use body parser
const bodyParser = require("body-parser");
//use corrs
const cors = require("cors");
const http = require("http");
const { Server } = require("socket.io");
const app = express();
app.use(cors());
app.use(logger("dev"));
app.use(bodyParser.json());
const server = http.createServer(app);
const io = new Server(server, {
cors: {
origin: "http://localhost:3000",
methods: ["GET", "POST"],
},
});
//TODO: run in terminal first to setup credentials export GOOGLE_APPLICATION_CREDENTIALS="./speech-to-text-key.json"
const speechClient = new speech.SpeechClient();
io.on("connection", (socket) => {
let recognizeStream = null;
console.log("** a user connected - " + socket.id + " **\n");
socket.on("disconnect", () => {
console.log("** user disconnected ** \n");
});
socket.on("send_message", (message) => {
console.log("message: " + message);
setTimeout(() => {
io.emit("receive_message", "got this message" + message);
}, 1000);
});
socket.on("startGoogleCloudStream", function (data) {
startRecognitionStream(this, data);
});
socket.on("endGoogleCloudStream", function () {
console.log("** ending google cloud stream **\n");
stopRecognitionStream();
});
socket.on("send_audio_data", async (audioData) => {
io.emit("receive_message", "Got audio data");
if (recognizeStream !== null) {
try {
recognizeStream.write(audioData.audio);
} catch (err) {
console.log("Error calling google api " + err);
}
} else {
console.log("RecognizeStream is null");
}
});
function startRecognitionStream(client) {
console.log("* StartRecognitionStream\n");
try {
recognizeStream = speechClient
.streamingRecognize(request)
.on("error", console.error)
.on("data", (data) => {
const result = data.results[0];
const isFinal = result.isFinal;
const transcription = data.results
.map((result) => result.alternatives[0].transcript)
.join("\n");
console.log(`Transcription: `, transcription);
client.emit("receive_audio_text", {
text: transcription,
final: isFinal,
});
});
} catch (err) {
console.error("Error streaming google api " + err);
}
}
function stopRecognitionStream() {
if (recognizeStream) {
console.log("* StopRecognitionStream \n");
recognizeStream.end();
}
recognizeStream = null;
}
});
server.listen(8081, () => {
console.log("WebSocket server listening on port 8081.");
});
// =========================== GOOGLE CLOUD SETTINGS ================================ //
// The encoding of the audio file, e.g. 'LINEAR16'
// The sample rate of the audio file in hertz, e.g. 16000
// The BCP-47 language code to use, e.g. 'en-US'
const encoding = "LINEAR16";
const sampleRateHertz = 16000;
const languageCode = "en-US"; //en-US
const alternativeLanguageCodes = ["en-US", "ko-KR"];
const request = {
config: {
encoding: encoding,
sampleRateHertz: sampleRateHertz,
languageCode: languageCode,
//alternativeLanguageCodes: alternativeLanguageCodes,
enableWordTimeOffsets: true,
enableAutomaticPunctuation: true,
enableWordConfidence: true,
enableSpeakerDiarization: true,
diarizationSpeakerCount: 2,
model: "video",
//model: "command_and_search",
useEnhanced: true,
speechContexts: [
{
phrases: ["hello", "안녕하세요"],
},
],
},
interimResults: true,
};
I'm newer with Node.js and Express.js.
I want to upload first a image into the server (directory: uploads/spots), and then (synchronous) upload the rest of form data in MongoDB.
I'm using REST (Method Post)
app.route('/spots').post(users.requiresLogin, spots.create);
and I'm using Multer for updating the image into the server, and works.
app.use(multer(
{ dest: './public/uploads/spots',
onFileUploadStart: function (file) {
var imagePath = file.path;
gm(imagePath).resize(850, 850).quality(70).noProfile().write('public/uploads/spots/850x850/'+file.name, function (err) {
if (!err) {
gm(imagePath).resize(150, 150).quality(70).noProfile().write('public/uploads/spots/150x150/'+file.name, function (err) {
if (!err) {
}
else{
console.log('Error: '+err);
}
});
}
else{
console.log('Error: '+err);
}
});
}
}));
Is working, but is asynchronous , and returns the response to frontend before that the image will be upload into the server.
My question is how to do this but synchronous and how to return the response to the frontend after that the image was uploaded.
Thank you!
spots.server.routes.js
'use strict';
module.exports = function(app) {
var gm = require('gm');
var multer = require('multer');
var users = require('../controllers/users.server.controller.js');
var spots = require('../controllers/spots.server.controller.js');
//Upload image
app.use(multer(
{ dest: './public/uploads/spots',
onFileUploadStart: function (file) {
var imagePath = file.path;
gm(imagePath).resize(850, 850).quality(70).noProfile().write('public/uploads/spots/850x850/'+file.name, function (err) {
if (!err) {
gm(imagePath).resize(150, 150).quality(70).noProfile().write('public/uploads/spots/150x150/'+file.name, function (err) {
if (!err) {
}
else{
console.log('Error: '+err);
}
});
}
else{
console.log('Error: '+err);
}
});
}
}));
// Spots Routes
app.route('/spots')
.get(spots.list)
.post(users.requiresLogin, spots.create);
app.route('/spots/:spotId')
.get(spots.read)
.put(users.requiresLogin, spots.update)
.delete(users.requiresLogin, spots.hasAuthorization, spots.delete);
// Finish by binding the Spot middleware
app.param('spotId', spots.spotByID);
};
spots.server.controller.js (create method)
'use strict';
/**
* Module dependencies.
*/
var mongoose = require('mongoose'),
errorHandler = require('./errors.server.controller.js'),
Spot = mongoose.model('Spot'),
_ = require('lodash'),
fs = require('fs');
/**
* Create a Spot
*/
exports.create = function(req, res) {
var spot = new Spot(JSON.parse(req.body.spot));
spot.user = req.user;
if(req.files.file)
spot.image=req.files.file.name;
else
spot.image='default.jpg';
spot.save(function(err) {
if (err) {
fs.unlinkSync('public/uploads/spots/'+spot.image);
fs.unlinkSync('public/uploads/spots/850x850/'+spot.image);
fs.unlinkSync('public/uploads/spots/150x150/'+spot.image);
return res.status(400).send({
message: errorHandler.getErrorMessage(err)
});
} else {
var socketio = req.app.get('socketio'); // tacke out socket instance from the app container
socketio.sockets.emit('spot.created.'+spot.municipality, {spot:spot, user:req.user});
socketio.sockets.emit('spot.created.'+spot.province, {spot:spot, user:req.user});
socketio.sockets.emit('spot.created.'+spot.community, {spot:spot, user:req.user});
socketio.sockets.emit('spot.created.'+spot.country, {spot:spot, user:req.user});
res.jsonp(spot);
}
});
};
/**
* Spot authorization middleware
*/
exports.hasAuthorization = function(req, res, next) {
if (req.spot.user.id !== req.user.id) {
return res.status(403).send('User is not authorized');
}
next();
};
The solution is not use onFileUploadStart method and use a function with callback in the controller.
routes
// Spots Routes
app.route('/spots')
.get(spots.list)
.post(users.requiresLogin,multer({ dest: './public/uploads/spots'}), spots.create);
controller
exports.create = function(req, res) {
if (req.files.file)
exports.uploadImage(req.files.file,callback);
else
callback();
function callback(){
var spot = new Spot(JSON.parse(req.body.spot));
spot.user = req.user;
if (req.files.file)
spot.image = req.files.file.name;
else
spot.image = 'default.jpg';
spot.save(function (err) {
if (err) {
fs.unlink('public/uploads/spots/850x850/'+spot.image);
fs.unlink('public/uploads/spots/150x150/'+spot.image);
return res.status(400).send({
message: errorHandler.getErrorMessage(err)
});
} else {
var socketio = req.app.get('socketio'); // tacke out socket instance from the app container
socketio.sockets.emit('spot.created.' + spot.municipality, {spot: spot, user: req.user});
socketio.sockets.emit('spot.created.' + spot.province, {spot: spot, user: req.user});
socketio.sockets.emit('spot.created.' + spot.community, {spot: spot, user: req.user});
socketio.sockets.emit('spot.created.' + spot.country, {spot: spot, user: req.user});
req.spot = spot;
Feedback.subscribeSpot(req);
Notify.getLocalSubscriptors(spot.municipality,spot.province,spot.community,spot.country,function(subscriptions){
Notify.create(req,null,spot,null,null,null,subscriptions,'spots/'+spot._id,false,'SPOT_CREATED', function(){
res.jsonp(spot);
});
});
}
});
}
};
exports.uploadImage = function(file, fn){
var imagePath = file.path;
gm(imagePath).resize(850, 850).quality(70).noProfile().write('public/uploads/spots/850x850/'+file.name, function (err) {
if (!err) {
gm(imagePath).resize(150, 150).quality(70).noProfile().write('public/uploads/spots/150x150/'+file.name, function (err) {
if (!err) {
if(fn)fn();
}
else{
console.log('Error: '+err);
}
});
}
else{
console.log('Error: '+err);
}
});
};