I have implemented cloudflare on a live website, the website has a socket server that's setup with socket.io and express, everything were working fine before implementing cloudflare
Currently I'm using port: 2053 which i've allowed access to through Laravel forge
socket.js
var app = require('express')();
const fs = require('fs');
var server = require('https').createServer({
key: fs.readFileSync('/etc/nginx/ssl/mywebsite.com/1234/server.key'),
cert: fs.readFileSync('/etc/nginx/ssl/mywebsite.com/1234/server.crt'),
}, app);
var io = require('socket.io')(server, {
cors: {
origin: function(origin, fn) {
if (origin === "http://mywebsite.test" || origin === "https://mywebsite.com") {
return fn(null, origin);
}
return fn('Error Invalid domain');
},
methods: ['GET', 'POST'],
'reconnect': true
},
});
var Redis = require('ioredis');
var redis = new Redis();
redis.subscribe('asset-channel', () => {
console.log('asset-channel: started');
});
redis.on('message', function(channel, message) {
var message = JSON.parse(message);
io.to(message.data.id).emit(channel + ':' +message.event + ':'+ message.data.id, message.data);
});
io.on("connection", (socket) => {
socket.on("join:", (data) => {
socket.join(data.id);
});
socket.on("leave:", (data) => {
socket.leave(data.id);
});
});
server.listen(2053, () => {
console.log('Server is running!');
});
app.js
if (! window.hasOwnProperty('io')) {
// if (
// window.origin === "http://mywebsite.test" ||
// window.origin === "https://mywebsite.com" ||
// window.origin == "https://mywebsite.test"
// ) {
window.io = io.connect(`${window.origin}:2053`);
window.io.on('connection');
// }
}
As mentioned before everything were working fine before implementing cloudflare and i have tried to read some different documentation like:
https://developers.cloudflare.com/cloudflare-one/policies/zero-trust/cors
https://socket.io/docs/v4/handling-cors/
I found many different problems similar online, and tried several solutions but nothing seem to make the socket connection work
Tried to allow all cors like so:
var io = require('socket.io')(server, {
cors: {
origin: "*",
methods: ['GET', 'POST'],
'reconnect': true
},
});
Didn't work either, tried configure some stuff in nginx which didn't work either
Error
Access to XMLHttpRequest at 'https://mywebsite.com:2053/socket.io/?EIO=4&transport=polling&t=NurmHmi' from origin 'https://mywebsite.com' has been blocked by CORS policy: No 'Access-Control-Allow-Origin' header is present on the requested resource.
I think i might have to configure something in the cloudflare dashboard, i just dont know what and my googling skills could not take me to the finish line this time around.
Im not too experienced with sockets so it would be awesome if there are some skilled socket expert who have had this issue before who can guide me in the correct direction? :)
I made it run by adding this to the app.js:
window.io = io.connect(`${window.origin}:2053`, { transports: ["websocket"] });
Apparently it will try to use polling instead of websocket.
Related
I'm trying to build a scraping script to get a bunch of Discord server's total members. I actually did that with Puppeteer like below but I think my IP address has been banned because I'm getting "Invite Invalid" error from Discord even though invite links are working.
My question is that does Discord have APIs to get any server's total member count? Or is there any 3rd party library for that purpose? Or any other method?
const puppeteer = require('puppeteer')
const discordMembers = async ({ server, browser }) => {
if (!server) return
let totalMembers
const page = await browser.newPage()
try {
await page.goto(`https://discord.com/invite/${server}`, {
timeout: 3000
})
const selector = '.pill-qMtBTq'
await page.waitForSelector(selector, {
timeout: 3000
})
const totalMembersContent = await page.evaluate(selector => {
return document.querySelectorAll(selector)[1].textContent
}, selector)
if (totalMembersContent) {
totalMembers = totalMembersContent
.replace(/ Members/, '')
.replace(/,/g, '')
totalMembers = parseInt(totalMembers)
}
} catch (err) {
console.log(err.message)
}
await page.close()
if (totalMembers) return totalMembers
}
const asyncForEach = async (array, callback) => {
for (let i = 0; i < array.length; i++) {
await callback(array[i], i, array)
}
}
const run = async () => {
const browser = await puppeteer.launch({
headless: true,
args: ['--no-sandbox']
})
const servers = ['tQp4pSE', '3P5K3dzgdB']
await asyncForEach(servers, async server => {
const members = await discordMembers({ server, browser })
console.log({ server, members })
// result
// { server: 'tQp4pSE', members: 57600 }
// { server: '3P5K3dzgdB', members: 159106 }
})
await browser.close()
}
run()
Update: Mar 22, 2022
Thanks for #Vaviloff's answer we can actually access Discord's private APIs but the problem is it's only accessible over browser. I'm getting Request failed with status code 400 issue from Axios. Is it a CORS issue? How do we get the results in a Node.js app?
const axios = require('axios')
const discordMembers = async ({ server }) => {
try {
const apiResult = await axios({
data: {},
method: 'get',
url: `https://discord.com/api/v9/invites/${server}?with_counts=true&with_expiration=true`
})
console.log(apiResult)
} catch (err) {
console.log(err)
}
}
discordMembers({ server: 'tQp4pSE' })
A lot of modern web applications have their own internal APIs. Oftentimes you can spot frontend making requests to it, by using Networking tab in Devtools (filter by Fetch/XHR type):
Such API endpoints can change any time of course, but usually the last for a long time and is a rather convenient way of scraping
Currently Discord uses this URL for basic instance description:
https://discord.com/api/v9/invites/tQp4pSE?with_counts=true&with_expiration=true
By accessing it you get the desired data:
Update
To make your code work don't send any data in the request:
const apiResult = await axios({
method: 'get',
url: `https://discord.com/api/v9/invites/${server}?with_counts=true&with_expiration=true`
})
I have a Nest-Service with the following main.ts:
async function bootstrap() {
if (!!environment.production) {
const app = await NestFactory.create(AppModule, {
httpsOptions: {
key: fs.readFileSync(environment.ssl.SSL_KEY_PATH),
cert: fs.readFileSync(environment.ssl.SSL_CERT_PATH)
},
});
app.useWebSocketAdapter(new WsAdapter(app));
app.enableCors();
await app.listen(3077);
} else {
const app = await NestFactory.create(AppModule);
app.useWebSocketAdapter(new WsAdapter(app));
app.enableCors();
await app.listen(3077);
}
}
bootstrap();
And two Gateways within the Service:
#WebSocketGateway(3078)
export class ItemsGateway implements OnGatewayConnection, OnGatewayDisconnect { ... }
#WebSocketGateway(3079)
export class UnitsGateway implements OnGatewayConnection, OnGatewayDisconnect { ... }
Without SSL this is working, but when I use the prod mode I can´t establish a secure connection to domain.tld:3078 and :3079.
How can I get the service to listen on all 3 Ports? I think there is the problem, because certs are only attached to the Server listening on Port: 3077, where all my REST-API stuff goes.
Thx, Dom
Edit: This also worked as there was just on WebsocketServer on the same port as the API -> 3077.
Edit 2:
I also tried this, but then comes the error that address is in use on the second attempt to create() a server:
async function bootstrap() {
if (!!environment.production) {
const httpsOptions = {
key: fs.readFileSync(environment.ssl.SSL_KEY_PATH),
cert: fs.readFileSync(environment.ssl.SSL_CERT_PATH)
};
const server = express();
const app = await NestFactory.create(
AppModule,
new ExpressAdapter(server)
);
app.useWebSocketAdapter(new WsAdapter(app));
app.enableCors();
await app.init();
https.createServer(httpsOptions, server).listen(environment.app.port);
https.createServer(httpsOptions, server).listen(environment.websocketPorts.units);
https.createServer(httpsOptions, server).listen(environment.websocketPorts.items);
} else {
const app = await NestFactory.create(AppModule);
app.useWebSocketAdapter(new WsAdapter(app));
app.enableCors();
await app.listen(environment.app.port);
}
}
bootstrap();
You need to .create() a separate app for each port on which you listen for wss connections.
I have a node/graphql server running on sitename.com:3333
I've created another server that I'm running on sitename.com:3334
I'm able to make requests to the server at sitename.com:3333 from sitename.com as well as subdomain.sitename.com
But if I try to connect to sitename.com:3334 (just a different port) from subdomain.sitename.com it gives me a cors error:
Cross-Origin Request Blocked: The Same Origin Policy disallows reading the remote resource at https://sitename.com:3334/graphql. (Reason: CORS request did not succeed)
I've opened the ports in the firewall and setup ssl on the server and client.
Please help!
Client code is below:
import { ApolloClient } from 'apollo-client'
import { withClientState } from 'apollo-link-state'
import { HttpLink } from 'apollo-link-http'
import { Agent } from 'https'
import fs from 'fs'
import { InMemoryCache } from 'apollo-cache-inmemory'
import { setContext } from 'apollo-link-context'
import { onError } from 'apollo-link-error'
import { ApolloLink } from 'apollo-link'
import decode from 'jwt-decode'
import history from '../history'
import Cookies from 'universal-cookie'
import {
APP,
AUTH,
CLIENT_AUTH_REQUEST_TYPE,
CLIENT_AUTHENTICATION_METHOD,
JWT,
VERSION
} from '../environment'
import https from 'https'
import { defaults, resolvers } from '../api'
import { createUploadLink } from 'apollo-upload-client'
const { CONSTANTS: { UNAUTHORIZED, FORBIDDEN } = {} } = APP
const cookies = new Cookies()
const opts = {
credentials: 'same-origin',
headers: {
'frontend-version': VERSION,
[AUTH.STRATEGIES.CLIENT.AUTH_HEADER]: CLIENT_AUTH_REQUEST_TYPE
}
}
const useLocalStorage = CLIENT_AUTHENTICATION_METHOD.LOCAL_STORAGE
process.env['NODE_TLS_REJECT_UNAUTHORIZED'] = 0
// const apolloCache = new InMemoryCache();
const apolloCache = new InMemoryCache({
// dataIdFromObject: e => `${e.__typename}_${e.id}` || null // eslint-
disable-line no-underscore-dangle
})
// const watchedMutationLink = new WatchedMutationLink(apolloCache,
watchedMutations);
const stateLink = withClientState({
cache: apolloCache,
defaults,
resolvers
})
const uploadLink = createUploadLink({
// uri: 'http://localhost:3333/graphql',
uri: 'https://demo.MYSITE.in:3334/graphql',
fetchOptions: {
agent: new https.Agent()
}
})
const httpLink = new HttpLink({
uri: 'https://demo.MYSITE.in:3334/graphql',
...opts
})
const TOKEN_NAME = 'x-connector-token'
const authLink = new ApolloLink((operation, forward) => {
operation.setContext(({ headers = {} }) => {
const token = cookies.get('token')
if (token) {
headers = { ...headers, 'x-connector-token': token }
}
return { headers }
})
return forward(operation)
})
const errorLink = onError(({ graphQLErrors, networkError }) => {
if (graphQLErrors && graphQLErrors.filter(e => e).length > 0) {
graphQLErrors.map(({ message = '', status = 200 }) => {
if (UNAUTHORIZED === message || status === 401) {
if (
history &&
history.location &&
history.location.pathname !== '/login'
) {
history.push('/login')
}
}
if (FORBIDDEN === message || status === 403) {
history.push(`/error-page/403`)
}
return null
})
}
if (networkError && networkError.statusCode === 401) {
// eslint-disable-next-line
history.push('/login')
}
if (networkError && networkError.statusCode === 403) {
// Do something
}
if (networkError && networkError.statusCode === 400) {
// Do something
}
if (networkError && networkError.statusCode >= 500) {
// eslint-disable-next-line
history.push(`/error-page/${networkError.statusCode}`)
}
})
let links = [errorLink, stateLink, httpLink]
links = [
errorLink,
stateLink,
// afterwareLink,
// authMiddlewareLink,
authLink,
// watchedMutationLink,
// httpLink,
uploadLink
]
const link = ApolloLink.from(links)
export default new ApolloClient({
link,
cache: apolloCache,
connectToDevTools: true,
// opts: {
// agent
// },
fetchOptions: {
agent: new https.Agent()
// rejectUnauthorized: false
},
defaultOptions: {
query: {
errorPolicy: 'all'
}
},
onError: ({ networkError, graphQLErrors }) => {}
})
Server Code:
const app = express();
// tried this too
const corsOptions = {
origin: 'https://demo.MYSITE.in',
}
// also tried app.use(cors)
app.use(cors({
'allowedHeaders': ['Content-Type'],
'origin': '*',
'preflightContinue': true
}));
app.use(helmet());
// app.use(cors());
The browser will not make requests to a server of different origin than the origin that the web page itself came from (and a different port constitutes a different origin) unless you specifically enable that request for that new origin on your server. This is a garden variety CORs issue of which there are millions of posts and articles on how to handle. Since you show NONE of your code in your question, we can't recommend a specific code fix to your code.
Your server needs to support the specific CORS request you are trying to do. If you're using Express, then the CORS module does a lot of the work for you if properly implemented. CORS is there for your site's protection so Javascript in other people's web pages run from a browser can't arbitrarily use your APIs so be careful in exactly what you open up to CORS requests.
And, since this seems like a new issue to you, I would strongly suggest you read and learn about what CORs is and how it works.
Also, note that there are "Simple" CORS requests and "Pre-flighted Requests" (non-simple requests) and more work is required to enable Pre-flighted requests. The browser decides whether a given cross origin request is simple or requires pre-flight based on the exact parameters of the request and your server has to do more things to enable pre-flighted requests.
Edit: It definitely isn't actually CORS. It is like it is just ignoring my attempts to write the tokens into cookies... I am also having trouble getting it to throw an error that I can find useful... I will keep throwing darts at the wall until one makes sense.
I have a graphql-yoga server with an apollo client frontend. I am using cookie-parser on the server to store Microsoft Graph authentication tokens in the browser. I am getting an error that shows up as CORS but I can't figure out why.
Access to fetch at 'https://api.mydomain.store/' from origin 'https://mydomain.store' has been blocked by CORS policy: No 'Access-Control-Allow-Origin' header is present on the requested resource. If an opaque response serves your needs, set the request's mode to 'no-cors' to fetch the resource with CORS disabled.
Error: Network error: Failed to fetch
This is strange because I can make other queries to the server with no CORS issues. I thought it had to do with the way I was setting the options on the cookies, but I think that I am doing that correctly:
//saveCookies
function saveValuesToCookie(token: any, res: Response) {
// Parse the identity token
const user: any = jwt.decode(token.token.id_token);
const isDev = process.env.NODE_ENV === 'development';
console.log('isDev: ', isDev);
const cookieOpts: CookieOptions = {
domain: isDev ? 'localhost' : 'mydomain.store',
maxAge: 1000 * 60 * 60 * 24 * 365,
httpOnly: true,
sameSite: isDev ? 'lax' : true,
secure: isDev ? false : true,
};
// Save the access token in a cookie
res.cookie('graph_access_token', token.token.access_token, cookieOpts);
//save the users email to a cookie
res.cookie('graph_user_email', user.preferred_username, cookieOpts);
// Save the refresh token in a cookie
res.cookie('graph_refresh_token', token.token.refresh_token, cookieOpts);
res.cookie('graph_user_id', user.oid, cookieOpts);
// Save the token expiration tiem in a cookie
res.cookie(
'graph_token_expires',
token.token.expires_at.getTime(),
cookieOpts
);
}
Based on the resources I have seen so far, this seems correct with current browser rules.
So I look at where I am calling the mutation:
//apollo react authorize mutation
const AUTHORIZE_MUTATION = gql`
mutation($code: String!) {
authorize(code: $code) {
id
email
displayName
studentFaculty
}
}
`;
function AuthorizePage() {
const router = useRouter();
const { code } = router.query; //grab the code from the query params
const [authorize, { loading }] = useMutation(AUTHORIZE_MUTATION, {
variables: { code },
// refetchQueries: [{ query: ME_QUERY }],
onError: (error) => {
console.error(error); //**this is where the CORS error originates**
return Router.push('/');
},
update: (_, { data }) => {
console.log(data); //** never gets this far **
},
});
useEffect(() => {
if (router.query.code) {
authorize();
}
}, [router.query]);
if (loading) return <Loading />;
return <Loading />;
}
I am at a loss. Can anyone spot what is wrong or point me towards other places to look? The hardest part for me is that the code works perfectly on localhost serving client on port 3000 and server on port 4000.
I don't think this error is a genuine CORS issue because of the fact that I can make unauthenticated queries no problem.
Thanks for your time!
I'm developing a Vue.js application which has only frontend (no server) and send a lot of requests to different APIs. The originally quite simple app became more complex. And there are problems with some APIs, because browsers do not accept the responses due to CORS. That is why I'm trying to test, if I can migrate the app to Nuxt.js.
My approach is as follows (inspired by this comment), but I expect, that there is probably a better way to send the requests from the client over the server.
pages/test-page.vue
methods: {
async sendRequest(testData) {
const response = await axios.post('api', testData)
// Here can I use the response on the page.
}
}
nuxt.config.js
serverMiddleware: [
{ path: '/api', handler: '~/server-middleware/postRequestHandler.js' }
],
server-middleware/postRequestHandler.js
import axios from 'axios'
const configs = require('../store/config.js')
module.exports = function(req, res, next) {
let body = ''
req.on('data', (data) => {
body += data
})
req.on('end', async () => {
if (req.hasOwnProperty('originalUrl') && req.originalUrl === '/api') {
const parsedBody = JSON.parse(body)
// Send the request from the server.
const response = await axios.post(
configs.state().testUrl,
body
)
req.body = response
}
next()
})
}
middleware/test.js (see: API: The Context)
export default function(context) {
// Universal keys
const { store } = context
// Server-side
if (process.server) {
const { req } = context
store.body = req.body
}
}
pages/api.vue
<template>
{{ body }}
</template>
<script>
export default {
middleware: 'test',
computed: {
body() {
return this.$store.body
}
}
}
</script>
When the user makes an action on the page "test", which will initiate the method "sendRequest()", then the request "axios.post('api', testData)" will result in a response, which contains the HTML code of the page "api". I can then extract the JSON "body" from the HTML.
I find the final step as suboptimal, but I have no idea, how can I send just the JSON and not the whole page. But I suppose, that there must be a much better way to get the data to the client.
There are two possible solutions:
Proxy (see: https://nuxtjs.org/faq/http-proxy)
API (see: https://medium.com/#johnryancottam/running-nuxt-in-parallel-with-express-ffbd1feef83c)
Ad 1. Proxy
The configuration of the proxy can look like this:
nuxt.config.js
module.exports = {
...
modules: [
'#nuxtjs/axios',
'#nuxtjs/proxy'
],
proxy: {
'/proxy/packagist-search/': {
target: 'https://packagist.org',
pathRewrite: {
'^/proxy/packagist-search/': '/search.json?q='
},
changeOrigin: true
}
},
...
}
The request over proxy can look like this:
axios
.get('/proxy/packagist-search/' + this.search.phpLibrary.searchPhrase)
.then((response) => {
console.log(
'Could get the values packagist.org',
response.data
)
}
})
.catch((e) => {
console.log(
'Could not get the values from packagist.org',
e
)
})
Ad 2. API
Select Express as the project’s server-side framework, when creating the new Nuxt.js app.
server/index.js
...
app.post('/api/confluence', confluence.send)
app.use(nuxt.render)
...
server/confluence.js (simplified)
const axios = require('axios')
const config = require('../nuxt.config.js')
exports.send = function(req, res) {
let body = ''
let page = {}
req.on('data', (data) => {
body += data
})
req.on('end', async () => {
const parsedBody = JSON.parse(body)
try {
page = await axios.get(
config.api.confluence.url.api + ...,
config.api.confluence.auth
)
} catch (e) {
console.log('ERROR: ', e)
}
}
res.json({
page
})
}
The request over API can look like this:
this.$axios
.post('api/confluence', postData)
.then((response) => {
console.log('Wiki response: ', response.data)
})
.catch((e) => {
console.log('Could not update the wiki page. ', e)
})
Now with nuxtjs3 :
nuxtjs3 rc release
you have fetch or useFetch no need to import axios or other libs, what is great, automatic parsing of body, automatic detection of head
fetching data
you have middleware and server api on same application, you can add headers on queries, hide for example token etc
server layer
a quick example here in vue file i call server api :
const { status } = await $fetch.raw( '/api/newsletter', { method: "POST", body: this.form.email } )
.then( (response) => ({
status: response.status,
}) )
.catch( (error) => ({
status: error?.response?.status || 500,
}) );
it will call a method on my server, to init the server on root directory i created a folder name server then api, and a file name newsletter.ts (i use typescript)
then in this file :
export default defineEventHandler(async (event) => {
const {REST_API, MAILINGLIST_UNID, MAILINGLIST_TOKEN} = useRuntimeConfig();
const subscriber = await readBody(event);
console.log("url used for rest call" + REST_API);
console.log("token" + MAILINGLIST_TOKEN);
console.log("mailing list unid" + MAILINGLIST_UNID);
let recipientWebDTO = {
email: subscriber,
subscriptions: [{
"mailingListUnid": MAILINGLIST_UNID
}]
};
const {status} = await $fetch.raw(REST_API, {
method: "POST",
body: recipientWebDTO,
headers: {
Authorization: MAILINGLIST_TOKEN,
},
}).then((response) => ({
status: response.status,
}))
.catch((error) => ({
status: error?.response?.status || 500,
}));
event.res.statusCode = status;
return "";
})
What are the benefits ?
REST_API,MAILING_LIST_UNID, MAILING_LIST_TOKEN are not exposed on
client and even file newsletter.ts is not available on debug browser.
You can add log only on server side You event not expose api url to avoid some attacks
You don't have to create a new backend just to hide some criticals token or datas
then it is up to you to choose middleware route or server api. You don't have to import new libs, h3 is embedded via nitro with nuxtjs3 and fetch with vuejs3
for proxy you have also sendProxy offered by h3 : sendProxy H3
When you build in dev server and client build in same time(and nothing to implement or configure in config file), and with build to o, just don deploy your project in static way (but i think you can deploy front in static and server in node i don't know)