Market data routing to frontend: Alpaca WebSocket -> Node.js WebSocket -> React Frontend - express

I'm trying to use the websocket example from:
https://github.com/alpacahq/alpaca-trade-api-js/blob/master/examples/websocket_example_datav2.js
In order to connect to the Alpaca V2 data stream. Currently, my stream is working but I'm trying to route my data to the client side using Server Sent Events. My data flow seems like it should be:
Alpaca Data Stream API -> My Node.js server -> React Frontend.
The issue I have is using the DataStream object in the example in order to route the data to the frontend. Since, with the object alone, I don't have any route to subscribe to via Server Sent Events, does this mean that I should also be using either express, socket.io, or ws? Since the all of the ".on_xyz" methods are defined within the DataStream object, I'm not sure how to set up the endpoint properly to allow my frontend to subscribe to it. If anyone knows how to route this datastream information forward it would be greatly appreciated- I'm particularly trying to work with the .onStockQuote method but any of them is fine! I'm simply trying to use Node as an inbetween router so that I don't have to subscribe directly from the frontend (and not use the sdk), because that limits scalability of the API's use.
"use strict";
/**
* This examples shows how to use tha alpaca data v2 websocket to subscribe to events.
* You should use the alpaca api's data_steam_v2, also add feed besides the other parameters.
* For subscribing (and unsubscribing) to trades, quotes and bars you should call
* a function for each like below.
*/
import express from 'express';
const app = express()
const Alpaca = require("#alpacahq/alpaca-trade-api");
const API_KEY = "XYZ_Key";
const API_SECRET = "XYZ_Secret";
const PORT = 3000;
// Add a new message and send it to all subscribed clients
const addMessage = (req, res) => {
const message = req.body;
// Return the message as a response for the "/message" call
res.json(message);
return ;
};
class DataStream {
constructor({ apiKey, secretKey, feed }) {
this.alpaca = new Alpaca({
keyId: apiKey,
secretKey,
feed,
});
const socket = this.alpaca.data_stream_v2;
socket.onConnect(function () {
console.log("Connected");
socket.subscribeForQuotes(["AAPL"]);
// socket.subscribeForTrades(["FB"]);
// socket.subscribeForBars(["SPY"]);
// socket.subscribeForStatuses(["*"]);
});
socket.onError((err) => {
console.log(err);
});
socket.onStockTrade((trade) => {
console.log(trade);
});
socket.onStockQuote((quote) => {
console.log(quote);
});
socket.onStockBar((bar) => {
console.log(bar);
});
socket.onStatuses((s) => {
console.log(s);
});
socket.onStateChange((state) => {
console.log(state);
});
socket.onDisconnect(() => {
console.log("Disconnected");
});
socket.connect();
// unsubscribe from FB after a second
// setTimeout(() => {
// socket.unsubscribeFromTrades(["FB"]);
// }, 1000);
}
}
app.post("/message", addMessage);
let stream = new DataStream({
apiKey: API_KEY,
secretKey: API_SECRET,
feed: "sip",
paper: false,
});
app.listen(PORT, () => {
console.log(`App listening on port ${PORT}`);
});

Related

refetch or poll an external api with fetch in svelte

New to Svelte and am running into some issues.
Currently doing the following in +page.server.js
I would like to poll this API every couple hundred milliseconds, I am unsure how to do that. I have tried using set Interval here to no avail.
export async function load({params}) {
const response = await fetch(
`http://localhost:9595/api/v1/chrysalis/example?uid=${params.uid}`
)
const site = await response.json()
const siteData = site[0]
console.log(siteData)
return {
uid: params.uid,
partitions: siteData.partitions,
zones: siteData.zones,
zTypes: siteData.zTypes,
zStates: siteData.zStates,
zNames: siteData.zNames
}
}
For example, I've built this in next.Js using SWR with refreshInterval: 1.
const {data, error, isLoading} = useSWR(
'http://localhost:9595/api/v1/chrysalis/example',
(url) => {
const searchParams = new URLSearchParams();
searchParams.append("uid", body.uid)
const newUrl = `${url}?${searchParams.toString()}`
const options = {
method: 'GET',
headers: {'Content-Type': 'application/json'},
}
return fetch(newUrl, options).then(res => res.json())
},
{
refreshInterval: 1
}
);
I have also tried to do the following onMount of the +page.svelte but when trying to hit the API from the client I get CORS error.( ran into this before if +page.js was not +page.server.js
let x;
onMount(async () => {
setInterval(async () => {
const response = await fetch(
`http://localhost:9595/api/v1/chrysalis/example?uid=${data.uid}`
)
const site = await response.json()
x = site[0]
console.log(x)
}, 3000)
})
The CORS error results because +page.svelte/+page.js are run in the browser. So you need to proxy the call through a service that allows being called from the browser. (Or relax the CORS restrictions on http://localhost:9595)
You can use SvelteKit itself to proxy the call by creating an internal endpoint. So:
The internal endpoint simply fetches http://localhost:9595/... and returns the results. (You can just forward the response object from fetch())
+page.svelte calls that internal endpoint from setInterval().

Forward decoded POST data to other URL using Express

Some background information:
I have a temperature sensor which is connected via LoRaWAN.
The data from the sensor is encoded, but the manufacturer of the sensor provides a decoder (node app).
I want to show the data in Thingsboard.
Now I was thinking to build a simple express API (I'm new to Express) as a kind of middleware.
The only purpose is to receive the encoded data, decode it (output is in JSON format), and pass it on to Thingsboard.
With the last step I need some help. This is my route:
const decoder = require('../helpers/decoder');
module.exports = app => {
app.post('/', (req, res) => {
const data = {
type: req.body.productType,
payload: req.body.payloadValue
};
const jsonData = decoder.decode(data.type, data.payload);
// jsonData needs to be forwarded/redirected Thingsboard
});
};
Can anyone point me in the right direction, how to pass on the "handled" POST data and pass it on to another URL (Thingsboard HTTP endpoint URL)
Thanks in advance!
💡 This is not the best practice, but if you want to read some data from some route in another route without passing them when the route was called then you can try this code below: 👇
const express = require('express');
const app = express();
app.post('/', (req, res) => {
app.locals['decoder'] = { data: "This is my decoder data"};
res.send('Successfully to add decoder data');
});
app.get('/', (req, res) => {
res.send(app.locals['decoder']);
} )
app.listen(3000, () => {
console.log('Server is up');
});
⚠️ It should be noted that the code above is only an example of how to read data from one route on another route.
Another option:
👨‍🏫 You can create a middleware that is used to decode your payload and use the middleware on each route that requires the results of the decoded payload.
⚠️ But, it should be noted that for this problem, you must leave a payload on each route that requires decoded results, so that it can be decoded.
Updated: Passing Data To External Endpoint
💡 If You want to pass the data to an external endpoint or another website, see code below: 👇
const axios = require('axios');
app.post('/', (req, res) => {
const decoder = { data: "This is my decoder data"};
// method of external-endpoint: e.g: POST
axios.post('https://www.example-website.com/api/v1/example-endpoint', decoder)
.then(response => {
console.log(response);
}).catch(ex => {
console.log(ex);
})
})
You can use that route above in the example express server in addition to this answer.
I hope it's can help you 🙏.

Here-API 401 : "Invalid app_id app_code combination"

I am using an Angular front-end with a Nodejs backend. Im currently proxying all my front-end requests through my express server. However when I make my http request to the Here API I am rejected due to an invalid combination of app_id and app_code.
angular service
import { Injectable } from '#angular/core';
import { HttpClient } from '#angular/common/http'
import { HttpParams } from '#angular/common/http'
#Injectable({
providedIn: 'root'
})
export class GetReqPlaces {
constructor(private http: HttpClient) { }
getPlaces(wLong,sLat,eLong,nLat){
// let obj = {params: {westLong: wLong, southLat: sLat, eastLong:eLong, northLat:nLat }};
let params = new HttpParams().set("westLong" , '-97.783').set("southLat", '30.231').set( "eastLong" , '-97.740').set("northLat", '30.329');
return this.http.get( 'api/find/places', { params : params}).subscribe(res=>console.log(res))
}
}
server.js
const express = require("express")
const bodyParser = require("body-parser")
const cors = require("cors")
const path = require("path")
const app = express();
const request = require("request")
const environment= require('./keys')
app.use(cors());
app.use(bodyParser.urlencoded({extended: true}));
app.use(bodyParser.json());
let reqPath = __dirname.substring(0,__dirname.length-7)
app.use(express.static(path.join(reqPath, '/dist/angular-places-search')));
app.get('/api/find/places', (req, res) => {
let appId = environment.environment.appId;
let appCode = environment.environment.appCode;
let URL= `https://places.cit.api.here.com/places/v1/discover/search?app_id={${appId}}&app_code={${appCode}}&in=${req.query.westLong},${req.query.southLat},${req.query.eastLong},${req.query.northLat}&pretty`;
console.log(URL)
request(URL, function (error, response, body) {
let data={
body:body,
};
console.log(error,response)
res.send(data);
});
});
app.get('/test', (req, res) => res.send('Well this route was a hit! Bada....tsss'));
// CATCH ALL
app.get('*', (req, res) => {
res.sendFile(path.join(reqPath, 'dist/angular-places-search/index.html'));
});
app.listen(4000, () => console.log(`Express server running on port 4000`));
Before this I was running into CORS and request issues but I think I sorted those out. Based on my research on this same error code (In the context of the framework that Im working in), people overwhelmingly suggest to wait for tokens to register with Here API. Waiting two days is enough I think, still doesnt work. Then there is the very popular solution of just scratching the Here freemium and starting a new project, which I did, and which did not solve my issue. Very few things I have 100% certainty on but I did copy my keys correctly and the URL path built is according to the required Here syntax.
If anyone has any insight you will be my Hero, and also the catalyst for my continued learning :D. Happy Sunday!
In addition the incoming message I get through express is :
method: 'GET',
path: '/places/v1/discover/search?app_id=%notmyid%7D&app_code=%normycode%7D&in=-97.783,30.231,-97.740,30.329&pretty'
However i dont know why it is setting the app_id=% instead of using {}, when i console log the URL it is correct, with my app_id and app_code
The %7D is the url encoded value of the symbol } (urlencoding) which is done by most libraries. For using the HERE API you should not enclose the app_id/app_code between {}. They should be provided directly as strings, check the examples

How to send data from redis subscriber to express route

I have a redis pubsub client where publisher is in one file and subscriber is in another file working perfectly
I have 2 controllers a home controller that handles '/' route and a data controller that handles '/data' route
Inside my redis subscriber I want to update the state of a variable that I continuously get from publisher
How do I send this state to both controllers when they do a request
I was doing
app.get('/', (req, res) => {
c = redis.createClient()
c.on("message", (channel, message) => {
// Send data here
})
})
This does not look like a good idea, it is creating a new CLIENT for every request to the '/' endpoint
I want to be able to do
// home controller file
app.get('/', (req, res) => {
res.json(state)
})
// data controller file
app.get('/data', (req, res) => {
res.json(state)
})
How to implement this state
After doing some research, I decided to use Node's native events module to solve this. This example uses ioredis rather than node_redis, but the principle is the same.
First, I instantiate three redis clients. One for regular db work, a publisher and a subscriber
/* redis.js */
const Redis = require('ioredis');
const redis = new Redis();
const publisher = new Redis();
const subscriber = new Redis();
// redis is the defaut export
// publisher and subscriber are "named" exports
const client = (module.exports = redis);
client.publisher = publisher;
client.subscriber = subscriber;
Next we create an EventEmitter in node that will emit an event every time the subscriber receives a message from a channel in redis.
/* emitter.js */
const EventEmitter = require('events');
const { subscriber } = require('./redis');
const eventEmitter = new EventEmitter();
subscriber.subscribe('my-channel', err => {
if (err) { return console.log('Unable to subscribe to my-event channel') };
console.log('Subscription to my-event channel successful');
});
subscriber.on('message', (channel, message) => {
eventEmitter.emit('my-event', message);
});
module.exports = eventEmitter;
Here we have two routes. The first handles a PUT request that sets a field in redis and then publishes a message to a channel with the key of the hash that was updated. The second route handles a GET request that stays open (as an EventSource for an SSE connection for example). It listens to an event from emitter and then sends the data of the updated key from redis
/* route.js*/
const express = require('express');
const redis = require('./redis');
const { publisher } = require('./redis');
const { eventEmitter } = require('./emitter');
const router = express.Router();
router.put('/content', async (req, res) => {
const { key, field, content } = req.body;
try {
await redis.hset(key, field, content);
res.sendStatus(200);
return publisher.publish('my-channel', key);
} catch(err) {
res.status(500).send(err.message);
}
});
router.get('/content-stream', (req, res) => {
res.writeHead(200, {
'Content-Type': 'text/event-stream',
'Cache-Control': 'no-cache',
Connection: 'keep-alive'
});
res.write('\n');
const handleEvent = async key => {
try {
const query = await redis.hgetall(key);
res.write(`data: ${JSON.stringify(query)}\n\n`);
} catch(err) {
console.log('Something went wrong');
}
}
eventEmitter.addListener('my-event', handleEvent);
req.on('close', eventEmitter.removeListener('my-event', handleEvent));
module.exports = router;
This will effectively allow you to avoid instantiating new redis clients with every connection. There may be better ways of doing this, but this is what worked for me.

how to get socketid in socket.io(nodejs)

In my nodejs application, i am using socket.io for sockets connection.
I am configuring my server side code like this
socket.io configuration in separate file.
//socket_io.js
var socket_io = require('socket.io');
var io = socket_io();
var socketApi = {};
socketApi.io = io;
module.exports = socketApi;
below is my server.js file in which i am attaching my socket io to the server like this
var socketApi = require('./server/socket_io');
// Create HTTP server.
const server = http.createServer(app);
// Attach Socket IO
var io = socketApi.io;
io.attach(server);
// Listen on provided port, on all network interfaces.
server.listen(port, () => console.log(`API running on localhost:${port}`));
and then i am using socket.io in my game.js file to emit updated user coins like this.
//game.js
var socketIO = require('../socket_io');
function updateUserCoins(userBet) {
userId = mongoose.Types.ObjectId(userBet.user);
User.findUserWithId(userId).then((user) => {
user.coins = user.coins - userBet.betAmount;
user.save((err, updatedUser) => {
socketIO.io.sockets.emit('user coins', {
userCoins: updatedUser.coins,
});
});
})
}
and then in my client side, i am doing something like this,
socket.on('user coins', (data) => {
this.coins = data.userCoins;
});
but with the above implementation, updating coins of any user, updates all user coins at the client side, since all the clients are listening to the same socket user coins.
To solve the above problem, i know that i have to do something like this,
// sending to individual socketid (private message)
socketIO.io.sockets.to(<socketid>).emit('user coins', {
userCoins: updatedUser.coins,
});
but my concern is that how will get <socketid> with my current implementation.
You can get it by listening to connection to your socket.io server :
io.on('connection', function (socket) {
socket.emit('news', { hello: 'world' });
socket.on('my other event', function (data) {
console.log(data);
});
});
Here you have a socket object ( io.on('connection', function (socket) ) and you can get his id with socket.id
So you'll probably need to wrap your code with the connection listener.
Source of the exemple for the connection listener
Socket object description