I wanted to make the word count streaming with only showing which word I would like to see with Twitter.
So, I made the cords as like below
import java.util.Properties
import org.apache.spark.SparkConf
import org.apache.spark.streaming.StreamingContext
import org.apache.spark.streaming.Seconds
import twitter4j.conf.ConfigurationBuilder
import twitter4j.auth.OAuthAuthorization
import twitter4j.Status
import org.apache.spark.streaming.twitter.TwitterUtils
import org.apache.spark.streaming._
import org.apache.log4j._
import org.apache.spark.streaming.StreamingContext._
import org.apache.spark.streaming.{Seconds, StreamingContext}
import org.apache.spark.streaming.twitter._
import twitter4j.TwitterFactory
import twitter4j.conf.ConfigurationBuilder
import java.util.Properties
import org.apache.spark.storage.StorageLevel
import twitter4j.auth.OAuthAuthorization
val appName = "TwitterData"
val ssc = new StreamingContext(sc, Seconds(10))
val hashTags = "XRP"
val cb = new ConfigurationBuilder
val prop = new Properties()
cb.setDebugEnabled(true).setOAuthConsumerKey("key number").setOAuthConsumerSecret("key number").setOAuthAccessToken("key number").setOAuthAccessTokenSecret("key number")
val bld = cb.build()
val tf = new TwitterFactory(bld)
val twitter = tf.getInstance()
val filters = Array(hashTags).toSeq
val auth = new OAuthAuthorization(bld)
val twitterStream = TwitterUtils.createStream(ssc, Some(auth), filters, StorageLevel.MEMORY_ONLY)
twitterStream.cache()
val lines = twitterStream.map(status => status.getText)
lines.print()
val words = lines.flatMap(_.split(" "))
val pairs = words.map(x => {
if (x == "xrp" || x == "ripple"){
(x, 1)
} else {
}
})
pairs.print()
ssc.start()
It works fine with Spark Streaming with Twitter, but as follow the result, I want to drop all the empty blanks except for the result which I wanted to get.
-------------------------------------------
Time: 1603866040000 ms
-------------------------------------------
#RuleXRP I need 15to25 usd per xrp
RT #Grayscale: 10/27/20 UPDATE: Net Assets Under Management, Holdings per Share, and Market Price per Share for our Investment Products.
T....
-------------------------------------------
Time: 1603866040000 ms
-------------------------------------------
()
()
()
()
()
()
(xrp,1)
()
()
()
...
How can I do that? And if is there any ways to take the only results what I want to get better then my cords, please let me know. I need your help.
I deeply appreciative for your advice.
Thanks
val pairs = words.map(x => {
if (x == "xrp" || x == "ripple"){
(x, 1)
} else {
}
})
this maps your result
instead you could use a filter before mapping it which would reduce your code a little bit:
val pairs = words
.filter(x => x == "xrp" || x == "ripple")
.map(x => (x, 1))
Related
I have created a model which applied Mobilenet V2 for the convolutional base layers in Google colab. Then I converted it by using this command:
path_to_h5 = working_dir + '/Tensorflow_PY_Model/SavedModel.h5'
path_tfjs = working_dir + '/TensorflowJS'
!tensorflowjs_converter --input_format keras \
{path_to_h5} \
{path_tfjs}
I used an image to test classify it on both. In python, I use this code below to do the prediction:
from google.colab import files
from io import BytesIO
from PIL import Image
import matplotlib.pyplot as plt
uploaded = files.upload()
last_uploaded = list(uploaded.keys())[-1]
im = Image.open(BytesIO(uploaded[last_uploaded]))
im = im.resize(size=(224,224))
img = np.array(im)
img = img / 255.0
prediction1 = model.predict(img[None,:,:])
print(prediction1)
That code above returns this array:
[6.1504150e-05 4.8508531e-11 5.1813848e-15 2.1887154e-12 9.9993849e-01
8.4171114e-13 1.4638757e-08 3.4268971e-14 7.5719299e-15 1.0649443e-16]]
After that I try to predict in Javascript with this code below:
async function predict(image) {
var model = await tf.loadLayersModel('./TFJs/model.json');
let predictions = model.predict(preprocessImage(image)).dataSync();
console.log(predictions);
return results;
}
function preprocessImage(image) {
let tensor = tf.browser.fromPixels(image);
const resizedImage = tensor.resizeNearestNeighbor([224,224]);
const batchedImage = resizedImage.expandDims(0);
return batchedImage.toFloat().div(tf.scalar(255)).sub(tf.scalar(1));
}
document.querySelector('input[type="file"]').addEventListener("change", async function () {
if (this.files && this.files[0]) {
img = document.getElementById("uploaded-img");
img.onload = () => {
URL.revokeObjectURL(img.src); // no longer needed, free memory
};
img.src = URL.createObjectURL(this.files[0]);
predictionResult = await predict(model, img);
displayResult(predictionResult);
}
});
However, with the same image as that I used when predicting on Python, it returns this result and it never change no matter I change the image.
Float32Array(10) [0.9489052295684814, 0.0036257198080420494, 0.000009185552698909305,
0.000029705168344662525, 0.04141413792967796, 1.4301890782775217e-9, 0.006003820803016424,
2.8357267645162665e-9, 0.000011812648153863847, 4.0659190858605143e-7]
So how to fix this problem? What more should I do? Thanks in advance for the answers and suggestions!
After I debug some possible causes, I realized that the problem is in this block code:
document.querySelector('input[type="file"]').addEventListener("change", async function () {
if (this.files && this.files[0]) {
img = document.getElementById("uploaded-img");
img.onload = () => {
URL.revokeObjectURL(img.src); // no longer needed, free memory
};
img.src = URL.createObjectURL(this.files[0]);
predictionResult = await predict(model, img);
displayResult(predictionResult);
}
});
Firstly, I wanted to make it automated so it will just instantly display the picked image and predict in a pipeline. But it can't be done, because the src attribute of img would still be the same value as before the whole block executed.
In my case, it executed the whole block until the prediction and result then the uploaded and wrong predicted ones appears altogether. So I finally made a change like adding another button only for predicting and take out the prediction lines from that block and putting them in another function. It works well at the end.
document.querySelector('input[type="file"]').addEventListener("change", async function () {
if (this.files && this.files[0]) {
img = document.getElementById("uploaded-img");
img.onload = () => {
URL.revokeObjectURL(img.src); // no longer needed, free memory
};
img.src = URL.createObjectURL(this.files[0]);
}
});
document.querySelector('#predict-btn').addEventListener("click", async function () {
img = document.getElementById("uploaded-img");
predictionResult = await predict(model, img);
displayResult(predictionResult);
});
Well, I am still curious if I can get these functions into a pipeline process so there would be only one upload button and the rest of works done by system.
it("Provides Station Details", () => {
const result = compareIssueDate('2021-03-01', '2020-03-09');
console.log('Result: ', result)
expect(result).to.have.status(200);
console.log('Tested successfully..!!!')
});
export function compareIssueDate(a: any, b: any) {
const bandA = a.mlIssueDt;
const bandB = b.mlIssueDt;
let comparison = 0;
if (bandA > bandB) {
comparison = -1;
} else if (bandA < bandB) {
comparison = 1;
}
return comparison;
}
Path of compareIssueDate is as: getting-started/src/controllers/hello.controller.ts
Path of app.test.js (where above test is defined) is as: getting-started/test/app.test.js
import { compareIssueDate } from '../src/controllers/hello.controller';
If I use import statement, it gives error as: Cannot use import statement outside a module
const compareIssueDate = require("../src/controllers/hello.controller");
If I use require statement, it gives error as: Cannot find module '../src/controllers/hello.controller
How to access above function inside app.test.js. Please help.
Thanks.
I use react native through firebase database
I have a database creating products each product has a number
I want to take a number and compare it with the product number
And if there is then I want to get a product
the function its give me my correct name but where i use it on render its not found the variable (name)
getAllContact = async key => {
let barCodeData2 = this.props.navigation.state.params.barcodeData
let self = this;
let contactRef = firebase.database().ref()
contactRef.on("value", dataSnapsot => {
if (dataSnapsot.val()) {
let contactResult = Object.values(dataSnapsot.val())
let contactKey = Object.keys(dataSnapsot.val())
contactKey.forEach((value, key) => {
contactResult[key]["key"] = value
})
self.setState({
fname: contactResult.fname,
data: contactResult.sort((a, b) => {
var nameA = a.barcode
var nameB = barCodeData2
const name = a.fname
console.log(`${nameA} What numers issssssss`);
if (nameA == nameB) {
alert(`${name} ........`)
console.log(`${nameA == nameB}is Equqlqlqlql`);
return name
}
}),
})
}
})
}
render() {
let t=this.state.name
alert(`${t} how?`)// is give Not found
// let d = this.props.navigation.state.params.barcodeData
return (
)
}
When you try such a comparison query i.e.
let ref = firebase.firestore();
ref.collection('zoo')
.where("id", "==", myID)
.get()
.then((snapshot) => {
console.log(snap.empty); //this will denote if results are empty
snapshot.forEach(snap => {
console.log(snap.exists); //alternatively this will also tell you if it is empty
})
})
well what you can do is run query based on you product no and if there's a product you will a product if there's none you will get an empty array.
read firebase documentation on queries
https://firebase.google.com/docs/reference/js/firebase.database.Query
Is there a way to apply transforms to multiple keys of an object in Ramda? I am aware this is achievable by R.evolve, but I am interested in knowing if this can be achieved by some modification of lenses.
E.g.:
const data = {
a: "100",
b: "non_numeric_string",
c: "0.5"
}
const numerize = x => +x
const mapping = {
a: numerize,
c: numerize
}
magicFunction(mapping, data)
output:
{
a: 100,
b: "non_numeric_string",
c: 0.5
}
The whole point of a lens is to focus on one part of a data structure. While it is not hard to write something using lensProp to achieve this, I'm don't think it's either very satisfying or a particularly appropriate use of lenses. Here's one Ramda solution:
const magicFunction = (mapping, data) =>
reduce
( (o, [k, fn]) => over (lensProp(k), fn, o)
, data
, toPairs (mapping)
)
const numerize = x => Number (x)
const mapping = {
a: numerize,
c: numerize
}
const data = {a: "100", b: "non_numeric_string", c: "0.5"}
console .log (
magicFunction (mapping, data)
)
<script src="//cdnjs.cloudflare.com/ajax/libs/ramda/0.26.1/ramda.js"></script>
<script> const { lensProp, over, reduce, toPairs } = R </script>
But note that a plain ES6 function does the job just as simply, without using lenses:
const magicFunction = (mapping, data) =>
Object.entries (mapping). reduce
( (o, [k, fn]) => ({...o, [k]: fn (o [k]) })
, data
)
Lenses simply don't gain you much here.
I do have two objects containing keys like
var a = {bar:[1,2], foo:[7,9]}
var b = {bar:[2,2], foo:[3,1]}
I want to get the fallowing results:
var c = {bar:[3,4], foo:[10,10]}
I already have a for logic like:
for (let key in b) {
if (a[key]) {
a[key][0] += b[key][0];
a[key][1] += b[key][1];
}
else a[key] = b[key];
}
But I would like to make this logic in a lodash way. How can I Do it?
You can use create a function that takes n objects, and collects them to an array using rest parameters. Now you can spread the array into _.mergeWith() to combine the objects, and in the customizer function sum the items in the arrays using Array.map() or lodash's _.map() and _.add():
const { mergeWith, isArray, map, add } = _
const fn = (...rest) => _.mergeWith({}, ...rest, (o = [], s) =>
map(s, (n, i) => add(n, o[i]))
)
const a = {bar:[1,2], foo:[7,9]}
const b = {bar:[2,2], foo:[3,1]}
const c = {bar:[3,2], foo:[5,6]}
const d = {bar:[4,2], foo:[5,4]}
const result = fn(a, b, c, d)
console.log(result)
<script src="https://cdnjs.cloudflare.com/ajax/libs/lodash.js/4.17.11/lodash.min.js"></script>
You can also use lodash/fp to create a function that merges all values to a multidimensional array with _.mergeAllWith(), then transpose the arrays using _.zipAll(), and sums each array:
const { rest, flow, mergeAllWith, isArray, head, mapValues, zipAll, map, sum } = _
const fn = rest(flow(
mergeAllWith((o, s) => [...isArray(head(o)) ? o : [o], s]), // combine to a multidimensional array
mapValues(flow(
zipAll,
map(sum)
)),
))
const a = {bar:[1,2], foo:[7,9]}
const b = {bar:[2,2], foo:[3,1]}
const c = {bar:[3,2], foo:[5,6]}
const d = {bar:[4,2], foo:[5,4]}
const result = fn(a, b, c, d)
console.log(result)
<script src='https://cdn.jsdelivr.net/g/lodash#4(lodash.min.js+lodash.fp.min.js)'></script>
You can accomplish this using plain JavaScript with Object.entries, concat and reduce:
const a = { bar: [1,2], foo: [7,9] };
const b = { bar: [2,2], foo: [3,1] };
const entries = Object.entries(a).concat(Object.entries(b));
const result = entries.reduce((accum, [key, val]) => {
accum[key] = accum[key] ? accum[key].map((x, i) => x + val[i]) : val;
return accum;
}, { });
console.log(result);