my environment:
ubuntu 18.04
rtx 2080ti
cuda 10.1
node v12.16.3
tfjs 1.7.4
the saved_model is efficientdet-d0,
and the step of inference is in inference step
for parsing image data with js,i convert img.png to img.jpg,and the result of saved_model is same with saved_model result
the command convert saved_model to tfjs_graph_model is
tensorflowjs_converter --input_format=tf_saved_model /tmp/saved_model ~/DATA/http_models/specDetection/
and my test code is
var tfc = require("#tensorflow/tfjs-converter");
var tf = require("#tensorflow/tfjs-core");
var jpeg_js = require("jpeg-js");
var fs = require("fs");
async function loadModel() {
var modelUrl = "http://localhost:8000/model.json"
var model = await tfc.loadGraphModel(modelUrl);
return model;
}
async function detect() {
var model = await loadModel();
var img = fs.readFileSync("~/SRC/automl_test/efficientdet/img.jpg");
const input = jpeg_js.decode(img,{useTArray:true,formatAsRGBA:false});
const batched = tf.tidy(() => {
const img = tf.browser.fromPixels(input);
// Reshape to a single-element batch so we can pass it to executeAsync.
return img.expandDims(0);
});
const result = await model.executeAsync({'image_arrays:0':batched},['detections:0']);
console.log(result);
}
detect();
when detect object in img.jpg with my test code,nothing detected --- the size of result is 0
what do i do to sovle this problem?
thanks for any cue
edit:
code 1:
var img = fs.readFileSync("~/DATA/http_models/specDetection/test.jpg");
var dataJpegJs = jpeg_js.decode(img,{useTArray:true,formatAsRGBA:false})
var batched = tf.browser.fromPixels({data:dataJpegJs.data, width: dataJpegJs.width, height:dataJpegJs.height},3);
batched = batched.slice([0,0,0],[-1,-1,3]);
var result = await model.executeAsync({'image_arrays:0':batched.expandDims(0)},['detections:0']);
result = tf.slice(result,[0,0,1],[1,-1,4]);
code 2:
var img = fs.readFileSync("~/DATA/http_models/specDetection/test.jpg");
var dataJpegJs = jpeg_js.decode(img,{useTArray:true,formatAsRGBA:true})
var batched = tf.browser.fromPixels({data:dataJpegJs.data, width: dataJpegJs.width, height:dataJpegJs.height},4);
batched = batched.slice([0,0,0],[-1,-1,3]);
var result = await model.executeAsync({'image_arrays:0':batched.expandDims(0)},['detections:0']);
result = tf.slice(result,[0,0,1],[1,-1,4]);
code 1 got a bad result and code 2 got a correct result.
code 2 decode jpg with formatAsRGBA:true,and set numChannels=4 in tf.browser.fromPixels. jpeg-js must decode jpg to RGBA to work correctly.
i think it is a bug of jpeg-js.or i am not familiar with jpg encoding?
The tensor is not well generated. fromPixels is mostly used to get a tensor from an htmlImageElement. Printing a summary of the tensor and compare it with the one generated for python can suffice to tell that.
Is there an issue with jpeg-js ?
First we need to know how the imageData works. An image Data pixel is a 4 numerical values R, G, B, A. When using the data decoded by jpeg_js.decode as argument of tf.browser.fromPixel with 3 channels (formatAsRGBA:false), it is considered as an image data. Let's consider the data [a, b, c, d, e, f] = jpeg_js.decode("path", {formatAsRGBA:false}) and the tensor t created from it
t = tf.browser.fromPixels({data, width: 2, height: 1}). How it is interpreted ? tf.browser.fromPixels, will create an ImageDate of height: 1 and of width: 2. Consequently, the imageData will be of size 1 * 2 * 4 (instead of 1 * 2 * 3) and has all its values set to 0. Then it will copy the data decoded to the imageData. So imageData = [a, b, c, d, e, f, 0, 0].
As a result, the slice (t.slice([0, 0, 0], [-1, -1, 3]) will be [a, b, c, e, f, 0].
Neither is jpeg_js the issue, nor tf.browser.fromPixels. This is how imageData works
What can be done ?
keep the alpha channel of the decoded image formatAsRGBA:true
Instead of using tf.browser.fromPixels, use directly tf.tensor to create the tensor
const img = tf.tensor(input.data, [input.height, input.width, 3])
Another option is to usetensorflow-node. And tf.node.decodeImage can decode an image from a tensor.
const img = fs.readFileSync("path/of/image");
const tensor = tf.node.decodeImage(img)
// use the tensor for prediction
Unlike jpeg-js that works only for image in jpeg encoding format, it can decode a wider range of images
Related
I have a model.json generated from tensorflow via tensorflow.js coverter
In the original implementation of model in tensorflow in python, it is built like this:
model = models.Sequential([
base_model,
layers.Dropout(0.2),
layers.Flatten(),
layers.Dense(128, activation='relu'),
layers.Dense(num_classes)
])
In tensorflow, the probability can be generated by score = tf.nn.softmax(predictions[0]), according to the tutorial on official website.
How do I get this probability in tensorflow.js?
I have copied the codes template as below:
$("#predict-button").click(async function () {
if (!modelLoaded) { alert("The model must be loaded first"); return; }
if (!imageLoaded) { alert("Please select an image first"); return; }
let image = $('#selected-image').get(0);
// Pre-process the image
console.log( "Loading image..." );
let tensor = tf.browser.fromPixels(image, 3)
.resizeNearestNeighbor([224, 224]) // change the image size
.expandDims()
.toFloat()
// RGB -> BGR
let predictions = await model.predict(tensor).data();
console.log(predictions);
let top5 = Array.from(predictions)
.map(function (p, i) { // this is Array.map
return {
probability: p,
className: TARGET_CLASSES[i] // we are selecting the value from the obj
};
}).sort(function (a, b) {
return b.probability - a.probability;
}).slice(0, 2);
console.log(top5);
$("#prediction-list").empty();
top5.forEach(function (p) {
$("#prediction-list").append(`<li>${p.className}: ${p.probability.toFixed(6)}</li>`);
});
How should I modify the above code?
The output is just the same as the value of variable 'predictions':
Float32Array(5)
0: -2.5525975227355957
1: 7.398464679718018
2: -3.252196788787842
3: 4.710395812988281
4: -4.636396408081055
buffer: (...)
byteLength: (...)
byteOffset: (...)
length: (...)
Symbol(Symbol.toStringTag): (...)
__proto__: TypedArray
0: {probability: 7.398464679718018, className: "Sunflower"}
1: {probability: 4.710395812988281, className: "Rose"}
length: 2
__proto__: Array(0)
Please help!!!
Thanks!
In order to extract the probabilities from the logits of the model using a softmax function you can do the following:
This is the array of logits that are also the predictions you get from the model
const logits = [-2.5525975227355957, 7.398464679718018, -3.252196788787842, 4.710395812988281, -4.636396408081055]
You can call tf.softmax() on the array of values
const probabilities = tf.softmax(logits)
Result:
[0.0000446, 0.9362511, 0.0000222, 0.0636765, 0.0000056]
Then if you wanted to get the index with the highest probability you can make use of tf.argMax():
const results = tf.argMax(probabilities).dataSync()[0]
Result:
1
Edit
I am not too familiar with jQuery so this might not be correct. But here is how I would get the probabilities of the outputs in descending order:
let probabilities = tf.softmax(predictions).dataSync();
$("#prediction-list").empty();
probabilities.forEach(function(p, i) {
$("#prediction-list").append(
`<li>${TARGET_CLASSES[i]}: ${p.toFixed(6)}</li>`
);
});
For the pre-trained model in python we can reset input/output shapes:
from tensorflow import keras
# Load the model
model = keras.models.load_model('models/generator.h5')
# Define arbitrary spatial dims, and 3 channels.
inputs = keras.Input((None, None, 3))
# Trace out the graph using the input:
outputs = model(inputs)
# Override the model:
model = keras.models.Model(inputs, outputs)
The source code
I'm trying to do the same in TFJS:
// Load the model
this.model = await tf.loadLayersModel('/assets/fast_srgan/model.json');
// Define arbitrary spatial dims, and 3 channels.
const inputs = tf.layers.input({shape: [null, null, 3]});
// Trace out the graph using the input.
const outputs = this.model.apply(inputs) as tf.SymbolicTensor;
// Override the model.
this.model = tf.model({inputs: inputs, outputs: outputs});
TFJS does not support one of the layers in the model:
...
u = keras.layers.Conv2D(filters, kernel_size=3, strides=1, padding='same')(layer_input)
u = tf.nn.depth_to_space(u, 2) # <- TFJS does not support this layer
u = keras.layers.PReLU(shared_axes=[1, 2])(u)
...
I wrote my own:
import * as tf from '#tensorflow/tfjs';
export class DepthToSpace extends tf.layers.Layer {
constructor() {
super({});
}
computeOutputShape(shape: Array<number>) {
// I think the issue is here
// because the error occurs during initialization of the model
return [null, ...shape.slice(1, 3).map(x => x * 2), 32];
}
call(input): tf.Tensor {
const result = tf.depthToSpace(input[0], 2);
return result;
}
static get className() {
return 'TensorFlowOpLayer';
}
}
Using the model:
tf.tidy(() => {
let img = tf.browser.fromPixels(this.imgLr.nativeElement, 3);
img = tf.div(img, 255);
img = tf.expandDims(img, 0);
let sr = this.model.predict(img) as tf.Tensor;
sr = tf.mul(tf.div(tf.add(sr, 1), 2), 255).arraySync()[0];
tf.browser.toPixels(sr as tf.Tensor3D, this.imgSrCanvas.nativeElement);
});
but I get the error:
Error: Input 0 is incompatible with layer p_re_lu: expected axis 1 of input shape to have value 96 but got shape 1,128,128,32.
The pre-trained model was trained with 96x96 pixels images. If I use the 96x96 image, it works. But if I try to use other sizes (for example 128x128), It doesn't work. In python, we can easily reset input/output shapes. Why it doesn't work in JS?
To define a new model from the layers of the previous model, you need to use tf.model
this.model = tf.model({inputs: inputs, outputs: outputs});
I tried to debug this class:
import * as tf from '#tensorflow/tfjs';
export class DepthToSpace extends tf.layers.Layer {
constructor() {
super({});
}
computeOutputShape(shape: Array<number>) {
return [null, ...shape.slice(1, 3).map(x => x * 2), 32];
}
call(input): tf.Tensor {
const result = tf.depthToSpace(input[0], 2);
return result;
}
static get className() {
return 'TensorFlowOpLayer';
}
}
and saw: when I do not try to rewrite the size, the computeOutputShape, method works only twice, and it works 4 times when I try to reset inputs/outputs. Well, then I opened the model's JSON file and changed inputs from [null, 96, 96, 32] to [null, 128, 128, 32] and removed these lines:
// Define arbitrary spatial dims, and 3 channels.
const inputs = tf.layers.input({shape: [null, null, 3]});
// Trace out the graph using the input.
const outputs = this.model.apply(inputs) as tf.SymbolicTensor;
// Override the model.
this.model = tf.model({inputs: inputs, outputs: outputs});
And now it works with 128x128 images. It looks like the piece of code above, adds the layers instead of rewriting them.
I am using p5 to return the vector path of a drawn line. All the vectors in the line are pushed into an array that holds all the vectors. I'm trying to use this as a tensor but I keep getting an error saying
Error when checking model input: the Array of Tensors that you are passing to your model is not the size the model expected. Expected to see 1 Tensor(s), but instead got the following list of Tensor(s):
When I opened the array on the dev tool, each vector was printed like this:
0: Vector {p5: p5, x: 0.5150300601202404, y: -0.25450901803607207, z: 0}
could it be the p5 text in the vector array that's giving me the error? Here's my model and fit code:
let vectorpath = []; //vector path array
// model, setting layers till next '-----'
const model = tf.sequential();
model.add(tf.layers.dense({units: 4, inputShape: [2, 2], activation: 'sigmoid'}));
model.add(tf.layers.dense({units: 2, activation: 'sigmoid'}));
console.log(JSON.stringify(model.outputs[0].shape));
model.weights.forEach(w => {
console.log(w.name, w.shape);
});
// -----
//this is under the draw function so it is continually updated
const labels = tf.randomUniform([0, 1]);
function onBatchEnd(batch, logs) {
console.log('Accuracy', logs.acc);
}
model.fit(vectorpath, labels, {
epochs: 5,
batchSize: 32,
callbacks: {onBatchEnd}
}).then(info => {
console.log('Final accuracy', info.history.acc);
});
What could be causing the error? and how can I fix it?
The question's pretty vague but I'm really just not sure.
I trained my model using Keras in Python and I converted my model to a tfjs model to use it in my webapp. I also wrote a small prediction script in python to validate my model on unseen data. In python it works perfectly, but when I'm trying to predict in my webapp it goes wrong.
This is the code I use in Python to create tensors and predict based on these created tensors:
input_dict = {name: tf.convert_to_tensor([value]) for name, value in sample_v.items()}
predictions = model.predict(input_dict)
classes = predictions.argmax(axis=-1)
In TFJS however it seems I can't pass a dict (or object) to the predict function, but if I write code to convert it to a tensor array (like I found on some places online), it still doesn't seem to work.
Object.keys(input).forEach((k) => {
input[k] = tensor1d([input[k]]);
});
console.log(Object.values(input));
const prediction = await model.executeAsync(Object.values(input));
console.log(prediction);
If I do the above, I get the following error: The shape of dict['key_1'] provided in model.execute(dict) must be [-1,1], but was [1]
If I then convert it to this code:
const input = { ...track.audioFeatures };
Object.keys(input).forEach((k) => {
input[k] = tensor2d([input[k]], [1, 1]);
});
console.log(Object.values(input));
I get the error that some dtypes have to be int32 but are float32. No problem, I can set the dtype manually:
const input = { ...track.audioFeatures };
Object.keys(input).forEach((k) => {
if (k === 'int_key') {
input[k] = tensor2d([input[k]], [1, 1], 'int32');
} else {
input[k] = tensor2d([input[k]], [1, 1]);
}
});
console.log(Object.values(input));
I still get the same error, but if I print it, I can see the datatype is set to int32.
I'm really confused as to why this is and why I can't just do like python and just put a dict (or object) in TFJS, and how to fix the issues I'm having.
Edit 1: Complete Prediction Snippet
const model = await loadModel();
const input = { ...track.audioFeatures };
Object.keys(input).forEach((k) => {
if (k === 'time_signature') {
input[k] = tensor2d([parseInt(input[k], 10)], [1, 1], 'int32');
} else {
input[k] = tensor2d([input[k]], [1, 1]);
}
});
console.log(Object.values(input));
const prediction = model.predict(Object.values(input));
console.log(prediction);
Edit 2: added full errormessage
How do I mutate value of a tensor in Tensorflow.js? For example if I have a tensor created like this:
const a = tf.tensor1d([1,2,3,4])
How do I change the value of the third element of the tensor? I know that tensors are immutable and variables are mutable.
Doing this: const a = tf.variable(tf.tensor1d([1,2,3,4])) doesn't seem to solve the problem. I cannot do:
const a = a[0].assign(5)
I am able to do this in python tensorflow like this:
a = tf.Variable([1,2,3,4])
a = a[0].assign(100)
with tf.Session() as sess:
sess.run(tf.global_variables_iniliazer())
print sess.run(a)
This outputs [100, 2,3,4]
Does tf.buffer work for you?
// Create a buffer and set values at particular indices.
const a = tf.tensor1d([1, 2, 3, 4]);
const buffer = tf.buffer(a.shape, a.dtype, a.dataSync());
buffer.set(5, 0);
const b = buffer.toTensor();
// Convert the buffer back to a tensor.
b.print();
Tensor
[5, 2, 3, 4]
I had to do this using mulStrict and addStrict which do element-wise multiplication and addition.
const a = tf.tensor1d([1,2,3,4]);
tf.mulStrict(a, tf.tensor1d([0,1,1,1]))
.addStrict(tf.tensor1d([100, 0, 0, 0]);
This was based on asnwer
here