The error looks like this and occurs after successful run of the model for some part of the first epoch:
Epoch 1/50
937/938 [============================>.] - ETA: 0s - loss: 0.0089 - accuracy: 0.9913
---------------------------------------------------------------------------
KeyError Traceback (most recent call last)
/usr/local/lib/python3.7/dist-packages/pandas/core/indexes/base.py in get_loc(self, key, method, tolerance)
2897 try:
-> 2898 return self._engine.get_loc(casted_key)
2899 except KeyError as err:
pandas/_libs/index.pyx in pandas._libs.index.IndexEngine.get_loc()
pandas/_libs/index.pyx in pandas._libs.index.IndexEngine.get_loc()
pandas/_libs/hashtable_class_helper.pxi in pandas._libs.hashtable.Int64HashTable.get_item()
pandas/_libs/hashtable_class_helper.pxi in pandas._libs.hashtable.Int64HashTable.get_item()
KeyError: 0
The above exception was the direct cause of the following exception:
KeyError Traceback (most recent call last)
10 frames
/usr/local/lib/python3.7/dist-packages/pandas/core/indexes/base.py in get_loc(self, key, method, tolerance)
2898 return self._engine.get_loc(casted_key)
2899 except KeyError as err:
-> 2900 raise KeyError(key) from err
2901
2902 if tolerance is not None:
KeyError: 0
My custom data generator looks as follows:
class CustomDataGenerator(ks.utils.Sequence):
def __init__(self, dataframe, x_col, y_col, img_h, img_w, batch_size):
self.dataframe = dataframe
self.x_col = x_col
self.y_col = y_col
self.img_h = img_h
self.img_w = img_w
self.batch_size = batch_size
def __len__(self):
return math.ceil(self.dataframe.shape[0] / self.batch_size)
def __getitem__(self, index):
X = np.empty(shape=(self.batch_size, self.img_w, self.img_h, 3), dtype='float32')
Y = np.empty(shape=(self.batch_size, self.img_w, self.img_h, 1), dtype='float32')
for i in range(self.batch_size):
img_path = self.dataframe[self.x_col][index * self.batch_size + i]
img = cv.imread(img_path)
img = cv.cvtColor(img, cv.COLOR_BGR2RGB)
img_np = np.array(img, dtype='float32')
img_np = img_np.reshape(-1, self.img_h, self.img_w, 3)
img_np = img_np / 255.
mask_path = self.dataframe[self.y_col][index * self.batch_size + i]
mask = cv.imread(mask_path, 0)
mask_np = np.array(mask, dtype='float32')
mask_np = mask_np.reshape(-1, self.img_h, self.img_w, 1)
mask_np = mask_np / 255.
X[i, :, :, :] = img_np
Y[i, :, :, :] = mask_np
return X, Y
def on_epoch_end(self):
self.dataframe = self.dataframe.sample(frac=1)
self.dataframe.reset_index(inplace=True, drop=True)
size = 16
train_gen = CustomDataGenerator(dataframe=train_df, x_col='Images', y_col='Masks', img_h=128, img_w=128, batch_size=size)
val_gen = CustomDataGenerator(dataframe=val_df, x_col='Images', y_col='Masks', img_h=128, img_w=128, batch_size=size)
test_gen = CustomDataGenerator(dataframe=test_df, x_col='Images', y_col='Masks', img_h=128, img_w=128, batch_size=size)
The dataframe consists of 2 columns; one containing input images and another containing output masks. The dataset can be found here:
https://www.kaggle.com/hngngn/portrait-segmentation-128x128
The KeyError might have occurred because 0 does not exist in the index.
For integer-location based indexing of a data frame use .iloc
img_path = self.dataframe[self.x_col].iloc[index * self.batch_size + i]
mask_path = self.dataframe[self.y_col].iloc[index * self.batch_size + i]
Related
I am trying to code a neural network using only numpy and pandas. I am having issues with the dimension of my data. I am getting the error "ValueError: operands could not be broadcast together with shapes (150,) (150,3)
." Not sure what the alternative is here, as we are trying to predict one of the three types of flower based on 4 numerical values. Here is my code:
import pandas as pd
class NeuralNet():
def __init__(self, i_dim, h_dim, o_dim, lr):
self.i_dim = i_dim
self.h_dim = h_dim
self.o_dim = o_dim
self.lr = lr
self.weights1 = np.random.randn(self.i_dim, self.h_dim) / np.sqrt(self.i_dim)
self.bias1 = np.zeros((1, self.h_dim))
self.weights2 = np.random.randn(self.h_dim, self.o_dim) / np.sqrt(self.h_dim)
self.bias2 = np.zeros((1, self.o_dim))
def sigmoid(self, x):
return 1 / (1 + np.exp(-x))
def softmax(self, x):
exps = np.exp(x - np.max(x, axis=1, keepdims=True))
return exps / np.sum(exps, axis=1, keepdims=True)
def forward(self, X):
self.layer1 = self.sigmoid(np.dot(X, self.weights1) + self.bias1)
self.layer2 = self.softmax(np.dot(self.layer1, self.weights2) + self.bias2)
return self.layer2
def sigmoid_derivative(self, x):
return x * (1 - x)
def softmax_derivative(self, x):
s = x.reshape(-1, 1)
return np.diagflat(s) - np.dot(s, s.T)
def backward(self, X, y, y_hat):
d_softmax = self.softmax_derivative(y_hat)
d_sigmoid = self.sigmoid_derivative(self.layer1)
d_weights2 = np.dot(self.layer1.T, (2 * (y - y_hat) * d_softmax))
d_bias2 = np.sum(2 * (y - y_hat) * d_softmax, axis=0, keepdims=True)
d_weights1 = np.dot(X.T, (np.dot(2 * (y - y_hat) * d_softmax, self.weights2.T) * d_sigmoid))
d_bias1 = np.sum(np.dot(2 * (y - y_hat) * d_softmax, self.weights2.T) * d_sigmoid, axis=0)
self.weights1 -= self.lr * d_weights1
self.bias1 -= self.lr * d_bias1
self.weights2 -= self.lr * d_weights2
self.bias2 -= self.lr * d_bias2
def cross_ent_loss(self):
sample_losses = - self.y * np.log(self.y_hat) - (1 - self.y) * np.log(1 - self.y_hat)
loss = np.mean(sample_losses)
return loss
def train(self, X, y, epochs):
for epoch in range(epochs):
y_hat = self.forward(X)
self.backward(X, y, y_hat)
loss = self.cross_ent_loss()
print(f"Epoch {epoch}: Loss = {loss}")
if epoch % 10 == 0:
print(f"Epoch {epoch}: Loss = {loss}")
def predict(self, X):
return self.forward(X)
df = pd.read_csv('/Users/brasilgu/PycharmProjects/NNfs/venv/lib/iris.data.txt', header=None)
X_train = df.iloc[:, :4].values
y_train = df.iloc[:, -1].values
nn = NeuralNet(4, 5, 3, 0.1)
nn.train(X_train, y_train, 1000)
y_pred = nn.predict(X_train)
y_pred_labels = np.argmax(y_pred, axis=1)
print(y_pred) ```
The stacktrace of the error:
``` Traceback (most recent call last):
File "/Users/brasilgu/PycharmProjects/NNfs/venv/lib/neural_net.py", line 72, in <module>
nn.train(X_train, y_train, 1000)
File "/Users/brasilgu/PycharmProjects/NNfs/venv/lib/neural_net.py", line 57, in train
self.backward(X, y, y_hat)
File "/Users/brasilgu/PycharmProjects/NNfs/venv/lib/neural_net.py", line 39, in backward
d_weights2 = np.dot(self.layer1.T, (2 * (y - y_hat) * d_softmax))
ValueError: operands could not be broadcast together with shapes (150,) (150,3)```
I saw the publicly available iris dataset and according to your code, the y seems to be a rank one matrix with shape (150, ).
So modify your y_train as y_train = y_train.reshape(-1, 1) to make it a proper matrix before creating the NeuralNet
I'm trying to iterate on a custom dataset, and it's failed on image transform.
transform = transforms.Compose([
transforms.ToPILImage(),
transforms.Resize((255, 255)),
# transforms.PILToTensor()])
transforms.ToTensor(),
transforms.Normalize(mean_img, std_img),
])
class img_dataset_fun(Dataset):
def __init__(self, csv_file, transform):
self.csv_file = pd.read_csv(csv_file)
self.transform = transform
def __len__(self):
return len(self.csv_file)
def __getitem__(self, index):
if torch.is_tensor(index):
index = index.tolist()
img_path = self.csv_file.iloc[index, 1]
image = io.imread(img_path)
if self.transform is not None:
image = self.transform(image)
return image
train_img_dataset = img_dataset_fun(csv_file="data.csv", transform=transform)
train_img_loader = torch.utils.data.DataLoader(
train_img_dataset,
batch_size=1,
num_workers=0,
shuffle=False,
)
it = iter(train_img_loader)
images_iter = next(it)
images_iter fails with the error:
TypeError Traceback (most recent call last)
<ipython-input-345-31cbe584d6de> in <module>()
7 shuffle=False,)
8 it = iter(train_img_loader)
----> 9 images_iter =next(it)
4 frames
<ipython-input-335-55277e02141a> in __getitem__(self, index)
17 if self.transform is not None:
---> 18 image=self.transform(image)
19
20
TypeError: 'module' object is not callable
Any idea what might be the problem?
This is my code:
import tensorflow as tf
import tensorflow_datasets as tfds
import tensorflow.keras.applications.vgg16 as vgg16
tf.enable_eager_execution()
def resize_image(image, shape = (224,224)):
target_width = shape[0]
target_height = shape[1]
initial_width = tf.shape(image)[0]
initial_height = tf.shape(image)[1]
im = image
ratio = 0
if(initial_width < initial_height):
ratio = tf.cast(256 / initial_width, tf.float32)
h = tf.cast(initial_height, tf.float32) * ratio
im = tf.image.resize(im, (256, h), method="bicubic")
else:
ratio = tf.cast(256 / initial_height, tf.float32)
w = tf.cast(initial_width, tf.float32) * ratio
im = tf.image.resize(im, (w, 256), method="bicubic")
width = tf.shape(im)[0]
height = tf.shape(im)[1]
startx = width//2 - (target_width//2)
starty = height//2 - (target_height//2)
im = tf.image.crop_to_bounding_box(im, startx, starty, target_width, target_height)
return im
def scale16(image, label):
im = resize_image(image)
im = vgg16.preprocess_input(im)
return (im, label)
data_dir = "/content/"
model = vgg16.VGG16(weights='imagenet', include_top=True)
datasets, info = tfds.load(name="imagenet2012",
with_info=True,
as_supervised=True,
download=False,
data_dir=data_dir, shuffle_files=False
)
train = datasets['train'].map(scale16).batch(2).take(1)
#tf.function
def predict_batch(b, m):
return m.predict_on_batch(b)
sample_imgs = train
for x in sample_imgs:
print("batch prediction", predict_batch(x[0], model))
When I run the code without #tf.function, i get the expected results (the prediction for the batch). When I run it with #tf.function, I get this error:
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
<ipython-input-1-ba7af933ed07> in <module>()
49 sample_imgs = train
50 for x in sample_imgs:
---> 51 print("batch prediction", predict_batch(x[0], model))
7 frames
/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/framework/func_graph.py in wrapper(*args, **kwargs)
903 except Exception as e: # pylint:disable=broad-except
904 if hasattr(e, "ag_error_metadata"):
--> 905 raise e.ag_error_metadata.to_exception(e)
906 else:
907 raise
ValueError: in converted code:
<ipython-input-1-ba7af933ed07>:47 predict_batch *
return m.predict_on_batch(b)
/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/engine/training.py:1155 predict_on_batch
self._make_predict_function()
/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/engine/training.py:2179 _make_predict_function
**kwargs)
/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/backend.py:3669 function
return EagerExecutionFunction(inputs, outputs, updates=updates, name=name)
/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/backend.py:3553 __init__
raise ValueError('Unknown graph. Aborting.')
ValueError: Unknown graph. Aborting.
I'm running the code on google Colaboratory. Why do I get this error? shouldn't the training step usually be withing a #tf.function? Why can't I use the model within this method?
Facing the same issue. I've reported it here: https://github.com/tensorflow/tensorflow/issues/33997
I am trying to build an image classifier with keras and the size of my dataset requires me to use the ImageDataGenerator class along with its flow_from_dataframe method. This is the code I am using.
training = pd.read_csv("dataset/train/training.csv")
training = (training[(training["view_type"].isin(["view_4","view_5"]))])
training = (training[["id", "class"]]).reset_index(drop = True)
datagen = ImageDataGenerator(rescale = 1. /255)
training.head()
id class
0 image_view4_0.jpg lace_up
1 image_view4_1.jpg lace_up
2 image_view4_2.jpg zipper
3 image_view4_3.jpg lace_up
4 image_view4_4.jpg hook&look
train_generaor = datagen.flow_from_dataframe(dataframe = training,
directory = "dataset/train/images/",
xcol = "id",
ycol = "class",
has_ext = True,
class_mode = "categorical",
target_size = (224, 224))
However, when I run the train generator in colab, I get the following error
KeyError Traceback (most recent call last)
/usr/local/lib/python3.6/dist-packages/pandas/core/indexes/base.py in get_loc(self, key, method, tolerance)
2524 try:
-> 2525 return self._engine.get_loc(key)
2526 except KeyError:
pandas/_libs/index.pyx in pandas._libs.index.IndexEngine.get_loc()
pandas/_libs/index.pyx in pandas._libs.index.IndexEngine.get_loc()
pandas/_libs/hashtable_class_helper.pxi in pandas._libs.hashtable.PyObjectHashTable.get_item()
pandas/_libs/hashtable_class_helper.pxi in pandas._libs.hashtable.PyObjectHashTable.get_item()
KeyError: 'filename'
During handling of the above exception, another exception occurred:
KeyError Traceback (most recent call last)
<ipython-input-39-e1787553cadc> in <module>()
7 target_size = (224, 224),
8 color_mode = "rgb",
----> 9 validate_filenames = True)
/usr/local/lib/python3.6/dist-packages/keras_preprocessing/image/image_data_generator.py in flow_from_dataframe(self, dataframe, directory, x_col, y_col, target_size, color_mode, classes, class_mode, batch_size, shuffle, seed, save_to_dir, save_prefix, save_format, subset, interpolation, drop_duplicates, **kwargs)
664 subset=subset,
665 interpolation=interpolation,
--> 666 drop_duplicates=drop_duplicates
667 )
668
/usr/local/lib/python3.6/dist-packages/keras_preprocessing/image/dataframe_iterator.py in __init__(self, dataframe, directory, image_data_generator, x_col, y_col, target_size, color_mode, classes, class_mode, batch_size, shuffle, seed, data_format, save_to_dir, save_prefix, save_format, subset, interpolation, dtype, drop_duplicates)
118 self.dtype = dtype
119 # check that inputs match the required class_mode
--> 120 self._check_params(df, x_col, y_col, classes)
121 if drop_duplicates:
122 df.drop_duplicates(x_col, inplace=True)
/usr/local/lib/python3.6/dist-packages/keras_preprocessing/image/dataframe_iterator.py in _check_params(self, df, x_col, y_col, classes)
162 .format(self.class_mode, self.allowed_class_modes))
163 # check that filenames/filepaths column values are all strings
--> 164 if not all(df[x_col].apply(lambda x: isinstance(x, str))):
165 raise ValueError('All values in column x_col={} must be strings.'
166 .format(x_col))
/usr/local/lib/python3.6/dist-packages/pandas/core/frame.py in __getitem__(self, key)
2137 return self._getitem_multilevel(key)
2138 else:
-> 2139 return self._getitem_column(key)
2140
2141 def _getitem_column(self, key):
/usr/local/lib/python3.6/dist-packages/pandas/core/frame.py in _getitem_column(self, key)
2144 # get column
2145 if self.columns.is_unique:
-> 2146 return self._get_item_cache(key)
2147
2148 # duplicate columns & possible reduce dimensionality
/usr/local/lib/python3.6/dist-packages/pandas/core/generic.py in _get_item_cache(self, item)
1840 res = cache.get(item)
1841 if res is None:
-> 1842 values = self._data.get(item)
1843 res = self._box_item_values(item, values)
1844 cache[item] = res
/usr/local/lib/python3.6/dist-packages/pandas/core/internals.py in get(self, item, fastpath)
3841
3842 if not isna(item):
-> 3843 loc = self.items.get_loc(item)
3844 else:
3845 indexer = np.arange(len(self.items))[isna(self.items)]
/usr/local/lib/python3.6/dist-packages/pandas/core/indexes/base.py in get_loc(self, key, method, tolerance)
2525 return self._engine.get_loc(key)
2526 except KeyError:
-> 2527 return self._engine.get_loc(self._maybe_cast_indexer(key))
2528
2529 indexer = self.get_indexer([key], method=method, tolerance=tolerance)
pandas/_libs/index.pyx in pandas._libs.index.IndexEngine.get_loc()
pandas/_libs/index.pyx in pandas._libs.index.IndexEngine.get_loc()
pandas/_libs/hashtable_class_helper.pxi in pandas._libs.hashtable.PyObjectHashTable.get_item()
pandas/_libs/hashtable_class_helper.pxi in pandas._libs.hashtable.PyObjectHashTable.get_item()
KeyError: 'filename'
What is going wrong? I have tried multiple things to fix this but cannot figure out why this is happening
Just looking at the keras documentation, 'filename' is what it expects in the argument xcol. xcol refers to a column in dataframe supposed to contain the image path relative to what you provided in the argument directory.
So you are just providing the wrong columns as arguments. Change the column names of your 2-column dataframe:
training.columns = ['path', 'class']
Then change your arguments to what is required:
train_generaor = datagen.flow_from_dataframe(dataframe = training,
directory = "dataset/train/images/",
xcol = "path",
ycol = "class",
has_ext = True,
class_mode = "categorical",
target_size = (224, 224))
here is my own layer code and the model can compile and predict fine, but when I use the model method model.fit(x,y) it turn out an error about none tensor error, and I can not find the reason
class CenterPointClassifierLayer(Layer):
def __init__(self, c, **kwargs):
self.c = c
super(CenterPointClassifierLayer, self).__init__(**kwargs)
def build(self, input_shape):
# check input_shape
if len(input_shape) != 2:
raise 'input should be in 1 dimension'
self.kernel = self.add_weight(name='kernel',
shape=(self.c, input_shape[1]),
initializer='uniform',
trainable=True)
self.one = K.constant(np.ones((self.c, 1)))
super(CenterPointClassifierLayer, self).build(input_shape)
def call(self, x):
def elem_op(pre, x_input):
x_shape = K.int_shape(x_input)
e = K.reshape(x_input, (1, x_shape[0]))
_x = K.dot(self.one, e)
del_x = K.square(tf.subtract(self.kernel, _x))
distance = K.sum(del_x, axis=1)
_c = K.argmin(distance)
_class = K.one_hot(_c, self.c)
return _class
y_pred = tf.scan(elem_op, x, initializer=K.one_hot(1, self.c))
return y_pred
def compute_output_shape(self, input_shape):
out_shape = (input_shape[0], self.c)
return out_shape
and here is the error i got when use fit method:
File "\tensorflow\python\ops\math_ops.py", line 412, in square
return gen_math_ops.square(x, name=name)
File "\tensorflow\python\ops\gen_math_ops.py", line 2585, in square
result = _op_def_lib.apply_op("Square", x=x, name=name)
File "\tensorflow\python\framework\op_def_library.py", line 509, in apply_op
(input_name, err))
ValueError: Tried to convert 'x' to a tensor and failed. Error: None values not supported.
please help, how can i fix the error when fit!!!
I do not know where the x come from and got the none values, and why there a square op in tf