TypeError: 'module' object is not callable self.transform - iterator

I'm trying to iterate on a custom dataset, and it's failed on image transform.
transform = transforms.Compose([
transforms.ToPILImage(),
transforms.Resize((255, 255)),
# transforms.PILToTensor()])
transforms.ToTensor(),
transforms.Normalize(mean_img, std_img),
])
class img_dataset_fun(Dataset):
def __init__(self, csv_file, transform):
self.csv_file = pd.read_csv(csv_file)
self.transform = transform
def __len__(self):
return len(self.csv_file)
def __getitem__(self, index):
if torch.is_tensor(index):
index = index.tolist()
img_path = self.csv_file.iloc[index, 1]
image = io.imread(img_path)
if self.transform is not None:
image = self.transform(image)
return image
train_img_dataset = img_dataset_fun(csv_file="data.csv", transform=transform)
train_img_loader = torch.utils.data.DataLoader(
train_img_dataset,
batch_size=1,
num_workers=0,
shuffle=False,
)
it = iter(train_img_loader)
images_iter = next(it)
images_iter fails with the error:
TypeError Traceback (most recent call last)
<ipython-input-345-31cbe584d6de> in <module>()
7 shuffle=False,)
8 it = iter(train_img_loader)
----> 9 images_iter =next(it)
4 frames
<ipython-input-335-55277e02141a> in __getitem__(self, index)
17 if self.transform is not None:
---> 18 image=self.transform(image)
19
20
TypeError: 'module' object is not callable
Any idea what might be the problem?

Related

making tf custom layer with ragged tensor inputs to tensor outputs

class CropLayer(layers.Layer):
def __init__(self, crop_t, **kwargs):
super().__init__(**kwargs)
self.crop_t = crop_t
def get_config(self):
config = super().get_config()
config.update({'crop_t': self.crop_t})
return config
def call(self, inputs):
outputs = []
for i, x in enumerate(inputs):
t = tf.shape(x[0])[0]
start = tf.experimental.numpy.random.randint(0, t-self.crop_t, dtype='int32')
end = start + self.crop_t
outputs.append(inputs[i, :, start:end].to_list())
return tf.constant(outputs)
def get_cropper(crop_t):
return keras.Sequential(
[
keras.Input(shape=(N, None), ragged=True),
CropLayer(crop_t)
]
)
cropper = get_cropper(crop_t)
I want to make a custom layer that the ragged tensor as input and tensor as output.
The layer crop ragged tensors to fit the size, so it can convert to tensor format. But when I run this code, the following error occurs.
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
<ipython-input-96-6430b3383331> in <module>()
----> 1 cropperr = get_cropper(crop_t)
3 frames
/usr/local/lib/python3.7/dist-packages/tensorflow/python/autograph/impl/api.py in wrapper(*args, **kwargs)
690 except Exception as e: # pylint:disable=broad-except
691 if hasattr(e, 'ag_error_metadata'):
--> 692 raise e.ag_error_metadata.to_exception(e)
693 else:
694 raise
ValueError: Exception encountered when calling layer "crop_layer_7" (type CropLayer).
in user code:
File "<ipython-input-93-419907fac9d0>", line 31, in call *
outputs.append(inputs[i, :, start:end].to_list())
ValueError: to_list can only be used in eager mode.

KeyError: 0 in CustomDataGenerator class I created

The error looks like this and occurs after successful run of the model for some part of the first epoch:
Epoch 1/50
937/938 [============================>.] - ETA: 0s - loss: 0.0089 - accuracy: 0.9913
---------------------------------------------------------------------------
KeyError Traceback (most recent call last)
/usr/local/lib/python3.7/dist-packages/pandas/core/indexes/base.py in get_loc(self, key, method, tolerance)
2897 try:
-> 2898 return self._engine.get_loc(casted_key)
2899 except KeyError as err:
pandas/_libs/index.pyx in pandas._libs.index.IndexEngine.get_loc()
pandas/_libs/index.pyx in pandas._libs.index.IndexEngine.get_loc()
pandas/_libs/hashtable_class_helper.pxi in pandas._libs.hashtable.Int64HashTable.get_item()
pandas/_libs/hashtable_class_helper.pxi in pandas._libs.hashtable.Int64HashTable.get_item()
KeyError: 0
The above exception was the direct cause of the following exception:
KeyError Traceback (most recent call last)
10 frames
/usr/local/lib/python3.7/dist-packages/pandas/core/indexes/base.py in get_loc(self, key, method, tolerance)
2898 return self._engine.get_loc(casted_key)
2899 except KeyError as err:
-> 2900 raise KeyError(key) from err
2901
2902 if tolerance is not None:
KeyError: 0
My custom data generator looks as follows:
class CustomDataGenerator(ks.utils.Sequence):
def __init__(self, dataframe, x_col, y_col, img_h, img_w, batch_size):
self.dataframe = dataframe
self.x_col = x_col
self.y_col = y_col
self.img_h = img_h
self.img_w = img_w
self.batch_size = batch_size
def __len__(self):
return math.ceil(self.dataframe.shape[0] / self.batch_size)
def __getitem__(self, index):
X = np.empty(shape=(self.batch_size, self.img_w, self.img_h, 3), dtype='float32')
Y = np.empty(shape=(self.batch_size, self.img_w, self.img_h, 1), dtype='float32')
for i in range(self.batch_size):
img_path = self.dataframe[self.x_col][index * self.batch_size + i]
img = cv.imread(img_path)
img = cv.cvtColor(img, cv.COLOR_BGR2RGB)
img_np = np.array(img, dtype='float32')
img_np = img_np.reshape(-1, self.img_h, self.img_w, 3)
img_np = img_np / 255.
mask_path = self.dataframe[self.y_col][index * self.batch_size + i]
mask = cv.imread(mask_path, 0)
mask_np = np.array(mask, dtype='float32')
mask_np = mask_np.reshape(-1, self.img_h, self.img_w, 1)
mask_np = mask_np / 255.
X[i, :, :, :] = img_np
Y[i, :, :, :] = mask_np
return X, Y
def on_epoch_end(self):
self.dataframe = self.dataframe.sample(frac=1)
self.dataframe.reset_index(inplace=True, drop=True)
size = 16
train_gen = CustomDataGenerator(dataframe=train_df, x_col='Images', y_col='Masks', img_h=128, img_w=128, batch_size=size)
val_gen = CustomDataGenerator(dataframe=val_df, x_col='Images', y_col='Masks', img_h=128, img_w=128, batch_size=size)
test_gen = CustomDataGenerator(dataframe=test_df, x_col='Images', y_col='Masks', img_h=128, img_w=128, batch_size=size)
The dataframe consists of 2 columns; one containing input images and another containing output masks. The dataset can be found here:
https://www.kaggle.com/hngngn/portrait-segmentation-128x128
The KeyError might have occurred because 0 does not exist in the index.
For integer-location based indexing of a data frame use .iloc
img_path = self.dataframe[self.x_col].iloc[index * self.batch_size + i]
mask_path = self.dataframe[self.y_col].iloc[index * self.batch_size + i]

I'm getting "Tensor.op is meaningless when eager execution is enabled." in my simple encoder model. (TF 2.0)

The code of my encoder model is given below, I have made it using functional API(TF 2.0)
embed_obj = EndTokenLayer()
def encoder_model(inp):
input_1 = embed_obj(inp)
h = Masking([(lambda x: x*0)(x) for x in range(128)])(input_1)
lstm1 , state_h, state_c = LSTM(512, return_sequences=True, return_state=True)(h)
model = Model(inputs=input_1, outputs=[lstm1, state_h, state_c])
return model
And when I'm calling my model:
for x,y in train.take(1):
k = x
model = encoder_model(k)
I'm getting the following error:
---------------------------------------------------------------------------
AttributeError Traceback (most recent call last)
<ipython-input-98-46e9c9596137> in <module>()
2 for x,y in train.take(1):
3 k = x
----> 4 model = encoder_model(k)
7 frames
/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/ops.py in op(self)
1111 def op(self):
1112 raise AttributeError(
-> 1113 "Tensor.op is meaningless when eager execution is enabled.")
1114
1115 #property
AttributeError: Tensor.op is meaningless when eager execution is enabled.
In TF2, static graph (preventing eager execution with dynamic graph) can be constructed by using a decorator.
Try #tf.function decorator
#tf.function
def encoder_model(inp):
input_1 = embed_obj(inp)
h = Masking([(lambda x: x*0)(x) for x in range(128)])(input_1)
lstm1 , state_h, state_c = LSTM(512, return_sequences=True, return_state=True)(h)
model = Model(inputs=input_1, outputs=[lstm1, state_h, state_c])
return model
Then call the function
for x,y in train.take(1):
k = x
model = encoder_model(k)

"Unknown graph" error when using keras application model with tf.functions

This is my code:
import tensorflow as tf
import tensorflow_datasets as tfds
import tensorflow.keras.applications.vgg16 as vgg16
tf.enable_eager_execution()
def resize_image(image, shape = (224,224)):
target_width = shape[0]
target_height = shape[1]
initial_width = tf.shape(image)[0]
initial_height = tf.shape(image)[1]
im = image
ratio = 0
if(initial_width < initial_height):
ratio = tf.cast(256 / initial_width, tf.float32)
h = tf.cast(initial_height, tf.float32) * ratio
im = tf.image.resize(im, (256, h), method="bicubic")
else:
ratio = tf.cast(256 / initial_height, tf.float32)
w = tf.cast(initial_width, tf.float32) * ratio
im = tf.image.resize(im, (w, 256), method="bicubic")
width = tf.shape(im)[0]
height = tf.shape(im)[1]
startx = width//2 - (target_width//2)
starty = height//2 - (target_height//2)
im = tf.image.crop_to_bounding_box(im, startx, starty, target_width, target_height)
return im
def scale16(image, label):
im = resize_image(image)
im = vgg16.preprocess_input(im)
return (im, label)
data_dir = "/content/"
model = vgg16.VGG16(weights='imagenet', include_top=True)
datasets, info = tfds.load(name="imagenet2012",
with_info=True,
as_supervised=True,
download=False,
data_dir=data_dir, shuffle_files=False
)
train = datasets['train'].map(scale16).batch(2).take(1)
#tf.function
def predict_batch(b, m):
return m.predict_on_batch(b)
sample_imgs = train
for x in sample_imgs:
print("batch prediction", predict_batch(x[0], model))
When I run the code without #tf.function, i get the expected results (the prediction for the batch). When I run it with #tf.function, I get this error:
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
<ipython-input-1-ba7af933ed07> in <module>()
49 sample_imgs = train
50 for x in sample_imgs:
---> 51 print("batch prediction", predict_batch(x[0], model))
7 frames
/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/framework/func_graph.py in wrapper(*args, **kwargs)
903 except Exception as e: # pylint:disable=broad-except
904 if hasattr(e, "ag_error_metadata"):
--> 905 raise e.ag_error_metadata.to_exception(e)
906 else:
907 raise
ValueError: in converted code:
<ipython-input-1-ba7af933ed07>:47 predict_batch *
return m.predict_on_batch(b)
/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/engine/training.py:1155 predict_on_batch
self._make_predict_function()
/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/engine/training.py:2179 _make_predict_function
**kwargs)
/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/backend.py:3669 function
return EagerExecutionFunction(inputs, outputs, updates=updates, name=name)
/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/backend.py:3553 __init__
raise ValueError('Unknown graph. Aborting.')
ValueError: Unknown graph. Aborting.
I'm running the code on google Colaboratory. Why do I get this error? shouldn't the training step usually be withing a #tf.function? Why can't I use the model within this method?
Facing the same issue. I've reported it here: https://github.com/tensorflow/tensorflow/issues/33997

when define my own keras layer occur a none tensor object

here is my own layer code and the model can compile and predict fine, but when I use the model method model.fit(x,y) it turn out an error about none tensor error, and I can not find the reason
class CenterPointClassifierLayer(Layer):
def __init__(self, c, **kwargs):
self.c = c
super(CenterPointClassifierLayer, self).__init__(**kwargs)
def build(self, input_shape):
# check input_shape
if len(input_shape) != 2:
raise 'input should be in 1 dimension'
self.kernel = self.add_weight(name='kernel',
shape=(self.c, input_shape[1]),
initializer='uniform',
trainable=True)
self.one = K.constant(np.ones((self.c, 1)))
super(CenterPointClassifierLayer, self).build(input_shape)
def call(self, x):
def elem_op(pre, x_input):
x_shape = K.int_shape(x_input)
e = K.reshape(x_input, (1, x_shape[0]))
_x = K.dot(self.one, e)
del_x = K.square(tf.subtract(self.kernel, _x))
distance = K.sum(del_x, axis=1)
_c = K.argmin(distance)
_class = K.one_hot(_c, self.c)
return _class
y_pred = tf.scan(elem_op, x, initializer=K.one_hot(1, self.c))
return y_pred
def compute_output_shape(self, input_shape):
out_shape = (input_shape[0], self.c)
return out_shape
and here is the error i got when use fit method:
File "\tensorflow\python\ops\math_ops.py", line 412, in square
return gen_math_ops.square(x, name=name)
File "\tensorflow\python\ops\gen_math_ops.py", line 2585, in square
result = _op_def_lib.apply_op("Square", x=x, name=name)
File "\tensorflow\python\framework\op_def_library.py", line 509, in apply_op
(input_name, err))
ValueError: Tried to convert 'x' to a tensor and failed. Error: None values not supported.
please help, how can i fix the error when fit!!!
I do not know where the x come from and got the none values, and why there a square op in tf