In Gradient Tape Tensorflow gradients are coming as 0.s - tensorflow2.0

Here is part of a model I am working. Being new in calculation gradients in tf, I got confused when I found all the gradient values are coming as 0. Here is the code:
class A:
def __init__(self, inputA_dim, inputB_dim):
self.inputA_dim = (35, 35, 1)
self.inputB_dim = 2
self.model = self.nn_model()
self.opt = tf.keras.optimizers.Adam()
# print(self.model.summary())
def nn_model(self):
inputA = Input(self.inputA_dim)
conv1 = Conv2D(10, 3, padding="same", activation="relu")(inputA)
pool1 = MaxPool2D(padding='same')(conv1)
conv2 = Conv2D(10, 3, padding="same", activation="relu")(pool1)
pool2 = MaxPool2D(padding='same')(conv2)
conv3 = Conv2D(10, 3, padding="same", activation="relu")(pool2)
pool3 = MaxPool2D(padding='same')(conv3)
flatten = Flatten()(pool3)
s2 = Dense(32, activation="relu")(flatten)
s3 = Dense(32, activation="relu")(s2)
s4 = Dense(2, activation="relu")(s3)
inputB = Input((self.inputB_dim,))
a1 = Dense(2, activation="relu")(inputB)
c1 = concatenate([s2, a1], axis=-1)
c2 = Dense(4, activation="relu")(c1)
outputs = Dense(1, activation="linear")(c2)
return tf.keras.Model([inputA, inputB], outputs)
def predict(self, inputs):
return self.model.predict(inputs)
def gradients(self, inputA, inputB):
inputB = tf.convert_to_tensor(inputB)
with tf.GradientTape() as tape:
tape.watch(inputB)
values = self.model([inputA, inputB])
values = tf.squeeze(values)
g = tape.gradient(values, inputB)
print(g)
return g
Later I found there is another method called jacobian which I also used here, still giving 0.s asgrad values. Can anyone tell me what to do. Thanks.

Related

Unable to interpret an argument of type tensorflow.python.data.ops.dataset_ops.PrefetchDataset as a TFF value in iterative process

I'm trying to run a classification simulation in tff, but I'm getting this error:
TypeError: Unable to interpret an argument of type tensorflow.python.data.ops.dataset_ops.PrefetchDataset as a TFF value.
Here is the code I'm using
client_lr = 1e-3
server_lr = 1e-1
NUM_ROUNDS = 200
NUM_EPOCHS = 5
BATCH_SIZE = 2048
EPOCHS = 400
TH = 0.5
def base_model():
return Sequential([
Dense(256, activation='relu', input_shape=(x_train.shape[-1],)),
Dropout(0.5),
Dense(256, activation='relu'),
Dropout(0.5),
Dense(256, activation='relu'),
Dropout(0.5),
Dense(1, activation='sigmoid'),
])
client_train_dataset = collections.OrderedDict()
for i in range(1, total_clients+1):
client_name = "client_" + str(i)
start = samples_per_set * (i-1)
end = samples_per_set * i
data = collections.OrderedDict((('y', y_train[start:end]), ('x', x_train[start:end])))
client_train_dataset[client_name] = data
train_dataset = tff.simulation.FromTensorSlicesClientData(client_train_dataset)
sample_dataset = train_dataset.create_tf_dataset_for_client(train_dataset.client_ids[0])
sample_element = next(iter(sample_dataset))
PREFETCH_BUFFER = 10
SHUFFLE_BUFFER = samples_per_set
def preprocess(dataset):
def batch_format_fn(element):
return collections.OrderedDict(
x=element['x'],
y=tf.reshape(element['y'], [-1, 1]))
return dataset.repeat(NUM_EPOCHS).shuffle(SHUFFLE_BUFFER).batch(BATCH_SIZE).map(batch_format_fn).prefetch(PREFETCH_BUFFER)
preprocessed_sample_dataset = preprocess(sample_dataset)
sample_batch = tf.nest.map_structure(lambda x: x.numpy(), next(iter(preprocessed_sample_dataset)))
def make_federated_data(client_data, client_ids):
return [preprocess(client_data.create_tf_dataset_for_client(x)) for x in client_ids]
federated_train_data = make_federated_data(train_dataset, train_dataset.client_ids)
def model_tff():
model = base_model()
return tff.learning.from_keras_model(
model,
input_spec=preprocessed_sample_dataset.element_spec,
loss=tf.keras.losses.BinaryCrossentropy(),
metrics=[
tfa.metrics.F1Score(num_classes=1, threshold=TH),
keras.metrics.Precision(name="precision", thresholds=TH),
keras.metrics.Recall(name="recall", thresholds=TH)
])
iterative_process = tff.learning.build_federated_averaging_process(
model_tff,
client_optimizer_fn=lambda: optimizers.Adam(learning_rate=client_lr),
server_optimizer_fn=lambda: optimizers.SGD(learning_rate=server_lr))
state = iterative_process.initialize()
federated_model = None
for round_num in range(1, NUM_ROUNDS+1):
state, tff_metrics = iterative_process.next(state, federated_train_data) # THE ERROR IS HERE
federated_model = base_model()
federated_model.compile(optimizer=optimizers.Adam(learning_rate=client_lr),
loss=tf.keras.losses.BinaryCrossentropy(),
metrics=[
tfa.metrics.F1Score(num_classes=1, threshold=TH),
keras.metrics.Precision(name="precision", thresholds=TH),
keras.metrics.Recall(name="recall", thresholds=TH)
])
state.model.assign_weights_to(model=federated_model)
federated_result = federated_model.evaluate(x_val, y_val, verbose=1, return_dict=True)
federated_test = federated_model.evaluate(x_test, y_test, verbose=1, return_dict=True)
I'm using this creditcard dataset: https://www.kaggle.com/mlg-ulb/creditcardfraud
The federated_train_data is a list of <PrefetchDataset shapes: OrderedDict([(x, (None, 29)), (y, (None, 1))]), types: OrderedDict([(x, tf.float64), (y, tf.int64)])>, just like the tutorial from the Tensorflow Federated website Federated Learning for Image Classification.
This might be issue#918. Does this only occur when running in Google Colab? What version of TFF is being used?
Commit#4e57386 is believed to have fixed this, which is now part of the tensorflow-federated-nightly pip package.

keras gives error when using two loss function for two outputs

I have a network. the one before last layer is a dense layer. I want the last layer to return both the max value from the layer before, and the index of that max value.
so if the output of the dense layer is [0,4,5,120,1], the last layer should return [120, 3].
the loss I need the network to work with is calculated only based on the max value, not the index. therefore, I wrote a loss function for the second output, the index, that always returns zero - but if there is a better solution I would like to hear it, in addition to how to fix this error.
the code is:
def ignor_loss(preds, trues):
return 0
# build deep q network
inputs = Input((240,256,3))
pool0 = MaxPooling2D()(inputs)
conv1 = Conv2D(30,3,activation='relu')(pool0)
pool1 = MaxPooling2D()(conv1)
conv2 = Conv2D(40,3,activation='relu')(pool1)
pool2 = MaxPooling2D()(conv2)
conv3 = Conv2D(50,3,activation='relu')(pool2)
pool3 = MaxPooling2D()(conv3)
conv4 = Conv2D(60,3,activation='relu')(pool3)
pool4 = MaxPooling2D()(conv4)
conv5 = Conv2D(80,3,activation='relu')(pool4)
flat = Flatten()(conv5)
dense1 = Dense(70)(flat)
dense2 = Dense(40)(dense1)
values = Dense(env.action_space.n)(dense2)
max_, ind = Lambda(lambda x : [K.max(x),K.argmax(x)])(values)
m = Model(inputs, [max_, ind])
m.compile('adam', ['mse',ignor_loss])
and the error is:
TypeError: unsupported operand type(s) for -: 'int' and 'NoneType'
any ideas?
EDIT:
here is my updated code:
# build deep q network
inputs = Input((240,256,3))
pool0 = MaxPooling2D()(inputs)
conv1 = Conv2D(30,3,activation='relu')(pool0)
pool1 = MaxPooling2D()(conv1)
conv2 = Conv2D(40,3,activation='relu')(pool1)
pool2 = MaxPooling2D()(conv2)
conv3 = Conv2D(50,3,activation='relu')(pool2)
pool3 = MaxPooling2D()(conv3)
conv4 = Conv2D(60,3,activation='relu')(pool3)
pool4 = MaxPooling2D()(conv4)
conv5 = Conv2D(80,3,activation='relu')(pool4)
flat = Flatten()(conv5)
dense1 = Dense(70)(flat)
dense2 = Dense(40)(dense1)
q_values = Dense(env.action_space.n)(dense2)
max_q = Lambda(lambda x : K.max(x), name='max')(q_values)
ind = Lambda(lambda x : K.argmax(x), name='ind')(q_values)
m = Model(inputs, [max_q,ind])
m.compile('adam', {'max':'mse','ind':'mse'}, loss_weights=[1., 0.0])
I still get the same error:
unsupported operand type(s) for -: 'int' and 'NoneType'
I need to know why this error heppens? any ideas?
EDIT 2:
now I added the keepdims=True to the max function and K.expand_dims to the argmax func, like this:
q_values = Dense(env.action_space.n)(dense2)
max_q = Lambda(lambda x : K.max(x, keepdims=True), name='max')(q_values)
ind = Lambda(lambda x : K.expand_dims(K.argmax(x)), name='ind')(q_values)
m = Model(inputs, [max_q,ind])
m.compile('adam', {'max':'mse','ind':'mse'}, loss_weights=[1., 0.0])
but I get a different error:
TypeError: Expected int64, got 0.0 of type 'float' instead.
I think that this is a cleaner solution
1 step: fit the model on the max
X = np.random.uniform(0,1, (2,240,256,3))
y = np.random.uniform(0,1, 2)
inputs = Input((240,256,3))
pool0 = MaxPooling2D()(inputs)
conv1 = Conv2D(30,3,activation='relu')(pool0)
pool1 = MaxPooling2D()(conv1)
conv2 = Conv2D(40,3,activation='relu')(pool1)
pool2 = MaxPooling2D()(conv2)
conv3 = Conv2D(50,3,activation='relu')(pool2)
pool3 = MaxPooling2D()(conv3)
conv4 = Conv2D(60,3,activation='relu')(pool3)
pool4 = MaxPooling2D()(conv4)
conv5 = Conv2D(80,3,activation='relu')(pool4)
flat = Flatten()(conv5)
dense1 = Dense(70)(flat)
dense2 = Dense(40)(dense1)
values = Dense(10)(dense2) # in my case env.action_space.n is 10
max_ = Lambda(lambda x: tf.reduce_max(x, axis=1, keepdims=True))(values)
m = Model(inputs, max_)
m.compile('adam', 'mse')
m.fit(X,y, epochs=3)
2 step: make inference with the fitted model returning max and argmax (this simply require to build a new model)
ind = Lambda(lambda x: tf.expand_dims(tf.argmax(x, axis=1),-1))(values)
final_model = Model(inputs, [max_, ind])
final_model.predict(X) this return max and argmax
EDIT: here a compact model which operate all the operation. if u have two outputs u need to pass to keras two targets. for this reason, the second target is generated by me as an array of 0 (it has no impact)
def ignor_loss(trues, preds):
return 0.
X = np.random.uniform(0,1, (2,240,256,3))
y = np.random.uniform(0,1, 2)
inputs = Input((240,256,3))
pool0 = MaxPooling2D()(inputs)
conv1 = Conv2D(30,3,activation='relu')(pool0)
pool1 = MaxPooling2D()(conv1)
conv2 = Conv2D(40,3,activation='relu')(pool1)
pool2 = MaxPooling2D()(conv2)
conv3 = Conv2D(50,3,activation='relu')(pool2)
pool3 = MaxPooling2D()(conv3)
conv4 = Conv2D(60,3,activation='relu')(pool3)
pool4 = MaxPooling2D()(conv4)
conv5 = Conv2D(80,3,activation='relu')(pool4)
flat = Flatten()(conv5)
dense1 = Dense(70)(flat)
dense2 = Dense(40)(dense1)
values = Dense(10)(dense2) # in my case env.action_space.n is 10
max_ = Lambda(lambda x: tf.reduce_max(x, axis=1, keepdims=True), name='max')(values)
ind = Lambda(lambda x: tf.expand_dims(tf.argmax(x, axis=1),-1), name='ind')(values)
m = Model(inputs, [max_,ind])
m.compile('adam', loss={'max':'mse', 'ind':ignor_loss},
loss_weights={'max':1., 'ind':0.})
m.fit(X, {'max':y, 'ind':np.zeros_like(y)}, epochs=3)
m.predict(X)

pixel wise softmax with crossentropy for multiclass segmentation

I am trying to implement a multiclass semantic segmentation model with 2
classes ( human, car). here is my modified implementation of unet architecture. I number of output channels to 3 (3 classes - human, car, background). How do i get pixel wise classification?
here are 2 examples from my ground truth masks.
i am using 1 channel for each object class ie.
channel 1 for class=car
channel 2 for class=background
channel 3 for class=human
def conv_block(tensor, nfilters, size=3, padding='same', initializer="he_normal"):
x = Conv2D(filters=nfilters, kernel_size=(size, size), padding=padding, kernel_initializer=initializer)(tensor)
x = BatchNormalization()(x)
x = Activation("relu")(x)
x = Conv2D(filters=nfilters, kernel_size=(size, size), padding=padding, kernel_initializer=initializer)(x)
x = BatchNormalization()(x)
x = Activation("relu")(x)
return x
def deconv_block(tensor, residual, nfilters, size=3, padding='same', strides=(2, 2)):
y = Conv2DTranspose(nfilters, kernel_size=(size, size), strides=strides, padding=padding)(tensor)
y = concatenate([y, residual], axis=3)
y = conv_block(y, nfilters)
return y
def Unet(img_height, img_width, nclasses=3, filters=64):
input_layer = Input(shape=(img_height, img_width, 3), name='image_input')
conv1 = conv_block(input_layer, nfilters=filters)
conv1_out = MaxPooling2D(pool_size=(2, 2))(conv1)
conv2 = conv_block(conv1_out, nfilters=filters*2)
conv2_out = MaxPooling2D(pool_size=(2, 2))(conv2)
conv3 = conv_block(conv2_out, nfilters=filters*4)
conv3_out = MaxPooling2D(pool_size=(2, 2))(conv3)
conv4 = conv_block(conv3_out, nfilters=filters*8)
conv4_out = MaxPooling2D(pool_size=(2, 2))(conv4)
conv4_out = Dropout(0.5)(conv4_out)
conv5 = conv_block(conv4_out, nfilters=filters*16)
conv5 = Dropout(0.5)(conv5)
deconv6 = deconv_block(conv5, residual=conv4, nfilters=filters*8)
deconv6 = Dropout(0.5)(deconv6)
deconv7 = deconv_block(deconv6, residual=conv3, nfilters=filters*4)
deconv7 = Dropout(0.5)(deconv7)
deconv8 = deconv_block(deconv7, residual=conv2, nfilters=filters*2)
deconv9 = deconv_block(deconv8, residual=conv1, nfilters=filters)
output_layer = Conv2D(filters=nclasses, kernel_size=(1, 1))(deconv9)
output_layer = BatchNormalization()(output_layer)
output_layer = Reshape((img_height*img_width, nclasses), input_shape=(img_height, img_width, nclasses))(output_layer)
output_layer = Activation('softmax')(output_layer)
model = Model(inputs=input_layer, outputs=output_layer, name='Unet')
return model
You are almost done, now backpropagate the network error with:
loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=output_layer, labels=labels))
tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(loss)
You don't have to convert your ground truth into the one-hot format, sparse_softmax will dot it for you.

Tensorflow model doesn't reduce loss value on MNIST example

I'm trying Tensorflow and tried to re-write a simple MNIST example with minor changes. I expect to see a reduction in the value of loss function after running the code while this does not happen.
I compared my code to many examples but was not able to figure out the problem.
Here is my code:
import numpy as np
import tensorflow as tf
BATCH_SIZE = 100
# Data Placeholders
t = tf.placeholder(tf.bool, name='IfTrain_placeholder') # if we are in training phase
X = tf.placeholder(dtype=tf.float32, shape=[None, 28, 28, 1], name='Data_placeholder')
y = tf.placeholder(dtype=tf.int32, shape=[None], name='Label_placeholder')
# Use Datasets to manage data
X_data = tf.data.Dataset.from_tensor_slices(X).batch(BATCH_SIZE)
y_data = tf.data.Dataset.from_tensor_slices(y).batch(BATCH_SIZE)
X_iter = X_data.make_initializable_iterator()
X_batch = X_iter.get_next()
y_iter = y_data.make_initializable_iterator()
y_batch = y_iter.get_next()
oh_y = tf.one_hot(indices=y_batch, depth=10)
# Model structure here
c1 = tf.layers.conv2d(inputs=X_batch,
filters=32,
kernel_size=[5,5],
padding='same',
activation=tf.nn.relu,
name='CNN1')
m1 = tf.layers.max_pooling2d(inputs=c1,
pool_size=[2,2],
strides=2,
padding='same',
name='MaxPool1')
c2 = tf.layers.conv2d(inputs=m1,
filters=64,
kernel_size=[5,5],
padding='same',
activation=tf.nn.relu,
name='CNN2')
m2 = tf.layers.max_pooling2d(inputs=c2,
pool_size=[2,2],
strides=2,
padding='same',
name='MaxPool2')
f1 = tf.reshape(tensor=m2, shape=[-1, 7*7*64], name='Flat1')
d1 = tf.layers.dense(inputs=f1,
units=1024,
activation=tf.nn.softmax,
name='Dense1')
dr1 = tf.layers.dropout(inputs=d1, rate=0.4, training=t, name='Dropout1')
d2 = tf.layers.dense(inputs=dr1,
units=10,
activation=tf.nn.softmax,
name='Dense2')
# Loss and otimization
loss = tf.losses.softmax_cross_entropy(onehot_labels=oh_y, logits=d2)
classes = tf.argmax(input=d2, axis=1, name='ArgMax1')
init = tf.global_variables_initializer()
optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.003, name='GD1')
train_op = optimizer.minimize(loss=loss, global_step=tf.train.get_global_step(), name='Optimizer1')
# Get data
mnist = tf.contrib.learn.datasets.load_dataset("mnist")
X_train = np.reshape(mnist.train.images, (-1, 28, 28, 1))
y_train = np.asarray(mnist.train.labels, dtype=np.int32)
X_test = np.reshape(mnist.test.images, (-1, 28, 28, 1))
y_test = np.asarray(mnist.test.labels, dtype=np.int32)
# Run session
with tf.Session() as sess:
sess.run(init)
sess.run(X_iter.initializer, feed_dict={X:X_train})
sess.run(y_iter.initializer, feed_dict={y:y_train})
while True:
try:
out = sess.run({'accuracy': accuracy, 'loss': loss, 'train optimizer': train_op}, feed_dict={t:True})
print(out['loss'])
except:
break
I appreciate if anyone can help me find the problem.

TensorFlow, the output of AlexNet are all zero, after training several steps, and sometime the output is all the same in each dimension

here is the output of AlexNet, I mean it's the output of the third full-connection
I don't know why all the output is zero. And I try to reduce the layer, while cutting the last two layer, the output is all the same in each dimension, after training several steps. Any idea will be appreciated.
Here is the main Inference code and initial value:
import tensorflow as tf
import numpy as np
import os
import csv
import cifar10
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
IMAGE_SIZES = 32
IMAGE_CHANELS = 3
NUM_CLASSES = 10
FIRST_CONV_NUM = 64
SECOND_CONV_NUM = 192
THIRD_CONV_NUM = 384
FOURTH_CONV_NUM = 256
FIFTH_CONV_NUM = 256
MAX_POOL_SIZE = 3
BATCH_SIZE = 100
FIRST_FC_UNIT_NUM = 4096
SECOND_FC_UNIT_NUM = 1000
DROP_OUT_PRO = 0.5
THIRD_FC_UNIT_NUM = NUM_CLASSES
TRAIN_EPOCH = 10
NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN = 50000
NUM_EXAMPLES_PER_EPOCH_FOR_EVAL = 10000
NUM_EPOCHS_PER_DECAY = 350.0
LEARNING_RATE_DECAY_FACTOR = 0.1
INITIAL_LEARNING_RATE = 0.01
DISPLAY_STEPS = 5
def leaky_relu(x, alpha=0.0):
return tf.nn.relu(x) - alpha * tf.nn.relu(-x)
def activation(x,alpha=0.0):
if alpha > 0:
return leaky_relu(x,alpha)
else:
return tf.nn.relu(x)
def Alex_Weight(weight_name, weight_shape, weight_stddev, weight_type):
Weight = tf.truncated_normal(shape=weight_shape, stddev=weight_stddev,type=weight_type)
return tf.Variable(initial_value=Weight, trainable=True, name=weight_name)
def Alex_Bias(bias_name, bias_shape, bias_type, bias_init=0.1):
initial = tf.constant(bias_init, shape=bias_shape)
return tf.Variable(initial, trainable=True, dtype=bias_type,name=bias_name)
def Alex_AddActivationSummary(out):
tf.summary.histogram('/activations',out)
tf.summary.scalar('/sparsity',tf.nn.zero_fraction(out))
def Alex_Conv(conv_name, input, weight, bias, strides, alpha=0.1,padding="SAME", activation=activation, act_name="ReLU"):
with tf.name_scope(conv_name):
conv = tf.nn.conv2d(input, weight, [1, strides, strides, 1], padding)
pre_activation = tf.nn.bias_add(conv, bias)
with tf.name_scope(act_name):
conv = activation(pre_activation,alpha=alpha)
return conv
def Alex_Pool(conv, ksize, strides, pool_fuction=tf.nn.max_pool, padding="SAME"):
return pool_fuction(conv, [1, ksize, ksize, 1], [1, strides,strides, 1], padding)
def Alex_Fully_Connect( input, weight, bias, activation=tf.nn.relu, act_name="ReLU"):
with tf.name_scope("Wx_b"):
y = tf.add(tf.matmul(input, weight), bias)
with tf.name_scope(act_name):
fc = activation(y, act_name)
return fc
def Alex_Norm(norm_name, pool):
with tf.name_scope(norm_name):
norm = tf.nn.lrn(pool, 4, bias=1.0, alpha=0.001 / 9.0, beta=0.75,
name=norm_name)
return norm
def Alex_Inference(images):
with tf.name_scope("First_Conv"):
W1 = Alex_Weight("Fist_Conv_Weight", [5, 5, IMAGE_CHANELS, FIRST_CONV_NUM], weight_stddev=0.01,
weight_type=tf.float32)
bias1 = Alex_Bias("First_Conv_Bias", [FIRST_CONV_NUM], tf.float32,bias_init=0.0)
first_conv = Alex_Conv("First_Conv", images, W1, bias1, strides=1, padding="SAME")
Alex_AddActivationSummary(first_conv)
with tf.name_scope('lrn1') as scope:
lrn1 = tf.nn.local_response_normalization(first_conv,
alpha=1e-4,
beta=0.75,
depth_radius=2,
bias=2.0)
with tf.name_scope("First_Pool"):
first_pool = Alex_Pool(lrn1, MAX_POOL_SIZE, strides=2, padding="VALID")
with tf.name_scope("Second_Conv"):
W2 = Alex_Weight("Second_Conv_Weight", [5, 5, FIRST_CONV_NUM, SECOND_CONV_NUM], weight_stddev=0.01,
weight_type=tf.float32)
bias2 = Alex_Bias("Second_Conv_Bias", [SECOND_CONV_NUM], tf.float32,bias_init=1.0)
second_conv = Alex_Conv("Second_Conv", first_pool, W2, bias2, strides=1, padding="SAME")
Alex_AddActivationSummary(second_conv)
with tf.name_scope('lrn2') as scope:
lrn2 = tf.nn.local_response_normalization(second_conv,
alpha=1e-4,
beta=0.75,
depth_radius=2,
bias=2.0)
with tf.name_scope("Second_Pool"):
second_pool = Alex_Pool(lrn2, MAX_POOL_SIZE, strides=2, padding="VALID")
with tf.name_scope("Third_Conv"):
W3 = Alex_Weight("Third_Conv_Weight", [3, 3, SECOND_CONV_NUM, THIRD_CONV_NUM], weight_stddev=0.01,
weight_type=tf.float32)
bias3 = Alex_Bias("Third_Conv_Bias", [THIRD_CONV_NUM], tf.float32,bias_init=0.0)
third_conv = Alex_Conv("Third_Conv", second_pool, W3, bias3, strides=1, padding="SAME")
Alex_AddActivationSummary(third_conv)
with tf.name_scope("Fourth_Conv"):
W4 = Alex_Weight("Fourth_Conv_Weight", [3, 3, THIRD_CONV_NUM, FOURTH_CONV_NUM], weight_stddev=0.01,
weight_type=tf.float32)
bias4 = Alex_Bias("Fourth_Conv_Bias", [FOURTH_CONV_NUM], tf.float32,bias_init=1.0)
fourth_conv = Alex_Conv("Fourth_Conv", third_conv, W4, bias4, strides=1, padding="SAME")
Alex_AddActivationSummary(fourth_conv)
with tf.name_scope("Fifth_Conv"):
W5 = Alex_Weight("Fifth_Conv_Weight", [3, 3, FOURTH_CONV_NUM, FIFTH_CONV_NUM], weight_stddev=0.01,
weight_type=tf.float32)
bias5 = Alex_Bias("Fifth_Conv_Bias", [FIFTH_CONV_NUM], tf.float32,bias_init=1.0)
fifth_conv = Alex_Conv("Fifth_Conv", fourth_conv, W5, bias5, strides=1, padding="SAME")
Alex_AddActivationSummary(fifth_conv)
with tf.name_scope("Third_Pool"):
third_pool = Alex_Pool(fifth_conv, MAX_POOL_SIZE, strides=2, padding="VALID")
with tf.name_scope("Flatten"):
flatten = tf.reshape(third_pool, [BATCH_SIZE, -1])
flatten_dim = flatten.get_shape()[1].value
with tf.name_scope("First_Fully_Connection"):
W = Alex_Weight("Fist_FC_Weight", [flatten_dim, FIRST_FC_UNIT_NUM], weight_stddev=4e-2, weight_type=tf.float32)
bias = Alex_Bias("First_FC_Bias", [FIRST_FC_UNIT_NUM], tf.float32, bias_init=1.0)
fc1 = Alex_Fully_Connect(flatten, W, bias, activation=tf.nn.relu, act_name="ReLU")
Alex_AddActivationSummary(fc1)
with tf.name_scope("Drop_Out_1"):
drop_out_1 = tf.nn.dropout(fc1, DROP_OUT_PRO)
with tf.name_scope("Second_Fully_Connection"):
W = Alex_Weight("Second_FC_Weight", [FIRST_FC_UNIT_NUM, SECOND_FC_UNIT_NUM], weight_stddev=4e-2,
weight_type=tf.float32)
bias = Alex_Bias("Second_FC_Bias", [SECOND_FC_UNIT_NUM], tf.float32, bias_init=1.0)
fc2 = Alex_Fully_Connect(drop_out_1, W, bias, activation=tf.nn.relu, act_name="ReLU")
Alex_AddActivationSummary(fc2)
with tf.name_scope("Drop_Out_2"):
drop_out_2 = tf.nn.dropout(fc2, DROP_OUT_PRO)
with tf.name_scope("Third_Fully_Connection"):
W = Alex_Weight("Third_FC_Weight", [SECOND_FC_UNIT_NUM, THIRD_FC_UNIT_NUM], weight_stddev=1/SECOND_FC_UNIT_NUM,
weight_type=tf.float32)
bias = Alex_Bias("Third_FC_Bias", [THIRD_FC_UNIT_NUM], tf.float32,bias_init=1.0)
fc3 = Alex_Fully_Connect(drop_out_2, W, bias, activation=tf.nn.relu, act_name="ReLU")
Alex_AddActivationSummary(fc3)
return fc3