How to obtain the encoder from the make_csv_dataset? - tensorflow

Question

I used this code from the tutorial:
def get_train_dataset(file_path, **kwargs):
dataset = tf.data.experimental.make_csv_dataset(
file_path,
batch_size=10, # Artificially small to make examples easier to show.
label_name=LABEL_COLUMN,
na_value="?",
num_epochs=1,
ignore_errors=True,
select_columns= CSV_COLUMNS,
**kwargs)
return dataset
Then I created the train set:
raw_train_data = get_train_dataset(train_file_path)
to train the model.
The question is how to get the encoder used for the training data for encoding new text?
I loaded the new data, but this doesn't use the same encoder as the training data:
raw_test_data = get_test_dataset(new_data_file_path)
How to obtain the original encoder when using tf.data.experimental.make_csv_dataset?
EDIT:
train_file_path = "./train.csv"
test_file_path = "./test.csv"
LABEL_COLUMN = 'target'
CSV_COLUMNS = ['text', 'target']
def get_train_dataset(file_path, **kwargs):
dataset = tf.data.experimental.make_csv_dataset(
file_path,
batch_size=10, # Artificially small to make examples easier to show.
label_name=LABEL_COLUMN,
na_value="?",
num_epochs=1,
ignore_errors=True,
select_columns= CSV_COLUMNS,
**kwargs)
return dataset
def get_test_dataset(file_path, **kwargs):
dataset = tf.data.experimental.make_csv_dataset(
file_path,
batch_size=10, # Artificially small to make examples easier to show.
na_value="?",
num_epochs=1,
ignore_errors=True,
**kwargs)
return dataset
sample_submission = pd.read_csv("./sample_submission.csv")
raw_train_data = get_train_dataset(train_file_path)
raw_test_data = get_test_dataset(test_file_path)
def extract_train_tensor(example, label):
print(example)
return example['text'], label
def extract_test_tensor(example):
print(example)
return example['text']
test_data = raw_test_data.map(lambda ex: extract_test_tensor(ex))
test_data_size = len(list(test_data))
print("test size: ", test_data_size)
train_data_all = raw_train_data.map(lambda ex, label: extract_train_tensor(ex, label))
train_data_all = train_data_all.shuffle(10000)
print(train_data_all)
train_data_size = len(list(train_data_all))
print("train size: ", train_data_size)
train_size = int(0.7 * train_data_size)
val_size = int(0.3 * train_data_size)
train_data = train_data_all.take(train_size)
val_data = train_data_all.skip(train_size)
embedding = "https://tfhub.dev/google/tf2-preview/gnews-swivel-20dim/1"
hub_layer = hub.KerasLayer(embedding, input_shape=[],
dtype=tf.string, trainable=True)
model = tf.keras.Sequential()
model.add(hub_layer)
model.add(tf.keras.layers.Dense(16, activation='relu'))
model.add(tf.keras.layers.Dense(1, activation='sigmoid'))
model.summary()
model.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=['accuracy'])
history = model.fit(train_data,
epochs=20,
validation_data=val_data,
verbose=1)
import numpy as np
predictions = model.predict(test_data)
predictions = np.where(predictions > 0.5, 1, 0)
sample_submission['target'] = predictions
print(predictions)
The two calls to get_train_dataset() and get_test_dataset() to generate train and test data. The train data is split into train and validation sets and the accuracy is great. However, the test data accuracy is very low. Both data sets are strings of text and I didn't do any encoding.

Answer 1

tf.data.experimental.make_csv_dataset does not make any encoding.
It is about:
Reads CSV files into a dataset, where each element is a (features,
labels) tuple that corresponds to a batch of CSV rows. The features
dictionary maps feature column names to Tensors containing the
corresponding feature data, and labels is a Tensor containing the
batch's label data.
So your get_test_dataset() function should not care about the get_train_dataset() function dataset generation procedure.
Regarding the low test performance:
You trained and validated your model with random combination of samples of the same data with repetition:
train_data = train_data_all.take(train_size)
val_data = train_data_all.skip(train_size)
Thus probably you will have identical samples in them, so your validation will not resulted in a real measurement of the accuracy of model.
In contrary the model never saw the test set's samples, so prediction on this set is a reliable measurement of performance.