Webcam Frame to model.predict_generator - Jupyter Notebook & CV2 - tensorflow

Question

I have created this custom CNN, trained it, and now wish to try and pass frames from my webcam in real-time for testing the predictions.
The webcam video playback starts to capture the frame by frame, however, I am unsure what to do to the frame in order to get it working with the CNN model
Any advice would be appreciated
I have provided the full code of what I am trying to achieve
#imported necessities
import os
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import cv2
from matplotlib.image import imread
from IPython.display import clear_output
import time
import PIL.Image
from io import StringIO
import IPython.display
import numpy as np
from io import BytesIO
import tensorflow as tf
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Activation, Dense, Conv2D, MaxPool2D, Dropout, Flatten, MaxPooling2D
from tensorflow.keras.callbacks import EarlyStopping
#Data Paths
data_dir = 'C:\\Users\\User\\Desktop\\DATAWeather'
test_path = data_dir+'\\Test\\'
train_path = data_dir+'\\Train\\'
#Variable to resize all of the images
image_shape = (224,224,3) #224*224*3 = 150528 Data Points : thats why we need image batch
#Apply a generator so it does not always get the same format of picture (recognizes different things)
image_gen = ImageDataGenerator(rotation_range=20, width_shift_range=0.1, height_shift_range=0.1, rescale=1/255, shear_range=0.1, zoom_range=0.1,horizontal_flip=True,fill_mode='nearest')
#setting up a base convolutional layer
model = Sequential()
model.add(Conv2D(filters=32, kernel_size=(3,3),input_shape=image_shape, activation='relu',))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(filters=64, kernel_size=(3,3),input_shape=image_shape, activation='relu',))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(filters=64, kernel_size=(3,3),input_shape=image_shape, activation='relu',))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(128))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(4))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy',optimizer='adam', metrics=['accuracy']), #model.summary()
#Create an early EPOCH stoppage based on the validation loss based off TWO epochs
early_stop=EarlyStopping(monitor='val_loss', patience=2)
#TRAINING MODEL - use two to the power
batch_size=32
#TWO generators
train_image_gen = image_gen.flow_from_directory(train_path, target_size=image_shape[:2], color_mode='rgb', batch_size = batch_size, class_mode='categorical', shuffle=True)
test_image_gen = image_gen.flow_from_directory(test_path, target_size=image_shape[:2], color_mode='rgb', batch_size = batch_size, class_mode='categorical', shuffle=False)
results = model.fit_generator(train_image_gen, epochs=1, validation_data=test_image_gen, callbacks=[early_stop])
***
**def showarray(a, fmt='jpeg'):
f = BytesIO()
PIL.Image.fromarray(a).save(f, fmt)
IPython.display.display(IPython.display.Image(data=f.getvalue()))
def get_frame(cam):
# Capture frame-by-frame
ret, frame = cam.read()
#flip image for natural viewing
frame = cv2.flip(frame, 1)
return frame
cam = cv2.VideoCapture(0)
def make_1080p():
cam.set(3, 224)
cam.set(4, 224)
def change_res(width, height):
cam.set(3, width)
cam.set(4, height)
change_res(224, 224)
try:
while(True):
t1 = time.time()
frame = get_frame(cam)
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
showarray(frame)
t2 = time.time()
print("%f FPS" % (1/(t2-t1)))
# Display the frame until new frame is available
clear_output(wait=True)
Weather_Prediction_Cell = (frame)
#Weather_Prediction_Cell /= 255
model.predict_generator(frame)
#print(Weather_Prediction_Cell)
print(pred)**
except KeyboardInterrupt:
cam.release()
print("Stream stopped")
***

Answer 1

Keras model has a method called "predict". It takes a single np.array or a list of np.arrays as input (which should have the exact same shape as your neural net. input, including the batch part: (batch_count, width, height, channels) for example). You input this to model.predict, then it returns you back the result as an np.array again, with the shape your neural network output layer has. I'm not used to webcam applications with opencv, but if you get the frame data in np.array somehow, you can feed it to your neural net. as well. Just be sure about its shape, and reshape it if needed.