I’m currently training a DL model that uses image recognition and everything seems fine at first until i get to the training cell and running it stopped at epoch 2 with a warning that says: Epoch 1/50 475/475 [==============================] - 33s 49ms/step - loss: 13.8371 - age_output_loss: 2.5835 - race_output_loss: 2.2759 - gender_output_loss: 0.8926 - age_output_mean_absolute_error: 1.2590 - race_output_accuracy: 0.0000e+00 - gender_output_accuracy: 0.0000e+00 - val_loss: 4.0951 - val_age_output_loss: 0.2312 - val_race_output_loss: 2.0730 - val_gender_output_loss: 0.6076 - val_age_output_mean_absolute_error: 0.3847 - val_race_output_accuracy: 0.0000e+00 - val_gender_output_accuracy: 0.0000e+00 Epoch 2/50 WARNING:tensorflow:Your input ran out of data; interrupting training. Make sure that your dataset or generator can generate at least steps_per_epoch * epochsbatches (in this case, 23750 batches). You may need to use the repeat() function when building your dataset. WARNING:tensorflow:Your input ran out of data; interrupting training. Make sure that your dataset or generator can generate at leaststeps_per_epoch * epochs batches (in this case, 119 batches). You may need to use the repeat() function when building your dataset. 475/475 [==============================] - 0s 227us/step - loss: 0.0000e+00 - age_output_loss: 0.0000e+00 - race_output_loss: 0.0000e+00 - gender_output_loss: 0.0000e+00 - age_output_mean_absolute_error: 0.0000e+00 - race_output_accuracy: 0.0000e+00 - gender_output_accuracy: 0.0000e+00

from tensorflow.keras.utils import to_categorical
TRAIN_TEST_SPLIT = 0.80
IM_WIDTH = 48
IM_HEIGHT = 48

class UtkFaceDataGenerator():
    """
    Data generator for the UTKFace dataset. This class should be used when training our Keras multi-output model.
    """
    def __init__(self, df):
        self.df = df

    def generate_split_indexes(self):
        p = np.random.permutation(len(self.df))
        train_up_to = int(len(self.df) * TRAIN_TEST_SPLIT)
        train_idx = p[:train_up_to]
        test_idx = p[train_up_to:]

        train_up_to = int(train_up_to * TRAIN_TEST_SPLIT)
        train_idx, valid_idx = train_idx[:train_up_to], train_idx[train_up_to:]

        # converts alias to id
        self.df['gender_id'] = self.df['Gender'].map(lambda gender: dataset_dict['gender_alias'][gender])
        self.df['race_id'] = self.df['Race'].map(lambda race: dataset_dict['race_alias'][race])

        self.max_age = self.df['Age'].max()

        return train_idx, valid_idx, test_idx

    def preprocess_image(self, img_path):
        """
        Used to perform some minor preprocessing with openCV on the image
        before inputting into the network.
        """
        image = cv2.imread(img_path)
        image = cv2.resize(image,(IM_HEIGHT, IM_WIDTH))
        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
        image = np.array(image) / 255.0

        return image

    def generate_images(self, image_idx, is_training, batch_size=16):

    # Shuffle the image indices if training
        if is_training:
          np.random.shuffle(image_idx)

    # Initialize batch lists
        images, ages, races, genders = [], [], [], []

    # Iterate over image indices
        for idx in image_idx:
          person = self.df.iloc[idx]

        age = person['Age']
        race = person['race_id']
        gender = person['gender_id']
        file = person['File']

        # Preprocess image
        im = self.preprocess_image(file)

        # Append data to batch lists
        ages.append(age / self.max_age)
        races.append(to_categorical(race, len(dataset_dict['race_id'])))
        genders.append(to_categorical(gender, len(dataset_dict['gender_id'])))
        images.append(im)

        # Yield batch if batch size reached
        if len(images) == batch_size:
            yield np.array(images), [np.array(ages), np.array(races), np.array(genders)]
            images, ages, races, genders = [], [], [], []

       # Yield remaining data if any
        if len(images) > 0:
            yield np.array(images), [np.array(ages), np.array(races), np.array(genders)]


data_generator = UtkFaceDataGenerator(df)
train_idx, valid_idx, test_idx = data_generator.generate_split_indexes()

and here is the training cell

from tensorflow.keras.losses import MeanSquaredError, CategoricalCrossentropy, BinaryCrossentropy
from tensorflow.keras.metrics import MeanAbsoluteError, Accuracy


# Instantiate the UtkMultiOutputModel class and assemble the full model
# Define the input shape
input_shape = (IM_HEIGHT, IM_WIDTH, 3)

# Instantiate the UtkMultiOutputModel class 
#model = UtkMultiOutputModel(input_shape=input_shape)
model = UtkMultiOutputModel(input_shape=input_shape, num_races=len(dataset_dict['race_alias']))

# Assemble the full model
#model = model.assemble_full_model((IM_HEIGHT, IM_WIDTH, 3), num_race=len(dataset_dict['race_alias']))
model = model.assemble_full_model()



# Compile the model
init_lr = 1e-4
opt = Adam(learning_rate=init_lr)

model.compile(optimizer=opt,
              loss={
                  'age_output': MeanSquaredError(),
                  'race_output': CategoricalCrossentropy(),
                  'gender_output': BinaryCrossentropy()},
              loss_weights={
                  'age_output': 4.,
                  'race_output': 1.5,
                  'gender_output': 0.1},
              metrics={
                  'age_output': MeanAbsoluteError(),
                  'race_output': Accuracy(),
                  'gender_output': Accuracy()})

# number of epochs
epochs = 50
batch_size = 32
valid_batch_size = 32
train_generator = data_generator.generate_images(train_idx, is_training=True, batch_size=batch_size)
val_generator = data_generator.generate_images(valid_idx, is_training=True, batch_size=valid_batch_size)

#Training 
history = model.fit(
    train_generator,
    epochs=epochs,
    validation_data=val_generator,
    verbose=1
)

even though i’m using UTKFace dataset which is over 20k images so i don’t want to use repeat() function and tried to focus on my data generator that i provided here, no matter how many times i change the data generator i still get the same warning that i start to doubt it maybe my GPU is fully utilized and can’t handle it? (i’m using google colab btw)

I’m not very experienced in this field but I’ll appreciate your help thank you!