I am trying to perform binary classification on image data (mammographs). The dataset is perfectly balanced. I am using EfficientnetV2S model for the classification, but the training and validation accuracy both are stuck at 50%. The following is the code:

# Loading the Data
train_df = new_df.sample(frac=0.7, random_state=0)  # 70% of the data for training
valid_df = new_df.drop(train_df.index)  # remaining 30% for validation and testing

batch_size = 32

# Create a data generator for training data
train_datagen = ImageDataGenerator(
    # rescale=1./255,
    rotation_range=20,
    width_shift_range=0.1,
    height_shift_range=0.1,
    zoom_range=0.2,
    horizontal_flip=True,
    vertical_flip=False,  # Typically not used for mammography
    fill_mode='nearest'
    )
train_generator = train_datagen.flow_from_dataframe(
    dataframe=train_df,
    x_col='images_filepath',
    y_col='labels',
    target_size=(224, 224),  # adjust this to the size of your images
    batch_size=batch_size,
    shuffle = True,
    class_mode='binary'  # use 'categorical' for multi-class problems
)

# Create a data generator for test data
valid_datagen = ImageDataGenerator(
    # rescale=1./255,
    rotation_range=20,
    width_shift_range=0.1,
    height_shift_range=0.1,
    zoom_range=0.2,
    horizontal_flip=True,
    vertical_flip=False,  # Typically not used for mammography
    fill_mode='nearest'
    )

valid_generator = valid_datagen.flow_from_dataframe(
    dataframe=valid_df,
    x_col='images_filepath',
    y_col='labels',
    target_size=(224, 224),  # adjust this to the size of your images
    batch_size=batch_size,
    shuffle = True,
    class_mode='binary'  # use 'categorical' for multi-class problems
)

# Train the model

efficientnet_model = keras.applications.EfficientNetV2S(
    include_top=False,
    weights="imagenet",
    input_tensor=Input(shape=(224, 224, 3))
)
x = Flatten()(efficientnet_model.output)
x_class = Dense(256, activation='relu')(x)
x_class = Dropout(0.5)(x_class)
output = Dense(1, activation='sigmoid')(x)

model = Model(inputs=efficientnet_model.input, outputs=output)

optimizer = Adam(learning_rate=1e-6, beta_1=0.9, beta_2=0.999)

model.compile(
    loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),
    optimizer=optimizer,
    metrics=["accuracy"],
)

history = model.fit(train_generator, epochs=10, validation_data=valid_generator)

I tried changing the hyperparameters but its still the same.