I am trying to fine-tune a DebertaV3 model using the new keras_nlp models from_preset and running into an strange error that I cannot quite understand.

Th error is indicating the “metric” is not built, but I am using a standard metric. The result is the same if I just use metrics=['accuracy']

The raw data are:

• list of texts from a essays.
• scores of the essays (ranging from 1 – 6)

TensorFlow: 2.15.0
Keras: 3.3.3
keras-nlp:  0.11.1

preprocessor = keras_nlp.src.models.deberta_v3.deberta_v3_classifier.DebertaV3Preprocessor.from_preset('deberta_v3_base_en',
                                               sequence_length=512)

classifier = keras_nlp.src.models.deberta_v3.deberta_v3_classifier.DebertaV3Classifier.from_preset('deberta_v3_base_en',
                                          preprocessor=None,
                                          load_weights=True,
                                          num_classes=6)

# Train Data
train_texts = train_df.full_text.tolist()  # Extract training texts
train_labels = train_df.score.tolist()  # Extract training labels
train_encoded = preprocessor(train_texts)
train_ds = tf.data.Dataset.from_tensor_slices((train_encoded, train_labels))
train_ds = train_ds.batch(CFG.batch_size).prefetch(tf.data.AUTOTUNE)

# Valid Data
valid_texts = valid_df.full_text.tolist()  # Extract validation texts
valid_labels = valid_df.score.tolist()  # Extract validation labels
valid_encoded = preprocessor(valid_texts)
valid_ds = tf.data.Dataset.from_tensor_slices((valid_encoded, valid_labels))
valid_ds = valid_ds.batch(CFG.batch_size).prefetch(tf.data.AUTOTUNE)

# Model creation
classifier.backbone.trainable = False

inputs = classifier.input
#input_layer = tf.keras.Input(shape=(512,), dtype=tf.int32)

logits = classifier(inputs)
outputs = keras.layers.Activation("softmax")(logits)

model = tf.keras.Model(inputs, outputs, name='model_' + CFG.preset)

# Set the head of the model as trainable
for layer in model.layers:
    layer.trainable = True

model.compile(
    loss=keras.losses.SparseCategoricalCrossentropy(from_logits=True),
    optimizer=keras.optimizers.AdamW(5e-5),
    metrics=[tf.keras.metrics.SparseCategoricalAccuracy(name='accuracy')],
    jit_compile=True,
)

model.fit(
    train_ds,
    validation_data=valid_ds,
    epochs=5
)

Model: "model_deberta_v3_base_en"
┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━┓
┃ Layer (type)        ┃ Output Shape      ┃    Param # ┃ Connected to      ┃
┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━┩
│ padding_mask        │ (None, None)      │          0 │ -                 │
│ (InputLayer)        │                   │            │                   │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ token_ids           │ (None, None)      │          0 │ -                 │
│ (InputLayer)        │                   │            │                   │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ deberta_v3_classif… │ (None, 6)         │ 184,426,7… │ padding_mask[0][… │
│ (DebertaV3Classifi… │                   │            │ token_ids[0][0]   │
├─────────────────────┼───────────────────┼────────────┼───────────────────┤
│ activation_2        │ (None, 6)         │          0 │ deberta_v3_class… │
│ (Activation)        │                   │            │                   │
└─────────────────────┴───────────────────┴────────────┴───────────────────┘
 Total params: 184,426,758 (703.53 MB)
 Trainable params: 595,206 (2.27 MB)
 Non-trainable params: 183,831,552 (701.26 MB)

Error:

ValueError                                Traceback (most recent call last)
Cell In[46], line 1
----> 1 model.fit(
      2     train_ds,
      3     validation_data=valid_ds,
      4     epochs=CFG.epochs,
      5     #callbacks=[lr_cb, ckpt_cb]
      6 )

File /opt/conda/lib/python3.10/site-packages/keras/src/utils/traceback_utils.py:122, in filter_traceback.<locals>.error_handler(*args, **kwargs)
    119     filtered_tb = _process_traceback_frames(e.__traceback__)
    120     # To get the full stack trace, call:
    121     # `keras.config.disable_traceback_filtering()`
--> 122     raise e.with_traceback(filtered_tb) from None
    123 finally:
    124     del filtered_tb

File /opt/conda/lib/python3.10/site-packages/keras/src/trainers/compile_utils.py:356, in CompileMetrics.result(self)
    354 def result(self):
    355     if not self.built:
--> 356         raise ValueError(
    357             "Cannot get result() since the metric has not yet been built."
    358         )
    359     results = {}
    360     unique_name_counters = {}

ValueError: Cannot get result() since the metric has not yet been built.