I’m basically trying to fine-tune my model with Neftune. Model is based on Turkish Language. But there I’m receiving zero training lose. I’ve tried to another model like Turkish-GPT2 there is no issue everything is okay. I think probably there is problem with model. I don’t know how to handle with this issue.

Loading Model:

from transformers import AutoTokenizer, AutoModelForCausalLM

tokenizer = AutoTokenizer.from_pretrained("asafaya/kanarya-750m")
model = AutoModelForCausalLM.from_pretrained("asafaya/kanarya-750m")

v3_prompt = """Aşağıda, daha fazla bağlam sağlayan bir girdiyle eşleştirilmiş, bir görevi açıklayan bir talimat bulunmaktadır. İsteği uygun şekilde tamamlayan bir yanıt yazın.

### Input:
{}

### Instructions:
{}

### Response:
{}
"""

Changing format prompts:

EOS_TOKEN = tokenizer.eos_token # Must add EOS_TOKEN
def formatting_prompts_func(examples):
    inputs       = examples["input"]
    instructions = examples['instructions']
    outputs      = examples["response"]
    texts = []
    for input, instructions, output in zip(inputs, instructions, outputs):
        # Must add EOS_TOKEN, otherwise your generation will go on forever!
        text = v3_prompt.format(input, instructions, output) + EOS_TOKEN
        texts.append(text)
    return { "text" : texts, }
pass

from datasets import Dataset

dataset = Dataset.from_pandas(data[:40000])
dataset = dataset.map(formatting_prompts_func, batched=True)

Neftune:

from trl import SFTTrainer
from transformers import TrainingArguments

trainer_2 = SFTTrainer(
    model=model,
    train_dataset=dataset,
    dataset_text_field="text",
    max_seq_length=512,
    neftune_noise_alpha=5,
    packing=False,
    args = TrainingArguments(
        per_device_train_batch_size = 1, #  batch size
        gradient_accumulation_steps = 2, #  gradient accumulation steps
        warmup_steps = 5,
        max_steps = 80,
        learning_rate = 2e-4,
        fp16 = False,
        bf16 = torch.cuda.is_bf16_supported(),
        logging_steps = 1,
        optim = "adamw_8bit",
        weight_decay = 0.01,
        lr_scheduler_type = "linear",
        seed = 3407,
        output_dir = "outputs",
    ),
)
trainer_2.train()

Output:

 [80/80 00:25, Epoch 0/1]
Step    Training Loss
1   0.000000
2   0.000000
3   0.000000
4   0.000000
5   0.000000
6   0.000000
7   0.000000
8   0.000000
9   0.000000
10  0.000000
11  0.000000
12  0.000000
13  0.000000
14  0.000000
15  0.000000
16  0.000000
17  0.000000
18  0.000000
19  0.000000
20  0.000000
21  0.000000
22  0.000000
23  0.000000
24  0.000000
25  0.000000
26  0.000000
27  0.000000
28  0.000000
29  0.000000
30  0.000000
31  0.000000
32  0.000000
33  0.000000
34  0.000000
35  0.000000
36  0.000000
37  0.000000
38  0.000000
39  0.000000
40  0.000000
41  0.000000
42  0.000000
43  0.000000
44  0.000000
45  0.000000
46  0.000000
47  0.000000
48  0.000000
49  0.000000
50  0.000000
51  0.000000
52  0.000000
53  0.000000
54  0.000000
55  0.000000
56  0.000000
57  0.000000
58  0.000000
59  0.000000
60  0.000000
61  0.000000
62  0.000000
63  0.000000
64  0.000000
65  0.000000
66  0.000000
67  0.000000
68  0.000000
69  0.000000
70  0.000000
71  0.000000
72  0.000000
73  0.000000
74  0.000000
75  0.000000
76  0.000000
77  0.000000
78  0.000000
79  0.000000
80  0.000000
TrainOutput(global_step=80, training_loss=0.0, metrics={'train_runtime': 25.3789, 'train_samples_per_second': 6.304, 'train_steps_per_second': 3.152, 'total_flos': 117937578909696.0, 'train_loss': 0.0, 'epoch': 0.004})