import math import torch import torch.nn as nn import torch.optim as optim import torch.nn.functional as F from torch.utils.data import Dataset, DataLoader from torch.optim.lr_scheduler import ReduceLROnPlateau from tqdm import tqdm import sentencepiece as spm # Define PositionalEncoding, MultiHeadAttention, PositionwiseFeedforward, # TransformerEncoderLayer, TransformerDecoderLayer, and Transformer classes class PositionalEncoding(nn.Module): def __init__(self, d_model, dropout=0.1, max_len=5000): super(PositionalEncoding, self).__init__() self.dropout = […]

%Run ai.py
/home/shaykhul/.local/lib/python3.11/site-packages/torch/optim/lr_scheduler.py:28: UserWarning: The verbose parameter is deprecated. Please use get_last_lr() to access the learning rate.
warnings.warn(“The verbose parameter is deprecated. Please use get_last_lr() ”
Epoch 1/10: 0%| | 0/2 [00:00<?, ?it/s]
Traceback (most recent call last):
File “/home/shaykhul/Desktop/ai.py”, line 272, in
train_model(model, dataloader, num_epochs, learning_rate, vocab_size)
File “/home/shaykhul/Desktop/ai.py”, line 196, in train_model
output = model(src, tgt_input, src_mask=src_mask, tgt_mask=tgt_mask)
File “/home/shaykhul/.local/lib/python3.11/site-packages/torch/nn/modules/module.py”, line 1532, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File “/home/shaykhul/.local/lib/python3.11/site-packages/torch/nn/modules/module.py”, line 1541, in _call_impl
return forward_call(*args, **kwargs)
File “/home/shaykhul/Desktop/ai.py”, line 137, in forward
src = layer(src, src_mask)
File “/home/shaykhul/.local/lib/python3.11/site-packages/torch/nn/modules/module.py”, line 1532, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File “/home/shaykhul/.local/lib/python3.11/site-packages/torch/nn/modules/module.py”, line 1541, in _call_impl
return forward_call(*args, **kwargs)
File “/home/shaykhul/Desktop/ai.py”, line 87, in forward
src2, _ = self.self_attn(src, src, src, src_mask)
File “/home/shaykhul/.local/lib/python3.11/site-packages/torch/nn/modules/module.py”, line 1532, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File “/home/shaykhul/.local/lib/python3.11/site-packages/torch/nn/modules/module.py”, line 1541, in _call_impl
return forward_call(*args, **kwargs)
File “/home/shaykhul/Desktop/ai.py”, line 55, in forward
scores = scores.masked_fill(mask == 0, float(‘-inf’))
RuntimeError: The size of tensor a (13) must match the size of tensor b (2) at non-singleton dimension 4

I am using the code found in this tutorial and the GitHub to visualize the inner mechanisms of the ViT model. z