My training loss and testing loss do not descend. Moreover, my Acc is very low, just the same as the random classification.
enter image description here
enter image description here
Below is part of my code.
datasets.py
import cupy as np
import struct
import glob
def load_mnist(path, kind='train'):
image_path = glob.glob(path + '/%s*3-ubyte' % kind)[0]
label_path = glob.glob(path + '/%s*1-ubyte' % kind)[0]
with open(label_path, "rb") as lbpath:
magic, n = struct.unpack('>II', lbpath.read(8))
labels = np.fromfile(lbpath, dtype=np.uint8)
with open(image_path, "rb") as impath:
magic, num, rows, cols = struct.unpack('>IIII', impath.read(16))
images = np.fromfile(impath, dtype=np.uint8).reshape(len(labels), 28 * 28)
return images, labels
module.py
import cupy as np
class Conv2d:
def __init__(self, in_channels: int, out_channels: int, kernel_size: int,
stride: int = 1, padding: int = 0, dtype=None):
self.in_channels = in_channels
self.out_channels = out_channels
self.kernel_size = kernel_size
self.stride = stride
self.padding = padding
self.dtype = dtype if dtype is not None else np.float16
# Initialize weights and bias
self.weight = np.random.randn(out_channels, in_channels, kernel_size, kernel_size).astype(dtype)
self.bias = np.random.randn(out_channels).astype(dtype)
# Initialize gradients
self.w_grad = np.zeros_like(self.weight)
self.b_grad = np.zeros_like(self.bias)
self.cache = None
def forward(self, x):
"""
x - shape (N, C, H, W)
return the result of Conv2d with shape (N, O, H', W')
"""
n, c, h, w = x.shape
# Calculate output dimensions
h_out = (h - self.kernel_size + 2 * self.padding) // self.stride + 1
w_out = (w - self.kernel_size + 2 * self.padding) // self.stride + 1
# Apply padding to the input
if self.padding > 0:
x_padded = np.pad(x, ((0, 0), (0, 0), (self.padding, self.padding), (self.padding, self.padding)),
mode='constant')
else:
x_padded = x
# Initialize output
out = np.zeros((n, self.out_channels, h_out, w_out), dtype=self.dtype)
# Perform the convolution
batch_stride, c_stride, h_stride, w_stride = x_padded.strides
strided_windows = np.lib.stride_tricks.as_strided(
x_padded,
shape = (n, c, h_out, w_out, self.kernel_size, self.kernel_size),
strides = (batch_stride, c_stride, h_stride * self.stride, w_stride * self.stride, h_stride, w_stride)
)
out = np.einsum('bihwkl,oikl->bohw', strided_windows, self.weight)
out += self.bias[None, :, None, None]
self.cache = x, strided_windows
return out
def backward(self, dy, lr):
"""
dy - the gradient of last layer with shape (N, O, H', W')
lr - learning rate
calculate self.w_grad to update self.weight,
calculate self.b_grad to update self.bias,
return the result of gradient dx with shape (N, C, H, W)
"""
x, strided_windows = self.cache
_, _, h_out, w_out = x.shape
b_out, c_out, _, _ = dy.shape
dy_changed = dy.copy()
if self.stride > 1:
dy_changed = np.insert(dy_changed, range(1, dy.shape[2]), 0, axis=2)
dy_changed = np.insert(dy_changed, range(1, dy.shape[3]), 0, axis=3)
if self.padding > 0:
dy_changed = np.pad(dy_changed, ((0,0), (0,0), (self.padding, self.padding), (self.padding, self.padding)),
'constant')
batch_stride, channel_stride, h_stride, w_stride = dy_changed.strides
# Initialize gradients
self.w_grad.fill(0)
self.b_grad.fill(0)
# Compute gradients
dout_windows = np.lib.stride_tricks.as_strided(dy_changed,
shape = (b_out, c_out, h_out, w_out, self.kernel_size, self.kernel_size),
strides = (batch_stride, channel_stride, h_stride * 1, w_stride * 1, h_stride, w_stride)
)
rotate_weights = np.rot90(self.weight, 2, (2, 3))
self.b_grad = np.sum(dy, axis=(0, 2, 3))
self.w_grad = np.einsum('bihwkl, bohw -> oikl', strided_windows, dy)
dx = np.einsum('bohwkl, oikl -> bihw', dout_windows, rotate_weights)
# Update weights and biases
self.weight -= lr * self.w_grad
self.bias -= lr * self.b_grad
return dx
class MaxPool2d:
def __init__(self, kernel_size: int, stride=None, padding=0):
self.kernel_size = kernel_size
self.stride = stride if stride is not None else kernel_size
self.padding = padding
self.out = None
def forward(self, x):
"""
x - shape (N, C, H, W)
return the result of MaxPool2d with shape (N, C, H', W')
"""
n, c, h, w = x.shape
kh, kw = self.kernel_size, self.kernel_size
sh, sw = self.stride, self.stride
ph, pw = self.padding, self.padding
h_out = (h + 2 * ph - kh) // sh + 1
w_out = (w + 2 * pw - kw) // sw + 1
x_padded = np.pad(x, ((0, 0), (0, 0), (ph, ph), (pw, pw)), mode='constant')
x_reshaped = x_padded.reshape(n, c, h_out, sh, w_out, sw)
out = np.max(x_reshaped, axis=(3, 5))
self.out = (x, x_padded)
return out
def backward(self, dy):
"""
dy - shape (N, C, H', W')
return the result of gradient dx with shape (N, C, H, W)
"""
x, x_padded = self.out
kh, kw = self.kernel_size, self.kernel_size
sh, sw = self.stride, self.stride
ph, pw = self.padding, self.padding
h_out, w_out = dy.shape[2], dy.shape[3]
dx_padded = np.zeros_like(x_padded)
dx_reshaped = dx_padded.reshape(x.shape[0], x.shape[1], h_out, sh, w_out, sw)
dy_reshaped = dy.reshape(x.shape[0], x.shape[1], h_out, 1, w_out, 1)
dx_reshaped += np.maximum_gradient(x_padded, dx_reshaped, axis=(3, 5))
if ph == 0 and pw == 0:
return dx_padded
return dx_padded[:, :, ph:-ph, pw:-pw] if ph > 0 and pw > 0 else dx_padded
class AvgPool2d:
def __init__(self, kernel_size: int, stride=None, padding=0):
self.kernel_size = kernel_size
self.stride = stride if stride is not None else kernel_size
self.padding = padding
self.out = None
def forward(self, x):
"""
x - shape (N, C, H, W)
return the result of AvgPool2d with shape (N, C, H', W')
"""
n, c, h, w = x.shape
kh, kw = self.kernel_size, self.kernel_size
sh, sw = self.stride, self.stride
ph, pw = self.padding, self.padding
h_out = (h + 2 * ph - kh) // sh + 1
w_out = (w + 2 * pw - kw) // sw + 1
x_padded = np.pad(x, ((0, 0), (0, 0), (ph, ph), (pw, pw)), mode='constant')
x_reshaped = x_padded.reshape(n, c, h_out, sh, w_out, sw)
out = x_reshaped.mean(axis=(3, 5))
self.out = (x, x_padded)
return out
def backward(self, dy):
"""
dy - shape (N, C, H', W')
return the result of gradient dx with shape (N, C, H, W)
"""
x, x_padded = self.out
kh, kw = self.kernel_size, self.kernel_size
sh, sw = self.stride, self.stride
ph, pw = self.padding, self.padding
h_out, w_out = dy.shape[2], dy.shape[3]
dx_padded = np.zeros_like(x_padded)
dx_reshaped = dx_padded.reshape(x.shape[0], x.shape[1], h_out, sh, w_out, sw)
dy_reshaped = dy.reshape(x.shape[0], x.shape[1], h_out, 1, w_out, 1)
dx_reshaped += dy_reshaped / (kh * kw)
if ph == 0 and pw == 0:
return dx_padded
return dx_padded[:, :, ph:-ph, pw:-pw] if ph > 0 and pw > 0 else dx_padded
class Linear:
def __init__(self, in_features: int, out_features: int, bias: bool = True):
self.in_features = in_features
self.out_features = out_features
self.bias = bias
self.input = None
self.W = np.random.randn(in_features, out_features)
if self.bias:
self.b = np.random.randn(1, out_features)
else:
self.b = None
def forward(self, x):
"""
x - shape (N, C)
return the result of Linear layer with shape (N, O)
"""
self.input = x
output = np.dot(self.input, self.W)
if self.bias:
output += self.b
return output
def backward(self, dy, lr):
"""
dy - shape (N, O)
return the result of gradient dx with shape (N, C)
"""
dx = np.dot(dy, self.W.T)
dw = np.dot(self.input.T, dy)
self.W -= lr * dw
if self.bias:
self.b -= lr * np.sum(dy, axis=0, keepdims=True)
return dx
network.py
from module import Conv2d, AvgPool2d, Linear
from tools import Sigmoid
class LeNet5:
def __init__(self):
self.conv1 = Conv2d(1, 6, 5, 1, 2)
self.relu1 = Sigmoid()
self.pool1 = AvgPool2d(2)
self.conv2 = Conv2d(6, 16, 5)
self.relu2 = Sigmoid()
self.pool2 = AvgPool2d(2)
self.fc1 = Linear(16 * 5 * 5, 120)
self.relu3 = Sigmoid()
self.fc2 = Linear(120, 84)
self.relu4 = Sigmoid()
self.fc3 = Linear(84, 10)
def forward(self, x):
x = self.conv1.forward(x)
x = self.relu1.forward(x)
x = self.pool1.forward(x)
x = self.conv2.forward(x)
x = self.relu2.forward(x)
x = self.pool2.forward(x)
x = x.reshape(-1, 400)
x = self.fc1.forward(x)
x = self.relu3.forward(x)
x = self.fc2.forward(x)
x = self.relu4.forward(x)
x = self.fc3.forward(x)
return x
def backward(self, dy, lr):
dy = self.fc3.backward(dy, lr)
dy = self.relu4.backward(dy)
dy = self.fc2.backward(dy, lr)
dy = self.relu3.backward(dy)
dy = self.fc1.backward(dy, lr)
dy = dy.reshape(-1, 16, 5, 5)
dy = self.pool2.backward(dy)
dy = self.relu2.backward(dy)
dy = self.conv2.backward(dy, lr)
dy = self.pool1.backward(dy)
dy = self.relu1.backward(dy)
dy = self.conv1.backward(dy, lr)
return dy
tools.py
import cupy as np
class Relu:
def __init__(self):
self.mask = None
self.input = None
def forward(self, x):
self.input = x
self.mask = (self.input <= 0)
out = self.input.copy()
out[self.mask] = 0
return out
def backward(self, d_out):
d_out[self.mask] = 0
dx = d_out
return dx
class Tanh:
def __init__(self):
self.input = None
def forward(self, x):
self.input = x
return np.tanh(self.input)
def backward(self, dy):
s = self.forward(self.input)
return dy * (1 - s ** 2)
class Sigmoid:
def __init__(self):
self.input = None
def forward(self, x):
self.input = x
return 1.0 / (1 + np.exp(-self.input))
def backward(self, dy):
s = self.forward(self.input)
return dy * s * (1 - s)
class CrossEntropyLoss:
def __call__(self, x, label):
x = np.array(x)
num_samples = x.shape[0]
# Clip the values to prevent division by zero
x = np.clip(x, 1e-12, 1. - 1e-12)
one_hot_labels = np.zeros_like(x)
one_hot_labels[np.arange(num_samples), label] = 1
loss = -np.sum(one_hot_labels * np.log(x)) / num_samples
return loss
train.py
import cupy as np
import tqdm
import logging
import pickle
from network import LeNet5
from datasets import load_mnist
from config import Config
from tools import CrossEntropyLoss
def create_batches(data, labels, batch_size):
"""
Creates batches of data and labels.
"""
for num in range(0, len(data), batch_size):
yield data[num:num + batch_size], labels[num:num + batch_size]
if __name__ == '__main__':
# Initialize logging
logging.basicConfig(filename='training3.log', level=logging.INFO, format='%(asctime)s - %(message)s')
opt = Config()
path = "./MNIST/MNIST-Dataset"
train_images, train_labels = load_mnist(path + "/Train", kind="train")
test_images, test_labels = load_mnist(path + "/Test", kind="t10k")
train_images = train_images.astype(np.float16) / 256
test_images = test_images.astype(np.float16) / 256
model = LeNet5()
lr = opt.lr
epochs = opt.epoch
batch_size = opt.batch
criterion = CrossEntropyLoss()
logging.info(f"Parameters: lr={lr}, epochs={epochs}")
for epoch in range(epochs):
total_loss = 0
batches = create_batches(train_images, train_labels, batch_size)
for x_batch, y_batch in tqdm.tqdm(batches):
batch_loss = 0
dy_batch = []
for x, y in zip(x_batch, y_batch):
x = x.reshape(1, 1, 28, 28)
output = model.forward(x)
loss = criterion(output, y)
batch_loss += loss
dy = np.zeros_like(output)
dy[0, y] = -1 / output[0, y]
dy_batch.append(dy)
total_loss += batch_loss
for x, dy in zip(x_batch, dy_batch):
x = x.reshape(1, 1, 28, 28)
model.backward(dy, lr)
avg_loss = total_loss / len(train_images)
test_loss = 0
test_batches = create_batches(test_images, test_labels, batch_size)
for x_batch, y_batch in test_batches:
batch_loss = 0
for x, y in zip(x_batch, y_batch):
x = x.reshape(1, 1, 28, 28)
output = model.forward(x) # No dropout during evaluation
loss = criterion(output, y)
batch_loss += loss
test_loss += batch_loss
avg_test_loss = test_loss / len(test_images)
logging.info(f"Epoch [{epoch + 1}/{epochs}], Average Loss: {avg_loss}, Testing Loss: {avg_test_loss}")
with open('lenet5_model3.pkl', 'wb') as f:
pickle.dump(model, f)
logging.info("Model saved as lenet5_model3.pkl")
test.py
import cupy as np
import tqdm
import logging
import pickle
from network import LeNet5
from datasets import load_mnist
from config import Config
from tools import CrossEntropyLoss
opt = Config()
path = "./MNIST/MNIST-Dataset"
train_images, train_labels = load_mnist(path + "/Train", kind="train")
test_images, test_labels = load_mnist(path + "/Test", kind="t10k")
train_images = train_images.astype(np.float16) / 256
test_images = test_images.astype(np.float16) / 256
model = LeNet5()
lr = opt.lr
epochs = opt.epoch
criterion = CrossEntropyLoss()
logging.basicConfig(filename='testing3.log', level=logging.INFO, format='%(asctime)s - %(message)s')
correct = 0
with open('lenet5_model3.pkl', 'rb') as f:
model = pickle.load(f)
logging.info("Model loaded from lenet5_model3.pkl")
for i in tqdm.tqdm(range(len(test_images))):
x = test_images[i]
x = x.reshape(1, 1, 28, 28)
y = test_labels[i]
output = model.forward(x)
prediction = np.argmax(output)
if prediction == y:
correct += 1
accuracy = correct / len(test_images)
logging.info(f"Test Accuracy: {accuracy}")
logging.info("*---------------------------------------------*")
logging.info(f"Test Accuracy: {accuracy}")
I think the possible problem is overfitting caused by incorrect parameter settings, or there is a loophole in the network layer propagation in module.py. However, I’ve tried to change the learning rate(0.05, 0.001, 0.0001) and the batch size(30, 60), but no improvement.
New contributor
chusong is a new contributor to this site. Take care in asking for clarification, commenting, and answering.
Check out our Code of Conduct.