Thiết kế website giá rẻ

Question

I have a small python program that builds a neural network in Python using numpy and another in C++. The Numpy version learns fast and generally in the correct direction, but the C++ never makes it beyond 20% accuracy. It typically oscillates up and down sometimes making large jumps in the correct direction.

data = pd.read_csv('data.csv', header=None)
data = np.array(data)

m, n = data.shape
#np.random.shuffle(data) # shuffle before splitting into dev and training sets

#data_dev = data[0:1000].T
#Y_dev = data_dev[0]
#X_dev = data_dev[1:n]
#X_dev = X_dev / 255.

data_train = data.T
Y_train = data_train[0]
X_train = data_train[1:n]
X_train = X_train /255


_,m_train = X_train.shape

def get_predictions(A2):
    return np.argmax(A2, 0)

def get_accuracy(predictions, Y):
    return np.sum(predictions == Y) / Y.size
def one_hot(Y):
    one_hot_Y = np.zeros((Y.size, Y.max() + 1))
    one_hot_Y[np.arange(Y.size), Y] = 1
    one_hot_Y = one_hot_Y.T
    return one_hot_Y
def init_params(n_classes, n_features, n_hidden):
    W1 = np.random.rand(n_hidden, n_features) - 0.5
    b1 = np.random.rand(n_hidden, 1) - 0.5
    W2 = np.random.rand(n_classes, n_hidden) - 0.5
    b2 = np.random.rand(n_classes, 1) - 0.5
    return W1, b1, W2, b2
def init_params(n_classes, n_features, n_hidden, precise=False):
    W1 = np.random.rand(n_hidden, n_features) - 0.5
    b1 = np.random.rand(n_hidden, 1) - 0.5
    W2 = np.random.rand(n_classes, n_hidden) - 0.5
    b2 = np.random.rand(n_classes, 1) - 0.5
    if precise:
        return  W1, b1, W2, b2
    return  W1.astype(np.float32), b1.astype(np.float32), W2.astype(np.float32), b2.astype(np.float32) 
def init_params_non_random(n_classes, n_features, n_hidden, precise = False):
    W1 = np.zeros( (n_hidden, n_features)) + .1
    
    b1 = np.zeros( (n_hidden, 1) ) + .1
    W2 = np.zeros( (n_classes, n_hidden) ) + .1
    b2 = np.zeros( (n_classes, 1)) + .1
    if precise:
        return  W1, b1, W2, b2
    return  W1.astype(np.float32), b1.astype(np.float32), W2.astype(np.float32), b2.astype(np.float32) 
    
def ReLU(Z):
    return np.maximum(Z, 0)

def softmax(Z):
    A = np.exp(Z) / sum(np.exp(Z))
    return A
    
def ReLU_deriv(Z):
    return Z > 0
    
W1, b1, W2, b2 = init_params(10,784, 10)
alpha = .9
one_hot_y = one_hot(Y_train)

num_iterations = 1
i = 0

W1, b1, W2, b2 = init_params(10,784, 10, True)

i = 0
while i <10: # Forward
    i+=1
    Z1 = W1.dot(X_train) + b1
    A1 = ReLU(Z1)
    Z2 = W2.dot(A1) + b2
    A2 = softmax(Z2)
    
    dZ2 = A2 - one_hot_y ### absolute difference between the prediction and the target. 
    db2 = 1 / m * np.sum(dZ2)
    dW2 = 1 / m * dZ2.dot(A1.T)
    dZ1 = W2.T.dot(dZ2) * ReLU_deriv(Z1)
    dW1 = 1 / m * dZ1.dot(X_train.T)
    db1 = 1 / m * np.sum(dZ1)
    
    W1 = W1 - alpha * dW1
    b1 = b1 - alpha * db1    
    #W2 = W2 - alpha * dW2  
    b2 = b2 - alpha * db2    
    predictions = get_predictions(A2)
    print(get_accuracy(predictions, Y_train))

output:

0.07441666666666667
0.11155
0.15333333333333332
0.16476666666666667
0.20651666666666665
0.23701666666666665
0.25858333333333333
0.27908333333333335
0.29013333333333335
0.3076833333333333

C++ version:

#include <bits/stdc++.h>
#include <strings.h>

#include <cmath>
#include <csignal>
#include <fstream>
#include <iostream>
#include <ostream>
#include <random>
#include <sstream>
#include <string>

using namespace std;

#define ROW 60000
#define COL 785
#define FEAT COL - 1
#define CLASSES 10
#define HIDDEN 10
#define MAX 60000
#define NORM 255  /// max value for normalization
const double euler = 2.71828182845904523536;

typedef float precision;
precision (*x_train)[ROW] = new precision[FEAT][ROW];
int *y = new int[ROW];

precision w1[HIDDEN][FEAT];
precision w2[CLASSES][HIDDEN];
precision w2T[HIDDEN][CLASSES];
precision b1[HIDDEN];
precision b2[CLASSES];
precision (*z1)[ROW] = new precision[HIDDEN][ROW];
precision (*a1)[ROW] = new precision[HIDDEN][ROW];

precision (*z2)[ROW] = new precision[CLASSES][ROW];
precision (*a2)[ROW] = new precision[CLASSES][ROW];

precision (*dz1)[ROW] = new precision[HIDDEN][ROW];
precision (*dz2)[ROW] = new precision[CLASSES][ROW];

precision dw2[CLASSES][HIDDEN];

precision db2;
precision db1;

precision (*dw1)[FEAT] = new precision[HIDDEN][FEAT];
int (*one_hot_y)[ROW] = new int[CLASSES][ROW];

int n_correct = 0;  // the number of correct predictions
precision alpha = 0.9;

int main() {
  std::ifstream file("data.csv");
  if (!file.is_open()) {
    std::cerr << "Error opening file data.csv" << endl;
  }
  std::string line;
  int row = 0;
  int col = 0;
  while (std::getline(file, line) && row < ROW) {
    std::istringstream iss(line);

    string s;
    while (getline(iss, s, ',') && col < COL) {
      if (col == 0) {
        y[row] = stoi(s);
      } else {
        x_train[col - 1][row] = stod(s) / (NORM);
      }
      col++;
    }
    col = 0;
    row++;
  }

  file.close();
  // one hot encoding
  //
  for (int i = 0; i < ROW; i++) {
    int index = y[i];
    for (int j = 0; j < CLASSES; j++) {
      one_hot_y[j][i] = (j == index) ? 1 : 0;
    }
  }

  // populate the weights;
  //
  std::random_device rd;
  std::mt19937 gen(rd());
  std::uniform_real_distribution<precision> dis(0, 1);

  bool test = false;
  for (int i = 0; i < HIDDEN; i++) {
    for (int j = 0; j < FEAT; j++) {
      w1[i][j] = test ? 0.1 : dis(gen) - .5;
      /// ;
    }
  }

  for (int i = 0; i < CLASSES; i++) {
    for (int j = 0; j < HIDDEN; j++) {
      w2[i][j] = test ? 0.1 : dis(gen) - .5;
    }
  }

  for (int i = 0; i < HIDDEN; i++) {
    b1[i] = test ? 0.1 : dis(gen) - .5;
  }
  for (int i = 0; i < CLASSES; i++) {
    b2[i] = test ? 0.1 : dis(gen) - .5;
  }

  while (n_correct / (precision)ROW < .95) {
    // Z1 = W1.dot(X_train) + b1
    // A1 = RelU(Z1)
    //

    for (int i = 0; i < HIDDEN; i++) {
      for (int j = 0; j < ROW; j++) {
        z1[i][j] = 0.0;
        for (int k = 0; k < FEAT; k++) {
          z1[i][j] += w1[i][k] * x_train[k][j];
        }
        z1[i][j] += b1[i];
        a1[i][j] = (z1[i][j] <= 0.0) ? 0.0 : z1[i][j];
      }
    }

    //
    // Z2 = W2.dot(A1) + b2
    //
    for (int i = 0; i < CLASSES; i++) {
      for (int j = 0; j < ROW; j++) {
        z2[i][j] = 0.0;  // initialize z2
        for (int k = 0; k < HIDDEN; k++) {
          z2[i][j] += (w2[i][k] * a1[k][j]);
        }
        z2[i][j] += b2[i];
      }
    }

    int n_correct = 0;
    for (int i = 0; i < ROW; i++) {
      precision exp_sum = 0.0;
      for (int j = 0; j < CLASSES; j++) {
        a2[j][i] = pow(euler, z2[j][i]);

        exp_sum += a2[j][i];
      }
      int prediction = 0;
      precision max = 0;
      for (int j = 0; j < CLASSES; j++) {
        a2[j][i] /= exp_sum;
        if (a2[j][i] > max) {
          prediction += 1;
          max = (a2[j][i]);
        }
      }

      if (y[i] == prediction) {
        n_correct += 1;
      }
    }

    db2 = 0.0;
    for (int i = 0; i < ROW; i++) {
      for (int j = 0; j < CLASSES; j++) {
        dz2[j][i] = a2[j][i] - one_hot_y[j][i];
        db2 += dz2[j][i];
      }
    }
    db2 /= MAX;

    for (int i = 0; i < CLASSES; i++) {
      for (int j = 0; j < HIDDEN; j++) {
        dw2[i][j] = 0.0;
        for (int k = 0; k < ROW; k++) {
          dw2[i][j] += (dz2[i][k] * a1[j][k]);
        }
        dw2[i][j] /= MAX;
      }
    }

    db1 = 0.0;
    // dZ1 = W2.T.dot(dZ2) * ReLU_deriv(Z1)
    //// this is incorrect
    ///
    ///
    ///
    ///
    for (int i = 0; i < CLASSES; i++) {
      for (int j = 0; j < HIDDEN; j++) {
        w2T[j][i] = w2[i][j];
      }
    }
    for (int i = 0; i < HIDDEN; i++) {
      for (int j = 0; j < ROW; j++) {
        dz1[i][j] = 0.0;

        for (int k = 0; k < CLASSES; k++) {
          dz1[i][j] += (w2T[i][k] * dz2[k][j]);
        }

        dz1[i][j] *= (z1[i][j] <= 0 ? 0 : 1);

        db1 += dz1[i][j];
      }
    }

    db1 /= MAX;
    for (int i = 0; i < HIDDEN; i++) {
      for (int j = 0; j < FEAT; j++) {
        dw1[i][j] = 0.0;
        for (int k = 0; k < ROW; k++) {
          dw1[i][j] += (dz1[i][k] * x_train[j][k]);
        }
        dw1[i][j] /= MAX;
      }
    }
    // dw2 appears to be correct, but not dW1
    // dw1 is incorrect because dz1 is incorrect
    // dz1 is based on dz2 and w2. dz2 appears to be correct. w2 must be
    // correct.
    // cout << w2[0][9] << endl;
    for (int i = 0; i < CLASSES; i++) {
      b2[i] -= (alpha * db2);
      for (int j = 0; j < HIDDEN; j++) {
        w2[i][j] -= (alpha * dw2[i][j]);
      }
    }

    for (int i = 0; i < HIDDEN; i++) {
      b1[i] -= (alpha * db1);
      for (int j = 0; j < FEAT; j++) {
        w1[i][j] -= (alpha * dw1[i][j]);
      }
    }

    cout << "Accuracy is " << n_correct / (double)ROW << endl;
  }
  return 0;
}

Accuracy is 0.0919833
Accuracy is 0.109233
Accuracy is 0.1223
Accuracy is 0.131617
Accuracy is 0.142067
Accuracy is 0.148883
Accuracy is 0.145183
Accuracy is 0.13635
Accuracy is 0.122533
Accuracy is 0.108967
Accuracy is 0.100317

My conclusion is either I am making a mistake with one of the jacobians or there is some kind of precision/numerical stability issue in the c++ implementation.

I am using -0fast but have also tried other optimization levels and all precision levels from float to long double.

Additionally I have tried different weights initialization.

I used the test boolean flag to generate predictable weights so I can compare them with the numpy version. The jacobian for dz1 appears to be incorrect (different from what numpy produces) but I cannot see why.

Thiết kế website giá rẻ

Danh mục

Why can’t I build a neural network in C++ with no dependencies, even though it works in Numpy?