I am currently doing some research with machine learning and I am facing some issues using pytorch with opacus. Starting at epoch one, I get an accuracy at roughly 0.61, and this number does not increasing matter how I choose the number of epochs and parameters like batchsize, learning rate, …
The accuracy reflects very much the distribution of my training data (binary classification), so I guess my model simply does not learn from data but from the probability of the target. I am pretty sure I made some mistakes during the implementation which leads me to not learn anything out of the data. In the best case, I want to keep the SGD as optimizer for some project related reasons.
I use the opacus module to get differential privacy. I want to predict the heart disease column based on the other attributes. These data are just dummy data with the encoding I will use later for different sets. I would really appreciate if you could help me 🙂

Here is how my data looks (I encoded them numerically, I split with skikitlearn‘s ‚train_test_split‘):

HeartDisease,BMI,Smoking,AlcoholDrinking,Stroke,PhysicalHealth,MentalHealth,DiffWalking,Sex,AgeCategory,Race,Diabetic,PhysicalActivity,GenHealth,SleepTime,Asthma,KidneyDisease,SkinCancer
0,16.6,1,0,0,3.0,30.0,0,0,7,0,1,1,0,5.0,1,0,1
1,28.87,1,0,0,6.0,0.0,1,0,11,1,0,0,1,12.0,0,0,0
1,34.3,1,0,0,30.0,0.0,1,1,8,0,1,0,3,15.0,1,0,0

And here is my code:

class DPOpacus(nn.Module):
    def __init__(self, input_size):
        super(DPOpacus, self).__init__()
        self.fc1 = nn.Linear(input_size, 64)
        self.fc2 = nn.Linear(64, 32)
        self.fc3 = nn.Linear(32, 1)
        self.fc4 = nn.Sigmoid()
    
    def forward(self, layer):
        layer = torch.relu(self.fc1(layer))
        layer = torch.relu(self.fc2(layer))
        layer = self.fc3(layer)
        return layer
    
class DPOpacusDataSet(Dataset):
    data = {}
    def __init__(self, feature, target, feature_test, target_test):
        self.data['feature'] = torch.tensor(feature, dtype=torch.float32)
        self.data['target'] = torch.tensor(target, dtype=torch.float32)
        self.data['feature_test'] = torch.tensor(feature_test, dtype=torch.float32)
        self.data['target_test'] = torch.tensor(target_test, dtype=torch.float32)

    def __len__(self):
        return len(self.data['feature'])
    
    def __getitem__(self, index):
        return self.data['feature'][index], self.data['target'][index]
    
        
    
def get_priv_engine():
    privacy_engine = PrivacyEngine()
    return privacy_engine

def binary_accuracy(preds, y):
    correct = torch.unsqueeze(torch.argmax(preds, dim=-1), dim=1)==y.long()
    acc = correct.sum() / len(correct)
    return acc

    
def train(model:DPOpacus, train_loader:DataLoader, optimizer:optim.SGD, epoch, priv_engine: PrivacyEngine, device="cuda" if torch.cuda.is_available else "cpu"):
    criterion = nn.BCEWithLogitsLoss()  #MSELoss() #TODO different loss
    losses = []
    accuracies = []
    model = model.train().to(device)

    for features, target in tqdm(train_loader):
        features = features.to(device)
        target = target.to(device)
        optimizer.zero_grad()
        predictions = model(features)
        loss = criterion(predictions, target)
        acc = binary_accuracy(predictions, target)

        loss.backward()
        optimizer.step()

        losses.append(loss.item())
        accuracies.append(acc.item())
    
    epsilon = priv_engine.accountant.get_epsilon(delta=0.02)
    print(
        f"Train epoch {epoch}: n"
        f"Loss: {np.mean(losses):.6f} -- Train-Accuracy {np.mean(accuracies):.6f}"
        f"(ε = {epsilon:.2f}, δ = {__DELTA__})"
        f"--------Model Parameters---------"
        f"{model.parameters}"
    )

def train_opacus(feature, target, feature_test, target_test):
    train_dataset = op.DPOpacusDataSet(feature, target, feature_test, target_test)
    train_loader = DataLoader(train_dataset, batch_size=__BATCH_SIZE__, shuffle=False)
    model = op.DPOpacus(17)
    optimizer = optim.SGD(model.parameters(), lr=0.01, weight_decay=1e-3)
    privacy_engine = op.get_priv_engine()
    model, optimizer, data_loader = privacy_engine.make_private(
        module=model,
        optimizer=optimizer,
        data_loader=train_loader,
        noise_multiplier= __NOISE_MUL__,
        max_grad_norm=__CLIPNORM__,
    )
    #TODO: train
    for epoch in range(1, 5): #TODO: __EPOCHS__
        op.train(
            model=model, 
            train_loader=data_loader, 
            optimizer=optimizer, 
            epoch=epoch, 
            priv_engine=privacy_engine, 
            device=__DEVICE__)

I tried different optimizers, different parameter and some other steps while learning but most turned out to be not appropriate (they threw errors) or did not change a thing. I am new to machine learning and I would really appreciate some help to find my problem, suggestions how to reorder/change my code or even a tutorial that fits my data – everything would help since I am stuck now for 2-3 weeks on this.