def ActivityNetV3(n_sensors, window_size):
    #inputs = Input(shape=(n_sensors, window_size, 1))
    inputs = Input(shape=(n_sensors, window_size, 1))
    print("Inside the model")
    print(inputs.shape)
    x = Conv2D(
            filters=1, kernel_size=(1, 1), weights=[np.array([[[[1.0]]]])], padding='same',
            activation=None, trainable=False, use_bias=False,
        )(inputs)
    x = BatchNormalization(center=False, scale=True,epsilon=1e-5)(x)

    x = ZeroPadding2D(padding=((0,0),(1,1)))(x) # (inputs)
    x = Conv2D(
        32,
        kernel_size=(n_sensors, tconfig.KERNEL_SIZE),
        strides=1,
        padding="valid",
        use_bias=False,
        name="Conv00",
    )(x)
    x = BatchNormalization(
        name="Conv00/BatchNorm"
    )(x)
    x = Activation(tconfig.ACTIVATION)(x)
    x = PrintLayer("After initial Conv2D and BatchNorm:")(x)
    x = check_numerics(x, "After initial Conv2D and BatchNorm")

    x = stack_fn(x, None)
    last_conv_ch = _depth(K.int_shape(x)[tconfig.CHANNEL_AXIS] * 3) # 3
    x = Conv2D(
        last_conv_ch,
        kernel_size=1,
        padding="same",
        use_bias=False,
        name="Conv_1",
    )(x)
    x = BatchNormalization(
        name="Conv_1/BatchNorm"
    )(x)
    x = Activation(tconfig.ACTIVATION)(x)
    x = PrintLayer("After stack_fn and additional Conv2D:")(x)
    x = check_numerics(x, "After initial Conv2D and BatchNorm")

    x = GlobalAvgPool2D(keepdims=True)(x)
    x = Conv2D(
        tconfig.LAST_POINT_CH,
        kernel_size=1,
        padding="same",
        use_bias=True,
        name="Conv_2",
    )(x)
    x = Activation(tconfig.ACTIVATION)(x)
    x = Dropout(tconfig.DROPOUT_RATE)(x) 
    
    x = Conv2D(
        5, kernel_size=1, padding="same", name="Logits"
    )(x)
    x = Flatten()(x)
    outputs = x

    model = Model(inputs=inputs, outputs=outputs)
    return model


def get_siamese_model(n_sensors, window_size):
    base_network = ActivityNetV3(n_sensors, window_size)
    input_a = Input(shape=(n_sensors, window_size,1))
    input_b = Input(shape=(n_sensors, window_size,1))

    encoded_a = base_network(input_a)
    encoded_b = base_network(input_b)

    L1_layer = Lambda(lambda tensors: K.abs(tensors[0] - tensors[1]))
    L1_distance = L1_layer([encoded_a, encoded_b])
    prediction = Dense(1, activation='sigmoid')(L1_distance)
    siamese_net = Model(inputs=[input_a, input_b], outputs=prediction)
    
    return siamese_net

def get_generator(X_train,X_val,y_train,y_val,s):
    while True:
        pairs,targets=get_batch(X_train,X_val,y_train,y_val,s)
        yield (pairs,targets)

def get_batch(X_train,X_val,y_train,y_val,s):
    rng=np.random.default_rng()
    if s == 'train':
        X=X_train.copy()
        categories = y_train.shape[0]
    elif s == 'val':
        X=X_val.copy()
        categories = y_val.shape[0]
    
    population = X.shape[0]
    n_sensors, timesteps, one = X.shape[1], X.shape[2], X.shape[3]
    #print(population, n_sensors, timesteps,one)
    categories = rng.choice(population,size=(tconfig.BATCH_SIZE,),replace=False)
    pairs=[np.zeros((tconfig.BATCH_SIZE,n_sensors,timesteps,1)) for _ in range(2)]
    targets = np.zeros((tconfig.BATCH_SIZE,))
    targets[tconfig.BATCH_SIZE//2:]=1
    for i in range(tconfig.BATCH_SIZE):
        category=categories[i]
#         print(category)
        idx_1=rng.integers(0,population-1)
#         print(idx_1)
#         print(X[category,idx_1])
        #pairs[0][i,:,:,0]=X[idx_1].reshape(n_sensors,timesteps)
        pairs[0][i,:,:,0]=X[idx_1,:,:,0]
        idx_2=rng.integers(0,population-1)
        if i>=tconfig.BATCH_SIZE//2:
            category_2=category
        else:
            category_2 = rng.choice(np.delete(np.arange(population), category))
        pairs[1][i,:,:,0]=X[category_2,:,:,0]
        #pairs[1][i,:,:,0]=X[category_2]#.reshape(n_sensors,timesteps)
    return pairs,targets

def train_q_model(X_train,X_val,y_train,y_val):
    # general training
    model = ActivityNetV3.get_siamese_model(X_train.shape[1], config.WINDOW_SIZE)
    model.compile(
        optimizer=Adam(learning_rate=1e-6),
        loss='sparse_categorical_crossentropy',
        metrics='accuracy',
    )
    history = model.fit(ActivityNetV3.get_generator(X_train,X_val,y_train,y_val,s="train"), 
                        epochs=1, 
                        class_weight=ActivityNetV3.get_sample_weights(y_train),
                        callbacks=ActivityNetV3.get_callback_info(),
                        validation_data=ActivityNetV3.get_generator(X_train,X_val,y_train,y_val,s="val")
                       ) 
    return model

Error

Not a number (NaN) or infinity (Inf) values detected in gradient. b’After initial Conv2D and BatchNorm’ : Tensor had NaN values

Tried and Tested

I have tried checking inf values and Nan values, also I have tried changing learning rate and other model parameter related stuff, but none worked for me.

I am using the shape of (None,3,100,1)