I trained a model on AAPL’s stock data from the beginning of 2010 to the beginning of 2021. However, I noticed that whenever I ask it to predict for the beginning of 2021 to the end of 2021, it somehow seems to come up with the exactly perfect answer, likely because some data is being leaked to the model. However, I can’t seem to find where the data is being leaked. Any help would be appreciated, thank you!

Training process:

# Download data
data = yf.download("AAPL", start="2010-01-01", end="2021-01-01")
data = data.dropna()
data = data[cols]

# Scale data between 0 and 1
scaler = MinMaxScaler(feature_range=(0, 1))
scaled_data = scaler.fit_transform(data)


window_size=60

# Create sequences for training and testing
# Create sequences for training and testing
def create_sequences(data, window_size):
  X, y = [], []
  for i in range(len(data) - window_size):
    X.append(data[i:(i + window_size), :])
    y.append(data[i + window_size, 0])  # We only predict the closing price
  return np.array(X), np.array(y)

X_train, y_train = create_sequences(scaled_data, window_size)


from keras.callbacks import EarlyStopping

# Define the early stopping criteria
early_stopping = EarlyStopping(monitor='loss', patience=10)

# Build LSTM model
model = Sequential()
model.add(LSTM(units=50, return_sequences=True, input_shape=(X_train.shape[1], X_train.shape[2])))
model.add(LSTM(units=50))
model.add(Dense(units=1))
model.compile(loss="mae", optimizer="adam")

# Add the early stopping callback to the fit function
model.fit(X_train, y_train, epochs=200, batch_size=64, callbacks=[early_stopping])

And here is where I test the model and it seems to be leaking data somehow:

data_custom_range = yf.download("AAPL", start="2021-01-01", end="2021-12-28")
data_custom_range = data_custom_range.dropna()
data_custom_range = data_custom_range[cols]  # Select closing price, volume

window_size = 60

# Create sequences for training and testing
def create_sequences(data, window_size):
  X, y = [], []
  for i in range(window_size, len(data)):  # Start from window_size
    X.append(data[(i - window_size):i, :])  # Use the previous window_size days of data
    y.append(data[i, 0])  # Predict the next day's price
  return np.array(X), np.array(y)

# Prepare the data
scaler = MinMaxScaler()
scaled_data = scaler.fit_transform(data_custom_range.values)

# Fit a separate scaler on just the closing prices
price_scaler = MinMaxScaler()
price_scaler.fit(data_custom_range.iloc[:, 0:1].values)  # Only use the closing prices

# Create sequences
X_custom, y_custom = create_sequences(scaled_data, window_size)

# Make predictions
predictions = []
for i in range(window_size, len(X_custom)):
  prediction = model.predict(X_custom[i:i+1])
  predictions.append(prediction[0, 0])

# Inverse transform the predictions using the price_scaler
predictions = price_scaler.inverse_transform(np.array(predictions).reshape(-1, 1))
actual = price_scaler.inverse_transform(y_custom.reshape(-1, 1))

And here is the graph of the output (shifted 60 days to the right):

import matplotlib.pyplot as plt

# Shift the predictions to the right by 80 units
shifted_predictions = np.append([None]*60, predictions)

# Plot the actual price and predicted price
plt.plot(actual, label='Actual Price')
plt.plot(shifted_predictions, label='Predicted Price')  # Use 'shifted_predictions' instead of 'predictions'
plt.xlabel('Time')
plt.ylabel('Price')
plt.title('Actual Price vs Predicted Price')
plt.legend()
plt.show()

If any other information is needed feel free to let me know, thank you so much in advance!

Note: Please let me know if I am accidentally giving it access to T+1 time data, I tried looking to see where I was accidentally doing that but can’t find it :/