This is the lambda function I use this inside a stepfunction and try to load the csv file to redshift table it requires layers , I too tried to install via pip still im getting the below error
import redshift_connector
import pandas as pd
import boto3
import sys
def lambda_handler(event, context):
print(event)
# Initialize Redshift Connector
host ='REDSHIFT ARN'
port = ******
database = '*********'
user = '*******'
password = '*******'
# Initialize Redshift Connector
conn = redshift_connector.connect(
host=host,
port=port,
database=database,
user=user,
password=password
)
print(f"connection success {host}")
# Access the values of the arguments
bucket_name = event['validation_result']['Payload']['bucket_name']
print(bucket_name)
source_folder = event['validation_result']['Payload']['folder_name']
file_name = event['validation_result']['Payload']['file_name']
result={}
result['bucket_name']=bucket_name
result['file_name']=file_name
# Load CSV file into a Pandas DataFrame
csv_file_path = f's3://{bucket_name}/{source_folder}/{file_name}'
print(csv_file_path)
df = pd.read_csv(csv_file_path)
# Define the Redshift table name
table_name = 'club_games' # Change this to your actual Redshift table name
# Insert data into the Redshift table
for _, row in df[columns].iterrows():
values = ', '.join([f"'{value}'" if isinstance(value, str) else str(value) for value in row])
insert_sql = f"INSERT INTO {table_name} VALUES ({values})"
cursor.execute(insert_sql)
conn.commit()
# Archive the processed CSV file
s3 = boto3.client('s3')
archive_key='archive' + '/' + file_name
object_key= folder_name + '/' + file_name
# Copy the file to the archive folder
s3.copy_object(Bucket=bucket_name, CopySource={'Bucket': bucket_name, 'Key': object_key}, Key=archive_key)
# Delete the original file from the input folder
s3.delete_object(Bucket=bucket_name, Key=object_key)
conn.commit()
# Close the Redshift connection
conn.close()
return result
However I tried installing the pip libraries for redshift_connector still it showing is there another way to solve this
{
"errorMessage": "Unable to import module 'lambda_function': No module named 'redshift_connector'",
"errorType": "Runtime.ImportModuleError",
"requestId": "c0aa62a9-e77d-44d3-b92b-acb9e7362ebd",
"stackTrace": []
}```
New contributor
user22331353 is a new contributor to this site. Take care in asking for clarification, commenting, and answering.
Check out our Code of Conduct.