In Apache Airflow, I have correctly setup the connection variables in the Connections UI. and saved the following DAG code in the DAG folder.
from airflow import DAG
from airflow.providers.amazon.aws.hooks.glue import GlueJobHook
from airflow.operators.empty import EmptyOperator
from datetime import datetime
# glue job
def get_glue(**kwargs):
hook = GlueJobHook(aws_conn_id='my_id', region_name='us-east-1')
response = hook.get_conn().get_jobs()
print("Table information:", response)
# Define the DAG
dag = DAG(dag_id='glue_catalog_hook_example',
default_args={'owner': 'airflow', 'start_date': datetime(2023, 1, 1)},
schedule_interval='@daily',
catchup=False)
start = EmptyOperator(task_id='start', dag=dag)
end = EmptyOperator(task_id='end', dag=dag)
get_table_info = PythonOperator(
task_id='get_glue_job_info',
python_callable=get_glue,
dag = dag
)
start>>get_table_info>>end
my problem is I encounter this error: botocore.exceptions.ClientError: An error occurred (UnrecognizedClientException) when calling the GetJobs operation: The security token included in the request is invalid.
where did I go wrong?
8