I am trying to create a custom module that download files from a AWS S3 bucket, since I am having problem with AWS CLI reading my credentials file with the module amazon.aws.s3_bucket. The idea is to download the file using boto3 library and them copy that file to the remote hosts using ansible.builtin.copy.

I have been trying different ways but I don’t get do make it work.

This is the code:

# library/fetch_s3.py
# Custom Ansible's module to download files stored on a AWS S3 bucket
# TODO: Test functionality
# TODO: What to do with result['message']
# TODO: Improve error message
# TODO: Implemented method to detect when changes happened

import os, boto3
from ansible.module_utils.basic import AnsibleModule, copy

DOCUMENTATION = r'''
---
module: a0.fetch_s3

description: | 
    Simple module to download files from a S3 container using AWS SDK (boto3). 
    This module doesn't accept AWS credentials as an input, so you will need to provide them either using environmental variables 
    or placing them on ~/.aws/credentials. See https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html for more information.

version_added: "0.0.1"

options:
    bucket:
        description: Name of the S3 bucket
        required: true
        type: str
    src:
        description: Key of the item/file to retrieve from the S3 bucket
        required: true
        type: str
    src:
        description: Remote host file destination
        required: true
        type: str
author:
    - (@nrk19)
'''

# Function responsible of retrieving the file from the S3 bucket and copy it to the remote hosts
def fetch_s3(bucket, src, dest, result):
    ERROR = True
    working_dir = os.getcwd()

    try:
        s3 = boto3.client("s3")
        local_file = os.path.join(working_dir, src)
        s3.download_file(bucket, src, local_file) 
        copy_result = copy(local_file, dest)  

        # Handle errors before removing local file
        if copy_result.get('failed', True):
            result['error_message'] = f"Errors happened when copying the file to remote host: {copy_result['stderr']}"
            result['failed'] = True
            return ERROR

        if copy_result.get('changed', True): result['changed'] = True  # Capture change state from copy_file func
        
        os.remove(local_file)  # Remove local file if no errors happened

    except Exception as e:
        result['error_message'] = f'Error happened when trying to retrieve the file {src} from bucket {bucket}: {e}'
        result['failed'] = True
        return ERROR

    return False

# Main module function
def run_module():
    
    is_error = False

    module_args = dict(
        bucket=dict(type='str', required=True),
        src=dict(type='str', required=True),
        dest=dict(type='str', required=True)
    )

    result = dict(
        message='',
        failed=False,
        changed=False,
        error_message='',
        original_message=''
    )

    module = AnsibleModule(
        argument_spec=module_args,
        supports_check_mode=False
    )

    result['original_message'] = module.params['name']  # Capture original message
    
    # fetch_s3 will return True if errors happened during execution
    is_error = fetch_s3(
        module_args['bucket'],
        module_args['src'],
        module_args['dest'],
        result
    )

    if is_error:
        module.fail_json(msg='Error: ', **result)

    module.exit_json(**result)

def main():
    run_module()

if __name__ == '__main__':
    main()
    

I would appreciate some help to understand the way of creating modules and using other modules inside my own.

Thank you!

I also tried to run ansible.builtin.copy using ansible_runner with no sucess.