I have a fairly simple nextflow process – it takes a file, does some simple processing, and uses sponge to write to the same file. At the end of the script, it uses gzip to compress the file. However, the file produced is empty. If I don’t use the gzip command at the end of the script, the uncompressed file is produced as expected.

However, if I change the sponge command to write to a named file instead of using sponge, gzip works as expected. Why does this happen, and is there any way I can still use sponge and also gzip my end file?

My original code, which produces an empty .txt.gz file:

process processFILE {
    container = params.docker_container
    publishDir "${params.output}", mode: 'copy'

    input:
    path file
    path output

    output:
    path "${file.baseName}.txt.gz"

    script:
    def output_file = "${file.baseName}.txt"

    """
    # Add header line to file
    sed -i '1i column1\tcolumn2\tcolumn3' ${output_file}

    # Add lineX to the header if the parameter is not null
    if [ -n "${params.criteria_1}" ]; then
        echo '# LineX' 
        | cat - ${output_file} 
        | sponge ${output_file}
    fi

    # Add lineZ to the header if the parameter is set to true
    if [ "${params.criteria_2}" = "true" ]; then
        echo '# LineZ' 
        | cat - ${output_file} 
        | sponge ${output_file}
    fi

    # Compress
    gzip ${output_file}
    """
}

However, when I change it to the following, it works:

process processFILE {
    container = params.docker_container
    publishDir "${params.output}", mode: 'copy'

    input:
    path file
    path output

    output:
    path "${file.baseName}.txt.gz"

    script:
    def output_file = "${file.baseName}.txt"
    def final_output_file = "${file.baseName}.txt.gz"

    """
    # Add header line to file
    sed -i '1i column1\tcolumn2\tcolumn3' ${output_file}

    # Add lineX to the header if the parameter is not null
    if [ -n "${params.criteria_1}" ]; then
        echo '# LineX' 
        | cat - ${output_file} 
        | sponge ${output_file}
    fi

    # Add lineZ to the header if the parameter is set to true
    if [ "${params.criteria_2}" = "true" ]; then
        echo '# LineZ' 
        | cat - ${output_file} > test.txt
    fi

    # Compress
    gzip -c test.txt > ${final_output_file}
    """
}