The following code runs successfully (I can print the results of the main
function), but on shutdown I get a bunch of exceptions:
from pathlib import Path
import asyncio
from tqdm.asyncio import tqdm_asyncio
from lib import get_gcs_fs, run, get_gcs_folder
async def check_sentinel_exists(cloud_fs, sample_id, sentinel):
exists = await cloud_fs._exists(sentinel)
return sample_id if exists else None
async def filter_samples(cloud_fs, sample_id_with_sentinel):
tasks = [check_sentinel_exists(cloud_fs, sample_id, sentinel)
for sample_id, sentinel in sample_id_with_sentinel]
results = await tqdm_asyncio.gather(*tasks, desc="Checking files")
return list(filter(None, results))
async def main():
cloud_fs = get_gcs_fs()
GCS_TSV = "..."
GCS_PREFIX = "..."
# Download TSV file if not exists
if not Path("/tmp/rows.tsv").exists():
run(f"gcloud storage cp {GCS_TSV} /tmp/rows.tsv")
# Prepare sample_id_with_sentinel list
sample_id_with_sentinel = []
for line in Path("/tmp/rows.tsv").read_text().splitlines():
if not line.strip():
continue
sample_id, _ = line.split("t")
WORK_SENTINEL = f"{get_gcs_folder(GCS_PREFIX, sample_id)}/stages/SOME_STAGE.DONE"
sample_id_with_sentinel.append((sample_id, WORK_SENTINEL))
filtered_sample_ids = await filter_samples(cloud_fs, sample_id_with_sentinel)
return filtered_sample_ids
if __name__ == "__main__":
sample_ids = asyncio.run(main())
print(sample_ids)
Exceptions:
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/ved/micromamba/envs/dev/lib/python3.10/asyncio/sslproto.py", line 684, in _process_write_backlog
self._transport.write(chunk)
File "/home/ved/micromamba/envs/dev/lib/python3.10/asyncio/selector_events.py", line 924, in write
self._fatal_error(exc, 'Fatal write error on socket transport')
File "/home/ved/micromamba/envs/dev/lib/python3.10/asyncio/selector_events.py", line 719, in _fatal_error
self._force_close(exc)
File "/home/ved/micromamba/envs/dev/lib/python3.10/asyncio/selector_events.py", line 731, in _force_close
self._loop.call_soon(self._call_connection_lost, exc)
File "/home/ved/micromamba/envs/dev/lib/python3.10/asyncio/base_events.py", line 745, in call_soon
self._check_closed()
File "/home/ved/micromamba/envs/dev/lib/python3.10/asyncio/base_events.py", line 510, in _check_closed
raise RuntimeError('Event loop is closed')
RuntimeError: Event loop is closed
Fatal error on SSL transport
protocol: <asyncio.sslproto.SSLProtocol object at 0x7f7a32eca5c0>
transport: <_SelectorSocketTransport closing fd=25>
Traceback (most recent call last):
File "/home/ved/micromamba/envs/dev/lib/python3.10/asyncio/selector_events.py", line 918, in write
n = self._sock.send(data)
OSError: [Errno 9] Bad file descriptor
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/ved/micromamba/envs/dev/lib/python3.10/asyncio/sslproto.py", line 684, in _process_write_backlog
self._transport.write(chunk)
File "/home/ved/micromamba/envs/dev/lib/python3.10/asyncio/selector_events.py", line 924, in write
self._fatal_error(exc, 'Fatal write error on socket transport')
File "/home/ved/micromamba/envs/dev/lib/python3.10/asyncio/selector_events.py", line 719, in _fatal_error
self._force_close(exc)
File "/home/ved/micromamba/envs/dev/lib/python3.10/asyncio/selector_events.py", line 731, in _force_close
self._loop.call_soon(self._call_connection_lost, exc)
File "/home/ved/micromamba/envs/dev/lib/python3.10/asyncio/base_events.py", line 745, in call_soon
self._check_closed()
File "/home/ved/micromamba/envs/dev/lib/python3.10/asyncio/base_events.py", line 510, in _check_closed
raise RuntimeError('Event loop is closed')
RuntimeError: Event loop is closed
Fatal error on SSL transport
protocol: <asyncio.sslproto.SSLProtocol object at 0x7f7a32eca770>
transport: <_SelectorSocketTransport closing fd=29>
Traceback (most recent call last):
File "/home/ved/micromamba/envs/dev/lib/python3.10/asyncio/selector_events.py", line 918, in write
n = self._sock.send(data)
OSError: [Errno 9] Bad file descriptor
Given that the code is completing, I’m not sure what the issue is. Any thoughts?