I’m having the following problem.
I’m running a Dagster instance on an EC2 machine. It’s in a Docker container. The Docker container itself can access the bucket, but when the code is running in Dagster, I’m getting the following error:
botocore.errorfactory.NoSuchBucket: An error occurred (NoSuchBucket) when calling the PutObject operation: The specified bucket does not exist File "/usr/local/lib/python3.11/site-packages/dagster/_grpc/impl.py", line 375, in get_external_sensor_execution with SensorEvaluationContext( File "/usr/local/lib/python3.11/site-packages/dagster/_core/definitions/sensor_definition.py", line 223, in __exit__ self._exit_stack.close() File "/usr/local/lib/python3.11/contextlib.py", line 597, in close self.__exit__(None, None, None) File "/usr/local/lib/python3.11/contextlib.py", line 589, in __exit__ raise exc_details[1] File "/usr/local/lib/python3.11/contextlib.py", line 574, in __exit__ if cb(*exc_details): ^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/dagster/_core/definitions/instigation_logger.py", line 118, in __exit__ self._exit_stack.close() File "/usr/local/lib/python3.11/contextlib.py", line 597, in close self.__exit__(None, None, None) File "/usr/local/lib/python3.11/contextlib.py", line 589, in __exit__ raise exc_details[1] File "/usr/local/lib/python3.11/contextlib.py", line 574, in __exit__ if cb(*exc_details): ^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/contextlib.py", line 144, in __exit__ next(self.gen) File "/usr/local/lib/python3.11/site-packages/dagster/_core/storage/cloud_storage_compute_log_manager.py", line 97, in open_log_stream self._on_capture_complete(log_key) File "/usr/local/lib/python3.11/site-packages/dagster/_core/storage/cloud_storage_compute_log_manager.py", line 101, in _on_capture_complete self.upload_to_cloud_storage(log_key, ComputeIOType.STDERR) File "/usr/local/lib/python3.11/site-packages/dagster_aws/s3/compute_log_manager.py", line 249, in upload_to_cloud_storage self._s3_session.upload_fileobj(data, self._s3_bucket, s3_key, ExtraArgs=extra_args) File "/usr/local/lib/python3.11/site-packages/boto3/s3/inject.py", line 642, in upload_fileobj return future.result() ^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/s3transfer/futures.py", line 103, in result return self._coordinator.result() ^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/s3transfer/futures.py", line 266, in result raise self._exception File "/usr/local/lib/python3.11/site-packages/s3transfer/tasks.py", line 139, in __call__ return self._execute_main(kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/s3transfer/tasks.py", line 162, in _execute_main return_value = self._main(**kwargs) ^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/s3transfer/upload.py", line 764, in _main client.put_object(Bucket=bucket, Key=key, Body=body, **extra_args) File "/usr/local/lib/python3.11/site-packages/botocore/client.py", line 565, in _api_call return self._make_api_call(operation_name, kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/botocore/client.py", line 1021, in _make_api_call raise error_class(parsed_response, operation_name)