I have launched docker container with Apache Airflow and it’s working. Now I am trying to extend all necessary dependencies and here’s the issue I’m faced with. My Dockerfile
FROM apache/airflow:2.9.2
COPY requirements.txt /requirements.txt
RUN pip install --upgrade pip
RUN pip install -U pip setuptools==66.0.0 wheel
RUN pip install -r /requirements.txt
RUN pip install --no-cache-dir -r /requirements.txt
My requirements
numpy == 1.21.1
scikit-learn==0.24.2
Here are full logs that I get after docker build . –tag extending_airflow:latest
DEPRECATED: The legacy builder is deprecated and will be removed in a future release.
Install the buildx component to build images with BuildKit:
https://docs.docker.com/go/buildx/
Sending build context to Docker daemon 2.097MB
Step 1/6 : FROM apache/airflow:2.9.2
---> 940b05c86097
Step 2/6 : COPY requirements.txt /requirements.txt
---> Using cache
---> cdbd62a1ecf5
Step 3/6 : RUN pip install --upgrade pip
---> Using cache
---> a37904cc8177
Step 4/6 : RUN pip install -U pip setuptools==66.0.0 wheel
---> Running in 7889e762ddd9
Requirement already satisfied: pip in /home/airflow/.local/lib/python3.12/site-packages (24.1.2)
Collecting setuptools==66.0.0
Downloading setuptools-66.0.0-py3-none-any.whl.metadata (6.2 kB)
Collecting wheel
Downloading wheel-0.43.0-py3-none-any.whl.metadata (2.2 kB)
Downloading setuptools-66.0.0-py3-none-any.whl (1.3 MB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.3/1.3 MB 1.4 MB/s eta 0:00:00
Downloading wheel-0.43.0-py3-none-any.whl (65 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 65.8/65.8 kB 4.5 MB/s eta 0:00:00
Installing collected packages: wheel, setuptools
Attempting uninstall: setuptools
Found existing installation: setuptools 66.1.1
Uninstalling setuptools-66.1.1:
Successfully uninstalled setuptools-66.1.1
Successfully installed setuptools-66.0.0 wheel-0.43.0
Removing intermediate container 7889e762ddd9
---> 846892fcfde1
Step 5/6 : RUN pip install -r /requirements.txt
---> Running in 719d1f76c572
Collecting numpy==1.21.1 (from -r /requirements.txt (line 1))
Downloading numpy-1.21.1.zip (10.3 MB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 10.3/10.3 MB 2.8 MB/s eta 0:00:00
Installing build dependencies: started
Installing build dependencies: finished with status 'done'
Getting requirements to build wheel: started
Getting requirements to build wheel: finished with status 'done'
ERROR: Exception:
Traceback (most recent call last):
File "/home/airflow/.local/lib/python3.12/site-packages/pip/_internal/cli/base_command.py", line 179, in exc_logging_wrapper
status = run_func(*args)
^^^^^^^^^^^^^^^
File "/home/airflow/.local/lib/python3.12/site-packages/pip/_internal/cli/req_command.py", line 67, in wrapper
return func(self, options, args)
^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/airflow/.local/lib/python3.12/site-packages/pip/_internal/commands/install.py", line 377, in run
requirement_set = resolver.resolve(
^^^^^^^^^^^^^^^^^
File "/home/airflow/.local/lib/python3.12/site-packages/pip/_internal/resolution/resolvelib/resolver.py", line 95, in resolve
result = self._result = resolver.resolve(
^^^^^^^^^^^^^^^^^
File "/home/airflow/.local/lib/python3.12/site-packages/pip/_vendor/resolvelib/resolvers.py", line 546, in resolve
state = resolution.resolve(requirements, max_rounds=max_rounds)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/airflow/.local/lib/python3.12/site-packages/pip/_vendor/resolvelib/resolvers.py", line 397, in resolve
self._add_to_criteria(self.state.criteria, r, parent=None)
File "/home/airflow/.local/lib/python3.12/site-packages/pip/_vendor/resolvelib/resolvers.py", line 173, in _add_to_criteria
if not criterion.candidates:
^^^^^^^^^^^^^^^^^^^^
File "/home/airflow/.local/lib/python3.12/site-packages/pip/_vendor/resolvelib/structs.py", line 156, in __bool__
return bool(self._sequence)
^^^^^^^^^^^^^^^^^^^^
File "/home/airflow/.local/lib/python3.12/site-packages/pip/_internal/resolution/resolvelib/found_candidates.py", line 174, in __bool__
return any(self)
^^^^^^^^^
File "/home/airflow/.local/lib/python3.12/site-packages/pip/_internal/resolution/resolvelib/found_candidates.py", line 162, in <genexpr>
return (c for c in iterator if id(c) not in self._incompatible_ids)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/airflow/.local/lib/python3.12/site-packages/pip/_internal/resolution/resolvelib/found_candidates.py", line 53, in _iter_built
candidate = func()
^^^^^^
File "/home/airflow/.local/lib/python3.12/site-packages/pip/_internal/resolution/resolvelib/factory.py", line 185, in _make_candidate_from_link
base: Optional[BaseCandidate] = self._make_base_candidate_from_link(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/airflow/.local/lib/python3.12/site-packages/pip/_internal/resolution/resolvelib/factory.py", line 231, in _make_base_candidate_from_link
self._link_candidate_cache[link] = LinkCandidate(
^^^^^^^^^^^^^^
File "/home/airflow/.local/lib/python3.12/site-packages/pip/_internal/resolution/resolvelib/candidates.py", line 303, in __init__
super().__init__(
File "/home/airflow/.local/lib/python3.12/site-packages/pip/_internal/resolution/resolvelib/candidates.py", line 158, in __init__
self.dist = self._prepare()
^^^^^^^^^^^^^^^
File "/home/airflow/.local/lib/python3.12/site-packages/pip/_internal/resolution/resolvelib/candidates.py", line 235, in _prepare
dist = self._prepare_distribution()
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/airflow/.local/lib/python3.12/site-packages/pip/_internal/resolution/resolvelib/candidates.py", line 314, in _prepare_distribution
return preparer.prepare_linked_requirement(self._ireq, parallel_builds=True)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/airflow/.local/lib/python3.12/site-packages/pip/_internal/operations/prepare.py", line 527, in prepare_linked_requirement
return self._prepare_linked_requirement(req, parallel_builds)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/airflow/.local/lib/python3.12/site-packages/pip/_internal/operations/prepare.py", line 642, in _prepare_linked_requirement
dist = _get_prepared_distribution(
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/airflow/.local/lib/python3.12/site-packages/pip/_internal/operations/prepare.py", line 72, in _get_prepared_distribution
abstract_dist.prepare_distribution_metadata(
File "/home/airflow/.local/lib/python3.12/site-packages/pip/_internal/distributions/sdist.py", line 56, in prepare_distribution_metadata
self._install_build_reqs(finder)
File "/home/airflow/.local/lib/python3.12/site-packages/pip/_internal/distributions/sdist.py", line 126, in _install_build_reqs
build_reqs = self._get_build_requires_wheel()
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/airflow/.local/lib/python3.12/site-packages/pip/_internal/distributions/sdist.py", line 103, in _get_build_requires_wheel
return backend.get_requires_for_build_wheel()
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/airflow/.local/lib/python3.12/site-packages/pip/_internal/utils/misc.py", line 709, in get_requires_for_build_wheel
return super().get_requires_for_build_wheel(config_settings=cs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/airflow/.local/lib/python3.12/site-packages/pip/_vendor/pyproject_hooks/_impl.py", line 166, in get_requires_for_build_wheel
return self._call_hook('get_requires_for_build_wheel', {
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/airflow/.local/lib/python3.12/site-packages/pip/_vendor/pyproject_hooks/_impl.py", line 321, in _call_hook
raise BackendUnavailable(data.get('traceback', ''))
pip._vendor.pyproject_hooks._impl.BackendUnavailable: Traceback (most recent call last):
File "/home/airflow/.local/lib/python3.12/site-packages/pip/_vendor/pyproject_hooks/_in_process/_in_process.py", line 77, in _build_backend
obj = import_module(mod_path)
^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/importlib/__init__.py", line 90, in import_module
return _bootstrap._gcd_import(name[level:], package, level)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "<frozen importlib._bootstrap>", line 1387, in _gcd_import
File "<frozen importlib._bootstrap>", line 1360, in _find_and_load
File "<frozen importlib._bootstrap>", line 1310, in _find_and_load_unlocked
File "<frozen importlib._bootstrap>", line 488, in _call_with_frames_removed
File "<frozen importlib._bootstrap>", line 1387, in _gcd_import
File "<frozen importlib._bootstrap>", line 1360, in _find_and_load
File "<frozen importlib._bootstrap>", line 1331, in _find_and_load_unlocked
File "<frozen importlib._bootstrap>", line 935, in _load_unlocked
File "<frozen importlib._bootstrap_external>", line 995, in exec_module
File "<frozen importlib._bootstrap>", line 488, in _call_with_frames_removed
File "/tmp/pip-build-env-bki2bnvu/overlay/lib/python3.12/site-packages/setuptools/__init__.py", line 18, in <module>
from setuptools.extern.six import PY3, string_types
ModuleNotFoundError: No module named 'setuptools.extern.six'
The command '/bin/bash -o pipefail -o errexit -o nounset -o nolog -c pip install -r /requirements.txt' returned a non-zero code: 2
DEPRECATED: The legacy builder is deprecated and will be removed in a future release.
Install the buildx component to build images with BuildKit:
https://docs.docker.com/go/buildx/
Sending build context to Docker daemon 2.097MB
Step 1/6 : FROM apache/airflow:2.9.2
---> 940b05c86097
Step 2/6 : COPY requirements.txt /requirements.txt
---> Using cache
---> cdbd62a1ecf5
Step 3/6 : RUN pip install --upgrade pip
---> Using cache
---> a37904cc8177
Step 4/6 : RUN pip install -U pip setuptools==66.0.0 wheel
---> Running in 7889e762ddd9
Requirement already satisfied: pip in /home/airflow/.local/lib/python3.12/site-packages (24.1.2)
Collecting setuptools==66.0.0
Downloading setuptools-66.0.0-py3-none-any.whl.metadata (6.2 kB)
Collecting wheel
Downloading wheel-0.43.0-py3-none-any.whl.metadata (2.2 kB)
Downloading setuptools-66.0.0-py3-none-any.whl (1.3 MB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.3/1.3 MB 1.4 MB/s eta 0:00:00
Downloading wheel-0.43.0-py3-none-any.whl (65 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 65.8/65.8 kB 4.5 MB/s eta 0:00:00
Installing collected packages: wheel, setuptools
Attempting uninstall: setuptools
Found existing installation: setuptools 66.1.1
Uninstalling setuptools-66.1.1:
Successfully uninstalled setuptools-66.1.1
Successfully installed setuptools-66.0.0 wheel-0.43.0
Removing intermediate container 7889e762ddd9
---> 846892fcfde1
Step 5/6 : RUN pip install -r /requirements.txt
---> Running in 719d1f76c572
Collecting numpy==1.21.1 (from -r /requirements.txt (line 1))
Downloading numpy-1.21.1.zip (10.3 MB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 10.3/10.3 MB 2.8 MB/s eta 0:00:00
Installing build dependencies: started
Installing build dependencies: finished with status 'done'
Getting requirements to build wheel: started
Getting requirements to build wheel: finished with status 'done'
ERROR: Exception:
Traceback (most recent call last):
File "/home/airflow/.local/lib/python3.12/site-packages/pip/_internal/cli/base_command.py", line 179, in exc_logging_wrapper
status = run_func(*args)
^^^^^^^^^^^^^^^
File "/home/airflow/.local/lib/python3.12/site-packages/pip/_internal/cli/req_command.py", line 67, in wrapper
return func(self, options, args)
^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/airflow/.local/lib/python3.12/site-packages/pip/_internal/commands/install.py", line 377, in run
requirement_set = resolver.resolve(
^^^^^^^^^^^^^^^^^
File "/home/airflow/.local/lib/python3.12/site-packages/pip/_internal/resolution/resolvelib/resolver.py", line 95, in resolve
result = self._result = resolver.resolve(
^^^^^^^^^^^^^^^^^
File "/home/airflow/.local/lib/python3.12/site-packages/pip/_vendor/resolvelib/resolvers.py", line 546, in resolve
state = resolution.resolve(requirements, max_rounds=max_rounds)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/airflow/.local/lib/python3.12/site-packages/pip/_vendor/resolvelib/resolvers.py", line 397, in resolve
self._add_to_criteria(self.state.criteria, r, parent=None)
File "/home/airflow/.local/lib/python3.12/site-packages/pip/_vendor/resolvelib/resolvers.py", line 173, in _add_to_criteria
if not criterion.candidates:
^^^^^^^^^^^^^^^^^^^^
File "/home/airflow/.local/lib/python3.12/site-packages/pip/_vendor/resolvelib/structs.py", line 156, in __bool__
return bool(self._sequence)
^^^^^^^^^^^^^^^^^^^^
File "/home/airflow/.local/lib/python3.12/site-packages/pip/_internal/resolution/resolvelib/found_candidates.py", line 174, in __bool__
return any(self)
^^^^^^^^^
File "/home/airflow/.local/lib/python3.12/site-packages/pip/_internal/resolution/resolvelib/found_candidates.py", line 162, in <genexpr>
return (c for c in iterator if id(c) not in self._incompatible_ids)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/airflow/.local/lib/python3.12/site-packages/pip/_internal/resolution/resolvelib/found_candidates.py", line 53, in _iter_built
candidate = func()
^^^^^^
File "/home/airflow/.local/lib/python3.12/site-packages/pip/_internal/resolution/resolvelib/factory.py", line 185, in _make_candidate_from_link
base: Optional[BaseCandidate] = self._make_base_candidate_from_link(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/airflow/.local/lib/python3.12/site-packages/pip/_internal/resolution/resolvelib/factory.py", line 231, in _make_base_candidate_from_link
self._link_candidate_cache[link] = LinkCandidate(
^^^^^^^^^^^^^^
File "/home/airflow/.local/lib/python3.12/site-packages/pip/_internal/resolution/resolvelib/candidates.py", line 303, in __init__
super().__init__(
File "/home/airflow/.local/lib/python3.12/site-packages/pip/_internal/resolution/resolvelib/candidates.py", line 158, in __init__
self.dist = self._prepare()
^^^^^^^^^^^^^^^
File "/home/airflow/.local/lib/python3.12/site-packages/pip/_internal/resolution/resolvelib/candidates.py", line 235, in _prepare
dist = self._prepare_distribution()
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/airflow/.local/lib/python3.12/site-packages/pip/_internal/resolution/resolvelib/candidates.py", line 314, in _prepare_distribution
return preparer.prepare_linked_requirement(self._ireq, parallel_builds=True)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/airflow/.local/lib/python3.12/site-packages/pip/_internal/operations/prepare.py", line 527, in prepare_linked_requirement
return self._prepare_linked_requirement(req, parallel_builds)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/airflow/.local/lib/python3.12/site-packages/pip/_internal/operations/prepare.py", line 642, in _prepare_linked_requirement
dist = _get_prepared_distribution(
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/airflow/.local/lib/python3.12/site-packages/pip/_internal/operations/prepare.py", line 72, in _get_prepared_distribution
abstract_dist.prepare_distribution_metadata(
File "/home/airflow/.local/lib/python3.12/site-packages/pip/_internal/distributions/sdist.py", line 56, in prepare_distribution_metadata
self._install_build_reqs(finder)
File "/home/airflow/.local/lib/python3.12/site-packages/pip/_internal/distributions/sdist.py", line 126, in _install_build_reqs
build_reqs = self._get_build_requires_wheel()
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/airflow/.local/lib/python3.12/site-packages/pip/_internal/distributions/sdist.py", line 103, in _get_build_requires_wheel
return backend.get_requires_for_build_wheel()
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/airflow/.local/lib/python3.12/site-packages/pip/_internal/utils/misc.py", line 709, in get_requires_for_build_wheel
return super().get_requires_for_build_wheel(config_settings=cs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/airflow/.local/lib/python3.12/site-packages/pip/_vendor/pyproject_hooks/_impl.py", line 166, in get_requires_for_build_wheel
return self._call_hook('get_requires_for_build_wheel', {
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/airflow/.local/lib/python3.12/site-packages/pip/_vendor/pyproject_hooks/_impl.py", line 321, in _call_hook
raise BackendUnavailable(data.get('traceback', ''))
pip._vendor.pyproject_hooks._impl.BackendUnavailable: Traceback (most recent call last):
File "/home/airflow/.local/lib/python3.12/site-packages/pip/_vendor/pyproject_hooks/_in_process/_in_process.py", line 77, in _build_backend
obj = import_module(mod_path)
^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/importlib/__init__.py", line 90, in import_module
return _bootstrap._gcd_import(name[level:], package, level)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "<frozen importlib._bootstrap>", line 1387, in _gcd_import
File "<frozen importlib._bootstrap>", line 1360, in _find_and_load
File "<frozen importlib._bootstrap>", line 1310, in _find_and_load_unlocked
File "<frozen importlib._bootstrap>", line 488, in _call_with_frames_removed
File "<frozen importlib._bootstrap>", line 1387, in _gcd_import
File "<frozen importlib._bootstrap>", line 1360, in _find_and_load
File "<frozen importlib._bootstrap>", line 1331, in _find_and_load_unlocked
File "<frozen importlib._bootstrap>", line 935, in _load_unlocked
File "<frozen importlib._bootstrap_external>", line 995, in exec_module
File "<frozen importlib._bootstrap>", line 488, in _call_with_frames_removed
File "/tmp/pip-build-env-bki2bnvu/overlay/lib/python3.12/site-packages/setuptools/__init__.py", line 18, in <module>
from setuptools.extern.six import PY3, string_types
ModuleNotFoundError: No module named 'setuptools.extern.six'
The command '/bin/bash -o pipefail -o errexit -o nounset -o nolog -c pip install -r /requirements.txt' returned a non-zero code: 2
I get an error, saying that ModuleNotFoundError: No module named ‘setuptools.extern.six’ . Trying to explicitly download setuptools in Dockerfile didn’t help. How do I fix this?