I am trying to install cpu and gpu version of torch and torchvision respectively, using poetry install --with cpu and poetry install --with gpu. I have the following in my pyproject.toml:

[tool.poetry.dependencies]
python = "^3.8"
filterpy = "^1.4.5"
gdown = "^5.1.0"
lapx = "^0.5.5"
loguru = "^0.7.2"
numpy = "1.24.4"
pyyaml = "^6.0.1"
regex = "^2024.0.0"
yacs = "^0.1.8"
scikit-learn = "^1.3.0"
pandas = "^2.0.0"
opencv-python = "^4.7.0"
ftfy = "^6.1.3"
gitpython = "^3.1.42"

[tool.poetry.group.gpu]
optional = true

[tool.poetry.group.gpu.dependencies]
torch = [
    {version = "^2.2.1", source="pypi", markers = "sys_platform == 'darwin'"},
    {version = "^2.2.1", source="torch_cuda121", markers = "sys_platform == 'linux'"},
    {version = "^2.2.1", source="torch_cuda121", markers = "sys_platform == 'win32'"},
]
torchvision = [
    {version = "^0.17.1", source="pypi", markers = "sys_platform == 'darwin'"},
    {version = "^0.17.1", source="torch_cuda121", markers = "sys_platform == 'linux'"},
    {version = "^0.17.1", source="torch_cuda121", markers = "sys_platform == 'win32'"},
]

[tool.poetry.group.cpu]
optional = true

[tool.poetry.group.cpu.dependencies]
torch = [
    {version = "^2.2.1", source="pypi", markers = "sys_platform == 'darwin'"},
    {version = "^2.2.1", source="torchcpu", markers = "sys_platform == 'linux'"},
    {version = "^2.2.1", source="torchcpu", markers = "sys_platform == 'win32'"},
]
torchvision = [
    {version = "^0.17.1", source="pypi", markers = "sys_platform == 'darwin'"},
    {version = "^0.17.1", source="torchcpu", markers = "sys_platform == 'linux'"},
    {version = "^0.17.1", source="torchcpu", markers = "sys_platform == 'win32'"},
]

[[tool.poetry.source]]
name = "torchcpu"
url = "https://download.pytorch.org/whl/cpu"
priority = "explicit"

[[tool.poetry.source]]
name = "torch_cuda121"
url = "https://download.pytorch.org/whl/cu121"
priority = "explicit"

When running poetry lock I get:

Incompatible constraints in requirements of boxmot (10.0.71):
torch (>=2.2.1,<3.0.0) ; sys_platform == "linux" or sys_platform == "win32" ; source=torchcpu
torch (>=2.2.1,<3.0.0) ; sys_platform == "linux" or sys_platform == "win32" ; source=torch_cuda121

What am I missing? How can this be achieved with poetry?