I’m using docker registry as a private image repository for a K8S cluster on my LAN, but the containerd on the K8S cluster can’t pull the image

version as follow

root@cp01:~# containerd --version
containerd github.com/containerd/containerd/v2 v2.0.0 207ad711eabd375a01713109a8a197d197ff6542
root@cp01:~# kubelet version
E1223 16:22:54.637029   35982 run.go:72] "command failed" err="unknown command version"
root@cp01:~# kubeadm version
kubeadm version: &version.Info{Major:"1", Minor:"32", GitVersion:"v1.32.0", GitCommit:"70d3cc986aa8221cd1dfb1121852688902d3bf53", GitTreeState:"clean", BuildDate:"2024-12-11T18:04:20Z", GoVersion:"go1.23.3", Compiler:"gc", Platform:"linux/arm64"}
root@cp01:~# kubectl version
Client Version: v1.32.0
Kustomize Version: v5.5.0
Server Version: v1.32.0

I’ve also referenced the same problem that already exists on github, as well as the official K8S docs and other articles, but I still can’t solve it, so these are the links I’ve looked at

Unable to pull image from insecure registry, http: server gave HTTP response to HTTPS client

Pull an Image from a Private Registry

How to Configure Private Registry for Kubernetes cluster running with containerd

/etc/containerd/config.toml as follow

version = 3
root = '/var/lib/containerd'
state = '/run/containerd'
temp = ''
plugin_dir = ''
disabled_plugins = []
required_plugins = []
oom_score = 0
imports = []

[grpc]
  address = '/run/containerd/containerd.sock'
  tcp_address = ''
  tcp_tls_ca = ''

  uid = 0
  gid = 0
  max_recv_message_size = 16777216
  max_send_message_size = 16777216

[ttrpc]
  address = ''
  uid = 0
  gid = 0

[debug]
  address = ''
  uid = 0
  gid = 0
  level = ''
  format = ''

[metrics]
  address = ''
  grpc_histogram = false

[plugins]
  [plugins.'io.containerd.cri.v1.images']
    snapshotter = 'overlayfs'
    disable_snapshot_annotations = true
    discard_unpacked_layers = false
    max_concurrent_downloads = 3
    image_pull_progress_timeout = '5m0s'
    image_pull_with_sync_fs = false
    stats_collect_period = 10

    [plugins.'io.containerd.cri.v1.images'.pinned_images]
      sandbox = 'registry.k8s.io/pause:3.10'

    [plugins.'io.containerd.cri.v1.images'.registry]
      config_path = ''
      [plugins."io.containerd.grpc.v1.cri".registry.auths]
      [plugins."io.containerd.grpc.v1.cri".registry.configs]      

        [plugins."io.containerd.grpc.v1.cri".registry.configs."172.26.22.242:5000".auth]
          auth = "a3Vyc2syMDIxOkRvY2tlckAyMDIx"
      [plugins."io.containerd.grpc.v1.cri".registry.headers]

      [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."172.26.22.242:5000"]
          endpoint = ["http://172.26.22.242:5000"]

    [plugins.'io.containerd.cri.v1.images'.image_decryption]
      key_model = 'node'

  [plugins.'io.containerd.cri.v1.runtime']
    enable_selinux = false
    selinux_category_range = 1024
    max_container_log_line_size = 16384
    disable_apparmor = false
    restrict_oom_score_adj = false
    disable_proc_mount = false
    unset_seccomp_profile = ''
    tolerate_missing_hugetlb_controller = true
    disable_hugetlb_controller = true
    device_ownership_from_security_context = false
    ignore_image_defined_volumes = false
    netns_mounts_under_state_dir = false
    enable_unprivileged_ports = true
    enable_unprivileged_icmp = true
    enable_cdi = true
    cdi_spec_dirs = ['/etc/cdi', '/var/run/cdi']
    drain_exec_sync_io_timeout = '0s'
    ignore_deprecation_warnings = []

    [plugins.'io.containerd.cri.v1.runtime'.containerd]
      default_runtime_name = 'runc'
      ignore_blockio_not_enabled_errors = false
      ignore_rdt_not_enabled_errors = false

      [plugins.'io.containerd.cri.v1.runtime'.containerd.runtimes]
        [plugins.'io.containerd.cri.v1.runtime'.containerd.runtimes.runc]
          runtime_type = 'io.containerd.runc.v2'
          runtime_path = ''
          pod_annotations = []
          container_annotations = []
          privileged_without_host_devices = false
          privileged_without_host_devices_all_devices_allowed = false
          base_runtime_spec = ''
          cni_conf_dir = ''
          cni_max_conf_num = 0
          snapshotter = ''
          sandboxer = 'podsandbox'
          io_type = ''

          [plugins.'io.containerd.cri.v1.runtime'.containerd.runtimes.runc.options]
            BinaryName = ''
            CriuImagePath = ''
            CriuWorkPath = ''
            IoGid = 0
            IoUid = 0
            NoNewKeyring = false
            Root = ''
            ShimCgroup = ''
        SystemdCgroup = true

    [plugins.'io.containerd.cri.v1.runtime'.cni]
      bin_dir = '/opt/cni/bin'
      conf_dir = '/etc/cni/net.d'
      max_conf_num = 1
      setup_serially = false
      conf_template = ''
      ip_pref = ''
      use_internal_loopback = false

  [plugins.'io.containerd.gc.v1.scheduler']
    pause_threshold = 0.02
    deletion_threshold = 0
    mutation_threshold = 100
    schedule_delay = '0s'
    startup_delay = '100ms'

  [plugins.'io.containerd.grpc.v1.cri']
    disable_tcp_service = true
    stream_server_address = '127.0.0.1'
    stream_server_port = '0'
    stream_idle_timeout = '4h0m0s'
    enable_tls_streaming = false

    [plugins.'io.containerd.grpc.v1.cri'.x509_key_pair_streaming]
      tls_cert_file = ''
      tls_key_file = ''

  [plugins.'io.containerd.image-verifier.v1.bindir']
    bin_dir = '/opt/containerd/image-verifier/bin'
    max_verifiers = 10
    per_verifier_timeout = '10s'

  [plugins.'io.containerd.internal.v1.opt']
    path = '/opt/containerd'

  [plugins.'io.containerd.internal.v1.tracing']

  [plugins.'io.containerd.metadata.v1.bolt']
    content_sharing_policy = 'shared'

  [plugins.'io.containerd.monitor.container.v1.restart']
    interval = '10s'

  [plugins.'io.containerd.monitor.task.v1.cgroups']
    no_prometheus = false

  [plugins.'io.containerd.nri.v1.nri']
    disable = false
    socket_path = '/var/run/nri/nri.sock'
    plugin_path = '/opt/nri/plugins'
    plugin_config_path = '/etc/nri/conf.d'
    plugin_registration_timeout = '5s'
    plugin_request_timeout = '2s'
    disable_connections = false

  [plugins.'io.containerd.runtime.v2.task']
    platforms = ['linux/arm64/v8']

  [plugins.'io.containerd.service.v1.diff-service']
    default = ['walking']
    sync_fs = false

  [plugins.'io.containerd.service.v1.tasks-service']
    blockio_config_file = ''
    rdt_config_file = ''

  [plugins.'io.containerd.shim.v1.manager']
    env = []

  [plugins.'io.containerd.snapshotter.v1.blockfile']
    root_path = ''
    scratch_file = ''
    fs_type = ''
    mount_options = []
    recreate_scratch = false

  [plugins.'io.containerd.snapshotter.v1.btrfs']
    root_path = ''

  [plugins.'io.containerd.snapshotter.v1.devmapper']
    root_path = ''
    pool_name = ''
    base_image_size = ''
    async_remove = false
    discard_blocks = false
    fs_type = ''
    fs_options = ''

  [plugins.'io.containerd.snapshotter.v1.native']
    root_path = ''

  [plugins.'io.containerd.snapshotter.v1.overlayfs']
    root_path = ''
    upperdir_label = false
    sync_remove = false
    slow_chown = false
    mount_options = []

  [plugins.'io.containerd.snapshotter.v1.zfs']
    root_path = ''

  [plugins.'io.containerd.tracing.processor.v1.otlp']

  [plugins.'io.containerd.transfer.v1.local']
    max_concurrent_downloads = 3
    max_concurrent_uploaded_layers = 3
    config_path = ''

[cgroup]
  path = ''

[timeouts]
  'io.containerd.timeout.bolt.open' = '0s'
  'io.containerd.timeout.metrics.shimstats' = '2s'
  'io.containerd.timeout.shim.cleanup' = '5s'
  'io.containerd.timeout.shim.load' = '5s'
  'io.containerd.timeout.shim.shutdown' = '3s'
  'io.containerd.timeout.task.state' = '2s'

[stream_processors]
  [stream_processors.'io.containerd.ocicrypt.decoder.v1.tar']
    accepts = ['application/vnd.oci.image.layer.v1.tar+encrypted']
    returns = 'application/vnd.oci.image.layer.v1.tar'
    path = 'ctd-decoder'
    args = ['--decryption-keys-path', '/etc/containerd/ocicrypt/keys']
    env = ['OCICRYPT_KEYPROVIDER_CONFIG=/etc/containerd/ocicrypt/ocicrypt_keyprovider.conf']

  [stream_processors.'io.containerd.ocicrypt.decoder.v1.tar.gzip']
    accepts = ['application/vnd.oci.image.layer.v1.tar+gzip+encrypted']
    returns = 'application/vnd.oci.image.layer.v1.tar+gzip'
    path = 'ctd-decoder'
    args = ['--decryption-keys-path', '/etc/containerd/ocicrypt/keys']
    env = ['OCICRYPT_KEYPROVIDER_CONFIG=/etc/containerd/ocicrypt/ocicrypt_keyprovider.conf']

Following the instructions in the github post on the same issue, I removed the following configuration items

  tcp_tls_cert = ''
  tcp_tls_key = ''

          [plugins."io.containerd.grpc.v1.cri".registry.configs."172.26.22.242:5000".tls]
             insecure_skip_verify = true

pod yaml as follow

root@cp01:~# cat private-nginx.yaml
apiVersion: v1
kind: Pod
metadata:
  name: private-nginx
spec:
  containers:
  - name: private-nginx-container
    image: 172.26.22.242:5000/nginx:latest
  imagePullSecrets:
  - name: regcred

pod configure and auth configure as follow

root@cp01:~# kubectl get secret regcred
NAME      TYPE                             DATA   AGE
regcred   kubernetes.io/dockerconfigjson   1      147m
root@cp01:~# kubectl get secret regcred --output=yaml
apiVersion: v1
data:
  .dockerconfigjson: ewoJImF1dGhzIjogewoJCSJodHRwczovL2luZGV4LmRvY2tlci5pby92MS8iOiB7CgkJCSJhdXRoIjogImEzVnljMnN5TURJeE9rUnZZMnRsY2tBeU1ESXgiCgkJfQoJfQp9
kind: Secret
metadata:
  creationTimestamp: "2024-12-23T06:17:57Z"
  name: regcred
  namespace: default
  resourceVersion: "294439"
  uid: 7130e5fc-ec44-4123-8027-25becb05f137
type: kubernetes.io/dockerconfigjson
root@cp01:~#
root@cp01:~# kubectl get secret regcred --output="jsonpath={.data..dockerconfigjson}" | base64 --decode
{
        "auths": {
                "https://index.docker.io/v1/": {
                        "auth": "a3Vyc2syMDIxOkRvY2tlckAyMDIx"
                }
        }
}

The value of auth for the secret resource regcred is also the same as the value of auth for the docker registry

root@privatehub:~# cat /root/.docker/config.json
{
        "auths": {
                "https://index.docker.io/v1/": {
                        "auth": "a3Vyc2syMDIxOkRvY2tlckAyMDIx"
                }
        }

After modifying the containerd configuration parameters, all nodes executed systemctl restart containerd

The domain name of the server where the private mirror is located is privatehub.ye, and its IP is 172.26.22.242.nignx is the image I’m pulling.

root@privatehub:~# docker images
REPOSITORY   TAG       IMAGE ID       CREATED         SIZE
nginx        latest    bdf62fd3a32f   3 weeks ago     197MB
registry     2         1c6adc34955d   14 months ago   25MB

But the container just fails to pull

root@cp01:~# kubectl events  private-nginx
LAST SEEN               TYPE      REASON      OBJECT              MESSAGE
44m (x88 over 65m)      Normal    BackOff     Pod/private-nginx   Back-off pulling image "172.26.22.242:5000/nginx:latest"
44m (x88 over 65m)      Warning   Failed      Pod/private-nginx   Error: ImagePullBackOff
44m                     Normal    Scheduled   Pod/private-nginx   Successfully assigned default/private-nginx to wn05
41m (x5 over 44m)       Normal    Pulling     Pod/private-nginx   Pulling image "172.26.22.242:5000/nginx:latest"
41m (x5 over 44m)       Warning   Failed      Pod/private-nginx   Failed to pull image "172.26.22.242:5000/nginx:latest": failed to pull and unpack image "172.26.22.242:5000/nginx:latest": failed to resolve reference "172.26.22.242:5000/nginx:latest": failed to do request: Head "https://172.26.22.242:5000/v2/nginx/manifests/latest": http: server gave HTTP response to HTTPS client
41m (x5 over 44m)       Warning   Failed      Pod/private-nginx   Error: ErrImagePull
4m25s (x176 over 44m)   Normal    BackOff     Pod/private-nginx   Back-off pulling image "172.26.22.242:5000/nginx:latest"
4m25s (x176 over 44m)   Warning   Failed      Pod/private-nginx   Error: ImagePullBackOff

root@cp01:~# kubectl describe pod  private-nginx
Name:             private-nginx
Namespace:        default
Priority:         0
Service Account:  default
Node:             wn05/172.26.11.64
Start Time:       Mon, 23 Dec 2024 16:13:54 +0800
Labels:           <none>
Annotations:      cni.projectcalico.org/containerID: 1cc07487c491164f36823d66d8a93287ff5bd41332e0051f8c4c9e77518e8b3f
                  cni.projectcalico.org/podIP: 10.224.228.7/32
                  cni.projectcalico.org/podIPs: 10.224.228.7/32
Status:           Pending
IP:               10.224.228.7
IPs:
  IP:  10.224.228.7
Containers:
  private-nginx-container:
    Container ID:
    Image:          172.26.22.242:5000/nginx:latest
    Image ID:
    Port:           <none>
    Host Port:      <none>
    State:          Waiting
      Reason:       ImagePullBackOff
    Ready:          False
    Restart Count:  0
    Environment:    <none>
    Mounts:
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-h9pkq (ro)
Conditions:
  Type                        Status
  PodReadyToStartContainers   True
  Initialized                 True
  Ready                       False
  ContainersReady             False
  PodScheduled                True
Volumes:
  kube-api-access-h9pkq:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   BestEffort
Node-Selectors:              <none>
Tolerations:                 node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:
  Type     Reason     Age                    From               Message
  ----     ------     ----                   ----               -------
  Normal   Scheduled  48m                    default-scheduler  Successfully assigned default/private-nginx to wn05
  Normal   Pulling    45m (x5 over 48m)      kubelet            Pulling image "172.26.22.242:5000/nginx:latest"
  Warning  Failed     45m (x5 over 48m)      kubelet            Failed to pull image "172.26.22.242:5000/nginx:latest": failed to pull and unpack image "172.26.22.242:5000/nginx:latest": failed to resolve reference "172.26.22.242:5000/nginx:latest": failed to do request: Head "https://172.26.22.242:5000/v2/nginx/manifests/latest": http: server gave HTTP response to HTTPS client
  Warning  Failed     45m (x5 over 48m)      kubelet            Error: ErrImagePull
  Normal   BackOff    3m35s (x199 over 48m)  kubelet            Back-off pulling image "172.26.22.242:5000/nginx:latest"
  Warning  Failed     3m35s (x199 over 48m)  kubelet            Error: ImagePullBackOff

Manual pull also fails

root@cp01:~# ctr -n k8s.io image pull --plain-http=true 172.26.22.242:5000/nginx:latest
ctr: rpc error: code = NotFound desc = failed to resolve image: 172.26.22.242:5000/nginx:latest: not found

I don’t know what the problem is and how to fix it. Please give me some guidance