Baike.dev
Log in
Back to tool/Back to issues
#13246·containerd

--hosts-dir worked but daemon doesn’t work

Author: easycodesnipperCreated Apr 17, 2026Updated Sep 16, 2026
Labelskind/bugarea/cri

Description

At Kubernetes initialize stage, kubeadm pull image failed.

bash
user@k8s-controller-1:~$ sudo journalctl -u containerd -f
Apr 17 14:40:23 k8s-controller-1 containerd[5176]: time="2026-04-17T14:40:23.575448298Z" level=info msg=serving... address=/run/containerd/containerd.sock
Apr 17 14:40:23 k8s-controller-1 containerd[5176]: time="2026-04-17T14:40:23.575450768Z" level=info msg="Start cni network conf syncer for default"
Apr 17 14:40:23 k8s-controller-1 containerd[5176]: time="2026-04-17T14:40:23.575454649Z" level=info msg="Start streaming server"
Apr 17 14:40:23 k8s-controller-1 containerd[5176]: time="2026-04-17T14:40:23.575459529Z" level=info msg="Registered namespace \"k8s.io\" with NRI"
Apr 17 14:40:23 k8s-controller-1 containerd[5176]: time="2026-04-17T14:40:23.575462100Z" level=info msg="runtime interface starting up..."
Apr 17 14:40:23 k8s-controller-1 containerd[5176]: time="2026-04-17T14:40:23.575464380Z" level=info msg="starting plugins..."
Apr 17 14:40:23 k8s-controller-1 containerd[5176]: time="2026-04-17T14:40:23.575474952Z" level=info msg="Synchronizing NRI (plugin) with current runtime state"
Apr 17 14:40:23 k8s-controller-1 containerd[5176]: time="2026-04-17T14:40:23.575525867Z" level=info msg="containerd successfully booted in 0.012365s"
Apr 17 14:40:23 k8s-controller-1 systemd[1]: Started containerd.service - containerd container runtime.
Apr 17 14:40:24 k8s-controller-1 containerd[5176]: time="2026-04-17T14:40:24.324792287Z" level=info msg="PullImage \"registry.k8s.io/kube-apiserver:v1.34.2\""
q^C
user@k8s-controller-1:~$ cat /etc/containerd/certs.d/
docker.io/       ghcr.io/         registry.k8s.io/
user@k8s-controller-1:~$ cat /etc/containerd/certs.d/registry.k8s.io/hosts.toml
# Generated by Ansible
server = "https://registry.k8s.io"

[host."https://k8s.m.daocloud.io"]
  capabilities = ["pull", "resolve"]
  skip_verify = true
user@k8s-controller-1:~$ sudo ctr image pull registry.k8s.io/kube-apiserver:v1.34.2
ctr: failed to resolve image: failed to do request: Head "https://us-west2-docker.pkg.dev/v2/k8s-artifacts-prod/images/kube-apiserver/manifests/v1.34.2": dial tcp 142.251.188.82:443: i/o timeout: context deadline exceeded
user@k8s-controller-1:~$ sudo ctr image pull --hosts-dir /etc/containerd/certs.d/ registry.k8s.io/kube-apiserver:v1.34.2
registry.k8s.io/kube apiserver:v1.34.2          saved
└──index (e009ef63deaf)                         complete        |++++++++++++++++++++++++++++++++++++++|
   ├──manifest (8002b35fedba)                   complete        |++++++++++++++++++++++++++++++++++++++|
   │  └──config (b684c782da7b)                  complete        |++++++++++++++++++++++++++++++++++++++|
   ├──manifest (f0e0dc00029a)                   complete        |++++++++++++++++++++++++++++++++++++++|
   │  ├──config (a5f569d49a97)                  complete        |++++++++++++++++++++++++++++++++++++++|
   │  ├──layer (fd4aa3667332)                   extracted       |++++++++++++++++++++++++++++++++++++++|
   │  ├──layer (3214acf345c0)                   extracted       |++++++++++++++++++++++++++++++++++++++|
   │  ├──layer (62de241dac5f)                   extracted       |++++++++++++++++++++++++++++++++++++++|
   │  ├──layer (bfb59b82a9b6)                   extracted       |++++++++++++++++++++++++++++++++++++++|
   │  ├──layer (2780920e5dbf)                   extracted       |++++++++++++++++++++++++++++++++++++++|
   │  ├──layer (017886f7e176)                   extracted       |++++++++++++++++++++++++++++++++++++++|
   │  ├──layer (4aa0ea1413d3)                   extracted       |++++++++++++++++++++++++++++++++++++++|
   │  ├──layer (7c12895b777b)                   extracted       |++++++++++++++++++++++++++++++++++++++|
   │  ├──layer (5664b15f108b)                   extracted       |++++++++++++++++++++++++++++++++++++++|
   │  ├──layer (ddf74a63f7d8)                   extracted       |++++++++++++++++++++++++++++++++++++++|
   │  ├──layer (045fc1c20da8)                   extracted       |++++++++++++++++++++++++++++++++++++++|
   │  ├──layer (da7816fa955e)                   extracted       |++++++++++++++++++++++++++++++++++++++|
   │  ├──layer (2f25b52ec3d9)                   extracted       |++++++++++++++++++++++++++++++++++++++|
   │  └──layer (30d14c5f03b2)                   extracted       |++++++++++++++++++++++++++++++++++++++|
   ├──manifest (9a94f333d6fe)                   complete        |++++++++++++++++++++++++++++++++++++++|
   │  └──config (b178af3d91f8)                  complete        |++++++++++++++++++++++++++++++++++++++|
   └──manifest (e1e4a34f7968)                   complete        |++++++++++++++++++++++++++++++++++++++|
      └──config (e34c03db3738)                  complete        |++++++++++++++++++++++++++++++++++++++|
application/vnd.docker.distribution.manifest.list.v2+json sha256:e009ef63deaf797763b5bd423d04a099a2fe414a081bf7d216b43bc9e76b9077
Completed pull from OCI Registry (registry.k8s.io/kube-apiserver:v1.34.2)       elapsed: 41.1s  total:  25.8 M  (643.0 KiB/s)
user@k8s-controller-1:~$ grep mirror -A1 /etc/containerd/config.toml |grep config_path -B1
user@k8s-controller-1:~$ grep mirror -A1 /etc/containerd/config.toml |grep config_path
user@k8s-controller-1:~$ grep mirror -A1 /etc/containerd/config.toml
user@k8s-controller-1:~$ grep config_path -B1 /etc/containerd/config.toml
    [plugins.'io.containerd.cri.v1.images'.registry]
      config_path = "/etc/containerd/certs.d"
--
    plugin_path = '/opt/nri/plugins'
    plugin_config_path = '/etc/nri/conf.d'
--
    check_platform_supported = false
    config_path = ''
user@k8s-controller-1:~$ sudo systemctl restart containerd.service
user@k8s-controller-1:~$ sudo ctr image pull registry.k8s.io/kube-apiserver:v1.34.2
ctr: failed to resolve image: failed to do request: Head "https://europe-west4-docker.pkg.dev/v2/k8s-artifacts-prod/images/kube-apiserver/manifests/v1.34.2": dial tcp 142.251.188.82:443: i/o timeout: context deadline exceeded
user@k8s-controller-1:~$ sudo journalctl -u containerd -n 50 --no-pager
Apr 17 14:51:59 k8s-controller-1 containerd[5504]: time="2026-04-17T14:51:59.840920975Z" level=info msg="loading plugin" id=io.containerd.grpc.v1.sandboxes type=io.containerd.grpc.v1
Apr 17 14:51:59 k8s-controller-1 containerd[5504]: time="2026-04-17T14:51:59.840926875Z" level=info msg="loading plugin" id=io.containerd.grpc.v1.snapshots type=io.containerd.grpc.v1
Apr 17 14:51:59 k8s-controller-1 containerd[5504]: time="2026-04-17T14:51:59.840932545Z" level=info msg="loading plugin" id=io.containerd.streaming.v1.manager type=io.containerd.streaming.v1
Apr 17 14:51:59 k8s-controller-1 containerd[5504]: time="2026-04-17T14:51:59.840953714Z" level=info msg="loading plugin" id=io.containerd.grpc.v1.streaming type=io.containerd.grpc.v1
Apr 17 14:51:59 k8s-controller-1 containerd[5504]: time="2026-04-17T14:51:59.840959395Z" level=info msg="loading plugin" id=io.containerd.grpc.v1.tasks type=io.containerd.grpc.v1
Apr 17 14:51:59 k8s-controller-1 containerd[5504]: time="2026-04-17T14:51:59.840965186Z" level=info msg="loading plugin" id=io.containerd.grpc.v1.transfer type=io.containerd.grpc.v1
Apr 17 14:51:59 k8s-controller-1 containerd[5504]: time="2026-04-17T14:51:59.840971995Z" level=info msg="loading plugin" id=io.containerd.grpc.v1.version type=io.containerd.grpc.v1
Apr 17 14:51:59 k8s-controller-1 containerd[5504]: time="2026-04-17T14:51:59.840976164Z" level=info msg="loading plugin" id=io.containerd.monitor.container.v1.restart type=io.containerd.monitor.container.v1
Apr 17 14:51:59 k8s-controller-1 containerd[5504]: time="2026-04-17T14:51:59.841000675Z" level=info msg="loading plugin" id=io.containerd.tracing.processor.v1.otlp type=io.containerd.tracing.processor.v1
Apr 17 14:51:59 k8s-controller-1 containerd[5504]: time="2026-04-17T14:51:59.841008605Z" level=info msg="skip loading plugin" error="skip plugin: tracing endpoint not configured" id=io.containerd.tracing.processor.v1.otlp type=io.containerd.tracing.processor.v1
Apr 17 14:51:59 k8s-controller-1 containerd[5504]: time="2026-04-17T14:51:59.841012714Z" level=info msg="loading plugin" id=io.containerd.internal.v1.tracing type=io.containerd.internal.v1
Apr 17 14:51:59 k8s-controller-1 containerd[5504]: time="2026-04-17T14:51:59.841018524Z" level=info msg="skip loading plugin" error="skip plugin: tracing endpoint not configured" id=io.containerd.internal.v1.tracing type=io.containerd.internal.v1
Apr 17 14:51:59 k8s-controller-1 containerd[5504]: time="2026-04-17T14:51:59.841023025Z" level=info msg="loading plugin" id=io.containerd.ttrpc.v1.otelttrpc type=io.containerd.ttrpc.v1
Apr 17 14:51:59 k8s-controller-1 containerd[5504]: time="2026-04-17T14:51:59.841050604Z" level=info msg="loading plugin" id=io.containerd.grpc.v1.healthcheck type=io.containerd.grpc.v1
Apr 17 14:51:59 k8s-controller-1 containerd[5504]: time="2026-04-17T14:51:59.841074774Z" level=info msg="loading plugin" id=io.containerd.grpc.v1.cri type=io.containerd.grpc.v1
Apr 17 14:51:59 k8s-controller-1 containerd[5504]: time="2026-04-17T14:51:59.841131793Z" level=info msg="Connect containerd service"
Apr 17 14:51:59 k8s-controller-1 containerd[5504]: time="2026-04-17T14:51:59.841150364Z" level=info msg="using experimental NRI integration - disable nri plugin to prevent this"
Apr 17 14:51:59 k8s-controller-1 containerd[5504]: time="2026-04-17T14:51:59.841386731Z" level=error msg="failed to load cni during init, please check CRI plugin status before setting up network for pods" error="cni config load failed: no network config found in /etc/cni/net.d: cni plugin not initialized: failed to load cni config"
Apr 17 14:51:59 k8s-controller-1 containerd[5504]: time="2026-04-17T14:51:59.845453028Z" level=info msg="Start subscribing containerd event"
Apr 17 14:51:59 k8s-controller-1 containerd[5504]: time="2026-04-17T14:51:59.845476378Z" level=info msg="Start recovering state"
Apr 17 14:51:59 k8s-controller-1 containerd[5504]: time="2026-04-17T14:51:59.845501637Z" level=info msg=serving... address=/run/containerd/containerd.sock.ttrpc
Apr 17 14:51:59 k8s-controller-1 containerd[5504]: time="2026-04-17T14:51:59.845537797Z" level=info msg=serving... address=/run/containerd/containerd.sock
Apr 17 14:51:59 k8s-controller-1 containerd[5504]: time="2026-04-17T14:51:59.848468894Z" level=info msg="Start event monitor"
Apr 17 14:51:59 k8s-controller-1 containerd[5504]: time="2026-04-17T14:51:59.848484574Z" level=info msg="Start cni network conf syncer for default"
Apr 17 14:51:59 k8s-controller-1 containerd[5504]: time="2026-04-17T14:51:59.848491664Z" level=info msg="Start streaming server"
Apr 17 14:51:59 k8s-controller-1 containerd[5504]: time="2026-04-17T14:51:59.848495484Z" level=info msg="Registered namespace \"k8s.io\" with NRI"
Apr 17 14:51:59 k8s-controller-1 containerd[5504]: time="2026-04-17T14:51:59.848499104Z" level=info msg="runtime interface starting up..."
Apr 17 14:51:59 k8s-controller-1 containerd[5504]: time="2026-04-17T14:51:59.848502964Z" level=info msg="starting plugins..."
Apr 17 14:51:59 k8s-controller-1 containerd[5504]: time="2026-04-17T14:51:59.848506814Z" level=info msg="Synchronizing NRI (plugin) with current runtime state"
Apr 17 14:51:59 k8s-controller-1 containerd[5504]: time="2026-04-17T14:51:59.848561263Z" level=info msg="containerd successfully booted in 0.014477s"
Apr 17 14:51:59 k8s-controller-1 systemd[1]: Started containerd.service - containerd container runtime.
Apr 17 14:52:09 k8s-controller-1 containerd[5504]: time="2026-04-17T14:52:09.944125027Z" level=info msg="PullImage \"registry.k8s.io/kube-apiserver:v1.34.2\""
Apr 17 14:52:11 k8s-controller-1 containerd[5504]: time="2026-04-17T14:52:11.394094622Z" level=info msg="ImageUpdate event name:\"registry.k8s.io/kube-apiserver:v1.34.2\" labels:{key:\"io.cri-containerd.image\" value:\"managed\"}"
Apr 17 14:52:11 k8s-controller-1 containerd[5504]: time="2026-04-17T14:52:11.397073647Z" level=info msg="stop pulling image registry.k8s.io/kube-apiserver:v1.34.2: active requests=0, bytes read=0"
Apr 17 14:52:11 k8s-controller-1 containerd[5504]: time="2026-04-17T14:52:11.398276378Z" level=info msg="Pulled image \"registry.k8s.io/kube-apiserver:v1.34.2\" with image id \"sha256:a5f569d49a979d9f62c742edf7a6b6ee8b3cf5855e05dacb0647445bb62ffb85\", repo tag \"registry.k8s.io/kube-apiserver:v1.34.2\", repo digest \"registry.k8s.io/kube-apiserver@sha256:e009ef63deaf797763b5bd423d04a099a2fe414a081bf7d216b43bc9e76b9077\", size \"27060130\" in 1.45413009s"
Apr 17 14:52:11 k8s-controller-1 containerd[5504]: time="2026-04-17T14:52:11.398295207Z" level=info msg="PullImage \"registry.k8s.io/kube-apiserver:v1.34.2\" returns image reference \"sha256:a5f569d49a979d9f62c742edf7a6b6ee8b3cf5855e05dacb0647445bb62ffb85\""
Apr 17 14:52:11 k8s-controller-1 containerd[5504]: time="2026-04-17T14:52:11.398559496Z" level=info msg="PullImage \"registry.k8s.io/kube-controller-manager:v1.34.2\""
Apr 17 14:52:12 k8s-controller-1 containerd[5504]: time="2026-04-17T14:52:12.541039571Z" level=info msg="ImageUpdate event name:\"registry.k8s.io/kube-controller-manager:v1.34.2\" labels:{key:\"io.cri-containerd.image\" value:\"managed\"}"
Apr 17 14:52:12 k8s-controller-1 containerd[5504]: time="2026-04-17T14:52:12.541465657Z" level=info msg="stop pulling image registry.k8s.io/kube-controller-manager:v1.34.2: active requests=0, bytes read=0"
Apr 17 14:52:12 k8s-controller-1 containerd[5504]: time="2026-04-17T14:52:12.542462460Z" level=info msg="Pulled image \"registry.k8s.io/kube-controller-manager:v1.34.2\" with image id \"sha256:01e8bacf0f50095b9b12daf485979dbcb454e08c405e42bde98e3d2198e475e8\", repo tag \"registry.k8s.io/kube-controller-manager:v1.34.2\", repo digest \"registry.k8s.io/kube-controller-manager@sha256:5c3998664b77441c09a4604f1361b230e63f7a6f299fc02fc1ebd1a12c38e3eb\", size \"22818657\" in 1.143858585s"
Apr 17 14:52:12 k8s-controller-1 containerd[5504]: time="2026-04-17T14:52:12.542475110Z" level=info msg="PullImage \"registry.k8s.io/kube-controller-manager:v1.34.2\" returns image reference \"sha256:01e8bacf0f50095b9b12daf485979dbcb454e08c405e42bde98e3d2198e475e8\""
Apr 17 14:52:12 k8s-controller-1 containerd[5504]: time="2026-04-17T14:52:12.542706468Z" level=info msg="PullImage \"registry.k8s.io/kube-scheduler:v1.34.2\""
Apr 17 14:52:25 k8s-controller-1 containerd[5504]: time="2026-04-17T14:52:25.500843430Z" level=info msg="ImageCreate event name:\"registry.k8s.io/kube-scheduler:v1.34.2\" labels:{key:\"io.cri-containerd.image\" value:\"managed\"}"
Apr 17 14:52:25 k8s-controller-1 containerd[5504]: time="2026-04-17T14:52:25.501190028Z" level=info msg="stop pulling image registry.k8s.io/kube-scheduler:v1.34.2: active requests=1, bytes read=14401472"
Apr 17 14:52:25 k8s-controller-1 containerd[5504]: time="2026-04-17T14:52:25.502037890Z" level=info msg="ImageCreate event name:\"sha256:88320b5498ff2caef2e5b089fc2c49c81d6529dcbba1481eb04badc3e40e5952\" labels:{key:\"io.cri-containerd.image\" value:\"managed\"}"
Apr 17 14:52:25 k8s-controller-1 containerd[5504]: time="2026-04-17T14:52:25.503020232Z" level=info msg="ImageCreate event name:\"registry.k8s.io/kube-scheduler@sha256:44229946c0966b07d5c0791681d803e77258949985e49b4ab0fbdff99d2a48c6\" labels:{key:\"io.cri-containerd.image\" value:\"managed\"}"
Apr 17 14:52:25 k8s-controller-1 containerd[5504]: time="2026-04-17T14:52:25.503168892Z" level=info msg="Pulled image \"registry.k8s.io/kube-scheduler:v1.34.2\" with image id \"sha256:88320b5498ff2caef2e5b089fc2c49c81d6529dcbba1481eb04badc3e40e5952\", repo tag \"registry.k8s.io/kube-scheduler:v1.34.2\", repo digest \"registry.k8s.io/kube-scheduler@sha256:44229946c0966b07d5c0791681d803e77258949985e49b4ab0fbdff99d2a48c6\", size \"17382272\" in 12.960448534s"
Apr 17 14:52:25 k8s-controller-1 containerd[5504]: time="2026-04-17T14:52:25.503180321Z" level=info msg="PullImage \"registry.k8s.io/kube-scheduler:v1.34.2\" returns image reference \"sha256:88320b5498ff2caef2e5b089fc2c49c81d6529dcbba1481eb04badc3e40e5952\""
Apr 17 14:52:25 k8s-controller-1 containerd[5504]: time="2026-04-17T14:52:25.503384949Z" level=info msg="PullImage \"registry.k8s.io/kube-proxy:v1.34.2\""
Apr 17 14:52:44 k8s-controller-1 containerd[5504]: time="2026-04-17T14:52:44.179197103Z" level=info msg="fetch failed" error="failed to do request: Head \"https://europe-west4-docker.pkg.dev/v2/k8s-artifacts-prod/images/kube-apiserver/manifests/v1.34.2\": dial tcp 142.251.188.82:443: i/o timeout" host=registry.k8s.io
user@k8s-controller-1:~$ containerd --version
containerd containerd.io v2.2.3 77c84241c7cbdd9b4eca2591793e3d4f4317c590

Steps to reproduce the issue

  1. sudo apt get install containerd
  2. configure above mirror and restart containerd
  3. sudo apt install kubelet=1.34.* kubeadm=1.34.* kubectl=1.34.*
  4. sudo kubeadm image pull # failed

Describe the results you received and expected

Expect whenever using --hosts-dir option or daemon（invoked by kubelet）to pull image, it works with the same result.

What version of containerd are you using?

containerd containerd.io v2.2.3 77c84241c7cbdd9b4eca2591793e3d4f4317c590

Any other relevant information

user@k8s-controller-1: cat /etc/os-release PRETTY_NAME="Ubuntu 24.04.4 LTS" NAME="Ubuntu" VERSION_ID="24.04" VERSION="24.04.4 LTS (Noble Numbat)" VERSION_CODENAME=noble ID=ubuntu ID_LIKE=debian HOME_URL="https://www.ubuntu.com/" SUPPORT_URL="https://help.ubuntu.com/" BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" UBUNTU_CODENAME=noble LOGO=ubuntu-logo

user@k8s-controller-1:$ tree /etc/containerd/ /etc/containerd/ ├── certs.d │   ├── docker.io │   │   └── hosts.toml │   ├── ghcr.io │   │   └── hosts.toml │   └── registry.k8s.io │   └── hosts.toml ├── config.toml ├── config.toml.3428.2026-04-17@14:32:31 └── config.toml.bak user@k8s-controller-1:~$ cat /etc/containerd/certs.d/registry.k8s.io/hosts.toml

Generated by Ansible

server = "https://registry.k8s.io"

[host."https://k8s.m.daocloud.io"] capabilities = ["pull", "resolve"] skip_verify = true user@k8s-controller-1:~$ diff /etc/containerd/config.toml /etc/containerd/config.toml.bak 54c54 < config_path = "/etc/containerd/certs.d"

  config_path = ''

Show configuration if it is related to CRI plugin.

version = 3 root = '/var/lib/containerd' state = '/run/containerd' temp = '' disabled_plugins = [] required_plugins = [] oom_score = 0 imports = ['/etc/containerd/conf.d/*.toml']

[grpc] address = '/run/containerd/containerd.sock' tcp_address = '' tcp_tls_ca = '' tcp_tls_cert = '' tcp_tls_key = '' uid = 0 gid = 0 max_recv_message_size = 16777216 max_send_message_size = 16777216 tcp_tls_common_name = ''

[ttrpc] address = '' uid = 0 gid = 0

[debug] address = '' uid = 0 gid = 0 level = '' format = ''

[metrics] address = '' grpc_histogram = false

[plugins] [plugins.'io.containerd.cri.v1.images'] snapshotter = 'overlayfs' disable_snapshot_annotations = true discard_unpacked_layers = false max_concurrent_downloads = 3 concurrent_layer_fetch_buffer = 0 image_pull_progress_timeout = '5m0s' image_pull_with_sync_fs = false stats_collect_period = 10 use_local_image_pull = false

[plugins.'io.containerd.cri.v1.images'.pinned_images]
  sandbox = 'registry.k8s.io/pause:3.10.1'

[plugins.'io.containerd.cri.v1.images'.registry]
  config_path = "/etc/containerd/certs.d"

[plugins.'io.containerd.cri.v1.images'.image_decryption]
  key_model = 'node'

[plugins.'io.containerd.cri.v1.runtime'] enable_selinux = false selinux_category_range = 1024 max_container_log_line_size = 16384 disable_apparmor = false restrict_oom_score_adj = false disable_proc_mount = false unset_seccomp_profile = '' tolerate_missing_hugetlb_controller = true disable_hugetlb_controller = true device_ownership_from_security_context

Source: containerd/containerd

View original on GitHubView discussion on GitHub