Skip to content

Commit 232443d

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request #17545 from hakman/automated-cherry-pick-of-#16593-upstream-release-1.33
Automated cherry pick of #16593: Add support for using ECR as pull-through image cache
2 parents de3ce4a + 3b49a11 commit 232443d

23 files changed

+489
-18
lines changed

‎docs/iam_roles.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,10 @@ The additional permissions are:
3434
"ecr:GetAuthorizationToken",
3535
"ecr:GetDownloadUrlForLayer",
3636
"ecr:GetRepositoryPolicy",
37+
"ecr:ReplicateImage",
38+
"ecr:BatchImportUpstreamImage",
39+
"ecr:CreateRepository",
40+
"ecr:TagResource",
3741
"ecr:ListImages"
3842
],
3943
"Resource": [

‎k8s/crds/kops.k8s.io_clusters.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1002,6 +1002,10 @@ spec:
10021002
description: State directory for execution state files (default
10031003
"/run/containerd").
10041004
type: string
1005+
useECRCredentialsForMirrors:
1006+
description: Enables Kubelet ECR Credential helper to pass credentials
1007+
to containerd mirrors, to use ECR as a pull-through cache
1008+
type: boolean
10051009
version:
10061010
description: Version used to pick the containerd package.
10071011
type: string

‎k8s/crds/kops.k8s.io_instancegroups.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -238,6 +238,10 @@ spec:
238238
description: State directory for execution state files (default
239239
"/run/containerd").
240240
type: string
241+
useECRCredentialsForMirrors:
242+
description: Enables Kubelet ECR Credential helper to pass credentials
243+
to containerd mirrors, to use ECR as a pull-through cache
244+
type: boolean
241245
version:
242246
description: Version used to pick the containerd package.
243247
type: string

‎nodeup/pkg/model/kubelet.go‎

Lines changed: 60 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -26,12 +26,15 @@ import (
2626
"path"
2727
"path/filepath"
2828
"strings"
29+
"time"
2930

3031
awsconfig "github.com/aws/aws-sdk-go-v2/config"
3132
"github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
3233
ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types"
34+
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
3335
"k8s.io/apimachinery/pkg/runtime"
3436
"k8s.io/apimachinery/pkg/runtime/serializer"
37+
3538
"k8s.io/klog/v2"
3639
"k8s.io/kops/pkg/apis/kops"
3740
"k8s.io/kops/pkg/flagbuilder"
@@ -41,6 +44,7 @@ import (
4144
"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
4245
"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"
4346
"k8s.io/kops/util/pkg/distributions"
47+
kubeletv1 "k8s.io/kubelet/config/v1"
4448
kubelet "k8s.io/kubelet/config/v1beta1"
4549
)
4650

@@ -467,25 +471,65 @@ func (b *KubeletBuilder) addECRCredentialProvider(c *fi.NodeupModelBuilderContex
467471
}
468472

469473
{
470-
configContent := `apiVersion: kubelet.config.k8s.io/v1
471-
kind: CredentialProviderConfig
472-
providers:
473-
- name: ecr-credential-provider
474-
matchImages:
475-
- "*.dkr.ecr.*.amazonaws.com"
476-
- "*.dkr.ecr.*.amazonaws.com.cn"
477-
- "*.dkr.ecr-fips.*.amazonaws.com"
478-
- "*.dkr.ecr.us-iso-east-1.c2s.ic.gov"
479-
- "*.dkr.ecr.us-isob-east-1.sc2s.sgov.gov"
480-
defaultCacheDuration: "12h"
481-
apiVersion: credentialprovider.kubelet.k8s.io/v1
482-
args:
483-
- get-credentials
484-
`
474+
475+
providerConfig := &kubeletv1.CredentialProviderConfig{}
476+
477+
// Build the list of container registry globs to match
478+
registryList := []string{
479+
"*.dkr.ecr.*.amazonaws.com",
480+
"*.dkr.ecr.*.amazonaws.com.cn",
481+
"*.dkr.ecr-fips.*.amazonaws.com",
482+
"*.dkr.ecr.us-iso-east-1.c2s.ic.gov",
483+
}
484+
485+
containerd := b.NodeupConfig.ContainerdConfig
486+
if containerd.UseECRCredentialsForMirrors {
487+
for name := range containerd.RegistryMirrors {
488+
registryList = append(registryList, name)
489+
}
490+
}
491+
492+
cacheDuration, err := time.ParseDuration("12h")
493+
if err != nil {
494+
return err
495+
}
496+
497+
providerConfig.Providers = []kubeletv1.CredentialProvider{
498+
{
499+
APIVersion: "credentialprovider.kubelet.k8s.io/v1",
500+
Name: "ecr-credential-provider",
501+
MatchImages: registryList,
502+
DefaultCacheDuration: &v1.Duration{Duration: cacheDuration},
503+
Args: []string{"get-credentials"},
504+
Env: []kubeletv1.ExecEnvVar{
505+
{
506+
Name: "AWS_REGION",
507+
Value: b.Cloud.Region(),
508+
},
509+
},
510+
},
511+
}
512+
513+
sch := runtime.NewScheme()
514+
if err := kubeletv1.AddToScheme(sch); err != nil {
515+
return err
516+
}
517+
518+
gv := kubeletv1.SchemeGroupVersion
519+
codecFactory := serializer.NewCodecFactory(sch)
520+
info, ok := runtime.SerializerInfoForMediaType(codecFactory.SupportedMediaTypes(), "application/yaml")
521+
if !ok {
522+
return fmt.Errorf("failed to find serializer")
523+
}
524+
encoder := codecFactory.EncoderForVersion(info.Serializer, gv)
525+
var w bytes.Buffer
526+
if err := encoder.Encode(providerConfig, &w); err != nil {
527+
return err
528+
}
485529

486530
t := &nodetasks.File{
487531
Path: credentialProviderConfigFilePath,
488-
Contents: fi.NewStringResource(configContent),
532+
Contents: fi.NewBytesResource(w.Bytes()),
489533
Type: nodetasks.FileType_File,
490534
Mode: s("0644"),
491535
}

‎pkg/apis/kops/containerdconfig.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,8 @@ type ContainerdConfig struct {
5454
SeLinuxEnabled bool `json:"selinuxEnabled,omitempty"`
5555
// NRI configures the Node Resource Interface.
5656
NRI *NRIConfig `json:"nri,omitempty"`
57+
// Enables Kubelet ECR Credential helper to pass credentials to containerd mirrors, to use ECR as a pull-through cache
58+
UseECRCredentialsForMirrors bool `json:"useECRCredentialsForMirrors,omitempty"`
5759
}
5860

5961
type NRIConfig struct {

‎pkg/apis/kops/v1alpha2/containerdconfig.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,8 @@ type ContainerdConfig struct {
5151
SeLinuxEnabled bool `json:"selinuxEnabled,omitempty"`
5252
// NRI configures the Node Resource Interface.
5353
NRI *NRIConfig `json:"nri,omitempty"`
54+
// Enables Kubelet ECR Credential helper to pass credentials to containerd mirrors, to use ECR as a pull-through cache
55+
UseECRCredentialsForMirrors bool `json:"useECRCredentialsForMirrors,omitempty"`
5456
}
5557

5658
type NRIConfig struct {

‎pkg/apis/kops/v1alpha2/zz_generated.conversion.go‎

Lines changed: 2 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pkg/apis/kops/v1alpha3/containerdconfig.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,8 @@ type ContainerdConfig struct {
5151
SeLinuxEnabled bool `json:"selinuxEnabled,omitempty"`
5252
// NRI configures the Node Resource Interface.
5353
NRI *NRIConfig `json:"nri,omitempty"`
54+
// Enables Kubelet ECR Credential helper to pass credentials to containerd mirrors, to use ECR as a pull-through cache
55+
UseECRCredentialsForMirrors bool `json:"useECRCredentialsForMirrors,omitempty"`
5456
}
5557

5658
type NRIConfig struct {

‎pkg/apis/kops/v1alpha3/zz_generated.conversion.go‎

Lines changed: 2 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pkg/model/iam/iam_builder.go‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -362,6 +362,10 @@ func (r *NodeRoleAPIServer) BuildAWSPolicy(b *PolicyBuilder) (*Policy, error) {
362362
addECRPermissions(p)
363363
}
364364

365+
if b.Cluster.Spec.Containerd != nil && b.Cluster.Spec.Containerd.UseECRCredentialsForMirrors {
366+
addECRPullThroughPermissions(p)
367+
}
368+
365369
if b.Cluster.Spec.Networking.AmazonVPC != nil {
366370
addAmazonVPCCNIPermissions(p)
367371
}
@@ -430,6 +434,10 @@ func (r *NodeRoleMaster) BuildAWSPolicy(b *PolicyBuilder) (*Policy, error) {
430434
addECRPermissions(p)
431435
}
432436

437+
if b.Cluster.Spec.Containerd != nil && b.Cluster.Spec.Containerd.UseECRCredentialsForMirrors {
438+
addECRPullThroughPermissions(p)
439+
}
440+
433441
if b.Cluster.Spec.Networking.AmazonVPC != nil {
434442
addAmazonVPCCNIPermissions(p)
435443
}
@@ -465,6 +473,10 @@ func (r *NodeRoleNode) BuildAWSPolicy(b *PolicyBuilder) (*Policy, error) {
465473
addECRPermissions(p)
466474
}
467475

476+
if b.Cluster.Spec.Containerd != nil && b.Cluster.Spec.Containerd.UseECRCredentialsForMirrors {
477+
addECRPullThroughPermissions(p)
478+
}
479+
468480
if b.Cluster.Spec.Networking.AmazonVPC != nil {
469481
addAmazonVPCCNIPermissions(p)
470482
}
@@ -776,6 +788,17 @@ func addECRPermissions(p *Policy) {
776788
)
777789
}
778790

791+
func addECRPullThroughPermissions(p *Policy) {
792+
// Permissions needed for ECR pull-through cache functionality
793+
// These permissions are only needed when UseECRCredentialsForMirrors is enabled
794+
p.unconditionalAction.Insert(
795+
"ecr:ReplicateImage",
796+
"ecr:BatchImportUpstreamImage",
797+
"ecr:CreateRepository",
798+
"ecr:TagResource",
799+
)
800+
}
801+
779802
func addCalicoSrcDstCheckPermissions(p *Policy) {
780803
p.unconditionalAction.Insert(
781804
"ec2:DescribeInstances",

0 commit comments

Comments
 (0)