diff --git a/.prow.yaml b/.prow.yaml index 8203b811956..a3e289ef438 100644 --- a/.prow.yaml +++ b/.prow.yaml @@ -64,9 +64,6 @@ presubmits: - image: quay.io/containers/buildah:v1.38.0 command: - hack/build-image.sh - env: - - name: DRY_RUN - value: "1" # docker-in-docker needs privileged mode securityContext: privileged: true diff --git a/hack/build-image-docker.sh b/hack/build-image-docker.sh new file mode 100755 index 00000000000..49d11dae09e --- /dev/null +++ b/hack/build-image-docker.sh @@ -0,0 +1,219 @@ +#!/usr/bin/env bash + +# Copyright 2025 The KCP Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Build container images for KCP using Docker +# +# This script builds container images using Docker (with or without buildx). +# +# Usage examples: +# # Build locally with default settings (uses current git commit hash) +# ./hack/build-image-docker.sh +# +# # Build locally with custom repository name +# REPOSITORY=my-registry/kcp ./hack/build-image-docker.sh +# +# # Build locally without pushing (dry run) +# DRY_RUN=1 ./hack/build-image-docker.sh +# +# # Build for specific architectures only +# ARCHITECTURES="amd64" ./hack/build-image-docker.sh +# +# Environment variables: +# REPOSITORY - Override default repository (default: ghcr.io/kcp-dev/kcp) +# ARCHITECTURES - Space-separated list of architectures (default: "amd64 arm64") +# DRY_RUN - Set to any value to build locally without pushing +# KCP_GHCR_USERNAME/KCP_GHCR_PASSWORD - Registry credentials for pushing +# +# Build tool support: +# - docker + buildx: Multi-arch support with intelligent platform handling +# - docker only: Single architecture fallback + +set -euo pipefail + +# make git available +if ! [ -x "$(command -v git)" ]; then + echo "Installing git ..." + yum install -y git +fi + +# Check if docker is available +if ! [ -x "$(command -v docker)" ]; then + echo "Error: Docker is not available" + exit 1 +fi + +echo "Using docker for container builds" +# Check if buildx is available for multi-arch builds +if docker buildx version >/dev/null 2>&1; then + echo "Docker buildx is available for multi-arch builds" + DOCKER_BUILDX=true +else + echo "Docker buildx not available, falling back to single-arch builds" + DOCKER_BUILDX=false +fi + + +if [ -z "${REPOSITORY:-}" ]; then + echo "Error: REPOSITORY environment variable is required" + exit 1 +fi +repository=$REPOSITORY +architectures=${ARCHITECTURES:-"amd64 arm64"} + +# when building locally, just tag with the current HEAD hash. +version="$(git rev-parse --short HEAD)" +branchName="" + +# deduce the tag from the Prow job metadata +if [ -n "${PULL_BASE_REF:-}" ]; then + version="$(git tag --list "$PULL_BASE_REF")" + + if [ -z "$version" ]; then + # if the base ref did not point to a tag, it's a branch name + version="$(git rev-parse --short "$PULL_BASE_REF")" + branchName="$PULL_BASE_REF" + else + # If PULL_BASE_REF is a tag, there is no branch available locally, plus + # there is no guarantee that vX.Y.Z is tagged _only_ in the release-X.Y + # branch; because of this we have to deduce the branch name from the tag + branchName="$(echo "$version" | sed -E 's/^v([0-9]+)\.([0-9]+)\..*/release-\1.\2/')" + fi +fi + +# Prefix with "pr-" if not on a tag or branch +if [ -n "${PULL_NUMBER:-}" ]; then + version="pr-$PULL_NUMBER-$version" + repository="$repository-prs" +fi + +image="$repository:$version" +echo "Building container image $image ..." + +# Function to build images with docker buildx +build_with_docker_buildx() { + echo "Building multi-arch image $image ..." + + # Create platforms string for buildx + platforms="" + for arch in $architectures; do + if [ -n "$platforms" ]; then + platforms="$platforms,linux/$arch" + else + platforms="linux/$arch" + fi + done + + # For push builds, use multi-platform; for local builds, build per arch + if [ -z "${DRY_RUN:-}" ]; then + # Building for push - use multi-platform with --push + docker buildx build \ + --file Dockerfile \ + --tag "$image" \ + --platform "$platforms" \ + --build-arg "TARGETOS=linux" \ + --push \ + . + else + # For local/dry-run builds, build each architecture separately with --load + for arch in $architectures; do + fullTag="$image-$arch" + echo "Building $fullTag ..." + docker buildx build \ + --file Dockerfile \ + --tag "$fullTag" \ + --platform "linux/$arch" \ + --build-arg "TARGETOS=linux" \ + --build-arg "TARGETARCH=$arch" \ + --load \ + . + done + # Tag the first architecture as the main image for local use + first_arch=$(echo $architectures | cut -d' ' -f1) + docker tag "$image-$first_arch" "$image" + fi +} + +# Function to build images with regular docker (single arch only) +build_with_docker() { + # Use only the first architecture for regular docker + arch=$(echo $architectures | cut -d' ' -f1) + fullTag="$image-$arch" + + echo "Building single-arch image $fullTag (docker without buildx) ..." + docker build \ + --file Dockerfile \ + --tag "$fullTag" \ + --platform "linux/$arch" \ + --build-arg "TARGETOS=linux" \ + --build-arg "TARGETARCH=$arch" \ + . + + # Tag it as the main image too + docker tag "$fullTag" "$image" +} + +# Build images based on available docker features +if [ "$DOCKER_BUILDX" = true ]; then + build_with_docker_buildx +else + build_with_docker +fi + +# Additionally to an image tagged with the Git tag, we also +# release images tagged with the current branch name, which +# is somewhere between a blanket "latest" tag and a specific +# tag. +if [ -n "$branchName" ] && [ -z "${PULL_NUMBER:-}" ]; then + branchImage="$repository:$branchName" + + if [ "$DOCKER_BUILDX" = true ]; then + echo "Tagging multi-arch image as $branchImage ..." + docker tag "$image" "$branchImage" + else + echo "Tagging single-arch image as $branchImage ..." + docker tag "$image" "$branchImage" + fi +fi + +# push images, except in dry runs +if [ -z "${DRY_RUN:-}" ]; then + echo "Logging into GHCR ..." + + if [ "$DOCKER_BUILDX" = true ]; then + # buildx with --push already pushed during build + echo "Images already pushed during buildx build" + else + # Regular docker - need to login and push + if [ -n "${GHCR_USERNAME:-}" ] && [ -n "${GHCR_PASSWORD:-}" ]; then + echo "$GHCR_PASSWORD" | docker login ghcr.io -u "$GHCR_USERNAME" --password-stdin + else + echo "Skipping login (GHCR_USERNAME/GHCR_PASSWORD not provided)" + fi + + echo "Pushing images ..." + docker push "$image" + + if [ -n "${branchImage:-}" ]; then + docker push "$branchImage" + + + fi + fi +else + echo "Not pushing images because \$DRY_RUN is set." +fi + +echo "Done." diff --git a/pkg/server/config.go b/pkg/server/config.go index dbc13f0313a..e8a612f1a62 100644 --- a/pkg/server/config.go +++ b/pkg/server/config.go @@ -136,6 +136,9 @@ type ExtraConfig struct { CacheDiscoveringDynamicSharedInformerFactory *informer.DiscoveringDynamicSharedInformerFactory CacheKcpSharedInformerFactory kcpinformers.SharedInformerFactory CacheKubeSharedInformerFactory kcpkubernetesinformers.SharedInformerFactory + + // If CABundleFile is set, read its contents into CABundleData + CABundleData []byte } type completedConfig struct { @@ -186,6 +189,14 @@ func NewConfig(ctx context.Context, opts kcpserveroptions.CompletedOptions) (*Co Options: opts, } + if opts.Extra.CABundleFile != "" { + data, err := os.ReadFile(opts.Extra.CABundleFile) + if err != nil { + return nil, fmt.Errorf("failed to read ca-bundle-file %q: %w", opts.Extra.CABundleFile, err) + } + c.CABundleData = data + } + if opts.Extra.ProfilerAddress != "" { //nolint:errcheck,gosec go http.ListenAndServe(opts.Extra.ProfilerAddress, nil) @@ -323,6 +334,15 @@ func NewConfig(ctx context.Context, opts kcpserveroptions.CompletedOptions) (*Co if err != nil { return nil, fmt.Errorf("failed to load the external logical cluster admin kubeconfig from %q: %w", c.Options.Extra.ExternalLogicalClusterAdminKubeconfig, err) } + if c.CABundleData != nil { + // Inject CA bundle into rest.Config + if c.ExternalLogicalClusterAdminConfig.TLSClientConfig.CAData == nil { + c.ExternalLogicalClusterAdminConfig.TLSClientConfig.CAData = c.CABundleData + } else { + // Append to existing CA data + c.ExternalLogicalClusterAdminConfig.TLSClientConfig.CAData = append(c.ExternalLogicalClusterAdminConfig.TLSClientConfig.CAData, c.CABundleData...) + } + } } // Setup apiextensions * informers @@ -345,12 +365,12 @@ func NewConfig(ctx context.Context, opts kcpserveroptions.CompletedOptions) (*Co return nil, err } var userToken string - if sets.New[string](opts.Extra.BatteriesIncluded...).Has(batteries.Admin) { + if sets.New(opts.Extra.BatteriesIncluded...).Has(batteries.Admin) { c.kcpAdminToken, c.shardAdminToken, userToken, c.shardAdminTokenHash, err = opts.AdminAuthentication.ApplyTo(c.GenericConfig) if err != nil { return nil, err } - if sets.New[string](opts.Extra.BatteriesIncluded...).Has(batteries.User) { + if sets.New(opts.Extra.BatteriesIncluded...).Has(batteries.User) { c.userToken = userToken } } diff --git a/pkg/server/options/options.go b/pkg/server/options/options.go index 17fd322c3cb..d134e80968b 100644 --- a/pkg/server/options/options.go +++ b/pkg/server/options/options.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "net" + "os" "path/filepath" "strings" "time" @@ -29,6 +30,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" genericapiserveroptions "k8s.io/apiserver/pkg/server/options" cliflag "k8s.io/component-base/cli/flag" + "k8s.io/klog/v2" controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver/options" etcdoptions "github.com/kcp-dev/embeddedetcd/options" @@ -74,6 +76,9 @@ type ExtraOptions struct { // --miniproxy-mapping-file flag of the front-proxy. Do NOT expose this flag to users via main server options. // It is overridden by the kcp start command. AdditionalMappingsFile string + + // If CABundleFile is set, it contains the path to a file containing a PEM-encoded CA bundle to validate client certificates presented to kcp when using client-go internally. + CABundleFile string } type completedOptions struct { @@ -114,8 +119,9 @@ func NewOptions(rootDir string) *Options { DiscoveryPollInterval: 60 * time.Second, ExperimentalBindFreePort: false, ConversionCELTransformationTimeout: time.Second, + CABundleFile: "", - BatteriesIncluded: sets.List[string](batteries.Defaults), + BatteriesIncluded: sets.List(batteries.Defaults), }, } @@ -168,6 +174,7 @@ func (o *Options) AddFlags(fss *cliflag.NamedFlagSets) { fs.StringVar(&o.Extra.LogicalClusterAdminKubeconfig, "logical-cluster-admin-kubeconfig", o.Extra.LogicalClusterAdminKubeconfig, "Kubeconfig holding system:kcp:logical-cluster-admin credentials for connecting to other shards. Defaults to the loopback client") fs.StringVar(&o.Extra.ExternalLogicalClusterAdminKubeconfig, "external-logical-cluster-admin-kubeconfig", o.Extra.ExternalLogicalClusterAdminKubeconfig, "Kubeconfig holding system:kcp:external-logical-cluster-admin credentials for connecting to the external address (e.g. the front-proxy). Defaults to the loopback client") fs.StringVar(&o.Extra.RootIdentitiesFile, "root-identities-file", "", "Path to a YAML file used to bootstrap APIExport identities inside the root workspace. The YAML file must be structured as {\"identities\": [ {\"export\": \"\", \"identity\": \"\"}, ... ]}. If a secret with matching APIExport name already exists inside kcp-system namespace, it will be left unchanged. Defaults to empty, i.e. no identities are bootstrapped.") + fs.StringVar(&o.Extra.CABundleFile, "ca-bundle-file", o.Extra.CABundleFile, "Path to a file containing a PEM-encoded CA bundle. If set, this CA bundle will be used to validate client certificates presented to the kcp when using client-go internally.") fs.BoolVar(&o.Extra.ExperimentalBindFreePort, "experimental-bind-free-port", o.Extra.ExperimentalBindFreePort, "Bind to a free port. --secure-port must be 0. Use the admin.kubeconfig to extract the chosen port.") fs.MarkHidden("experimental-bind-free-port") //nolint:errcheck @@ -228,7 +235,7 @@ func (o *CompletedOptions) Validate() []error { } } - batterySet := sets.New[string](o.Extra.BatteriesIncluded...) + batterySet := sets.New(o.Extra.BatteriesIncluded...) if batterySet.Has(batteries.User) && !batterySet.Has(batteries.Admin) { errs = append(errs, fmt.Errorf("battery %s enabled which requires %s as well", batteries.User, batteries.Admin)) } @@ -237,6 +244,20 @@ func (o *CompletedOptions) Validate() []error { errs = append(errs, fmt.Errorf("--shard-external-url is required if --logical-cluster-admin-kubeconfig is set")) } + if o.Extra.CABundleFile != "" { + data, err := os.ReadFile(o.Extra.CABundleFile) + if err != nil { + errs = append(errs, fmt.Errorf("ca-bundle-file %q could not be read: %w", o.Extra.CABundleFile, err)) + } else { + if len(data) == 0 { + errs = append(errs, fmt.Errorf("ca-bundle-file %q is empty", o.Extra.CABundleFile)) + } else { + // TODO: validate if this is valid PEM data? + klog.Infof("ca-bundle-file %q read successfully, %d bytes", o.Extra.CABundleFile, len(data)) + } + } + } + return errs } @@ -374,7 +395,7 @@ func (o *Options) Complete(ctx context.Context, rootDir string) (*CompletedOptio } } if differential { - bats := sets.New[string](sets.List[string](batteries.Defaults)...) + bats := sets.New(sets.List(batteries.Defaults)...) for _, b := range o.Extra.BatteriesIncluded { if strings.HasPrefix(b, "+") { bats.Insert(b[1:]) @@ -382,7 +403,7 @@ func (o *Options) Complete(ctx context.Context, rootDir string) (*CompletedOptio bats.Delete(b[1:]) } } - o.Extra.BatteriesIncluded = sets.List[string](bats) + o.Extra.BatteriesIncluded = sets.List(bats) } completedEmbeddedEtcd := o.EmbeddedEtcd.Complete(o.GenericControlPlane.Etcd) @@ -399,6 +420,16 @@ func (o *Options) Complete(ctx context.Context, rootDir string) (*CompletedOptio return nil, err } + if o.Extra.CABundleFile != "" { + if !filepath.IsAbs(o.Extra.CABundleFile) { + pwd, err := os.Getwd() + if err != nil { + return nil, err + } + o.Extra.CABundleFile = filepath.Join(pwd, o.Extra.CABundleFile) + } + } + return &CompletedOptions{ completedOptions: &completedOptions{ GenericControlPlane: completedGenericOptions,