From 08a4cb938f0b7de941b8d36dc19067194305224e Mon Sep 17 00:00:00 2001 From: Matheus Pimenta Date: Tue, 20 Oct 2026 12:44:24 +0100 Subject: [PATCH] [RFC-0014] ServiceAccount tokens for generic ImageRepository Signed-off-by: Matheus Pimenta --- .github/workflows/e2e.yaml | 28 ++ docs/spec/v1/imagerepositories.md | 83 +++++- hack/ci/oci-zot-workload-identity-e2e.sh | 365 +++++++++++++++++++++++ internal/registry/options.go | 59 ++-- internal/registry/options_test.go | 54 ++++ 5 files changed, 552 insertions(+), 37 deletions(-) create mode 100644 .github/workflows/e2e.yaml create mode 100755 hack/ci/oci-zot-workload-identity-e2e.sh diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml new file mode 100644 index 00000000..482920d9 --- /dev/null +++ b/.github/workflows/e2e.yaml @@ -0,0 +1,28 @@ +name: e2e +on: + workflow_dispatch: + pull_request: + push: + branches: + - 'main' + - 'release/**' +jobs: + zot-oci-workload-identity: + runs-on: ubuntu-latest + permissions: + contents: read # for reading the repository code. + steps: + - name: Test suite setup + uses: fluxcd/gha-workflows/.github/actions/setup-kubernetes@v0.16.0 + with: + go-version: 1.27.x + - name: Setup Flux CLI + uses: fluxcd/flux2/action@main + - name: Build container image + run: | + make docker-build IMG=test/image-reflector-controller:latest \ + BUILD_PLATFORMS=linux/amd64 + - name: Run Zot OCI workload identity e2e + run: hack/ci/oci-zot-workload-identity-e2e.sh + env: + IMG: test/image-reflector-controller:latest diff --git a/docs/spec/v1/imagerepositories.md b/docs/spec/v1/imagerepositories.md index 9098b289..513f0dd8 100644 --- a/docs/spec/v1/imagerepositories.md +++ b/docs/spec/v1/imagerepositories.md @@ -30,10 +30,11 @@ In the above example: - The image-reflector-controller scans the image repository for tags every hour, indicated by the `.spec.interval` field. - The registry authentication is done using a generic provider, indicated by the - `.spec.provider` field and referenced using `.spec.secretRef`. No - authentication is attempted when secret reference is not provided for generic - provider. See [Provider](#provider) for more details related to registry - authentication. + `.spec.provider` field and referenced using `.spec.secretRef`. For the generic + provider, authentication can also use the image pull secrets attached to + `.spec.serviceAccountName`, or a ServiceAccount token for object-level + workload identity. See [Provider](#provider) for more details related to + registry authentication. - The canonical form of the image set in `.spec.image` is used to scan the repository. The resolved canonical form of the image is reported in the `.status.canonicalImageName` field. @@ -193,8 +194,10 @@ reference. in the same namespace as ImageRepository with purpose depending on the value of the `.spec.provider` field: -- When `.spec.provider` is set to `generic`, the controller will fetch the image - pull secrets attached to the Service Account and use them for authentication. +- When `.spec.provider` is set to `generic`, the controller will use the image + pull secrets attached to the Service Account for static authentication, or + use the Service Account token for object-level workload identity when the + Service Account opts in with the credential annotations described below. - When `.spec.provider` is set to `aws`, `azure`, or `gcp`, the Service Account will be used for Workload Identity authentication. In this case, the controller feature gate `ObjectLevelWorkloadIdentity` must be enabled, otherwise the @@ -203,6 +206,68 @@ the `.spec.provider` field: **Note:** that for a publicly accessible image repository, you don't need to provide a `secretRef` nor `serviceAccountName`. +#### Generic provider ServiceAccount tokens + +For the `generic` provider, the controller can authenticate to a registry +using a short-lived Kubernetes ServiceAccount token. The token is presented +to the registry as a bearer token by default, or as the password in basic +authentication when `credential.auth.fluxcd.io/username` is set. + +The ServiceAccount must opt in with the following required annotations: + +- `credential.auth.fluxcd.io/provider`: must be set to `kubernetes`. +- `credential.auth.fluxcd.io/type`: must be set to `jwt`. + +The following annotations are optional: + +- `credential.auth.fluxcd.io/audiences`: a comma or newline separated list of + token audiences. Defaults to the registry host derived from `.spec.image`. +- `credential.auth.fluxcd.io/username`: the username to use when the registry + expects basic authentication. When unset, the token is sent as a bearer token. +- `credential.auth.fluxcd.io/expiration`: the requested token expiration as a + Go duration, e.g. `10m`. + +A ServiceAccount that does not set both required annotations does not opt in +and continues to use its `imagePullSecrets`, or anonymous access if it has none. +Setting only one of the two required annotations is a configuration error. The +credential annotations cannot be combined with `imagePullSecrets`. + +Object-level workload identity for the `generic` provider requires the +`ObjectLevelWorkloadIdentity` feature gate to be enabled: + +```bash +--feature-gates=ObjectLevelWorkloadIdentity=true +``` + +Example: + +```yaml +--- +apiVersion: image.toolkit.fluxcd.io/v1 +kind: ImageRepository +metadata: + name: app + namespace: apps +spec: + interval: 5m + image: registry.example.com/app + provider: generic + serviceAccountName: registry-puller +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: registry-puller + namespace: apps + annotations: + credential.auth.fluxcd.io/provider: kubernetes + credential.auth.fluxcd.io/type: jwt + # Optional. Defaults to the registry host from .spec.image. + credential.auth.fluxcd.io/audiences: registry.example.com + # Optional. Use when the registry expects basic authentication. + credential.auth.fluxcd.io/username: flux +``` + For a complete guide on how to set up authentication for cloud providers, see the integration [docs](/flux/integrations/). @@ -456,9 +521,11 @@ Supported options are: - `azure` - `gcp` -The `generic` provider can be used for public repositories or when +The `generic` provider can be used for public repositories, when static credentials are used for authentication, either with -`spec.secretRef` or `spec.serviceAccountName`. +`spec.secretRef` or `spec.serviceAccountName`, or when a Kubernetes +ServiceAccount token is used for object-level workload identity, as described +in the [ServiceAccount name](#serviceaccount-name) section. If you do not specify `.spec.provider`, it defaults to `generic`. For a complete guide on how to set up authentication for cloud providers, diff --git a/hack/ci/oci-zot-workload-identity-e2e.sh b/hack/ci/oci-zot-workload-identity-e2e.sh new file mode 100755 index 00000000..00ca82cd --- /dev/null +++ b/hack/ci/oci-zot-workload-identity-e2e.sh @@ -0,0 +1,365 @@ +#!/usr/bin/env bash +# End-to-end test for OCI registry authentication with Kubernetes ServiceAccount +# tokens, using Zot's OIDC bearer authentication. +# +# It creates a dedicated kind cluster with a ServiceAccount issuer reachable +# from a Zot container on the kind network, deploys image-reflector-controller, +# and verifies that ImageRepositories using the generic provider authenticate to +# Zot with projected ServiceAccount tokens. +set -euo pipefail + +IMG="${IMG:-test/image-reflector-controller:latest}" +ZOT_IMAGE="${ZOT_IMAGE:-ghcr.io/project-zot/zot-linux-amd64:v2.1.22}" +CLUSTER="${CLUSTER:-zot-oidc}" +ZOT_CONTAINER="${ZOT_CONTAINER:-zot-oidc}" +ZOT_PORT="${ZOT_PORT:-5000}" +ZOT_HOST_PORT="${ZOT_HOST_PORT:-5050}" +AUDIENCE="zot-registry" +NS="image-reflector-system" +APP_NS="zot-e2e" +SA_NAME="zot-puller" +SA_NAME_B="zot-puller-b" +WRONG_AUD_SA_NAME="zot-puller-wrong-aud" +REPO_A="zot-e2e-a" +REPO_B="zot-e2e-b" +IMAGEREPO_A="zot-oidc-a" +IMAGEREPO_B="zot-oidc-b" +REV="test" + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +WORKDIR="$(mktemp -d)" +CREATED_CLUSTER=false + +log() { echo -e "\n\033[1;34m==> $*\033[0m"; } +info() { echo " $*"; } +fail() { echo -e "\033[1;31mFAIL: $*\033[0m" >&2; exit 1; } + +cleanup() { + local code=$? + if [ "$code" -ne 0 ]; then + log "FAILURE (exit ${code}) — dumping diagnostics" + kubectl -n "${NS}" logs deploy/image-reflector-controller --tail=120 2>/dev/null || true + kubectl -n "${APP_NS}" get imagerepositories -o yaml 2>/dev/null || true + docker logs --tail=80 "${ZOT_CONTAINER}" 2>/dev/null || true + fi + if [ -n "${KEEP:-}" ]; then + log "KEEP set — leaving cluster, container and ${WORKDIR} for inspection" + exit "${code}" + fi + log "Cleaning up" + docker rm -f "${ZOT_CONTAINER}" >/dev/null 2>&1 || true + if [ "${CREATED_CLUSTER}" = "true" ]; then + kind delete cluster --name "${CLUSTER}" >/dev/null 2>&1 || true + fi + rm -rf "${WORKDIR}" 2>/dev/null || sudo rm -rf "${WORKDIR}" 2>/dev/null || true + exit "${code}" +} +trap cleanup EXIT + +require() { command -v "$1" >/dev/null 2>&1 || fail "required tool '$1' not found in PATH"; } +for bin in docker kind kubectl kustomize flux; do require "$bin"; done + +start_zot() { + local config="$1" + docker rm -f "${ZOT_CONTAINER}" >/dev/null 2>&1 || true + docker run -d --name "${ZOT_CONTAINER}" --network kind \ + -p "127.0.0.1:${ZOT_HOST_PORT}:${ZOT_PORT}" \ + -v "${config}:/etc/zot/config.json:ro" \ + -v "${WORKDIR}/zot-data:/var/lib/zot" \ + "${ZOT_IMAGE}" serve /etc/zot/config.json >/dev/null + + local code + for _ in $(seq 1 60); do + code="$(curl -s -o /dev/null -w '%{http_code}' "http://localhost:${ZOT_HOST_PORT}/v2/" 2>/dev/null || echo 000)" + case "${code}" in + 200|401) return 0 ;; + esac + sleep 1 + done + docker logs --tail=80 "${ZOT_CONTAINER}" 2>/dev/null || true + fail "Zot did not become ready (last HTTP status: ${code})" +} + +############################################################################## +log "Creating kind cluster '${CLUSTER}' with an externally reachable issuer" +############################################################################## +if kind get clusters 2>/dev/null | grep -qx "${CLUSTER}"; then + info "Deleting pre-existing cluster" + kind delete cluster --name "${CLUSTER}" +fi +cat </dev/null +kubectl wait --for=condition=Ready nodes --all --timeout=180s + +OIDC_ISSUER="https://${CLUSTER}-control-plane:6443" +info "OIDC issuer: ${OIDC_ISSUER}" + +log "Allowing unauthenticated OIDC discovery" +kubectl create clusterrolebinding oidc-reviewer \ + --clusterrole=system:service-account-issuer-discovery \ + --group=system:unauthenticated >/dev/null 2>&1 || true + +log "Exporting the kind CA certificate" +docker cp "${CLUSTER}-control-plane:/etc/kubernetes/pki/ca.crt" "${WORKDIR}/kind-ca.pem" +mkdir -p "${WORKDIR}/zot-data" +chmod 0777 "${WORKDIR}/zot-data" + +############################################################################## +log "Seeding Zot with artifacts while authentication is disabled" +############################################################################## +cat > "${WORKDIR}/zot-open.json" < "${WORKDIR}/artifact-a/index.txt" +echo "artifact for ${SA_NAME_B}" > "${WORKDIR}/artifact-b/index.txt" +flux push artifact "oci://localhost:${ZOT_HOST_PORT}/${REPO_A}:${REV}" \ + --path="${WORKDIR}/artifact-a" \ + --source=zot-oidc-e2e-a \ + --revision="${REV}" +flux push artifact "oci://localhost:${ZOT_HOST_PORT}/${REPO_B}:${REV}" \ + --path="${WORKDIR}/artifact-b" \ + --source=zot-oidc-e2e-b \ + --revision="${REV}" +docker rm -f "${ZOT_CONTAINER}" >/dev/null + +############################################################################## +log "Starting Zot with OIDC bearer authentication" +############################################################################## +cat > "${WORKDIR}/zot-auth.json" </dev/null + +for _ in $(seq 1 60); do + code="$(curl -s -o /dev/null -w '%{http_code}' "http://localhost:${ZOT_HOST_PORT}/v2/" 2>/dev/null || echo 000)" + [ "${code}" = "401" ] && break + sleep 1 +done +[ "${code}" = "401" ] || fail "Zot did not start with bearer auth (last HTTP status: ${code})" + +############################################################################## +log "Deploying image-reflector-controller" +############################################################################## +kind load docker-image "${IMG}" --name "${CLUSTER}" +make -C "${ROOT_DIR}" install >/dev/null +make -C "${ROOT_DIR}" dev-deploy IMG="${IMG}" >/dev/null +kubectl -n "${NS}" patch deploy/image-reflector-controller --type=json \ + -p='[{"op":"add","path":"/spec/template/spec/containers/0/args/-","value":"--feature-gates=ObjectLevelWorkloadIdentity=true"}]' >/dev/null +kubectl -n "${NS}" rollout status deploy/image-reflector-controller --timeout=2m + +############################################################################## +log "Creating ServiceAccounts and ImageRepositories" +############################################################################## +kubectl create namespace "${APP_NS}" --dry-run=client -o yaml | kubectl apply -f - >/dev/null +kubectl -n "${APP_NS}" apply -f - >/dev/null </dev/null </dev/null || true)" + msg="$(kubectl -n "${APP_NS}" get "imagerepository/${name}" \ + -o jsonpath='{.status.conditions[?(@.type=="Ready")].message}' 2>/dev/null || true)" + if [ "${status}" = "True" ]; then + fail "ImageRepository ${name} unexpectedly succeeded" + fi + if [ "${status}" = "False" ] && printf '%s' "${msg}" | grep -qiE '401|403|unauthorized|forbidden|denied|authentication'; then + info "ImageRepository ${name} correctly denied" + return 0 + fi + sleep 2 + done + fail "ImageRepository ${name} did not fail in time (status=${status:-unknown}; message=${msg:-})" +} + +log "Verifying cross-ServiceAccount access is denied" +expect_denied "${IMAGEREPO_A}-cross" +expect_denied "${IMAGEREPO_B}-cross" +expect_denied "${IMAGEREPO_A}-wrong-audience" + +log "SUCCESS: each ServiceAccount scanned only its own repository" diff --git a/internal/registry/options.go b/internal/registry/options.go index 7e4e637f..983e2c5a 100644 --- a/internal/registry/options.go +++ b/internal/registry/options.go @@ -85,22 +85,15 @@ func (r *AuthOptionsGetter) GetOptions(ctx context.Context, repo *imagev1.ImageR var authenticator authn.Authenticator provider := repo.GetProvider() - useProviderAuth := provider != "" && provider != "generic" - // Resolve the ServiceAccount used for provider authentication. The default - // ServiceAccount only applies to providers that support object-level - // workload identity; for the generic provider, only the ServiceAccount - // explicitly referenced by the ImageRepository is used for image pull - // secrets. + // Apply the default ServiceAccount when the object does not specify one. saName := repo.Spec.ServiceAccountName - if useProviderAuth && saName == "" { + if saName == "" { saName = r.DefaultServiceAccount } + var sa *corev1.ServiceAccount if saName != "" { - if useProviderAuth && !r.ObjectLevelWorkloadIdentity { - return nil, auth.ErrObjectLevelWorkloadIdentityNotEnabled - } sa = &corev1.ServiceAccount{} saRef := types.NamespacedName{ Name: saName, @@ -111,27 +104,35 @@ func (r *AuthOptionsGetter) GetOptions(ctx context.Context, repo *imagev1.ImageR } } - if useProviderAuth { - // Build the provider-agnostic authentication spec, validating the - // kubeconfig-equivalent inputs and the ServiceAccount annotations. - buildOpts := []cryptov1.Option{ - cryptov1.WithServiceProvider(provider), - cryptov1.WithOCI(), - } - if sa != nil { - buildOpts = append(buildOpts, cryptov1.WithAnnotations(sa.Annotations)) - if len(sa.ImagePullSecrets) > 0 { - buildOpts = append(buildOpts, cryptov1.WithImagePullSecrets()) - } - } - if repo.Spec.CertSecretRef != nil { - buildOpts = append(buildOpts, cryptov1.WithCertSecretRef()) - } - spec, err := cryptov1.BuildSpec(buildOpts...) - if err != nil { - return nil, fmt.Errorf("failed to validate configuration: %w", err) + // Build the provider-agnostic authentication spec. BuildSpec is safe to call + // even when there is no ServiceAccount or provider-specific configuration: + // it returns a spec with a nil Auth in that case. + buildOpts := []cryptov1.Option{ + cryptov1.WithServiceProvider(provider), + cryptov1.WithOCI(), + } + if sa != nil { + buildOpts = append(buildOpts, cryptov1.WithAnnotations(sa.Annotations)) + if len(sa.ImagePullSecrets) > 0 { + buildOpts = append(buildOpts, cryptov1.WithImagePullSecrets()) } + } + if repo.Spec.CertSecretRef != nil { + buildOpts = append(buildOpts, cryptov1.WithCertSecretRef()) + } + spec, err := cryptov1.BuildSpec(buildOpts...) + if err != nil { + return nil, fmt.Errorf("failed to validate configuration: %w", err) + } + + // Workload identity kicks in when the provider is not generic or the SA has + // opted into object-level workload identity. + useProviderAuth := provider != "generic" || spec.Auth.OptedIn() + if useProviderAuth && sa != nil && !r.ObjectLevelWorkloadIdentity { + return nil, auth.ErrObjectLevelWorkloadIdentityNotEnabled + } + if useProviderAuth { authOpts := []auth.Option{ auth.WithClient(r.Client), auth.WithObject(imagev1.GroupVersion.Group, imagev1.ImageRepositoryKind, repo), diff --git a/internal/registry/options_test.go b/internal/registry/options_test.go index 54b7c0cb..66418363 100644 --- a/internal/registry/options_test.go +++ b/internal/registry/options_test.go @@ -20,6 +20,7 @@ import ( "context" "testing" + cryptov1 "github.com/fluxcd/pkg/apis/crypto/v1" "github.com/fluxcd/pkg/apis/meta" "github.com/fluxcd/pkg/runtime/secrets" . "github.com/onsi/gomega" @@ -111,6 +112,17 @@ func TestNewAuthOptionsGetter_GetOptions(t *testing.T) { testServiceAccountWithSecret := testServiceAccount.DeepCopy() testServiceAccountWithSecret.ImagePullSecrets = []corev1.LocalObjectReference{{Name: testSecretName}} + // ServiceAccount opted into object-level workload identity. + testServiceAccountWithOptIn := testServiceAccount.DeepCopy() + testServiceAccountWithOptIn.Annotations = map[string]string{ + cryptov1.AnnotationCredentialProvider: cryptov1.CredentialProviderKubernetes, + cryptov1.AnnotationCredentialType: cryptov1.CredentialTypeJWT, + } + + // Opt-in ServiceAccount that also has image pull secrets. + testServiceAccountWithOptInAndSecret := testServiceAccountWithOptIn.DeepCopy() + testServiceAccountWithOptInAndSecret.ImagePullSecrets = []corev1.LocalObjectReference{{Name: testSecretName}} + tests := []struct { name string mockObjs []client.Object @@ -291,6 +303,48 @@ func TestNewAuthOptionsGetter_GetOptions(t *testing.T) { defaultServiceAccount: testServiceAccountName, objectLevelWorkloadIdentity: true, }, + { + name: "generic provider service account without opt-in", + mockObjs: []client.Object{testServiceAccount}, + imageRepoSpec: imagev1.ImageRepositorySpec{ + Image: testImg, + Provider: "generic", + ServiceAccountName: testServiceAccountName, + }, + objectLevelWorkloadIdentity: false, + }, + { + name: "generic provider service account without feature gate", + mockObjs: []client.Object{testServiceAccountWithOptIn}, + imageRepoSpec: imagev1.ImageRepositorySpec{ + Image: testImg, + Provider: "generic", + ServiceAccountName: testServiceAccountName, + }, + objectLevelWorkloadIdentity: false, + wantErr: true, + }, + { + name: "generic provider service account with feature gate", + mockObjs: []client.Object{testServiceAccountWithOptIn}, + imageRepoSpec: imagev1.ImageRepositorySpec{ + Image: testImg, + Provider: "generic", + ServiceAccountName: testServiceAccountName, + }, + objectLevelWorkloadIdentity: true, + }, + { + name: "generic provider opt-in service account with image pull secrets", + mockObjs: []client.Object{testServiceAccountWithOptInAndSecret, testSecret}, + imageRepoSpec: imagev1.ImageRepositorySpec{ + Image: testImg, + Provider: "generic", + ServiceAccountName: testServiceAccountName, + }, + objectLevelWorkloadIdentity: true, + wantErr: true, + }, { name: "missing default service account for provider", imageRepoSpec: imagev1.ImageRepositorySpec{