Always report components health in bootstrap

This is useful in case the `Kustomization` does not reconcile
successfully because for example the controller(s) are in a crash loop,
which is not visible in the resource itself.

Signed-off-by: Hidde Beydals <hello@hidde.co>

internal/bootstrap/bootstrap.go

@@ -58,11 +58,15 @@ type Reconciler interface {
 	// ReconcileSyncConfig reconciles the sync configuration by generating
 	// the sync manifests with the provided values, committing them to Git
 	// and pushing to remote if there are any changes.
-	ReconcileSyncConfig(ctx context.Context, options sync.Options, pollInterval, timeout time.Duration) error
+	ReconcileSyncConfig(ctx context.Context, options sync.Options) error
 
-	// ConfirmHealthy confirms that the components and extra components in
-	// install.Options are healthy.
-	ConfirmHealthy(ctx context.Context, options install.Options, timeout time.Duration) error
+	// ReportKustomizationHealth reports about the health of the
+	// Kustomization synchronizing the components.
+	ReportKustomizationHealth(ctx context.Context, options sync.Options, pollInterval, timeout time.Duration) error
+
+	// ReportComponentsHealth reports about the health for the components
+	// and extra components in install.Options.
+	ReportComponentsHealth(ctx context.Context, options install.Options, timeout time.Duration) error
 }
 
 type RepositoryReconciler interface {
@@ -89,11 +93,22 @@ func Run(ctx context.Context, reconciler Reconciler, manifestsBase string,
 	if err := reconciler.ReconcileSourceSecret(ctx, secretOpts); err != nil {
 		return err
 	}
-	if err := reconciler.ReconcileSyncConfig(ctx, syncOpts, pollInterval, timeout); err != nil {
+	if err := reconciler.ReconcileSyncConfig(ctx, syncOpts); err != nil {
 		return err
 	}
-	if err := reconciler.ConfirmHealthy(ctx, installOpts, timeout); err != nil {
-		return err
+
+	var healthErrCount int
+	if err := reconciler.ReportKustomizationHealth(ctx, syncOpts, pollInterval, timeout); err != nil {
+		healthErrCount++
+	}
+	if err := reconciler.ReportComponentsHealth(ctx, installOpts, timeout); err != nil {
+		healthErrCount++
+	}
+	if healthErrCount > 0 {
+		// Composing a "smart" error message here from the returned
+		// errors does not result in any useful information for the
+		// user, as both methods log the failures they run into.
+		err = fmt.Errorf("bootstrap failed with %d health check failure(s)", healthErrCount)
 	}
 
 	return err

internal/bootstrap/bootstrap_plain_git.go

@@ -34,7 +34,6 @@ import (
 	kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1beta1"
 
 	"github.com/fluxcd/flux2/internal/bootstrap/git"
-
 	"github.com/fluxcd/flux2/internal/utils"
 	"github.com/fluxcd/flux2/pkg/log"
 	"github.com/fluxcd/flux2/pkg/manifestgen/install"
@@ -207,7 +206,7 @@ func (b *PlainGitBootstrapper) ReconcileSourceSecret(ctx context.Context, option
 	return nil
 }
 
-func (b *PlainGitBootstrapper) ReconcileSyncConfig(ctx context.Context, options sync.Options, pollInterval, timeout time.Duration) error {
+func (b *PlainGitBootstrapper) ReconcileSyncConfig(ctx context.Context, options sync.Options) error {
 	// Confirm that sync configuration does not overwrite existing config
 	if curPath, err := kustomizationPathDiffers(ctx, b.kube, client.ObjectKey{Name: options.Name, Namespace: options.Namespace}, options.TargetPath); err != nil {
 		return fmt.Errorf("failed to determine if sync configuration would overwrite existing Kustomization: %w", err)
@@ -283,22 +282,35 @@ func (b *PlainGitBootstrapper) ReconcileSyncConfig(ctx context.Context, options
 	if _, err = utils.ExecKubectlCommand(ctx, utils.ModeStderrOS, b.kubeconfig, b.kubecontext, kubectlArgs...); err != nil {
 		return err
 	}
-	b.logger.Successf("applied sync manifests")
-
-	// Wait till Kustomization is reconciled
-	var k kustomizev1.Kustomization
-	expectRevision := fmt.Sprintf("%s/%s", options.Branch, commit)
-	if err := wait.PollImmediate(pollInterval, timeout, kustomizationReconciled(
-		ctx, b.kube, client.ObjectKey{Name: options.Name, Namespace: options.Namespace}, &k, expectRevision),
-	); err != nil {
-		return fmt.Errorf("failed waiting for Kustomization: %w", err)
-	}
-
 	b.logger.Successf("reconciled sync configuration")
+
 	return nil
 }
 
-func (b *PlainGitBootstrapper) ConfirmHealthy(ctx context.Context, install install.Options, timeout time.Duration) error {
+func (b *PlainGitBootstrapper) ReportKustomizationHealth(ctx context.Context, options sync.Options, pollInterval, timeout time.Duration) error {
+	head, err := b.git.Head()
+	if err != nil {
+		return err
+	}
+
+	objKey := client.ObjectKey{Name: options.Name, Namespace: options.Namespace}
+
+	b.logger.Waitingf("waiting for Kustomization %q to be reconciled", objKey.String())
+
+	expectRevision := fmt.Sprintf("%s/%s", options.Branch, head)
+	var k kustomizev1.Kustomization
+	if err := wait.PollImmediate(pollInterval, timeout, kustomizationReconciled(
+		ctx, b.kube, objKey, &k, expectRevision),
+	); err != nil {
+		b.logger.Failuref(err.Error())
+		return err
+	}
+
+	b.logger.Successf("Kustomization reconciled successfully")
+	return nil
+}
+
+func (b *PlainGitBootstrapper) ReportComponentsHealth(ctx context.Context, install install.Options, timeout time.Duration) error {
 	cfg, err := utils.KubeConfig(b.kubeconfig, b.kubecontext)
 	if err != nil {
 		return err

internal/bootstrap/bootstrap_provider.go

@@ -168,7 +168,7 @@ func (o sshHostnameOption) applyGitProvider(b *GitProviderBootstrapper) {
 	b.sshHostname = string(o)
 }
 
-func (b *GitProviderBootstrapper) ReconcileSyncConfig(ctx context.Context, options sync.Options, pollInterval, timeout time.Duration) error {
+func (b *GitProviderBootstrapper) ReconcileSyncConfig(ctx context.Context, options sync.Options) error {
 	repo, err := b.getRepository(ctx)
 	if err != nil {
 		return err
@@ -187,7 +187,7 @@ func (b *GitProviderBootstrapper) ReconcileSyncConfig(ctx context.Context, optio
 		}
 		options.URL = syncURL
 	}
-	return b.PlainGitBootstrapper.ReconcileSyncConfig(ctx, options, pollInterval, timeout)
+	return b.PlainGitBootstrapper.ReconcileSyncConfig(ctx, options)
 }
 
 // ReconcileRepository reconciles an organization or user repository with the

internal/bootstrap/git/git.go

@@ -47,5 +47,6 @@ type Git interface {
 	Commit(message Commit) (string, error)
 	Push(ctx context.Context) error
 	Status() (bool, error)
+	Head() (string, error)
 	Path() string
 }

internal/bootstrap/git/gogit/gogit.go

@@ -212,6 +212,17 @@ func (g *GoGit) Status() (bool, error) {
 	return status.IsClean(), nil
 }
 
+func (g *GoGit) Head() (string, error) {
+	if g.repository == nil {
+		return "", git.ErrNoGitRepository
+	}
+	head, err := g.repository.Head()
+	if err != nil {
+		return "", err
+	}
+	return head.Hash().String(), nil
+}
+
 func (g *GoGit) Path() string {
 	return g.path
 }