Skip to content

Commit 88a2e47

Browse files
stefanprodanstefanprodan
committed
Retry downloading the artifact on not found errors
- Extract the artifact operations such as download, verify, untar into a dedicated struct - Introduce a dedicated type for artifact not found errors - On artifact not found errors, log the error, update the ready status message and requeue the object - Retry the artifact download at the interval set with `--requeue-dependency` (defaults to 30s) Signed-off-by: Stefan Prodan <stefan.prodan@gmail.com>
1 parent db3c321 commit 88a2e47

4 files changed

Lines changed: 301 additions & 83 deletions

File tree

‎controllers/kustomization_controller.go‎

Lines changed: 15 additions & 82 deletions
Original file line numberDiff line numberDiff line change
@@ -19,19 +19,13 @@ package controllers
1919
import (
2020
"bytes"
2121
"context"
22-
"crypto/sha1"
23-
"crypto/sha256"
2422
"fmt"
25-
"io"
26-
"net/http"
27-
"net/url"
2823
"os"
2924
"sort"
3025
"strings"
3126
"time"
3227

3328
securejoin "github.com/cyphar/filepath-securejoin"
34-
"github.com/hashicorp/go-retryablehttp"
3529
corev1 "k8s.io/api/core/v1"
3630
apierrors "k8s.io/apimachinery/pkg/api/errors"
3731
apimeta "k8s.io/apimachinery/pkg/api/meta"
@@ -62,7 +56,6 @@ import (
6256
"github.com/fluxcd/pkg/runtime/metrics"
6357
"github.com/fluxcd/pkg/runtime/predicates"
6458
"github.com/fluxcd/pkg/ssa"
65-
"github.com/fluxcd/pkg/untar"
6659
sourcev1 "github.com/fluxcd/source-controller/api/v1beta2"
6760

6861
kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1beta2"
@@ -79,7 +72,7 @@ import (
7972
// KustomizationReconciler reconciles a Kustomization object
8073
type KustomizationReconciler struct {
8174
client.Client
82-
httpClient *retryablehttp.Client
75+
artifactDownloader *ArtifactDownloader
8376
requeueDependency time.Duration
8477
Scheme *runtime.Scheme
8578
EventRecorder kuberecorder.EventRecorder
@@ -122,15 +115,7 @@ func (r *KustomizationReconciler) SetupWithManager(mgr ctrl.Manager, opts Kustom
122115

123116
r.requeueDependency = opts.DependencyRequeueInterval
124117
r.statusManager = fmt.Sprintf("gotk-%s", r.ControllerName)
125-
126-
// Configure the retryable http client used for fetching artifacts.
127-
// By default it retries 10 times within a 3.5 minutes window.
128-
httpClient := retryablehttp.NewClient()
129-
httpClient.RetryWaitMin = 5 * time.Second
130-
httpClient.RetryWaitMax = 30 * time.Second
131-
httpClient.RetryMax = opts.HTTPRetry
132-
httpClient.Logger = nil
133-
r.httpClient = httpClient
118+
r.artifactDownloader = NewArtifactDownloader(opts.HTTPRetry)
134119

135120
return ctrl.NewControllerManagedBy(mgr).
136121
For(&kustomizev1.Kustomization{}, builder.WithPredicates(
@@ -268,6 +253,18 @@ func (r *KustomizationReconciler) Reconcile(ctx context.Context, req ctrl.Reques
268253

269254
// reconcile kustomization by applying the latest revision
270255
reconciledKustomization, reconcileErr := r.reconcile(ctx, *kustomization.DeepCopy(), source)
256+
257+
// requeue if the artifact is not found
258+
if reconcileErr == ArtifactNotFoundError {
259+
msg := fmt.Sprintf("Source is not ready, artifact not found, retrying in %s", r.requeueDependency.String())
260+
log.Info(msg)
261+
if err := r.patchStatus(ctx, req, kustomizev1.KustomizationProgressing(kustomization, msg).Status); err != nil {
262+
log.Error(err, "unable to update status for artifact not found")
263+
return ctrl.Result{Requeue: true}, err
264+
}
265+
return ctrl.Result{RequeueAfter: r.requeueDependency}, nil
266+
}
267+
271268
if err := r.patchStatus(ctx, req, reconciledKustomization.Status); err != nil {
272269
return ctrl.Result{Requeue: true}, err
273270
}
@@ -320,7 +317,7 @@ func (r *KustomizationReconciler) reconcile(
320317
defer os.RemoveAll(tmpDir)
321318

322319
// download artifact and extract files
323-
err = r.download(source.GetArtifact(), tmpDir)
320+
err = r.artifactDownloader.Fetch(source.GetArtifact(), tmpDir)
324321
if err != nil {
325322
return kustomizev1.KustomizationNotReady(
326323
kustomization,
@@ -526,70 +523,6 @@ func (r *KustomizationReconciler) checkDependencies(source sourcev1.Source, kust
526523
return nil
527524
}
528525

529-
func (r *KustomizationReconciler) download(artifact *sourcev1.Artifact, tmpDir string) error {
530-
artifactURL := artifact.URL
531-
if hostname := os.Getenv("SOURCE_CONTROLLER_LOCALHOST"); hostname != "" {
532-
u, err := url.Parse(artifactURL)
533-
if err != nil {
534-
return err
535-
}
536-
u.Host = hostname
537-
artifactURL = u.String()
538-
}
539-
540-
req, err := retryablehttp.NewRequest(http.MethodGet, artifactURL, nil)
541-
if err != nil {
542-
return fmt.Errorf("failed to create a new request: %w", err)
543-
}
544-
545-
resp, err := r.httpClient.Do(req)
546-
if err != nil {
547-
return fmt.Errorf("failed to download artifact, error: %w", err)
548-
}
549-
defer resp.Body.Close()
550-
551-
// check response
552-
if resp.StatusCode != http.StatusOK {
553-
return fmt.Errorf("failed to download artifact from %s, status: %s", artifactURL, resp.Status)
554-
}
555-
556-
var buf bytes.Buffer
557-
558-
// verify checksum matches origin
559-
if err := r.verifyArtifact(artifact, &buf, resp.Body); err != nil {
560-
return err
561-
}
562-
563-
// extract
564-
if _, err = untar.Untar(&buf, tmpDir); err != nil {
565-
return fmt.Errorf("failed to untar artifact, error: %w", err)
566-
}
567-
568-
return nil
569-
}
570-
571-
func (r *KustomizationReconciler) verifyArtifact(artifact *sourcev1.Artifact, buf *bytes.Buffer, reader io.Reader) error {
572-
hasher := sha256.New()
573-
574-
// for backwards compatibility with source-controller v0.17.2 and older
575-
if len(artifact.Checksum) == 40 {
576-
hasher = sha1.New()
577-
}
578-
579-
// compute checksum
580-
mw := io.MultiWriter(hasher, buf)
581-
if _, err := io.Copy(mw, reader); err != nil {
582-
return err
583-
}
584-
585-
if checksum := fmt.Sprintf("%x", hasher.Sum(nil)); checksum != artifact.Checksum {
586-
return fmt.Errorf("failed to verify artifact: computed checksum '%s' doesn't match advertised '%s'",
587-
checksum, artifact.Checksum)
588-
}
589-
590-
return nil
591-
}
592-
593526
func (r *KustomizationReconciler) getSource(ctx context.Context, kustomization kustomizev1.Kustomization) (sourcev1.Source, error) {
594527
var source sourcev1.Source
595528
sourceNamespace := kustomization.GetNamespace()

‎controllers/kustomization_downloader.go‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
/*
2+
Copyright 2022 The Flux authors
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package controllers
18+
19+
import (
20+
"bytes"
21+
"crypto/sha1"
22+
"crypto/sha256"
23+
"errors"
24+
"fmt"
25+
"io"
26+
"net/http"
27+
"net/url"
28+
"os"
29+
"time"
30+
31+
"github.com/fluxcd/pkg/untar"
32+
sourcev1 "github.com/fluxcd/source-controller/api/v1beta2"
33+
"github.com/hashicorp/go-retryablehttp"
34+
)
35+
36+
// ArtifactDownloader holds the HTTP client that reties with back off when
37+
// the artifact server is offline.
38+
type ArtifactDownloader struct {
39+
httpClient *retryablehttp.Client
40+
}
41+
42+
// ArtifactNotFoundError is an error type used to signal 404 HTTP status code responses.
43+
var ArtifactNotFoundError = errors.New("artifact not found")
44+
45+
// NewArtifactDownloader configures the retryable http client used for fetching artifacts.
46+
// By default, it retries 10 times within a 3.5 minutes window.
47+
func NewArtifactDownloader(retries int) *ArtifactDownloader {
48+
httpClient := retryablehttp.NewClient()
49+
httpClient.RetryWaitMin = 5 * time.Second
50+
httpClient.RetryWaitMax = 30 * time.Second
51+
httpClient.RetryMax = retries
52+
httpClient.Logger = nil
53+
54+
return &ArtifactDownloader{httpClient: httpClient}
55+
}
56+
57+
// Fetch downloads, verifies and extracts the artifact content to the specified directory.
58+
// If the artifact server responds with 5xx errors, the download operation is retried.
59+
// If the artifact server responds with 404, the returned error is of type ArtifactNotFoundError.
60+
// If the artifact server is unavailable for more than 3 minutes, the returned error contains the original status code.
61+
func (r *ArtifactDownloader) Fetch(artifact *sourcev1.Artifact, dir string) error {
62+
artifactURL := artifact.URL
63+
if hostname := os.Getenv("SOURCE_CONTROLLER_LOCALHOST"); hostname != "" {
64+
u, err := url.Parse(artifactURL)
65+
if err != nil {
66+
return err
67+
}
68+
u.Host = hostname
69+
artifactURL = u.String()
70+
}
71+
72+
req, err := retryablehttp.NewRequest(http.MethodGet, artifactURL, nil)
73+
if err != nil {
74+
return fmt.Errorf("failed to create a new request: %w", err)
75+
}
76+
77+
resp, err := r.httpClient.Do(req)
78+
if err != nil {
79+
return fmt.Errorf("failed to download artifact, error: %w", err)
80+
}
81+
defer resp.Body.Close()
82+
83+
if code := resp.StatusCode; code != http.StatusOK {
84+
if code == http.StatusNotFound {
85+
return ArtifactNotFoundError
86+
}
87+
return fmt.Errorf("failed to download artifact from %s, status: %s", artifactURL, resp.Status)
88+
}
89+
90+
var buf bytes.Buffer
91+
92+
// verify checksum matches origin
93+
if err := r.Verify(artifact, &buf, resp.Body); err != nil {
94+
return err
95+
}
96+
97+
// extract
98+
if _, err = untar.Untar(&buf, dir); err != nil {
99+
return fmt.Errorf("failed to untar artifact, error: %w", err)
100+
}
101+
102+
return nil
103+
}
104+
105+
// Verify computes the checksum of the tarball and returns an error if the computed value
106+
// does not match the artifact advertised checksum.
107+
func (r *ArtifactDownloader) Verify(artifact *sourcev1.Artifact, buf *bytes.Buffer, reader io.Reader) error {
108+
hasher := sha256.New()
109+
110+
// for backwards compatibility with source-controller v0.17.2 and older
111+
if len(artifact.Checksum) == 40 {
112+
hasher = sha1.New()
113+
}
114+
115+
// compute checksum
116+
mw := io.MultiWriter(hasher, buf)
117+
if _, err := io.Copy(mw, reader); err != nil {
118+
return err
119+
}
120+
121+
if checksum := fmt.Sprintf("%x", hasher.Sum(nil)); checksum != artifact.Checksum {
122+
return fmt.Errorf("failed to verify artifact: computed checksum '%s' doesn't match advertised '%s'",
123+
checksum, artifact.Checksum)
124+
}
125+
126+
return nil
127+
}

0 commit comments

Comments
 (0)
⚡