vbatts/registry
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Attempt to deal with eventual consistency by retrying

Browse source 
Rather than accept the resulting of a layer validation, we retry up to three
times, backing off 100ms after each try. The thought is that we allow s3 files
to make their way into the correct location increasing the liklihood the
verification can proceed, if possible.

Signed-off-by: Stephen J Day <stephen.day@docker.com>
This commit is contained in:
Stephen J Day 2015-04-20 18:43:19 -07:00
parent 3802701cdc
commit d4ce8f5ef8
1 changed files with 25 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

29
docs/storage/layerwriter.go
View file

@ -46,16 +46,37 @@ func (lw *layerWriter) StartedAt() time.Time {
// uploaded layer. The final size and checksum are validated against the // uploaded layer. The final size and checksum are validated against the
// contents of the uploaded layer. The checksum should be provided in the // contents of the uploaded layer. The checksum should be provided in the
// format <algorithm>:<hex digest>. // format <algorithm>:<hex digest>.
func (lw *layerWriter) Finish(digest digest.Digest) (distribution.Layer, error) { func (lw *layerWriter) Finish(dgst digest.Digest) (distribution.Layer, error) {
ctxu.GetLogger(lw.layerStore.repository.ctx).Debug("(*layerWriter).Finish") ctxu.GetLogger(lw.layerStore.repository.ctx).Debug("(*layerWriter).Finish")
if err := lw.bufferedFileWriter.Close(); err != nil { if err := lw.bufferedFileWriter.Close(); err != nil {
return nil, err return nil, err
} }
canonical, err := lw.validateLayer(digest) var (
if err != nil { canonical digest.Digest
err error
)
// HACK(stevvooe): To deal with s3's lack of consistency, attempt to retry
// validation on failure. Three attempts are made, backing off 100ms each
// time.
for retries := 0; ; retries++ {
canonical, err = lw.validateLayer(dgst)
if err == nil {
break
}
ctxu.GetLoggerWithField(lw.layerStore.repository.ctx, "retries", retries).
Errorf("error validating layer: %v", err)
if retries < 3 {
time.Sleep(100 * time.Millisecond)
continue
}
return nil, err return nil, err
} }
if err := lw.moveLayer(canonical); err != nil { if err := lw.moveLayer(canonical); err != nil {
@ -64,7 +85,7 @@ func (lw *layerWriter) Finish(digest digest.Digest) (distribution.Layer, error)
} }
// Link the layer blob into the repository. // Link the layer blob into the repository.
if err := lw.linkLayer(canonical, digest); err != nil { if err := lw.linkLayer(canonical, dgst); err != nil {
return nil, err return nil, err
} }