vbatts/registry
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

[Swift] add simple heuristic to detect incomplete DLOs during read ops

Browse source 
This is similar to waitForSegmentsToShowUp which is called during
Close/Commit. Intuitively, you wouldn't expect missing segments to be a
problem during read operations, since the previous Close/Commit
confirmed that all segments are there.

But due to the distributed nature of Swift, the read request could be
hitting a different storage node of the Swift cluster, where the
segments are still missing.

Load tests on my team's staging Swift cluster have shown this to occur
about once every 100-200 layer uploads when the Swift proxies are under
high load. The retry logic, borrowed from waitForSegmentsToShowUp, fixes
this temporary inconsistency.

Signed-off-by: Stefan Majewsky <stefan.majewsky@sap.com>
This commit is contained in:
Stefan Majewsky 2016-05-06 13:14:21 +02:00
parent 4e17ab5d31
commit 1f03d4e77d
1 changed files with 61 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

77
registry/storage/driver/swift/swift.go
View file

@ -302,14 +302,40 @@ func (d *driver) Reader(ctx context.Context, path string, offset int64) (io.Read
headers := make(swift.Headers)
headers["Range"] = "bytes=" + strconv.FormatInt(offset, 10) + "-"
file, _, err := d.Conn.ObjectOpen(d.Container, d.swiftPath(path), false, headers)
if err == swift.ObjectNotFound {
return nil, storagedriver.PathNotFoundError{Path: path}
waitingTime := readAfterWriteWait
endTime := time.Now().Add(readAfterWriteTimeout)
for {
file, headers, err := d.Conn.ObjectOpen(d.Container, d.swiftPath(path), false, headers)
if err != nil {
if err == swift.ObjectNotFound {
return nil, storagedriver.PathNotFoundError{Path: path}
}
if swiftErr, ok := err.(*swift.Error); ok && swiftErr.StatusCode == http.StatusRequestedRangeNotSatisfiable {
return ioutil.NopCloser(bytes.NewReader(nil)), nil
}
return file, err
}
//if this is a DLO and it is clear that segments are still missing,
//wait until they show up
_, isDLO := headers["X-Object-Manifest"]
size, err := file.Length()
if err != nil {
return file, err
}
if isDLO && size == 0 {
if time.Now().Add(waitingTime).After(endTime) {
return nil, fmt.Errorf("Timeout expired while waiting for segments of %s to show up", path)
}
time.Sleep(waitingTime)
waitingTime *= 2
continue
}
//if not, then this reader will be fine
return file, nil
}
if swiftErr, ok := err.(*swift.Error); ok && swiftErr.StatusCode == http.StatusRequestedRangeNotSatisfiable {
return ioutil.NopCloser(bytes.NewReader(nil)), nil
}
return file, err
}
// Writer returns a FileWriter which will store the content written to it
@ -389,17 +415,36 @@ func (d *driver) Stat(ctx context.Context, path string) (storagedriver.FileInfo,
//Don't trust an empty `objects` slice. A container listing can be
//outdated. For files, we can make a HEAD request on the object which
//reports existence (at least) much more reliably.
info, _, err := d.Conn.Object(d.Container, swiftPath)
if err != nil {
if err == swift.ObjectNotFound {
return nil, storagedriver.PathNotFoundError{Path: path}
waitingTime := readAfterWriteWait
endTime := time.Now().Add(readAfterWriteTimeout)
for {
info, headers, err := d.Conn.Object(d.Container, swiftPath)
if err != nil {
if err == swift.ObjectNotFound {
return nil, storagedriver.PathNotFoundError{Path: path}
}
return nil, err
}
return nil, err
//if this is a DLO and it is clear that segments are still missing,
//wait until they show up
_, isDLO := headers["X-Object-Manifest"]
if isDLO && info.Bytes == 0 {
if time.Now().Add(waitingTime).After(endTime) {
return nil, fmt.Errorf("Timeout expired while waiting for segments of %s to show up", path)
}
time.Sleep(waitingTime)
waitingTime *= 2
continue
}
//otherwise, accept the result
fi.IsDir = false
fi.Size = info.Bytes
fi.ModTime = info.LastModified
return storagedriver.FileInfoInternal{FileInfoFields: fi}, nil
}
fi.IsDir = false
fi.Size = info.Bytes
fi.ModTime = info.LastModified
return storagedriver.FileInfoInternal{FileInfoFields: fi}, nil
}
// List returns a list of the objects that are direct descendants of the given path.