[Swift] add simple heuristic to detect incomplete DLOs during read ops

This is similar to waitForSegmentsToShowUp which is called during
Close/Commit. Intuitively, you wouldn't expect missing segments to be a
problem during read operations, since the previous Close/Commit
confirmed that all segments are there.

But due to the distributed nature of Swift, the read request could be
hitting a different storage node of the Swift cluster, where the
segments are still missing.

Load tests on my team's staging Swift cluster have shown this to occur
about once every 100-200 layer uploads when the Swift proxies are under
high load. The retry logic, borrowed from waitForSegmentsToShowUp, fixes
this temporary inconsistency.

Signed-off-by: Stefan Majewsky <stefan.majewsky@sap.com>

registry/storage/driver/swift/swift.go

@@ -302,7 +302,12 @@ func (d *driver) Reader(ctx context.Context, path string, offset int64) (io.Read
 	headers := make(swift.Headers)
 	headers["Range"] = "bytes=" + strconv.FormatInt(offset, 10) + "-"
 
-	file, _, err := d.Conn.ObjectOpen(d.Container, d.swiftPath(path), false, headers)
+	waitingTime := readAfterWriteWait
+	endTime := time.Now().Add(readAfterWriteTimeout)
+
+	for {
+		file, headers, err := d.Conn.ObjectOpen(d.Container, d.swiftPath(path), false, headers)
+		if err != nil {
 			if err == swift.ObjectNotFound {
 				return nil, storagedriver.PathNotFoundError{Path: path}
 			}
@@ -312,6 +317,27 @@ func (d *driver) Reader(ctx context.Context, path string, offset int64) (io.Read
 			return file, err
 		}
 
+		//if this is a DLO and it is clear that segments are still missing,
+		//wait until they show up
+		_, isDLO := headers["X-Object-Manifest"]
+		size, err := file.Length()
+		if err != nil {
+			return file, err
+		}
+		if isDLO && size == 0 {
+			if time.Now().Add(waitingTime).After(endTime) {
+				return nil, fmt.Errorf("Timeout expired while waiting for segments of %s to show up", path)
+			}
+			time.Sleep(waitingTime)
+			waitingTime *= 2
+			continue
+		}
+
+		//if not, then this reader will be fine
+		return file, nil
+	}
+}
+
 // Writer returns a FileWriter which will store the content written to it
 // at the location designated by "path" after the call to Commit.
 func (d *driver) Writer(ctx context.Context, path string, append bool) (storagedriver.FileWriter, error) {
@@ -389,18 +415,37 @@ func (d *driver) Stat(ctx context.Context, path string) (storagedriver.FileInfo,
 	//Don't trust an empty `objects` slice. A container listing can be
 	//outdated. For files, we can make a HEAD request on the object which
 	//reports existence (at least) much more reliably.
-	info, _, err := d.Conn.Object(d.Container, swiftPath)
+	waitingTime := readAfterWriteWait
+	endTime := time.Now().Add(readAfterWriteTimeout)
+
+	for {
+		info, headers, err := d.Conn.Object(d.Container, swiftPath)
 		if err != nil {
 			if err == swift.ObjectNotFound {
 				return nil, storagedriver.PathNotFoundError{Path: path}
 			}
 			return nil, err
 		}
+
+		//if this is a DLO and it is clear that segments are still missing,
+		//wait until they show up
+		_, isDLO := headers["X-Object-Manifest"]
+		if isDLO && info.Bytes == 0 {
+			if time.Now().Add(waitingTime).After(endTime) {
+				return nil, fmt.Errorf("Timeout expired while waiting for segments of %s to show up", path)
+			}
+			time.Sleep(waitingTime)
+			waitingTime *= 2
+			continue
+		}
+
+		//otherwise, accept the result
 		fi.IsDir = false
 		fi.Size = info.Bytes
 		fi.ModTime = info.LastModified
 		return storagedriver.FileInfoInternal{FileInfoFields: fi}, nil
 	}
+}
 
 // List returns a list of the objects that are direct descendants of the given path.
 func (d *driver) List(ctx context.Context, path string) ([]string, error) {