swift: correct segment path generation
When uploading segments to Swift, the registry generates a random file, by taking the hash of the container path and 32-bytes of random data. The registry attempts to shard across multiple directory paths, by taking the first three hex characters as leader. The implementation in registry, unfortunately, takes the hash of nothing, and appends it to the path and random data. This results in all segments being created in one directory. Fixes: #2407 Fixes: #2311 Signed-off-by: Terin Stock <terinjokes@gmail.com>
This commit is contained in:
parent
79f6bcbe16
commit
b23dd1ef37
2 changed files with 42 additions and 3 deletions
|
@ -667,13 +667,16 @@ func (d *driver) swiftPath(path string) string {
|
||||||
return strings.TrimLeft(strings.TrimRight(d.Prefix+"/files"+path, "/"), "/")
|
return strings.TrimLeft(strings.TrimRight(d.Prefix+"/files"+path, "/"), "/")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// swiftSegmentPath returns a randomly generated path in the segments directory.
|
||||||
func (d *driver) swiftSegmentPath(path string) (string, error) {
|
func (d *driver) swiftSegmentPath(path string) (string, error) {
|
||||||
checksum := sha1.New()
|
checksum := sha1.New()
|
||||||
random := make([]byte, 32)
|
checksum.Write([]byte(path))
|
||||||
if _, err := rand.Read(random); err != nil {
|
|
||||||
|
if _, err := io.CopyN(checksum, rand.Reader, 32); err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
path = hex.EncodeToString(checksum.Sum(append([]byte(path), random...)))
|
|
||||||
|
path = hex.EncodeToString(checksum.Sum(nil))
|
||||||
return strings.TrimLeft(strings.TrimRight(d.Prefix+"/segments/"+path[0:3]+"/"+path[3:], "/"), "/"), nil
|
return strings.TrimLeft(strings.TrimRight(d.Prefix+"/segments/"+path[0:3]+"/"+path[3:], "/"), "/"), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -243,3 +243,39 @@ func TestFilenameChunking(t *testing.T) {
|
||||||
t.Fatal("expected error for size = -1")
|
t.Fatal("expected error for size = -1")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestSwiftSegmentPath(t *testing.T) {
|
||||||
|
d := &driver{
|
||||||
|
Prefix: "/test/segment/path",
|
||||||
|
}
|
||||||
|
|
||||||
|
s1, err := d.swiftSegmentPath("foo-baz")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected error generating segment path: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
s2, err := d.swiftSegmentPath("foo-baz")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected error generating segment path: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !strings.HasPrefix(s1, "test/segment/path/segments/") {
|
||||||
|
t.Fatalf("expected to be prefixed: %s", s1)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !strings.HasPrefix(s1, "test/segment/path/segments/") {
|
||||||
|
t.Fatalf("expected to be prefixed: %s", s2)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(s1) != 68 {
|
||||||
|
t.Fatalf("unexpected segment path length, %d != %d", len(s1), 68)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(s2) != 68 {
|
||||||
|
t.Fatalf("unexpected segment path length, %d != %d", len(s2), 68)
|
||||||
|
}
|
||||||
|
|
||||||
|
if s1 == s2 {
|
||||||
|
t.Fatalf("expected segment paths to differ, %s == %s", s1, s2)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
Loading…
Reference in a new issue