add better generate

Signed-off-by: Jess Frazelle <acidburn@microsoft.com>
This commit is contained in:
Jess Frazelle 2018-03-20 01:33:56 -04:00
parent 3fc6abf56b
commit cdd93563f5
5655 changed files with 1187011 additions and 392 deletions

View file

@ -0,0 +1,25 @@
package reference // import "github.com/docker/docker/reference"
type notFoundError string
func (e notFoundError) Error() string {
return string(e)
}
func (notFoundError) NotFound() {}
type invalidTagError string
func (e invalidTagError) Error() string {
return string(e)
}
func (invalidTagError) InvalidParameter() {}
type conflictingTagError string
func (e conflictingTagError) Error() string {
return string(e)
}
func (conflictingTagError) Conflict() {}

View file

@ -0,0 +1,343 @@
package reference // import "github.com/docker/docker/reference"
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"sort"
"sync"
"github.com/docker/distribution/reference"
"github.com/docker/docker/pkg/ioutils"
"github.com/opencontainers/go-digest"
"github.com/pkg/errors"
)
var (
// ErrDoesNotExist is returned if a reference is not found in the
// store.
ErrDoesNotExist notFoundError = "reference does not exist"
)
// An Association is a tuple associating a reference with an image ID.
type Association struct {
Ref reference.Named
ID digest.Digest
}
// Store provides the set of methods which can operate on a reference store.
type Store interface {
References(id digest.Digest) []reference.Named
ReferencesByName(ref reference.Named) []Association
AddTag(ref reference.Named, id digest.Digest, force bool) error
AddDigest(ref reference.Canonical, id digest.Digest, force bool) error
Delete(ref reference.Named) (bool, error)
Get(ref reference.Named) (digest.Digest, error)
}
type store struct {
mu sync.RWMutex
// jsonPath is the path to the file where the serialized tag data is
// stored.
jsonPath string
// Repositories is a map of repositories, indexed by name.
Repositories map[string]repository
// referencesByIDCache is a cache of references indexed by ID, to speed
// up References.
referencesByIDCache map[digest.Digest]map[string]reference.Named
}
// Repository maps tags to digests. The key is a stringified Reference,
// including the repository name.
type repository map[string]digest.Digest
type lexicalRefs []reference.Named
func (a lexicalRefs) Len() int { return len(a) }
func (a lexicalRefs) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a lexicalRefs) Less(i, j int) bool {
return a[i].String() < a[j].String()
}
type lexicalAssociations []Association
func (a lexicalAssociations) Len() int { return len(a) }
func (a lexicalAssociations) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a lexicalAssociations) Less(i, j int) bool {
return a[i].Ref.String() < a[j].Ref.String()
}
// NewReferenceStore creates a new reference store, tied to a file path where
// the set of references are serialized in JSON format.
func NewReferenceStore(jsonPath string) (Store, error) {
abspath, err := filepath.Abs(jsonPath)
if err != nil {
return nil, err
}
store := &store{
jsonPath: abspath,
Repositories: make(map[string]repository),
referencesByIDCache: make(map[digest.Digest]map[string]reference.Named),
}
// Load the json file if it exists, otherwise create it.
if err := store.reload(); os.IsNotExist(err) {
if err := store.save(); err != nil {
return nil, err
}
} else if err != nil {
return nil, err
}
return store, nil
}
// AddTag adds a tag reference to the store. If force is set to true, existing
// references can be overwritten. This only works for tags, not digests.
func (store *store) AddTag(ref reference.Named, id digest.Digest, force bool) error {
if _, isCanonical := ref.(reference.Canonical); isCanonical {
return errors.WithStack(invalidTagError("refusing to create a tag with a digest reference"))
}
return store.addReference(reference.TagNameOnly(ref), id, force)
}
// AddDigest adds a digest reference to the store.
func (store *store) AddDigest(ref reference.Canonical, id digest.Digest, force bool) error {
return store.addReference(ref, id, force)
}
func favorDigest(originalRef reference.Named) (reference.Named, error) {
ref := originalRef
// If the reference includes a digest and a tag, we must store only the
// digest.
canonical, isCanonical := originalRef.(reference.Canonical)
_, isNamedTagged := originalRef.(reference.NamedTagged)
if isCanonical && isNamedTagged {
trimmed, err := reference.WithDigest(reference.TrimNamed(canonical), canonical.Digest())
if err != nil {
// should never happen
return originalRef, err
}
ref = trimmed
}
return ref, nil
}
func (store *store) addReference(ref reference.Named, id digest.Digest, force bool) error {
ref, err := favorDigest(ref)
if err != nil {
return err
}
refName := reference.FamiliarName(ref)
refStr := reference.FamiliarString(ref)
if refName == string(digest.Canonical) {
return errors.WithStack(invalidTagError("refusing to create an ambiguous tag using digest algorithm as name"))
}
store.mu.Lock()
defer store.mu.Unlock()
repository, exists := store.Repositories[refName]
if !exists || repository == nil {
repository = make(map[string]digest.Digest)
store.Repositories[refName] = repository
}
oldID, exists := repository[refStr]
if exists {
// force only works for tags
if digested, isDigest := ref.(reference.Canonical); isDigest {
return errors.WithStack(conflictingTagError("Cannot overwrite digest " + digested.Digest().String()))
}
if !force {
return errors.WithStack(
conflictingTagError(
fmt.Sprintf("Conflict: Tag %s is already set to image %s, if you want to replace it, please use the force option", refStr, oldID.String()),
),
)
}
if store.referencesByIDCache[oldID] != nil {
delete(store.referencesByIDCache[oldID], refStr)
if len(store.referencesByIDCache[oldID]) == 0 {
delete(store.referencesByIDCache, oldID)
}
}
}
repository[refStr] = id
if store.referencesByIDCache[id] == nil {
store.referencesByIDCache[id] = make(map[string]reference.Named)
}
store.referencesByIDCache[id][refStr] = ref
return store.save()
}
// Delete deletes a reference from the store. It returns true if a deletion
// happened, or false otherwise.
func (store *store) Delete(ref reference.Named) (bool, error) {
ref, err := favorDigest(ref)
if err != nil {
return false, err
}
ref = reference.TagNameOnly(ref)
refName := reference.FamiliarName(ref)
refStr := reference.FamiliarString(ref)
store.mu.Lock()
defer store.mu.Unlock()
repository, exists := store.Repositories[refName]
if !exists {
return false, ErrDoesNotExist
}
if id, exists := repository[refStr]; exists {
delete(repository, refStr)
if len(repository) == 0 {
delete(store.Repositories, refName)
}
if store.referencesByIDCache[id] != nil {
delete(store.referencesByIDCache[id], refStr)
if len(store.referencesByIDCache[id]) == 0 {
delete(store.referencesByIDCache, id)
}
}
return true, store.save()
}
return false, ErrDoesNotExist
}
// Get retrieves an item from the store by reference
func (store *store) Get(ref reference.Named) (digest.Digest, error) {
if canonical, ok := ref.(reference.Canonical); ok {
// If reference contains both tag and digest, only
// lookup by digest as it takes precedence over
// tag, until tag/digest combos are stored.
if _, ok := ref.(reference.Tagged); ok {
var err error
ref, err = reference.WithDigest(reference.TrimNamed(canonical), canonical.Digest())
if err != nil {
return "", err
}
}
} else {
ref = reference.TagNameOnly(ref)
}
refName := reference.FamiliarName(ref)
refStr := reference.FamiliarString(ref)
store.mu.RLock()
defer store.mu.RUnlock()
repository, exists := store.Repositories[refName]
if !exists || repository == nil {
return "", ErrDoesNotExist
}
id, exists := repository[refStr]
if !exists {
return "", ErrDoesNotExist
}
return id, nil
}
// References returns a slice of references to the given ID. The slice
// will be nil if there are no references to this ID.
func (store *store) References(id digest.Digest) []reference.Named {
store.mu.RLock()
defer store.mu.RUnlock()
// Convert the internal map to an array for two reasons:
// 1) We must not return a mutable
// 2) It would be ugly to expose the extraneous map keys to callers.
var references []reference.Named
for _, ref := range store.referencesByIDCache[id] {
references = append(references, ref)
}
sort.Sort(lexicalRefs(references))
return references
}
// ReferencesByName returns the references for a given repository name.
// If there are no references known for this repository name,
// ReferencesByName returns nil.
func (store *store) ReferencesByName(ref reference.Named) []Association {
refName := reference.FamiliarName(ref)
store.mu.RLock()
defer store.mu.RUnlock()
repository, exists := store.Repositories[refName]
if !exists {
return nil
}
var associations []Association
for refStr, refID := range repository {
ref, err := reference.ParseNormalizedNamed(refStr)
if err != nil {
// Should never happen
return nil
}
associations = append(associations,
Association{
Ref: ref,
ID: refID,
})
}
sort.Sort(lexicalAssociations(associations))
return associations
}
func (store *store) save() error {
// Store the json
jsonData, err := json.Marshal(store)
if err != nil {
return err
}
return ioutils.AtomicWriteFile(store.jsonPath, jsonData, 0600)
}
func (store *store) reload() error {
f, err := os.Open(store.jsonPath)
if err != nil {
return err
}
defer f.Close()
if err := json.NewDecoder(f).Decode(&store); err != nil {
return err
}
for _, repository := range store.Repositories {
for refStr, refID := range repository {
ref, err := reference.ParseNormalizedNamed(refStr)
if err != nil {
// Should never happen
continue
}
if store.referencesByIDCache[refID] == nil {
store.referencesByIDCache[refID] = make(map[string]reference.Named)
}
store.referencesByIDCache[refID][refStr] = ref
}
}
return nil
}

View file

@ -0,0 +1,350 @@
package reference // import "github.com/docker/docker/reference"
import (
"bytes"
"io/ioutil"
"os"
"path/filepath"
"strings"
"testing"
"github.com/docker/distribution/reference"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
digest "github.com/opencontainers/go-digest"
)
var (
saveLoadTestCases = map[string]digest.Digest{
"registry:5000/foobar:HEAD": "sha256:470022b8af682154f57a2163d030eb369549549cba00edc69e1b99b46bb924d6",
"registry:5000/foobar:alternate": "sha256:ae300ebc4a4f00693702cfb0a5e0b7bc527b353828dc86ad09fb95c8a681b793",
"registry:5000/foobar:latest": "sha256:6153498b9ac00968d71b66cca4eac37e990b5f9eb50c26877eb8799c8847451b",
"registry:5000/foobar:master": "sha256:6c9917af4c4e05001b346421959d7ea81b6dc9d25718466a37a6add865dfd7fc",
"jess/hollywood:latest": "sha256:ae7a5519a0a55a2d4ef20ddcbd5d0ca0888a1f7ab806acc8e2a27baf46f529fe",
"registry@sha256:367eb40fd0330a7e464777121e39d2f5b3e8e23a1e159342e53ab05c9e4d94e6": "sha256:24126a56805beb9711be5f4590cc2eb55ab8d4a85ebd618eed72bb19fc50631c",
"busybox:latest": "sha256:91e54dfb11794fad694460162bf0cb0a4fa710cfa3f60979c177d920813e267c",
}
marshalledSaveLoadTestCases = []byte(`{"Repositories":{"busybox":{"busybox:latest":"sha256:91e54dfb11794fad694460162bf0cb0a4fa710cfa3f60979c177d920813e267c"},"jess/hollywood":{"jess/hollywood:latest":"sha256:ae7a5519a0a55a2d4ef20ddcbd5d0ca0888a1f7ab806acc8e2a27baf46f529fe"},"registry":{"registry@sha256:367eb40fd0330a7e464777121e39d2f5b3e8e23a1e159342e53ab05c9e4d94e6":"sha256:24126a56805beb9711be5f4590cc2eb55ab8d4a85ebd618eed72bb19fc50631c"},"registry:5000/foobar":{"registry:5000/foobar:HEAD":"sha256:470022b8af682154f57a2163d030eb369549549cba00edc69e1b99b46bb924d6","registry:5000/foobar:alternate":"sha256:ae300ebc4a4f00693702cfb0a5e0b7bc527b353828dc86ad09fb95c8a681b793","registry:5000/foobar:latest":"sha256:6153498b9ac00968d71b66cca4eac37e990b5f9eb50c26877eb8799c8847451b","registry:5000/foobar:master":"sha256:6c9917af4c4e05001b346421959d7ea81b6dc9d25718466a37a6add865dfd7fc"}}}`)
)
func TestLoad(t *testing.T) {
jsonFile, err := ioutil.TempFile("", "tag-store-test")
if err != nil {
t.Fatalf("error creating temp file: %v", err)
}
defer os.RemoveAll(jsonFile.Name())
// Write canned json to the temp file
_, err = jsonFile.Write(marshalledSaveLoadTestCases)
if err != nil {
t.Fatalf("error writing to temp file: %v", err)
}
jsonFile.Close()
store, err := NewReferenceStore(jsonFile.Name())
if err != nil {
t.Fatalf("error creating tag store: %v", err)
}
for refStr, expectedID := range saveLoadTestCases {
ref, err := reference.ParseNormalizedNamed(refStr)
if err != nil {
t.Fatalf("failed to parse reference: %v", err)
}
id, err := store.Get(ref)
if err != nil {
t.Fatalf("could not find reference %s: %v", refStr, err)
}
if id != expectedID {
t.Fatalf("expected %s - got %s", expectedID, id)
}
}
}
func TestSave(t *testing.T) {
jsonFile, err := ioutil.TempFile("", "tag-store-test")
assert.NilError(t, err)
_, err = jsonFile.Write([]byte(`{}`))
assert.NilError(t, err)
jsonFile.Close()
defer os.RemoveAll(jsonFile.Name())
store, err := NewReferenceStore(jsonFile.Name())
if err != nil {
t.Fatalf("error creating tag store: %v", err)
}
for refStr, id := range saveLoadTestCases {
ref, err := reference.ParseNormalizedNamed(refStr)
if err != nil {
t.Fatalf("failed to parse reference: %v", err)
}
if canonical, ok := ref.(reference.Canonical); ok {
err = store.AddDigest(canonical, id, false)
if err != nil {
t.Fatalf("could not add digest reference %s: %v", refStr, err)
}
} else {
err = store.AddTag(ref, id, false)
if err != nil {
t.Fatalf("could not add reference %s: %v", refStr, err)
}
}
}
jsonBytes, err := ioutil.ReadFile(jsonFile.Name())
if err != nil {
t.Fatalf("could not read json file: %v", err)
}
if !bytes.Equal(jsonBytes, marshalledSaveLoadTestCases) {
t.Fatalf("save output did not match expectations\nexpected:\n%s\ngot:\n%s", marshalledSaveLoadTestCases, jsonBytes)
}
}
func TestAddDeleteGet(t *testing.T) {
jsonFile, err := ioutil.TempFile("", "tag-store-test")
if err != nil {
t.Fatalf("error creating temp file: %v", err)
}
_, err = jsonFile.Write([]byte(`{}`))
jsonFile.Close()
defer os.RemoveAll(jsonFile.Name())
store, err := NewReferenceStore(jsonFile.Name())
if err != nil {
t.Fatalf("error creating tag store: %v", err)
}
testImageID1 := digest.Digest("sha256:9655aef5fd742a1b4e1b7b163aa9f1c76c186304bf39102283d80927c916ca9c")
testImageID2 := digest.Digest("sha256:9655aef5fd742a1b4e1b7b163aa9f1c76c186304bf39102283d80927c916ca9d")
testImageID3 := digest.Digest("sha256:9655aef5fd742a1b4e1b7b163aa9f1c76c186304bf39102283d80927c916ca9e")
// Try adding a reference with no tag or digest
nameOnly, err := reference.ParseNormalizedNamed("username/repo")
if err != nil {
t.Fatalf("could not parse reference: %v", err)
}
if err = store.AddTag(nameOnly, testImageID1, false); err != nil {
t.Fatalf("error adding to store: %v", err)
}
// Add a few references
ref1, err := reference.ParseNormalizedNamed("username/repo1:latest")
if err != nil {
t.Fatalf("could not parse reference: %v", err)
}
if err = store.AddTag(ref1, testImageID1, false); err != nil {
t.Fatalf("error adding to store: %v", err)
}
ref2, err := reference.ParseNormalizedNamed("username/repo1:old")
if err != nil {
t.Fatalf("could not parse reference: %v", err)
}
if err = store.AddTag(ref2, testImageID2, false); err != nil {
t.Fatalf("error adding to store: %v", err)
}
ref3, err := reference.ParseNormalizedNamed("username/repo1:alias")
if err != nil {
t.Fatalf("could not parse reference: %v", err)
}
if err = store.AddTag(ref3, testImageID1, false); err != nil {
t.Fatalf("error adding to store: %v", err)
}
ref4, err := reference.ParseNormalizedNamed("username/repo2:latest")
if err != nil {
t.Fatalf("could not parse reference: %v", err)
}
if err = store.AddTag(ref4, testImageID2, false); err != nil {
t.Fatalf("error adding to store: %v", err)
}
ref5, err := reference.ParseNormalizedNamed("username/repo3@sha256:58153dfb11794fad694460162bf0cb0a4fa710cfa3f60979c177d920813e267c")
if err != nil {
t.Fatalf("could not parse reference: %v", err)
}
if err = store.AddDigest(ref5.(reference.Canonical), testImageID2, false); err != nil {
t.Fatalf("error adding to store: %v", err)
}
// Attempt to overwrite with force == false
if err = store.AddTag(ref4, testImageID3, false); err == nil || !strings.HasPrefix(err.Error(), "Conflict:") {
t.Fatalf("did not get expected error on overwrite attempt - got %v", err)
}
// Repeat to overwrite with force == true
if err = store.AddTag(ref4, testImageID3, true); err != nil {
t.Fatalf("failed to force tag overwrite: %v", err)
}
// Check references so far
id, err := store.Get(nameOnly)
if err != nil {
t.Fatalf("Get returned error: %v", err)
}
if id != testImageID1 {
t.Fatalf("id mismatch: got %s instead of %s", id.String(), testImageID1.String())
}
id, err = store.Get(ref1)
if err != nil {
t.Fatalf("Get returned error: %v", err)
}
if id != testImageID1 {
t.Fatalf("id mismatch: got %s instead of %s", id.String(), testImageID1.String())
}
id, err = store.Get(ref2)
if err != nil {
t.Fatalf("Get returned error: %v", err)
}
if id != testImageID2 {
t.Fatalf("id mismatch: got %s instead of %s", id.String(), testImageID2.String())
}
id, err = store.Get(ref3)
if err != nil {
t.Fatalf("Get returned error: %v", err)
}
if id != testImageID1 {
t.Fatalf("id mismatch: got %s instead of %s", id.String(), testImageID1.String())
}
id, err = store.Get(ref4)
if err != nil {
t.Fatalf("Get returned error: %v", err)
}
if id != testImageID3 {
t.Fatalf("id mismatch: got %s instead of %s", id.String(), testImageID3.String())
}
id, err = store.Get(ref5)
if err != nil {
t.Fatalf("Get returned error: %v", err)
}
if id != testImageID2 {
t.Fatalf("id mismatch: got %s instead of %s", id.String(), testImageID3.String())
}
// Get should return ErrDoesNotExist for a nonexistent repo
nonExistRepo, err := reference.ParseNormalizedNamed("username/nonexistrepo:latest")
if err != nil {
t.Fatalf("could not parse reference: %v", err)
}
if _, err = store.Get(nonExistRepo); err != ErrDoesNotExist {
t.Fatal("Expected ErrDoesNotExist from Get")
}
// Get should return ErrDoesNotExist for a nonexistent tag
nonExistTag, err := reference.ParseNormalizedNamed("username/repo1:nonexist")
if err != nil {
t.Fatalf("could not parse reference: %v", err)
}
if _, err = store.Get(nonExistTag); err != ErrDoesNotExist {
t.Fatal("Expected ErrDoesNotExist from Get")
}
// Check References
refs := store.References(testImageID1)
if len(refs) != 3 {
t.Fatal("unexpected number of references")
}
// Looking for the references in this order verifies that they are
// returned lexically sorted.
if refs[0].String() != ref3.String() {
t.Fatalf("unexpected reference: %v", refs[0].String())
}
if refs[1].String() != ref1.String() {
t.Fatalf("unexpected reference: %v", refs[1].String())
}
if refs[2].String() != nameOnly.String()+":latest" {
t.Fatalf("unexpected reference: %v", refs[2].String())
}
// Check ReferencesByName
repoName, err := reference.ParseNormalizedNamed("username/repo1")
if err != nil {
t.Fatalf("could not parse reference: %v", err)
}
associations := store.ReferencesByName(repoName)
if len(associations) != 3 {
t.Fatal("unexpected number of associations")
}
// Looking for the associations in this order verifies that they are
// returned lexically sorted.
if associations[0].Ref.String() != ref3.String() {
t.Fatalf("unexpected reference: %v", associations[0].Ref.String())
}
if associations[0].ID != testImageID1 {
t.Fatalf("unexpected reference: %v", associations[0].Ref.String())
}
if associations[1].Ref.String() != ref1.String() {
t.Fatalf("unexpected reference: %v", associations[1].Ref.String())
}
if associations[1].ID != testImageID1 {
t.Fatalf("unexpected reference: %v", associations[1].Ref.String())
}
if associations[2].Ref.String() != ref2.String() {
t.Fatalf("unexpected reference: %v", associations[2].Ref.String())
}
if associations[2].ID != testImageID2 {
t.Fatalf("unexpected reference: %v", associations[2].Ref.String())
}
// Delete should return ErrDoesNotExist for a nonexistent repo
if _, err = store.Delete(nonExistRepo); err != ErrDoesNotExist {
t.Fatal("Expected ErrDoesNotExist from Delete")
}
// Delete should return ErrDoesNotExist for a nonexistent tag
if _, err = store.Delete(nonExistTag); err != ErrDoesNotExist {
t.Fatal("Expected ErrDoesNotExist from Delete")
}
// Delete a few references
if deleted, err := store.Delete(ref1); err != nil || !deleted {
t.Fatal("Delete failed")
}
if _, err := store.Get(ref1); err != ErrDoesNotExist {
t.Fatal("Expected ErrDoesNotExist from Get")
}
if deleted, err := store.Delete(ref5); err != nil || !deleted {
t.Fatal("Delete failed")
}
if _, err := store.Get(ref5); err != ErrDoesNotExist {
t.Fatal("Expected ErrDoesNotExist from Get")
}
if deleted, err := store.Delete(nameOnly); err != nil || !deleted {
t.Fatal("Delete failed")
}
if _, err := store.Get(nameOnly); err != ErrDoesNotExist {
t.Fatal("Expected ErrDoesNotExist from Get")
}
}
func TestInvalidTags(t *testing.T) {
tmpDir, err := ioutil.TempDir("", "tag-store-test")
assert.NilError(t, err)
defer os.RemoveAll(tmpDir)
store, err := NewReferenceStore(filepath.Join(tmpDir, "repositories.json"))
assert.NilError(t, err)
id := digest.Digest("sha256:470022b8af682154f57a2163d030eb369549549cba00edc69e1b99b46bb924d6")
// sha256 as repo name
ref, err := reference.ParseNormalizedNamed("sha256:abc")
assert.NilError(t, err)
err = store.AddTag(ref, id, true)
assert.Check(t, is.ErrorContains(err, ""))
// setting digest as a tag
ref, err = reference.ParseNormalizedNamed("registry@sha256:367eb40fd0330a7e464777121e39d2f5b3e8e23a1e159342e53ab05c9e4d94e6")
assert.NilError(t, err)
err = store.AddTag(ref, id, true)
assert.Check(t, is.ErrorContains(err, ""))
}