add better generate

Signed-off-by: Jess Frazelle <acidburn@microsoft.com>

container/image.go

@@ -0,0 +1,178 @@
+package container
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/docker/distribution/reference"
+	"github.com/docker/distribution/registry/storage"
+	"github.com/docker/distribution/registry/storage/driver/filesystem"
+	"github.com/docker/docker/pkg/archive"
+	"github.com/genuinetools/reg/repoutils"
+	bindata "github.com/jteeuwen/go-bindata"
+)
+
+// EmbedImage pulls a docker image locally. Creates a tarball of it's contents
+// and then embeds the tarball as binary data into an output bindata.go file.
+func EmbedImage(image string) error {
+	// Get the current working directory.
+	wd, err := os.Getwd()
+	if err != nil {
+		return err
+	}
+
+	// Create our output path
+	output := filepath.Join(wd, "bindata.go")
+
+	// Create the temporary directory for the image contents.
+	tmpd, err := ioutil.TempDir("", "container-lib")
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(tmpd) // Cleanup on complete.
+
+	// Create our tarball path.
+	tarball := filepath.Join(tmpd, DefaultTarballPath)
+
+	// Create our image root and state.
+	root := filepath.Join(tmpd, "root")
+	state := filepath.Join(tmpd, "state")
+
+	// Create the rootfs
+	if err := createRootFS(image, root, state); err != nil {
+		return err
+	}
+
+	// Create the tar.
+	tar, err := archive.Tar(root, archive.Gzip)
+	if err != nil {
+		return fmt.Errorf("create tar failed: %v", err)
+	}
+
+	// Create the tarball writer.
+	writer, err := os.Create(tarball)
+	if err != nil {
+		return err
+	}
+	defer writer.Close() // Close the writer.
+
+	if _, err := io.Copy(writer, tar); err != nil {
+		return fmt.Errorf("copy tarball failed: %v", err)
+	}
+
+	// Create the bindata config.
+	bc := bindata.NewConfig()
+	bc.Input = []bindata.InputConfig{
+		{
+			Path:      tarball,
+			Recursive: false,
+		},
+	}
+	bc.Output = output
+	bc.Package = "main"
+	bc.NoMetadata = true
+	bc.Prefix = filepath.Dir(tarball)
+
+	if err := bindata.Translate(bc); err != nil {
+		return fmt.Errorf("bindata failed: %v", err)
+	}
+
+	return nil
+}
+
+// createRootFS creates the base filesystem for a docker image.
+// It will pull the base image if it does not exist locally.
+// This function takes in a image name and the directory where the
+// rootfs should be created.
+func createRootFS(image, rootfs, state string) error {
+	// Create the context.
+	ctx := context.Background()
+
+	// Create the new local registry storage.
+	local, err := storage.NewRegistry(ctx, filesystem.New(filesystem.DriverParameters{
+		RootDirectory: state,
+		MaxThreads:    100,
+	}))
+	if err != nil {
+		return fmt.Errorf("creating new registry storage failed: %v", err)
+	}
+
+	// Parse the repository name.
+	name, err := reference.ParseNormalizedNamed(image)
+	if err != nil {
+		return fmt.Errorf("not a valid image %q: %v", image, err)
+	}
+	// Add latest to the image name if it is empty.
+	name = reference.TagNameOnly(name)
+
+	// Get the tag for the repo.
+	_, tag, err := repoutils.GetRepoAndRef(image)
+	if err != nil {
+		return err
+	}
+
+	// Create the local repository.
+	repo, err := local.Repository(ctx, name)
+	if err != nil {
+		return fmt.Errorf("creating local repository for %q failed: %v", reference.Path(name), err)
+	}
+
+	// Create the manifest service.
+	ms, err := repo.Manifests(ctx)
+	if err != nil {
+		return fmt.Errorf("creating manifest service failed: %v", err)
+	}
+
+	// Get the specific tag.
+	td, err := repo.Tags(ctx).Get(ctx, tag)
+	// Check if we got an unknown error, that means the tag does not exist.
+	if err != nil && strings.Contains(err.Error(), "unknown") {
+		log.Println("image not found locally, pulling the image")
+
+		// Pull the image.
+		if err := pull(ctx, local, name, tag); err != nil {
+			return fmt.Errorf("pulling failed: %v", err)
+		}
+
+		// Try to get the tag again.
+		td, err = repo.Tags(ctx).Get(ctx, tag)
+	}
+	if err != nil {
+		return fmt.Errorf("getting local repository tag %q failed: %v", tag, err)
+	}
+
+	// Get the specific manifest for the tag.
+	manifest, err := ms.Get(ctx, td.Digest)
+	if err != nil {
+		return fmt.Errorf("getting local manifest for digest %q failed: %v", td.Digest.String(), err)
+	}
+
+	blobStore := repo.Blobs(ctx)
+	for i, ref := range manifest.References() {
+		if i == 0 {
+			fmt.Printf("skipping config %v\n", ref.Digest.String())
+			continue
+		}
+		fmt.Printf("unpacking %v\n", ref.Digest.String())
+		layer, err := blobStore.Open(ctx, ref.Digest)
+		if err != nil {
+			return fmt.Errorf("getting blob %q failed: %v", ref.Digest.String(), err)
+		}
+
+		// Unpack the tarfile to the mount path.
+		// FROM: https://godoc.org/github.com/moby/moby/pkg/archive#TarOptions
+		if err := archive.Untar(layer, rootfs, &archive.TarOptions{
+			NoLchown: true,
+		}); err != nil {
+			return fmt.Errorf("error extracting tar for %q: %v", ref.Digest.String(), err)
+		}
+	}
+
+	return nil
+}

container/pull.go

@@ -0,0 +1,196 @@
+package container
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"runtime"
+
+	"github.com/docker/distribution"
+	"github.com/docker/distribution/manifest/manifestlist"
+	"github.com/docker/distribution/manifest/schema1"
+	"github.com/docker/distribution/manifest/schema2"
+	"github.com/docker/distribution/reference"
+	"github.com/genuinetools/reg/registry"
+	"github.com/genuinetools/reg/repoutils"
+	digest "github.com/opencontainers/go-digest"
+)
+
+func pull(ctx context.Context, dst distribution.Namespace, name reference.Named, tag string) error {
+	// Get the auth config.
+	auth, err := repoutils.GetAuthConfig("", "", reference.Domain(name))
+	if err != nil {
+		return err
+	}
+
+	// TODO: add flag to flip switch for turning off SSL verification
+	// Create a new registry client.
+	src, err := registry.New(auth, false)
+	if err != nil {
+		return fmt.Errorf("creating new registry api client failed: %v", err)
+	}
+
+	fmt.Println("pulling", name.String())
+
+	imgPath := reference.Path(name)
+
+	// Get the manifest.
+	manifest, err := src.Manifest(imgPath, tag)
+	if err != nil {
+		return fmt.Errorf("getting manifest for '%s:%s' failed: %v", imgPath, tag, err)
+	}
+
+	switch v := manifest.(type) {
+	case *schema1.SignedManifest:
+		return pullV1()
+	case *schema2.DeserializedManifest:
+		return pullV2(ctx, dst, src, v, name, imgPath, tag)
+	case *manifestlist.DeserializedManifestList:
+		return pullManifestList(ctx, dst, src, v, name, imgPath, tag)
+	}
+
+	return errors.New("unsupported manifest format")
+}
+
+func pullV1() error {
+	return errors.New("schema1 manifest not supported")
+}
+
+func pullV2(ctx context.Context, dst distribution.Namespace, src *registry.Registry, manifest *schema2.DeserializedManifest, name reference.Named, imgPath, tag string) error {
+	dstRepo, err := dst.Repository(ctx, name)
+	if err != nil {
+		return fmt.Errorf("creating the destination repository failed: %v", err)
+	}
+
+	dstBlobStore := dstRepo.Blobs(ctx)
+	for _, ref := range manifest.References() {
+		// TODO: make a progress bar
+		fmt.Printf("pulling layer %s\n", ref.Digest.String())
+
+		blob, err := src.DownloadLayer(imgPath, ref.Digest)
+		if err != nil {
+			return fmt.Errorf("getting remote blob %q failed failed: %v", ref.Digest.String(), err)
+		}
+
+		upload, err := dstBlobStore.Create(ctx)
+		if err != nil {
+			return fmt.Errorf("creating the local blob writer failed: %v", err)
+		}
+
+		if _, err := io.Copy(upload, blob); err != nil {
+			return fmt.Errorf("writing to the local blob failed: %v", err)
+		}
+
+		if _, err := upload.Commit(ctx, ref); err != nil {
+			return fmt.Errorf("commiting %q locally failed: %v", ref.Digest.String(), err)
+		}
+
+		upload.Close()
+	}
+
+	// Create the manifest service locally.
+	dms, err := dstRepo.Manifests(ctx)
+	if err != nil {
+		return fmt.Errorf("creating manifest service locally failed: %v", err)
+	}
+
+	// Put the manifest locally.
+	manDst, err := dms.Put(ctx, manifest, distribution.WithTag(tag))
+	if err != nil {
+		return fmt.Errorf("putting the manifest with tag %q locally failed: %v", tag, err)
+	}
+
+	// TODO: find a better way to get the manifest descriptor locally.
+	// Get the manifest descriptor.
+	mf, err := dms.Get(ctx, manDst)
+	if err != nil {
+		return fmt.Errorf("getting the manifest with digest %q locally failed: %v", manDst.String(), err)
+	}
+	mediatype, pl, err := mf.Payload()
+	if err != nil {
+		return fmt.Errorf("payload failed: %v", err)
+	}
+	_, desc, err := distribution.UnmarshalManifest(mediatype, pl)
+	if err != nil {
+		return fmt.Errorf("umarshal failed: %v", err)
+	}
+
+	// Put the tag locally.
+	if err := dstRepo.Tags(ctx).Tag(ctx, tag, desc); err != nil {
+		return fmt.Errorf("establishing a relationship between the tag %q and digest %q locally failed: %v", tag, manDst.String(), err)
+	}
+
+	return nil
+}
+
+func pullManifestList(ctx context.Context, dst distribution.Namespace, src *registry.Registry, mfstList *manifestlist.DeserializedManifestList, name reference.Named, imgPath, tag string) error {
+	if _, err := schema2ManifestDigest(name, mfstList); err != nil {
+		return err
+	}
+
+	log.Printf("%s resolved to a manifestList object with %d entries; looking for a %s/%s match", name, len(mfstList.Manifests), runtime.GOOS, runtime.GOARCH)
+
+	manifestMatches := filterManifests(mfstList.Manifests, runtime.GOOS)
+
+	if len(manifestMatches) == 0 {
+		return fmt.Errorf("no matching manifest for %s/%s in the manifest list entries", runtime.GOOS, runtime.GOARCH)
+	}
+
+	if len(manifestMatches) > 1 {
+		log.Printf("found multiple matches in manifest list, choosing best match %s", manifestMatches[0].Digest.String())
+
+	}
+	manifestDigest := manifestMatches[0].Digest
+
+	// Get the manifest.
+	manifest, err := src.Manifest(imgPath, manifestDigest.String())
+	if err != nil {
+		return fmt.Errorf("getting manifest for %s@%s failed: %v", imgPath, manifestDigest.String(), err)
+	}
+
+	switch v := manifest.(type) {
+	case *schema1.SignedManifest:
+		return pullV1()
+	case *schema2.DeserializedManifest:
+		return pullV2(ctx, dst, src, v, name, imgPath, tag)
+	}
+
+	return errors.New("unsupported manifest format")
+}
+
+// schema2ManifestDigest computes the manifest digest, and, if pulling by
+// digest, ensures that it matches the requested digest.
+func schema2ManifestDigest(ref reference.Named, mfst distribution.Manifest) (digest.Digest, error) {
+	_, canonical, err := mfst.Payload()
+	if err != nil {
+		return "", err
+	}
+
+	// If pull by digest, then verify the manifest digest.
+	if digested, isDigested := ref.(reference.Canonical); isDigested {
+		verifier := digested.Digest().Verifier()
+		if _, err := verifier.Write(canonical); err != nil {
+			return "", err
+		}
+		if !verifier.Verified() {
+			return "", fmt.Errorf("manifest verification failed for digest %s", digested.Digest())
+		}
+		return digested.Digest(), nil
+	}
+
+	return digest.FromBytes(canonical), nil
+}
+
+func filterManifests(manifests []manifestlist.ManifestDescriptor, os string) []manifestlist.ManifestDescriptor {
+	var matches []manifestlist.ManifestDescriptor
+	for _, manifestDescriptor := range manifests {
+		if manifestDescriptor.Platform.Architecture == runtime.GOARCH && manifestDescriptor.Platform.OS == os {
+			matches = append(matches, manifestDescriptor)
+
+			log.Printf("found match for %s/%s with media type %s, digest %s", os, runtime.GOARCH, manifestDescriptor.MediaType, manifestDescriptor.Digest.String())
+		}
+	}
+	return matches
+}

container/rootfs.go

@@ -0,0 +1,43 @@
+package container
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/docker/docker/pkg/archive"
+)
+
+const (
+	// DefaultTarballPath holds the default path for the embedded tarball.
+	DefaultTarballPath = "image.tar"
+)
+
+// UnpackRootfs unpacks the embedded tarball to the rootfs.
+func (c *Container) UnpackRootfs(rootfsDir string, asset func(string) ([]byte, error)) error {
+	// Make the rootfs directory.
+	if err := os.MkdirAll(rootfsDir, 0755); err != nil {
+		return err
+	}
+
+	// Get the embedded tarball.
+	data, err := asset(DefaultTarballPath)
+	if err != nil {
+		return fmt.Errorf("getting bindata asset image.tar failed: %v", err)
+	}
+
+	// Unpack the tarball.
+	r := bytes.NewReader(data)
+	if err := archive.Untar(r, rootfsDir, &archive.TarOptions{NoLchown: true}); err != nil {
+		return err
+	}
+
+	// Write a resolv.conf.
+	if err := ioutil.WriteFile(filepath.Join(rootfsDir, "etc", "resolv.conf"), []byte("nameserver 8.8.8.8\nnameserver 8.8.4.4"), 0755); err != nil {
+		return err
+	}
+
+	return nil
+}

container/seccomp_unsupported.go

@@ -0,0 +1,10 @@
+// +build !seccomp
+
+package container
+
+import (
+	specs "github.com/opencontainers/runtime-spec/specs-go"
+)
+
+// DefaultSeccompProfile defines the whitelist for the default seccomp profile.
+var DefaultSeccompProfile = &specs.LinuxSeccomp{}

container/spec.go

@@ -0,0 +1,54 @@
+package container
+
+import (
+	aaprofile "github.com/docker/docker/profiles/apparmor"
+	"github.com/opencontainers/runc/libcontainer/apparmor"
+	"github.com/opencontainers/runc/libcontainer/specconv"
+	specs "github.com/opencontainers/runtime-spec/specs-go"
+)
+
+const (
+	// DefaultApparmorProfile is the default apparmor profile for the containers.
+	DefaultApparmorProfile = "docker-default"
+)
+
+// SpecOpts defines the options available for a spec.
+type SpecOpts struct {
+	Rootless bool
+	Readonly bool
+	Terminal bool
+	Hooks    *specs.Hooks
+}
+
+// Spec returns a default oci spec with some options being passed.
+func Spec(opts SpecOpts) *specs.Spec {
+	// Initialize the spec.
+	spec := specconv.Example()
+
+	// Set the spec to be rootless.
+	if opts.Rootless {
+		specconv.ToRootless(spec)
+	}
+
+	// Setup readonly fs in spec.
+	spec.Root.Readonly = opts.Readonly
+
+	// Setup tty in spec.
+	spec.Process.Terminal = opts.Terminal
+
+	// Pass in any hooks to the spec.
+	spec.Hooks = opts.Hooks
+
+	// Set the default seccomp profile.
+	spec.Linux.Seccomp = DefaultSeccompProfile
+
+	// Install the default apparmor profile.
+	if apparmor.IsEnabled() {
+		// Check if we have the docker-default apparmor profile loaded.
+		if _, err := aaprofile.IsLoaded(DefaultApparmorProfile); err == nil {
+			spec.Process.ApparmorProfile = DefaultApparmorProfile
+		}
+	}
+
+	return spec
+}