From 5f08e609c0c4100deba19cc6ecd4264b571d09c1 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Fri, 27 Jan 2017 22:49:10 -0800 Subject: [PATCH 01/12] Vendor continuity Signed-off-by: Derek McGowan (github: dmcgowan) --- vendor.conf | 2 + vendor/github.com/stevvooe/continuity/LICENSE | 202 ++++++ .../github.com/stevvooe/continuity/context.go | 640 ++++++++++++++++++ .../stevvooe/continuity/devices_darwin.go | 15 + .../stevvooe/continuity/devices_linux.go | 15 + .../stevvooe/continuity/devices_unix.go | 55 ++ .../github.com/stevvooe/continuity/digests.go | 88 +++ .../github.com/stevvooe/continuity/driver.go | 158 +++++ .../stevvooe/continuity/driver_darwin.go | 7 + .../stevvooe/continuity/driver_unix.go | 107 +++ .../stevvooe/continuity/groups_unix.go | 113 ++++ .../stevvooe/continuity/hardlinks.go | 57 ++ .../stevvooe/continuity/hardlinks_unix.go | 34 + .../stevvooe/continuity/hardlinks_windows.go | 5 + .../github.com/stevvooe/continuity/ioutils.go | 39 ++ .../stevvooe/continuity/manifest.go | 144 ++++ .../continuity/manifest_test_darwin.go | 23 + .../stevvooe/continuity/proto/gen.go | 3 + .../stevvooe/continuity/proto/manifest.pb.go | 98 +++ .../stevvooe/continuity/proto/manifest.proto | 77 +++ .../stevvooe/continuity/resource.go | 578 ++++++++++++++++ .../github.com/stevvooe/continuity/sysx/asm.s | 10 + .../stevvooe/continuity/sysx/chmod_darwin.go | 18 + .../continuity/sysx/chmod_darwin_386.go | 25 + .../continuity/sysx/chmod_darwin_amd64.go | 25 + .../stevvooe/continuity/sysx/chmod_linux.go | 12 + .../stevvooe/continuity/sysx/sys.go | 37 + .../stevvooe/continuity/sysx/xattr.go | 64 ++ .../stevvooe/continuity/sysx/xattr_darwin.go | 71 ++ .../continuity/sysx/xattr_darwin_386.go | 111 +++ .../continuity/sysx/xattr_darwin_amd64.go | 111 +++ .../stevvooe/continuity/sysx/xattr_linux.go | 58 ++ .../continuity/sysx/xattr_linux_386.go | 111 +++ .../continuity/sysx/xattr_linux_amd64.go | 111 +++ .../continuity/sysx/xattr_linux_arm.go | 111 +++ .../continuity/sysx/xattr_linux_arm64.go | 111 +++ 36 files changed, 3446 insertions(+) create mode 100644 vendor/github.com/stevvooe/continuity/LICENSE create mode 100644 vendor/github.com/stevvooe/continuity/context.go create mode 100644 vendor/github.com/stevvooe/continuity/devices_darwin.go create mode 100644 vendor/github.com/stevvooe/continuity/devices_linux.go create mode 100644 vendor/github.com/stevvooe/continuity/devices_unix.go create mode 100644 vendor/github.com/stevvooe/continuity/digests.go create mode 100644 vendor/github.com/stevvooe/continuity/driver.go create mode 100644 vendor/github.com/stevvooe/continuity/driver_darwin.go create mode 100644 vendor/github.com/stevvooe/continuity/driver_unix.go create mode 100644 vendor/github.com/stevvooe/continuity/groups_unix.go create mode 100644 vendor/github.com/stevvooe/continuity/hardlinks.go create mode 100644 vendor/github.com/stevvooe/continuity/hardlinks_unix.go create mode 100644 vendor/github.com/stevvooe/continuity/hardlinks_windows.go create mode 100644 vendor/github.com/stevvooe/continuity/ioutils.go create mode 100644 vendor/github.com/stevvooe/continuity/manifest.go create mode 100644 vendor/github.com/stevvooe/continuity/manifest_test_darwin.go create mode 100644 vendor/github.com/stevvooe/continuity/proto/gen.go create mode 100644 vendor/github.com/stevvooe/continuity/proto/manifest.pb.go create mode 100644 vendor/github.com/stevvooe/continuity/proto/manifest.proto create mode 100644 vendor/github.com/stevvooe/continuity/resource.go create mode 100644 vendor/github.com/stevvooe/continuity/sysx/asm.s create mode 100644 vendor/github.com/stevvooe/continuity/sysx/chmod_darwin.go create mode 100644 vendor/github.com/stevvooe/continuity/sysx/chmod_darwin_386.go create mode 100644 vendor/github.com/stevvooe/continuity/sysx/chmod_darwin_amd64.go create mode 100644 vendor/github.com/stevvooe/continuity/sysx/chmod_linux.go create mode 100644 vendor/github.com/stevvooe/continuity/sysx/sys.go create mode 100644 vendor/github.com/stevvooe/continuity/sysx/xattr.go create mode 100644 vendor/github.com/stevvooe/continuity/sysx/xattr_darwin.go create mode 100644 vendor/github.com/stevvooe/continuity/sysx/xattr_darwin_386.go create mode 100644 vendor/github.com/stevvooe/continuity/sysx/xattr_darwin_amd64.go create mode 100644 vendor/github.com/stevvooe/continuity/sysx/xattr_linux.go create mode 100644 vendor/github.com/stevvooe/continuity/sysx/xattr_linux_386.go create mode 100644 vendor/github.com/stevvooe/continuity/sysx/xattr_linux_amd64.go create mode 100644 vendor/github.com/stevvooe/continuity/sysx/xattr_linux_arm.go create mode 100644 vendor/github.com/stevvooe/continuity/sysx/xattr_linux_arm64.go diff --git a/vendor.conf b/vendor.conf index 3dcb6e8..04d1079 100644 --- a/vendor.conf +++ b/vendor.conf @@ -68,3 +68,5 @@ github.com/opencontainers/go-digest 21dfd564fd89c944783d00d069f33e3e7123c448 golang.org/x/sys/unix d75a52659825e75fff6158388dddc6a5b04f9ba5 # image-spec master as of 1/17/2017 github.com/opencontainers/image-spec 0ff14aabcda3b2ee62621174f1b29fc157bdf335 +# continuity master as of 1/10/2017 +github.com/stevvooe/continuity 6c9282fa1546987eefc2b123fe087b818d821725 diff --git a/vendor/github.com/stevvooe/continuity/LICENSE b/vendor/github.com/stevvooe/continuity/LICENSE new file mode 100644 index 0000000..8f71f43 --- /dev/null +++ b/vendor/github.com/stevvooe/continuity/LICENSE @@ -0,0 +1,202 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/vendor/github.com/stevvooe/continuity/context.go b/vendor/github.com/stevvooe/continuity/context.go new file mode 100644 index 0000000..b897173 --- /dev/null +++ b/vendor/github.com/stevvooe/continuity/context.go @@ -0,0 +1,640 @@ +package continuity + +import ( + "bytes" + "fmt" + "io" + "log" + "os" + "path/filepath" + "strings" + "syscall" + + "github.com/opencontainers/go-digest" +) + +var ( + ErrNotFound = fmt.Errorf("not found") + ErrNotSupported = fmt.Errorf("not supported") +) + +// Context represents a file system context for accessing resources. The +// responsibility of the context is to convert system specific resources to +// generic Resource objects. Most of this is safe path manipulation, as well +// as extraction of resource details. +type Context interface { + Apply(Resource) error + Verify(Resource) error + Resource(string, os.FileInfo) (Resource, error) + Walk(filepath.WalkFunc) error +} + +// SymlinkPath is intended to give the symlink target value +// in a root context. Target and linkname are absolute paths +// not under the given root. +type SymlinkPath func(root, linkname, target string) (string, error) + +type ContextOptions struct { + Digester Digester + Driver Driver + Provider ContentProvider +} + +// context represents a file system context for accessing resources. +// Generally, all path qualified access and system considerations should land +// here. +type context struct { + driver Driver + root string + digester Digester + provider ContentProvider +} + +// NewContext returns a Context associated with root. The default driver will +// be used, as returned by NewDriver. +func NewContext(root string) (Context, error) { + return NewContextWithOptions(root, ContextOptions{}) +} + +// NewContextWithOptions returns a Context associate with the root. +func NewContextWithOptions(root string, options ContextOptions) (Context, error) { + // normalize to absolute path + root, err := filepath.Abs(filepath.Clean(root)) + if err != nil { + return nil, err + } + + driver := options.Driver + if driver == nil { + driver, err = NewSystemDriver() + if err != nil { + return nil, err + } + } + + digester := options.Digester + if digester == nil { + digester = simpleDigester{digest.Canonical} + } + + // Check the root directory. Need to be a little careful here. We are + // allowing a link for now, but this may have odd behavior when + // canonicalizing paths. As long as all files are opened through the link + // path, this should be okay. + fi, err := driver.Stat(root) + if err != nil { + return nil, err + } + + if !fi.IsDir() { + return nil, &os.PathError{Op: "NewContext", Path: root, Err: os.ErrInvalid} + } + + return &context{ + root: root, + driver: driver, + digester: digester, + provider: options.Provider, + }, nil +} + +// Resource returns the resource as path p, populating the entry with info +// from fi. The path p should be the path of the resource in the context, +// typically obtained through Walk or from the value of Resource.Path(). If fi +// is nil, it will be resolved. +func (c *context) Resource(p string, fi os.FileInfo) (Resource, error) { + fp, err := c.fullpath(p) + if err != nil { + return nil, err + } + + if fi == nil { + fi, err = c.driver.Lstat(fp) + if err != nil { + return nil, err + } + } + + // TODO(stevvooe): This need to be resolved for the container's root, + // where here we are really getting the host OS's value. We need to allow + // this be passed in and fixed up to make these uid/gid mappings portable. + // Either this can be part of the driver or we can achieve it through some + // other mechanism. + sys, ok := fi.Sys().(*syscall.Stat_t) + if !ok { + // TODO(stevvooe): This may not be a hard error for all platforms. We + // may want to move this to the driver. + return nil, fmt.Errorf("unable to resolve syscall.Stat_t from (os.FileInfo).Sys(): %#v", fi) + } + + base, err := newBaseResource(p, fi.Mode(), fmt.Sprint(sys.Uid), fmt.Sprint(sys.Gid)) + if err != nil { + return nil, err + } + + base.xattrs, err = c.resolveXAttrs(fp, fi, base) + if err == ErrNotSupported { + log.Printf("resolving xattrs on %s not supported", fp) + } else if err != nil { + return nil, err + } + + // TODO(stevvooe): Handle windows alternate data streams. + + if fi.Mode().IsRegular() { + dgst, err := c.digest(p) + if err != nil { + return nil, err + } + + return newRegularFile(*base, base.paths, fi.Size(), dgst) + } + + if fi.Mode().IsDir() { + return newDirectory(*base) + } + + if fi.Mode()&os.ModeSymlink != 0 { + // We handle relative links vs absolute links by including a + // beginning slash for absolute links. Effectively, the bundle's + // root is treated as the absolute link anchor. + target, err := c.driver.Readlink(fp) + if err != nil { + return nil, err + } + + return newSymLink(*base, target) + } + + if fi.Mode()&os.ModeNamedPipe != 0 { + return newNamedPipe(*base, base.paths) + } + + if fi.Mode()&os.ModeDevice != 0 { + deviceDriver, ok := c.driver.(DeviceInfoDriver) + if !ok { + log.Printf("device extraction not supported %s", fp) + return nil, ErrNotSupported + } + + // character and block devices merely need to recover the + // major/minor device number. + major, minor, err := deviceDriver.DeviceInfo(fi) + if err != nil { + return nil, err + } + + return newDevice(*base, base.paths, major, minor) + } + + log.Printf("%q (%v) is not supported", fp, fi.Mode()) + return nil, ErrNotFound +} + +func (c *context) verifyMetadata(resource, target Resource) error { + if target.Mode() != resource.Mode() { + return fmt.Errorf("resource %q has incorrect mode: %v != %v", target.Path(), target.Mode(), resource.Mode()) + } + + if target.UID() != resource.UID() { + return fmt.Errorf("unexpected uid for %q: %v != %v", target.Path(), target.UID(), resource.GID()) + } + + if target.GID() != resource.GID() { + return fmt.Errorf("unexpected gid for %q: %v != %v", target.Path(), target.GID(), target.GID()) + } + + if xattrer, ok := resource.(XAttrer); ok { + txattrer, tok := target.(XAttrer) + if !tok { + return fmt.Errorf("resource %q has xattrs but target does not support them", resource.Path()) + } + + // For xattrs, only ensure that we have those defined in the resource + // and their values match. We can ignore other xattrs. In other words, + // we only verify that target has the subset defined by resource. + txattrs := txattrer.XAttrs() + for attr, value := range xattrer.XAttrs() { + tvalue, ok := txattrs[attr] + if !ok { + return fmt.Errorf("resource %q target missing xattr %q", resource.Path(), attr) + } + + if !bytes.Equal(value, tvalue) { + return fmt.Errorf("xattr %q value differs for resource %q", attr, resource.Path()) + } + } + } + + switch r := resource.(type) { + case RegularFile: + // TODO(stevvooe): Another reason to use a record-based approach. We + // have to do another type switch to get this to work. This could be + // fixed with an Equal function, but let's study this a little more to + // be sure. + t, ok := target.(RegularFile) + if !ok { + return fmt.Errorf("resource %q target not a regular file", r.Path()) + } + + if t.Size() != r.Size() { + return fmt.Errorf("resource %q target has incorrect size: %v != %v", t.Path(), t.Size(), r.Size()) + } + case Directory: + t, ok := target.(Directory) + if !ok { + return fmt.Errorf("resource %q target not a directory", t.Path()) + } + case SymLink: + t, ok := target.(SymLink) + if !ok { + return fmt.Errorf("resource %q target not a symlink", t.Path()) + } + + if t.Target() != r.Target() { + return fmt.Errorf("resource %q target has mismatched target: %q != %q", t.Path(), t.Target(), r.Target()) + } + case Device: + t, ok := target.(Device) + if !ok { + return fmt.Errorf("resource %q is not a device", t.Path()) + } + + if t.Major() != r.Major() || t.Minor() != r.Minor() { + return fmt.Errorf("resource %q has mismatched major/minor numbers: %d,%d != %d,%d", t.Path(), t.Major(), t.Minor(), r.Major(), r.Minor()) + } + case NamedPipe: + t, ok := target.(NamedPipe) + if !ok { + return fmt.Errorf("resource %q is not a named pipe", t.Path()) + } + default: + return fmt.Errorf("cannot verify resource: %v", resource) + } + + return nil +} + +// Verify the resource in the context. An error will be returned a discrepancy +// is found. +func (c *context) Verify(resource Resource) error { + fp, err := c.fullpath(resource.Path()) + if err != nil { + return err + } + + fi, err := c.driver.Lstat(fp) + if err != nil { + return err + } + + target, err := c.Resource(resource.Path(), fi) + if err != nil { + return err + } + + if target.Path() != resource.Path() { + return fmt.Errorf("resource paths do not match: %q != %q", target.Path(), resource.Path()) + } + + if err := c.verifyMetadata(resource, target); err != nil { + return err + } + + if h, isHardlinkable := resource.(Hardlinkable); isHardlinkable { + hardlinkKey, err := newHardlinkKey(fi) + if err == errNotAHardLink { + if len(h.Paths()) > 1 { + return fmt.Errorf("%q is not a hardlink to %q", h.Paths()[1], resource.Path()) + } + } else if err != nil { + return err + } + + for _, path := range h.Paths()[1:] { + fpLink, err := c.fullpath(path) + if err != nil { + return err + } + + fiLink, err := c.driver.Lstat(fpLink) + if err != nil { + return err + } + + targetLink, err := c.Resource(path, fiLink) + if err != nil { + return err + } + + hardlinkKeyLink, err := newHardlinkKey(fiLink) + if err != nil { + return err + } + + if hardlinkKeyLink != hardlinkKey { + return fmt.Errorf("%q is not a hardlink to %q", path, resource.Path()) + } + + if err := c.verifyMetadata(resource, targetLink); err != nil { + return err + } + } + } + + switch r := resource.(type) { + case RegularFile: + t, ok := target.(RegularFile) + if !ok { + return fmt.Errorf("resource %q target not a regular file", r.Path()) + } + + // TODO(stevvooe): This may need to get a little more sophisticated + // for digest comparison. We may want to actually calculate the + // provided digests, rather than the implementations having an + // overlap. + if !digestsMatch(t.Digests(), r.Digests()) { + return fmt.Errorf("digests for resource %q do not match: %v != %v", t.Path(), t.Digests(), r.Digests()) + } + } + + return nil +} + +func (c *context) checkoutFile(fp string, rf RegularFile) error { + if c.provider == nil { + return fmt.Errorf("no file provider") + } + var ( + r io.ReadCloser + err error + ) + for _, dgst := range rf.Digests() { + r, err = c.provider.Reader(dgst) + if err == nil { + break + } + } + if err != nil { + return fmt.Errorf("file content could not be provided: %v", err) + } + defer r.Close() + + return atomicWriteFile(fp, r, rf) +} + +// Apply the resource to the contexts. An error will be returned if the +// operation fails. Depending on the resource type, the resource may be +// created. For resource that cannot be resolved, an error will be returned. +func (c *context) Apply(resource Resource) error { + fp, err := c.fullpath(resource.Path()) + if err != nil { + return err + } + + if !strings.HasPrefix(fp, c.root) { + return fmt.Errorf("resource %v escapes root", resource) + } + + var chmod = true + fi, err := c.driver.Lstat(fp) + if err != nil { + if !os.IsNotExist(err) { + return err + } + } + + switch r := resource.(type) { + case RegularFile: + if fi == nil { + if err := c.checkoutFile(fp, r); err != nil { + return fmt.Errorf("error checking out file %q: %v", resource.Path(), err) + } + chmod = false + } else { + if !fi.Mode().IsRegular() { + return fmt.Errorf("file %q should be a regular file, but is not", resource.Path()) + } + if fi.Size() != r.Size() { + if err := c.checkoutFile(fp, r); err != nil { + return fmt.Errorf("error checking out file %q: %v", resource.Path(), err) + } + } else { + for _, dgst := range r.Digests() { + f, err := os.Open(fp) + if err != nil { + return fmt.Errorf("failure opening file for read %q: %v", resource.Path(), err) + } + compared, err := dgst.Algorithm().FromReader(f) + if err == nil && dgst != compared { + if err := c.checkoutFile(fp, r); err != nil { + return fmt.Errorf("error checking out file %q: %v", resource.Path(), err) + } + break + } + if err1 := f.Close(); err == nil { + err = err1 + } + if err != nil { + return fmt.Errorf("error checking digest for %q: %v", resource.Path(), err) + } + } + } + } + case Directory: + if fi == nil { + if err := c.driver.Mkdir(fp, resource.Mode()); err != nil { + return err + } + } else if !fi.Mode().IsDir() { + return fmt.Errorf("%q should be a directory, but is not", resource.Path()) + } + + case SymLink: + var target string // only possibly set if target resource is a symlink + + if fi != nil { + if fi.Mode()&os.ModeSymlink != 0 { + target, err = c.driver.Readlink(fp) + if err != nil { + return err + } + } + } + + if target != r.Target() { + if err := c.driver.Remove(fp); err != nil { // RemoveAll? + return err + } + + if err := c.driver.Symlink(r.Target(), fp); err != nil { + return err + } + } + + // NOTE(stevvooe): Chmod on symlink is not supported on linux. We + // may want to maintain support for other platforms that have it. + chmod = false + + case Device: + if fi == nil { + if err := c.driver.Mknod(fp, resource.Mode(), int(r.Major()), int(r.Minor())); err != nil { + return err + } + } else if (fi.Mode() & os.ModeDevice) == 0 { + return fmt.Errorf("%q should be a device, but is not", resource.Path()) + } else { + major, minor, err := deviceInfo(fi) + if err != nil { + return err + } + if major != r.Major() || minor != r.Minor() { + if err := c.driver.Remove(fp); err != nil { + return err + } + + if err := c.driver.Mknod(fp, resource.Mode(), int(r.Major()), int(r.Minor())); err != nil { + return err + } + } + } + + case NamedPipe: + if fi == nil { + if err := c.driver.Mkfifo(fp, resource.Mode()); err != nil { + return err + } + } else if (fi.Mode() & os.ModeNamedPipe) == 0 { + return fmt.Errorf("%q should be a named pipe, but is not", resource.Path()) + } + } + + if h, isHardlinkable := resource.(Hardlinkable); isHardlinkable { + for _, path := range h.Paths() { + if path == resource.Path() { + continue + } + + lp, err := c.fullpath(path) + if err != nil { + return err + } + + if _, fi := c.driver.Lstat(lp); fi == nil { + c.driver.Remove(lp) + } + if err := c.driver.Link(fp, lp); err != nil { + return err + } + } + } + + // Update filemode if file was not created + if chmod { + if err := c.driver.Lchmod(fp, resource.Mode()); err != nil { + return err + } + } + + if err := c.driver.Lchown(fp, resource.UID(), resource.GID()); err != nil { + return err + } + + if xattrer, ok := resource.(XAttrer); ok { + // For xattrs, only ensure that we have those defined in the resource + // and their values are set. We can ignore other xattrs. In other words, + // we only set xattres defined by resource but never remove. + + if _, ok := resource.(SymLink); ok { + lxattrDriver, ok := c.driver.(LXAttrDriver) + if !ok { + return fmt.Errorf("unsupported symlink xattr for resource %q", resource.Path()) + } + if err := lxattrDriver.LSetxattr(fp, xattrer.XAttrs()); err != nil { + return err + } + } else { + xattrDriver, ok := c.driver.(XAttrDriver) + if !ok { + return fmt.Errorf("unsupported xattr for resource %q", resource.Path()) + } + if err := xattrDriver.Setxattr(fp, xattrer.XAttrs()); err != nil { + return err + } + } + } + + return nil +} + +// Walk provides a convenience function to call filepath.Walk correctly for +// the context. Otherwise identical to filepath.Walk, the path argument is +// corrected to be contained within the context. +func (c *context) Walk(fn filepath.WalkFunc) error { + return filepath.Walk(c.root, func(p string, fi os.FileInfo, err error) error { + contained, err := c.contain(p) + return fn(contained, fi, err) + }) +} + +// fullpath returns the system path for the resource, joined with the context +// root. The path p must be a part of the context. +func (c *context) fullpath(p string) (string, error) { + p = filepath.Join(c.root, p) + if !strings.HasPrefix(p, c.root) { + return "", fmt.Errorf("invalid context path") + } + + return p, nil +} + +// contain cleans and santizes the filesystem path p to be an absolute path, +// effectively relative to the context root. +func (c *context) contain(p string) (string, error) { + sanitized, err := filepath.Rel(c.root, p) + if err != nil { + return "", err + } + + // ZOMBIES(stevvooe): In certain cases, we may want to remap these to a + // "containment error", so the caller can decide what to do. + return filepath.Join("/", filepath.Clean(sanitized)), nil +} + +// digest returns the digest of the file at path p, relative to the root. +func (c *context) digest(p string) (digest.Digest, error) { + f, err := c.driver.Open(filepath.Join(c.root, p)) + if err != nil { + return "", err + } + defer f.Close() + + return c.digester.Digest(f) +} + +// resolveXAttrs attempts to resolve the extended attributes for the resource +// at the path fp, which is the full path to the resource. If the resource +// cannot have xattrs, nil will be returned. +func (c *context) resolveXAttrs(fp string, fi os.FileInfo, base *resource) (map[string][]byte, error) { + if fi.Mode().IsRegular() || fi.Mode().IsDir() { + xattrDriver, ok := c.driver.(XAttrDriver) + if !ok { + log.Println("xattr extraction not supported") + return nil, ErrNotSupported + } + + return xattrDriver.Getxattr(fp) + } + + if fi.Mode()&os.ModeSymlink != 0 { + lxattrDriver, ok := c.driver.(LXAttrDriver) + if !ok { + log.Println("xattr extraction for symlinks not supported") + return nil, ErrNotSupported + } + + return lxattrDriver.LGetxattr(fp) + } + + return nil, nil +} diff --git a/vendor/github.com/stevvooe/continuity/devices_darwin.go b/vendor/github.com/stevvooe/continuity/devices_darwin.go new file mode 100644 index 0000000..864162f --- /dev/null +++ b/vendor/github.com/stevvooe/continuity/devices_darwin.go @@ -0,0 +1,15 @@ +package continuity + +// from /usr/include/sys/types.h + +func getmajor(dev int32) uint64 { + return (uint64(dev) >> 24) & 0xff +} + +func getminor(dev int32) uint64 { + return uint64(dev) & 0xffffff +} + +func makedev(major int, minor int) int { + return ((major << 24) | minor) +} diff --git a/vendor/github.com/stevvooe/continuity/devices_linux.go b/vendor/github.com/stevvooe/continuity/devices_linux.go new file mode 100644 index 0000000..f47fd8a --- /dev/null +++ b/vendor/github.com/stevvooe/continuity/devices_linux.go @@ -0,0 +1,15 @@ +package continuity + +// from /usr/include/linux/kdev_t.h + +func getmajor(dev uint64) uint64 { + return dev >> 8 +} + +func getminor(dev uint64) uint64 { + return dev & 0xff +} + +func makedev(major int, minor int) int { + return ((major << 8) | minor) +} diff --git a/vendor/github.com/stevvooe/continuity/devices_unix.go b/vendor/github.com/stevvooe/continuity/devices_unix.go new file mode 100644 index 0000000..2b07d55 --- /dev/null +++ b/vendor/github.com/stevvooe/continuity/devices_unix.go @@ -0,0 +1,55 @@ +// +build linux darwin + +package continuity + +import ( + "fmt" + "os" + "syscall" +) + +func deviceInfo(fi os.FileInfo) (uint64, uint64, error) { + sys, ok := fi.Sys().(*syscall.Stat_t) + if !ok { + return 0, 0, fmt.Errorf("cannot extract device from os.FileInfo") + } + + return getmajor(sys.Rdev), getminor(sys.Rdev), nil +} + +// mknod provides a shortcut for syscall.Mknod +func mknod(p string, mode os.FileMode, maj, min int) error { + var ( + m = syscallMode(mode.Perm()) + dev int + ) + + if mode&os.ModeDevice != 0 { + dev = makedev(maj, min) + + if mode&os.ModeCharDevice != 0 { + m |= syscall.S_IFCHR + } else { + m |= syscall.S_IFBLK + } + } else if mode&os.ModeNamedPipe != 0 { + m |= syscall.S_IFIFO + } + + return syscall.Mknod(p, m, dev) +} + +// syscallMode returns the syscall-specific mode bits from Go's portable mode bits. +func syscallMode(i os.FileMode) (o uint32) { + o |= uint32(i.Perm()) + if i&os.ModeSetuid != 0 { + o |= syscall.S_ISUID + } + if i&os.ModeSetgid != 0 { + o |= syscall.S_ISGID + } + if i&os.ModeSticky != 0 { + o |= syscall.S_ISVTX + } + return +} diff --git a/vendor/github.com/stevvooe/continuity/digests.go b/vendor/github.com/stevvooe/continuity/digests.go new file mode 100644 index 0000000..355b080 --- /dev/null +++ b/vendor/github.com/stevvooe/continuity/digests.go @@ -0,0 +1,88 @@ +package continuity + +import ( + "fmt" + "io" + "sort" + + "github.com/opencontainers/go-digest" +) + +// Digester produces a digest for a given read stream +type Digester interface { + Digest(io.Reader) (digest.Digest, error) +} + +// ContentProvider produces a read stream for a given digest +type ContentProvider interface { + Reader(digest.Digest) (io.ReadCloser, error) +} + +type simpleDigester struct { + algorithm digest.Algorithm +} + +func (sd simpleDigester) Digest(r io.Reader) (digest.Digest, error) { + digester := sd.algorithm.Digester() + + if _, err := io.Copy(digester.Hash(), r); err != nil { + return "", err + } + + return digester.Digest(), nil +} + +// uniqifyDigests sorts and uniqifies the provided digest, ensuring that the +// digests are not repeated and no two digests with the same algorithm have +// different values. Because a stable sort is used, this has the effect of +// "zipping" digest collections from multiple resources. +func uniqifyDigests(digests ...digest.Digest) ([]digest.Digest, error) { + sort.Stable(digestSlice(digests)) // stable sort is important for the behavior here. + seen := map[digest.Digest]struct{}{} + algs := map[digest.Algorithm][]digest.Digest{} // detect different digests. + + var out []digest.Digest + // uniqify the digests + for _, d := range digests { + if _, ok := seen[d]; ok { + continue + } + + seen[d] = struct{}{} + algs[d.Algorithm()] = append(algs[d.Algorithm()], d) + + if len(algs[d.Algorithm()]) > 1 { + return nil, fmt.Errorf("conflicting digests for %v found", d.Algorithm()) + } + + out = append(out, d) + } + + return out, nil +} + +// digestsMatch compares the two sets of digests to see if they match. +func digestsMatch(as, bs []digest.Digest) bool { + all := append(as, bs...) + + uniqified, err := uniqifyDigests(all...) + if err != nil { + // the only error uniqifyDigests returns is when the digests disagree. + return false + } + + disjoint := len(as) + len(bs) + if len(uniqified) == disjoint { + // if these two sets have the same cardinality, we know both sides + // didn't share any digests. + return false + } + + return true +} + +type digestSlice []digest.Digest + +func (p digestSlice) Len() int { return len(p) } +func (p digestSlice) Less(i, j int) bool { return p[i] < p[j] } +func (p digestSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } diff --git a/vendor/github.com/stevvooe/continuity/driver.go b/vendor/github.com/stevvooe/continuity/driver.go new file mode 100644 index 0000000..6af7f02 --- /dev/null +++ b/vendor/github.com/stevvooe/continuity/driver.go @@ -0,0 +1,158 @@ +package continuity + +import ( + "errors" + "os" + "strconv" +) + +// Driver provides all of the system-level functions in a common interface. +// The context should call these with full paths and should never use the `os` +// package or any other package to access resources on the filesystem. This +// mechanism let's us carefully control access to the context and maintain +// path and resource integrity. It also gives us an interface to reason about +// direct resource access. +// +// Implementations don't need to do much other than meet the interface. For +// example, it is not required to wrap os.FileInfo to return correct paths for +// the call to Name(). +type Driver interface { + Open(path string) (*os.File, error) + Stat(path string) (os.FileInfo, error) + Lstat(path string) (os.FileInfo, error) + Readlink(p string) (string, error) + Mkdir(path string, mode os.FileMode) error + Remove(path string) error + + Link(oldname, newname string) error + Lchmod(path string, mode os.FileMode) error + Lchown(path, uid, gid string) error + Symlink(oldname, newname string) error + + // TODO(aaronl): These methods might move outside the main Driver + // interface in the future as more platforms are added. + Mknod(path string, mode os.FileMode, major int, minor int) error + Mkfifo(path string, mode os.FileMode) error + + // NOTE(stevvooe): We may want to actually include the path manipulation + // functions here, as well. They have been listed below to make the + // discovery process easier. + + // Join(path ...string) string + // IsAbs(string) bool + // Abs(string) (string, error) + // Rel(base, target string) (string, error) + // Walk(string, filepath.WalkFunc) error +} + +func NewSystemDriver() (Driver, error) { + // TODO(stevvooe): Consider having this take a "hint" path argument, which + // would be the context root. The hint could be used to resolve required + // filesystem support when assembling the driver to use. + return &driver{}, nil +} + +// XAttrDriver should be implemented on operation systems and filesystems that +// have xattr support for regular files and directories. +type XAttrDriver interface { + // Getxattr returns all of the extended attributes for the file at path. + // Typically, this takes a syscall call to Listxattr and Getxattr. + Getxattr(path string) (map[string][]byte, error) + + // Setxattr sets all of the extended attributes on file at path, following + // any symbolic links, if necessary. All attributes on the target are + // replaced by the values from attr. If the operation fails to set any + // attribute, those already applied will not be rolled back. + Setxattr(path string, attr map[string][]byte) error +} + +// LXAttrDriver should be implemented by drivers on operating systems and +// filesystems that support setting and getting extended attributes on +// symbolic links. If this is not implemented, extended attributes will be +// ignored on symbolic links. +type LXAttrDriver interface { + // LGetxattr returns all of the extended attributes for the file at path + // and does not follow symlinks. Typically, this takes a syscall call to + // Llistxattr and Lgetxattr. + LGetxattr(path string) (map[string][]byte, error) + + // LSetxattr sets all of the extended attributes on file at path, without + // following symbolic links. All attributes on the target are replaced by + // the values from attr. If the operation fails to set any attribute, + // those already applied will not be rolled back. + LSetxattr(path string, attr map[string][]byte) error +} + +type DeviceInfoDriver interface { + DeviceInfo(fi os.FileInfo) (maj uint64, min uint64, err error) +} + +// driver is a simple default implementation that sends calls out to the "os" +// package. Extend the "driver" type in system-specific files to add support, +// such as xattrs, which can add support at compile time. +type driver struct{} + +var _ Driver = &driver{} + +func (d *driver) Open(p string) (*os.File, error) { + return os.Open(p) +} + +func (d *driver) Stat(p string) (os.FileInfo, error) { + return os.Stat(p) +} + +func (d *driver) Lstat(p string) (os.FileInfo, error) { + return os.Lstat(p) +} + +func (d *driver) Readlink(p string) (string, error) { + return os.Readlink(p) +} + +func (d *driver) Mkdir(p string, mode os.FileMode) error { + return os.Mkdir(p, mode) +} + +// Remove is used to unlink files and remove directories. +// This is following the golang os package api which +// combines the operations into a higher level Remove +// function. If explicit unlinking or directory removal +// to mirror system call is required, they should be +// split up at that time. +func (d *driver) Remove(path string) error { + return os.Remove(path) +} + +func (d *driver) Link(oldname, newname string) error { + return os.Link(oldname, newname) +} + +func (d *driver) Lchown(name, uidStr, gidStr string) error { + uid, err := strconv.Atoi(uidStr) + if err != nil { + return err + } + gid, err := strconv.Atoi(gidStr) + if err != nil { + return err + } + return os.Lchown(name, uid, gid) +} + +func (d *driver) Symlink(oldname, newname string) error { + return os.Symlink(oldname, newname) +} + +func (d *driver) Mknod(path string, mode os.FileMode, major, minor int) error { + return mknod(path, mode, major, minor) +} + +func (d *driver) Mkfifo(path string, mode os.FileMode) error { + if mode&os.ModeNamedPipe == 0 { + return errors.New("mode passed to Mkfifo does not have the named pipe bit set") + } + // mknod with a mode that has ModeNamedPipe set creates a fifo, not a + // device. + return mknod(path, mode, 0, 0) +} diff --git a/vendor/github.com/stevvooe/continuity/driver_darwin.go b/vendor/github.com/stevvooe/continuity/driver_darwin.go new file mode 100644 index 0000000..96dbc93 --- /dev/null +++ b/vendor/github.com/stevvooe/continuity/driver_darwin.go @@ -0,0 +1,7 @@ +package continuity + +import "os" + +func (d *driver) DeviceInfo(fi os.FileInfo) (maj uint64, min uint64, err error) { + return deviceInfo(fi) +} diff --git a/vendor/github.com/stevvooe/continuity/driver_unix.go b/vendor/github.com/stevvooe/continuity/driver_unix.go new file mode 100644 index 0000000..337ab12 --- /dev/null +++ b/vendor/github.com/stevvooe/continuity/driver_unix.go @@ -0,0 +1,107 @@ +// +build linux darwin + +package continuity + +import ( + "fmt" + "os" + "path/filepath" + "sort" + + "github.com/stevvooe/continuity/sysx" +) + +// Lchmod changes the mode of an file not following symlinks. +func (d *driver) Lchmod(path string, mode os.FileMode) (err error) { + if !filepath.IsAbs(path) { + path, err = filepath.Abs(path) + if err != nil { + return + } + } + + return sysx.Fchmodat(0, path, uint32(mode), sysx.AtSymlinkNofollow) +} + +// Getxattr returns all of the extended attributes for the file at path p. +func (d *driver) Getxattr(p string) (map[string][]byte, error) { + xattrs, err := sysx.Listxattr(p) + if err != nil { + return nil, fmt.Errorf("listing %s xattrs: %v", p, err) + } + + sort.Strings(xattrs) + m := make(map[string][]byte, len(xattrs)) + + for _, attr := range xattrs { + value, err := sysx.Getxattr(p, attr) + if err != nil { + return nil, fmt.Errorf("getting %q xattr on %s: %v", attr, p, err) + } + + // NOTE(stevvooe): This append/copy tricky relies on unique + // xattrs. Break this out into an alloc/copy if xattrs are no + // longer unique. + m[attr] = append(m[attr], value...) + } + + return m, nil +} + +// Setxattr sets all of the extended attributes on file at path, following +// any symbolic links, if necessary. All attributes on the target are +// replaced by the values from attr. If the operation fails to set any +// attribute, those already applied will not be rolled back. +func (d *driver) Setxattr(path string, attrMap map[string][]byte) error { + for attr, value := range attrMap { + if err := sysx.Setxattr(path, attr, value, 0); err != nil { + return fmt.Errorf("error setting xattr %q on %s: %v", attr, path, err) + } + } + + return nil +} + +// LGetxattr returns all of the extended attributes for the file at path p +// not following symbolic links. +func (d *driver) LGetxattr(p string) (map[string][]byte, error) { + xattrs, err := sysx.LListxattr(p) + if err != nil { + return nil, fmt.Errorf("listing %s xattrs: %v", p, err) + } + + sort.Strings(xattrs) + m := make(map[string][]byte, len(xattrs)) + + for _, attr := range xattrs { + value, err := sysx.LGetxattr(p, attr) + if err != nil { + return nil, fmt.Errorf("getting %q xattr on %s: %v", attr, p, err) + } + + // NOTE(stevvooe): This append/copy tricky relies on unique + // xattrs. Break this out into an alloc/copy if xattrs are no + // longer unique. + m[attr] = append(m[attr], value...) + } + + return m, nil +} + +// LSetxattr sets all of the extended attributes on file at path, not +// following any symbolic links. All attributes on the target are +// replaced by the values from attr. If the operation fails to set any +// attribute, those already applied will not be rolled back. +func (d *driver) LSetxattr(path string, attrMap map[string][]byte) error { + for attr, value := range attrMap { + if err := sysx.LSetxattr(path, attr, value, 0); err != nil { + return fmt.Errorf("error setting xattr %q on %s: %v", attr, path, err) + } + } + + return nil +} + +func (d *driver) DeviceInfo(fi os.FileInfo) (maj uint64, min uint64, err error) { + return deviceInfo(fi) +} diff --git a/vendor/github.com/stevvooe/continuity/groups_unix.go b/vendor/github.com/stevvooe/continuity/groups_unix.go new file mode 100644 index 0000000..e15c14f --- /dev/null +++ b/vendor/github.com/stevvooe/continuity/groups_unix.go @@ -0,0 +1,113 @@ +package continuity + +import ( + "bufio" + "fmt" + "io" + "os" + "strconv" + "strings" +) + +// TODO(stevvooe): This needs a lot of work before we can call it useful. + +type groupIndex struct { + byName map[string]*group + byGID map[int]*group +} + +func getGroupIndex() (*groupIndex, error) { + f, err := os.Open("/etc/group") + if err != nil { + return nil, err + } + defer f.Close() + + groups, err := parseGroups(f) + if err != nil { + return nil, err + } + + return newGroupIndex(groups), nil +} + +func newGroupIndex(groups []group) *groupIndex { + gi := &groupIndex{ + byName: make(map[string]*group), + byGID: make(map[int]*group), + } + + for i, group := range groups { + gi.byGID[group.gid] = &groups[i] + gi.byName[group.name] = &groups[i] + } + + return gi +} + +type group struct { + name string + gid int + members []string +} + +func getGroupName(gid int) (string, error) { + f, err := os.Open("/etc/group") + if err != nil { + return "", err + } + defer f.Close() + + groups, err := parseGroups(f) + if err != nil { + return "", err + } + + for _, group := range groups { + if group.gid == gid { + return group.name, nil + } + } + + return "", fmt.Errorf("no group for gid") +} + +// parseGroups parses an /etc/group file for group names, ids and membership. +// This is unix specific. +func parseGroups(rd io.Reader) ([]group, error) { + var groups []group + scanner := bufio.NewScanner(rd) + + for scanner.Scan() { + if strings.HasPrefix(scanner.Text(), "#") { + continue // skip comment + } + + parts := strings.SplitN(scanner.Text(), ":", 4) + + if len(parts) != 4 { + return nil, fmt.Errorf("bad entry: %q", scanner.Text()) + } + + name, _, sgid, smembers := parts[0], parts[1], parts[2], parts[3] + + gid, err := strconv.Atoi(sgid) + if err != nil { + return nil, fmt.Errorf("bad gid: %q", gid) + } + + members := strings.Split(smembers, ",") + + groups = append(groups, group{ + name: name, + gid: gid, + members: members, + }) + } + + if scanner.Err() != nil { + return nil, scanner.Err() + } + + return groups, nil +} diff --git a/vendor/github.com/stevvooe/continuity/hardlinks.go b/vendor/github.com/stevvooe/continuity/hardlinks.go new file mode 100644 index 0000000..8b39bd0 --- /dev/null +++ b/vendor/github.com/stevvooe/continuity/hardlinks.go @@ -0,0 +1,57 @@ +package continuity + +import ( + "fmt" + "os" +) + +var ( + errNotAHardLink = fmt.Errorf("invalid hardlink") +) + +type hardlinkManager struct { + hardlinks map[hardlinkKey][]Resource +} + +func newHardlinkManager() *hardlinkManager { + return &hardlinkManager{ + hardlinks: map[hardlinkKey][]Resource{}, + } +} + +// Add attempts to add the resource to the hardlink manager. If the resource +// cannot be considered as a hardlink candidate, errNotAHardLink is returned. +func (hlm *hardlinkManager) Add(fi os.FileInfo, resource Resource) error { + if _, ok := resource.(Hardlinkable); !ok { + return errNotAHardLink + } + + key, err := newHardlinkKey(fi) + if err != nil { + return err + } + + hlm.hardlinks[key] = append(hlm.hardlinks[key], resource) + + return nil +} + +// Merge processes the current state of the hardlink manager and merges any +// shared nodes into hardlinked resources. +func (hlm *hardlinkManager) Merge() ([]Resource, error) { + var resources []Resource + for key, linked := range hlm.hardlinks { + if len(linked) < 1 { + return nil, fmt.Errorf("no hardlink entrys for dev, inode pair: %#v", key) + } + + merged, err := Merge(linked...) + if err != nil { + return nil, fmt.Errorf("error merging hardlink: %v", err) + } + + resources = append(resources, merged) + } + + return resources, nil +} diff --git a/vendor/github.com/stevvooe/continuity/hardlinks_unix.go b/vendor/github.com/stevvooe/continuity/hardlinks_unix.go new file mode 100644 index 0000000..5880f06 --- /dev/null +++ b/vendor/github.com/stevvooe/continuity/hardlinks_unix.go @@ -0,0 +1,34 @@ +package continuity + +import ( + "fmt" + "os" + "syscall" +) + +// hardlinkKey provides a tuple-key for managing hardlinks. This is system- +// specific. +type hardlinkKey struct { + dev uint64 + inode uint64 +} + +// newHardlinkKey returns a hardlink key for the provided file info. If the +// resource does not represent a possible hardlink, errNotAHardLink will be +// returned. +func newHardlinkKey(fi os.FileInfo) (hardlinkKey, error) { + sys, ok := fi.Sys().(*syscall.Stat_t) + if !ok { + return hardlinkKey{}, fmt.Errorf("cannot resolve (*syscall.Stat_t) from os.FileInfo") + } + + if sys.Nlink < 2 { + // NOTE(stevvooe): This is not always true for all filesystems. We + // should somehow detect this and provided a slow "polyfill" that + // leverages os.SameFile if we detect a filesystem where link counts + // is not really supported. + return hardlinkKey{}, errNotAHardLink + } + + return hardlinkKey{dev: uint64(sys.Dev), inode: sys.Ino}, nil +} diff --git a/vendor/github.com/stevvooe/continuity/hardlinks_windows.go b/vendor/github.com/stevvooe/continuity/hardlinks_windows.go new file mode 100644 index 0000000..9d55972 --- /dev/null +++ b/vendor/github.com/stevvooe/continuity/hardlinks_windows.go @@ -0,0 +1,5 @@ +package continuity + +// NOTE(stevvooe): Obviously, this is not yet implemented. However, the +// makings of an implementation are available in src/os/types_windows.go. More +// investigation needs to be done to figure out exactly how to do this. diff --git a/vendor/github.com/stevvooe/continuity/ioutils.go b/vendor/github.com/stevvooe/continuity/ioutils.go new file mode 100644 index 0000000..0e2cc5f --- /dev/null +++ b/vendor/github.com/stevvooe/continuity/ioutils.go @@ -0,0 +1,39 @@ +package continuity + +import ( + "io" + "io/ioutil" + "os" + "path/filepath" +) + +// atomicWriteFile writes data to a file by first writing to a temp +// file and calling rename. +func atomicWriteFile(filename string, r io.Reader, rf RegularFile) error { + f, err := ioutil.TempFile(filepath.Dir(filename), ".tmp-"+filepath.Base(filename)) + if err != nil { + return err + } + err = os.Chmod(f.Name(), rf.Mode()) + if err != nil { + f.Close() + return err + } + n, err := io.Copy(f, r) + if err == nil && n < rf.Size() { + f.Close() + return io.ErrShortWrite + } + if err != nil { + f.Close() + return err + } + if err := f.Sync(); err != nil { + f.Close() + return err + } + if err := f.Close(); err != nil { + return err + } + return os.Rename(f.Name(), filename) +} diff --git a/vendor/github.com/stevvooe/continuity/manifest.go b/vendor/github.com/stevvooe/continuity/manifest.go new file mode 100644 index 0000000..4315bcb --- /dev/null +++ b/vendor/github.com/stevvooe/continuity/manifest.go @@ -0,0 +1,144 @@ +package continuity + +import ( + "fmt" + "io" + "log" + "os" + "sort" + + "github.com/golang/protobuf/proto" + pb "github.com/stevvooe/continuity/proto" +) + +// Manifest provides the contents of a manifest. Users of this struct should +// not typically modify any fields directly. +type Manifest struct { + // Resources specifies all the resources for a manifest in order by path. + Resources []Resource +} + +func Unmarshal(p []byte) (*Manifest, error) { + var bm pb.Manifest + + if err := proto.Unmarshal(p, &bm); err != nil { + return nil, err + } + + var m Manifest + for _, b := range bm.Resource { + r, err := fromProto(b) + if err != nil { + return nil, err + } + + m.Resources = append(m.Resources, r) + } + + return &m, nil +} + +func Marshal(m *Manifest) ([]byte, error) { + var bm pb.Manifest + for _, resource := range m.Resources { + bm.Resource = append(bm.Resource, toProto(resource)) + } + + return proto.Marshal(&bm) +} + +func MarshalText(w io.Writer, m *Manifest) error { + var bm pb.Manifest + for _, resource := range m.Resources { + bm.Resource = append(bm.Resource, toProto(resource)) + } + + return proto.MarshalText(w, &bm) +} + +// BuildManifest creates the manifest for the given context +func BuildManifest(ctx Context) (*Manifest, error) { + resourcesByPath := map[string]Resource{} + hardlinks := newHardlinkManager() + + if err := ctx.Walk(func(p string, fi os.FileInfo, err error) error { + if err != nil { + return fmt.Errorf("error walking %s: %v", p, err) + } + + if p == "/" { + // skip root + return nil + } + + resource, err := ctx.Resource(p, fi) + if err != nil { + if err == ErrNotFound { + return nil + } + log.Printf("error getting resource %q: %v", p, err) + return err + } + + // add to the hardlink manager + if err := hardlinks.Add(fi, resource); err == nil { + // Resource has been accepted by hardlink manager so we don't add + // it to the resourcesByPath until we merge at the end. + return nil + } else if err != errNotAHardLink { + // handle any other case where we have a proper error. + return fmt.Errorf("adding hardlink %s: %v", p, err) + } + + resourcesByPath[p] = resource + + return nil + }); err != nil { + return nil, err + } + + // merge and post-process the hardlinks. + hardlinked, err := hardlinks.Merge() + if err != nil { + return nil, err + } + + for _, resource := range hardlinked { + resourcesByPath[resource.Path()] = resource + } + + var resources []Resource + for _, resource := range resourcesByPath { + resources = append(resources, resource) + } + + sort.Stable(ByPath(resources)) + + return &Manifest{ + Resources: resources, + }, nil +} + +// VerifyManifest verifies all the resources in a manifest +// against files from the given context. +func VerifyManifest(ctx Context, manifest *Manifest) error { + for _, resource := range manifest.Resources { + if err := ctx.Verify(resource); err != nil { + return err + } + } + + return nil +} + +// ApplyManifest applies on the resources in a manifest to +// the given context. +func ApplyManifest(ctx Context, manifest *Manifest) error { + for _, resource := range manifest.Resources { + if err := ctx.Apply(resource); err != nil { + return err + } + } + + return nil +} diff --git a/vendor/github.com/stevvooe/continuity/manifest_test_darwin.go b/vendor/github.com/stevvooe/continuity/manifest_test_darwin.go new file mode 100644 index 0000000..873ec31 --- /dev/null +++ b/vendor/github.com/stevvooe/continuity/manifest_test_darwin.go @@ -0,0 +1,23 @@ +// +build ignore + +package continuity + +import "os" + +var ( + devNullResource = resource{ + kind: chardev, + path: "/dev/null", + major: 3, + minor: 2, + mode: 0666 | os.ModeDevice | os.ModeCharDevice, + } + + devZeroResource = resource{ + kind: chardev, + path: "/dev/zero", + major: 3, + minor: 3, + mode: 0666 | os.ModeDevice | os.ModeCharDevice, + } +) diff --git a/vendor/github.com/stevvooe/continuity/proto/gen.go b/vendor/github.com/stevvooe/continuity/proto/gen.go new file mode 100644 index 0000000..8f26ff5 --- /dev/null +++ b/vendor/github.com/stevvooe/continuity/proto/gen.go @@ -0,0 +1,3 @@ +package proto + +//go:generate protoc --go_out=. manifest.proto diff --git a/vendor/github.com/stevvooe/continuity/proto/manifest.pb.go b/vendor/github.com/stevvooe/continuity/proto/manifest.pb.go new file mode 100644 index 0000000..edb1781 --- /dev/null +++ b/vendor/github.com/stevvooe/continuity/proto/manifest.pb.go @@ -0,0 +1,98 @@ +// Code generated by protoc-gen-go. +// source: manifest.proto +// DO NOT EDIT! + +/* +Package proto is a generated protocol buffer package. + +It is generated from these files: + manifest.proto + +It has these top-level messages: + Manifest + Resource +*/ +package proto + +import proto1 "github.com/golang/protobuf/proto" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto1.Marshal + +// Manifest specifies the entries in a container bundle, keyed and sorted by +// path. +type Manifest struct { + Resource []*Resource `protobuf:"bytes,1,rep,name=resource" json:"resource,omitempty"` +} + +func (m *Manifest) Reset() { *m = Manifest{} } +func (m *Manifest) String() string { return proto1.CompactTextString(m) } +func (*Manifest) ProtoMessage() {} + +func (m *Manifest) GetResource() []*Resource { + if m != nil { + return m.Resource + } + return nil +} + +type Resource struct { + // Path specifies the path from the bundle root. If more than one + // path is present, the entry may represent a hardlink, rather than using + // a link target. The path format is operating system specific. + Path []string `protobuf:"bytes,1,rep,name=path" json:"path,omitempty"` + // Uid specifies the user id for the resource. A string type is used for + // compatibility across different OS. + Uid string `protobuf:"bytes,2,opt,name=uid" json:"uid,omitempty"` + // Gid specifies the group id for the resource. A string type is used for + // compatibility across different OS. + Gid string `protobuf:"bytes,3,opt,name=gid" json:"gid,omitempty"` + // user and group are not currently used but their field numbers have been + // reserved for future use. As such, they are marked as deprecated. + User string `protobuf:"bytes,4,opt,name=user" json:"user,omitempty"` + Group string `protobuf:"bytes,5,opt,name=group" json:"group,omitempty"` + // Mode defines the file mode and permissions. We've used the same + // bit-packing from Go's os package, + // http://golang.org/pkg/os/#FileMode, since they've done the work of + // creating a cross-platform layout. + Mode uint32 `protobuf:"varint,6,opt,name=mode" json:"mode,omitempty"` + // Size specifies the size in bytes of the resource. This is only valid + // for regular files. + Size uint64 `protobuf:"varint,7,opt,name=size" json:"size,omitempty"` + // Digest specifies the content digest of the target file. Only valid for + // regular files. The strings are formatted as :. + // The digests are sorted in lexical order and implementations may choose + // which algorithms they prefer. + Digest []string `protobuf:"bytes,8,rep,name=digest" json:"digest,omitempty"` + // Target defines the target of a hard or soft link. Absolute links start + // with a slash and specify the resource relative to the bundle root. + // Relative links do not start with a slash and are relative to the + // resource path. + Target string `protobuf:"bytes,9,opt,name=target" json:"target,omitempty"` + // Major specifies the major device number for charactor and block devices. + Major uint64 `protobuf:"varint,10,opt,name=major" json:"major,omitempty"` + // Minor specifies the minor device number for charactor and block devices. + Minor uint64 `protobuf:"varint,11,opt,name=minor" json:"minor,omitempty"` + // Xattr provides storage for extended attributes for the target resource. + Xattr map[string][]byte `protobuf:"bytes,12,rep,name=xattr" json:"xattr,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value,proto3"` + // Ads stores one or more alternate data streams for the target resource. + Ads map[string][]byte `protobuf:"bytes,13,rep,name=ads" json:"ads,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value,proto3"` +} + +func (m *Resource) Reset() { *m = Resource{} } +func (m *Resource) String() string { return proto1.CompactTextString(m) } +func (*Resource) ProtoMessage() {} + +func (m *Resource) GetXattr() map[string][]byte { + if m != nil { + return m.Xattr + } + return nil +} + +func (m *Resource) GetAds() map[string][]byte { + if m != nil { + return m.Ads + } + return nil +} diff --git a/vendor/github.com/stevvooe/continuity/proto/manifest.proto b/vendor/github.com/stevvooe/continuity/proto/manifest.proto new file mode 100644 index 0000000..f7f5d3e --- /dev/null +++ b/vendor/github.com/stevvooe/continuity/proto/manifest.proto @@ -0,0 +1,77 @@ +syntax = "proto3"; + +package proto; + +// Manifest specifies the entries in a container bundle, keyed and sorted by +// path. +message Manifest { + repeated Resource resource = 1; +} + +message Resource { + // Path specifies the path from the bundle root. If more than one + // path is present, the entry may represent a hardlink, rather than using + // a link target. The path format is operating system specific. + repeated string path = 1; + + // NOTE(stevvooe): Need to define clear precedence for user/group/uid/gid precedence. + + // Uid specifies the user id for the resource. A string type is used for + // compatibility across different OS. + string uid = 2; + + // Gid specifies the group id for the resource. A string type is used for + // compatibility across different OS. + string gid = 3; + + // user and group are not currently used but their field numbers have been + // reserved for future use. As such, they are marked as deprecated. + string user = 4 [deprecated=true]; + string group = 5 [deprecated=true]; + + // Mode defines the file mode and permissions. We've used the same + // bit-packing from Go's os package, + // http://golang.org/pkg/os/#FileMode, since they've done the work of + // creating a cross-platform layout. + uint32 mode = 6; + + // NOTE(stevvooe): Beyond here, we start defining type specific fields. + + // Size specifies the size in bytes of the resource. This is only valid + // for regular files. + uint64 size = 7; + + // Digest specifies the content digest of the target file. Only valid for + // regular files. The strings are formatted as :. + // The digests are sorted in lexical order and implementations may choose + // which algorithms they prefer. + repeated string digest = 8; + + // Target defines the target of a hard or soft link. Absolute links start + // with a slash and specify the resource relative to the bundle root. + // Relative links do not start with a slash and are relative to the + // resource path. + string target = 9; + + // Major specifies the major device number for charactor and block devices. + uint64 major = 10; + + // Minor specifies the minor device number for charactor and block devices. + uint64 minor = 11; + + // TODO(stevvooe): The use of maps here may be problematic for + // deterministic generation. Check out this comment: + // https://developers.google.com/protocol-buffers/docs/proto3#backwards-compatibility + // Fortunately, the Go implementation correctly sorts the map keys to + // ensure deterministic generation, but this is not guaranteed for all + // implementations. If this is problem, we should generate that schema and + // sort by key. We can do this at any time and retain backwards + // compatibility. + + // Xattr provides storage for extended attributes for the target resource. + map xattr = 12; + + // Ads stores one or more alternate data streams for the target resource. + map ads = 13; + +} diff --git a/vendor/github.com/stevvooe/continuity/resource.go b/vendor/github.com/stevvooe/continuity/resource.go new file mode 100644 index 0000000..3ba5431 --- /dev/null +++ b/vendor/github.com/stevvooe/continuity/resource.go @@ -0,0 +1,578 @@ +package continuity + +import ( + "errors" + "fmt" + "os" + "reflect" + "sort" + + "github.com/opencontainers/go-digest" + pb "github.com/stevvooe/continuity/proto" +) + +// TODO(stevvooe): A record based model, somewhat sketched out at the bottom +// of this file, will be more flexible. Another possibly is to tie the package +// interface directly to the protobuf type. This will have efficiency +// advantages at the cost coupling the nasty codegen types to the exported +// interface. + +type Resource interface { + // Path provides the primary resource path relative to the bundle root. In + // cases where resources have more than one path, such as with hard links, + // this will return the primary path, which is often just the first entry. + Path() string + + // Mode returns the + Mode() os.FileMode + + UID() string + GID() string +} + +// ByPath provides the canonical sort order for a set of resources. Use with +// sort.Stable for deterministic sorting. +type ByPath []Resource + +func (bp ByPath) Len() int { return len(bp) } +func (bp ByPath) Swap(i, j int) { bp[i], bp[j] = bp[j], bp[i] } +func (bp ByPath) Less(i, j int) bool { return bp[i].Path() < bp[j].Path() } + +type XAttrer interface { + XAttrs() map[string][]byte +} + +// Hardlinkable is an interface that a resource type satisfies if it can be a +// hardlink target. +type Hardlinkable interface { + // Paths returns all paths of the resource, including the primary path + // returned by Resource.Path. If len(Paths()) > 1, the resource is a hard + // link. + Paths() []string +} + +type RegularFile interface { + Resource + XAttrer + Hardlinkable + + Size() int64 + Digests() []digest.Digest +} + +// Merge two or more Resources into new file. Typically, this should be +// used to merge regular files as hardlinks. If the files are not identical, +// other than Paths and Digests, the merge will fail and an error will be +// returned. +func Merge(fs ...Resource) (Resource, error) { + if len(fs) < 1 { + return nil, fmt.Errorf("please provide a resource to merge") + } + + if len(fs) == 1 { + return fs[0], nil + } + + var paths []string + var digests []digest.Digest + bypath := map[string][]Resource{} + + // The attributes are all compared against the first to make sure they + // agree before adding to the above collections. If any of these don't + // correctly validate, the merge fails. + prototype := fs[0] + xattrs := make(map[string][]byte) + + // initialize xattrs for use below. All files must have same xattrs. + if prototypeXAttrer, ok := prototype.(XAttrer); ok { + for attr, value := range prototypeXAttrer.XAttrs() { + xattrs[attr] = value + } + } + + for _, f := range fs { + h, isHardlinkable := f.(Hardlinkable) + if !isHardlinkable { + return nil, errNotAHardLink + } + + if f.Mode() != prototype.Mode() { + return nil, fmt.Errorf("modes do not match: %v != %v", f.Mode(), prototype.Mode()) + } + + if f.UID() != prototype.UID() { + return nil, fmt.Errorf("uid does not match: %v != %v", f.UID(), prototype.UID()) + } + + if f.GID() != prototype.GID() { + return nil, fmt.Errorf("gid does not match: %v != %v", f.GID(), prototype.GID()) + } + + if xattrer, ok := f.(XAttrer); ok { + fxattrs := xattrer.XAttrs() + if !reflect.DeepEqual(fxattrs, xattrs) { + return nil, fmt.Errorf("resource %q xattrs do not match: %v != %v", fxattrs, xattrs) + } + } + + for _, p := range h.Paths() { + pfs, ok := bypath[p] + if !ok { + // ensure paths are unique by only appending on a new path. + paths = append(paths, p) + } + + bypath[p] = append(pfs, f) + } + + if regFile, isRegFile := f.(RegularFile); isRegFile { + prototypeRegFile, prototypeIsRegFile := prototype.(RegularFile) + if !prototypeIsRegFile { + return nil, errors.New("prototype is not a regular file") + } + + if regFile.Size() != prototypeRegFile.Size() { + return nil, fmt.Errorf("size does not match: %v != %v", regFile.Size(), prototypeRegFile.Size()) + } + + digests = append(digests, regFile.Digests()...) + } else if device, isDevice := f.(Device); isDevice { + prototypeDevice, prototypeIsDevice := prototype.(Device) + if !prototypeIsDevice { + return nil, errors.New("prototype is not a device") + } + + if device.Major() != prototypeDevice.Major() { + return nil, fmt.Errorf("major number does not match: %v != %v", device.Major(), prototypeDevice.Major()) + } + if device.Minor() != prototypeDevice.Minor() { + return nil, fmt.Errorf("minor number does not match: %v != %v", device.Minor(), prototypeDevice.Minor()) + } + } else if _, isNamedPipe := f.(NamedPipe); isNamedPipe { + _, prototypeIsNamedPipe := prototype.(NamedPipe) + if !prototypeIsNamedPipe { + return nil, errors.New("prototype is not a named pipe") + } + } else { + return nil, errNotAHardLink + } + } + + sort.Stable(sort.StringSlice(paths)) + + // Choose a "canonical" file. Really, it is just the first file to sort + // against. We also effectively select the very first digest as the + // "canonical" one for this file. + first := bypath[paths[0]][0] + + resource := resource{ + paths: paths, + mode: first.Mode(), + uid: first.UID(), + gid: first.GID(), + xattrs: xattrs, + } + + switch typedF := first.(type) { + case RegularFile: + var err error + digests, err = uniqifyDigests(digests...) + if err != nil { + return nil, err + } + + return ®ularFile{ + resource: resource, + size: typedF.Size(), + digests: digests, + }, nil + case Device: + return &device{ + resource: resource, + major: typedF.Major(), + minor: typedF.Minor(), + }, nil + + case NamedPipe: + return &namedPipe{ + resource: resource, + }, nil + + default: + return nil, errNotAHardLink + } +} + +type Directory interface { + Resource + XAttrer + + // Directory is a no-op method to identify directory objects by interface. + Directory() +} + +type SymLink interface { + Resource + + // Target returns the target of the symlink contained in the . + Target() string +} + +type NamedPipe interface { + Resource + Hardlinkable + XAttrer + + // Pipe is a no-op method to allow consistent resolution of NamedPipe + // interface. + Pipe() +} + +type Device interface { + Resource + Hardlinkable + XAttrer + + Major() uint64 + Minor() uint64 +} + +type resource struct { + paths []string + mode os.FileMode + uid, gid string + xattrs map[string][]byte +} + +var _ Resource = &resource{} + +// newBaseResource returns a *resource, populated with data from p and fi, +// where p will be populated directly. +func newBaseResource(p string, mode os.FileMode, uid, gid string) (*resource, error) { + return &resource{ + paths: []string{p}, + mode: mode, + + uid: uid, + gid: gid, + + // NOTE(stevvooe): Population of shared xattrs field is deferred to + // the resource types that populate it. Since they are a property of + // the context, they must set there. + }, nil +} + +func (r *resource) Path() string { + if len(r.paths) < 1 { + return "" + } + + return r.paths[0] +} + +func (r *resource) Mode() os.FileMode { + return r.mode +} + +func (r *resource) UID() string { + return r.uid +} + +func (r *resource) GID() string { + return r.gid +} + +type regularFile struct { + resource + size int64 + digests []digest.Digest +} + +var _ RegularFile = ®ularFile{} + +// newRegularFile returns the RegularFile, using the populated base resource +// and one or more digests of the content. +func newRegularFile(base resource, paths []string, size int64, dgsts ...digest.Digest) (RegularFile, error) { + if !base.Mode().IsRegular() { + return nil, fmt.Errorf("not a regular file") + } + + base.paths = make([]string, len(paths)) + copy(base.paths, paths) + + // make our own copy of digests + ds := make([]digest.Digest, len(dgsts)) + copy(ds, dgsts) + + return ®ularFile{ + resource: base, + size: size, + digests: ds, + }, nil +} + +func (rf *regularFile) Paths() []string { + paths := make([]string, len(rf.paths)) + copy(paths, rf.paths) + return paths +} + +func (rf *regularFile) Size() int64 { + return rf.size +} + +func (rf *regularFile) Digests() []digest.Digest { + digests := make([]digest.Digest, len(rf.digests)) + copy(digests, rf.digests) + return digests +} + +func (rf *regularFile) XAttrs() map[string][]byte { + xattrs := make(map[string][]byte, len(rf.xattrs)) + + for attr, value := range rf.xattrs { + xattrs[attr] = append(xattrs[attr], value...) + } + + return xattrs +} + +type directory struct { + resource +} + +var _ Directory = &directory{} + +func newDirectory(base resource) (Directory, error) { + if !base.Mode().IsDir() { + return nil, fmt.Errorf("not a directory") + } + + return &directory{ + resource: base, + }, nil +} + +func (d *directory) Directory() {} + +func (d *directory) XAttrs() map[string][]byte { + xattrs := make(map[string][]byte, len(d.xattrs)) + + for attr, value := range d.xattrs { + xattrs[attr] = append(xattrs[attr], value...) + } + + return xattrs +} + +type symLink struct { + resource + target string +} + +var _ SymLink = &symLink{} + +func newSymLink(base resource, target string) (SymLink, error) { + if base.Mode()&os.ModeSymlink == 0 { + return nil, fmt.Errorf("not a symlink") + } + + return &symLink{ + resource: base, + target: target, + }, nil +} + +func (l *symLink) Target() string { + return l.target +} + +type namedPipe struct { + resource +} + +var _ NamedPipe = &namedPipe{} + +func newNamedPipe(base resource, paths []string) (NamedPipe, error) { + if base.Mode()&os.ModeNamedPipe == 0 { + return nil, fmt.Errorf("not a namedpipe") + } + + base.paths = make([]string, len(paths)) + copy(base.paths, paths) + + return &namedPipe{ + resource: base, + }, nil +} + +func (np *namedPipe) Pipe() {} + +func (np *namedPipe) Paths() []string { + paths := make([]string, len(np.paths)) + copy(paths, np.paths) + return paths +} + +func (np *namedPipe) XAttrs() map[string][]byte { + xattrs := make(map[string][]byte, len(np.xattrs)) + + for attr, value := range np.xattrs { + xattrs[attr] = append(xattrs[attr], value...) + } + + return xattrs +} + +type device struct { + resource + major, minor uint64 +} + +var _ Device = &device{} + +func newDevice(base resource, paths []string, major, minor uint64) (Device, error) { + if base.Mode()&os.ModeDevice == 0 { + return nil, fmt.Errorf("not a device") + } + + base.paths = make([]string, len(paths)) + copy(base.paths, paths) + + return &device{ + resource: base, + major: major, + minor: minor, + }, nil +} + +func (d *device) Paths() []string { + paths := make([]string, len(d.paths)) + copy(paths, d.paths) + return paths +} + +func (d *device) XAttrs() map[string][]byte { + xattrs := make(map[string][]byte, len(d.xattrs)) + + for attr, value := range d.xattrs { + xattrs[attr] = append(xattrs[attr], value...) + } + + return xattrs +} + +func (d device) Major() uint64 { + return d.major +} + +func (d device) Minor() uint64 { + return d.minor +} + +// toProto converts a resource to a protobuf record. We'd like to push this +// the individual types but we want to keep this all together during +// prototyping. +func toProto(resource Resource) *pb.Resource { + b := &pb.Resource{ + Path: []string{resource.Path()}, + Mode: uint32(resource.Mode()), + Uid: resource.UID(), + Gid: resource.GID(), + } + + if xattrer, ok := resource.(XAttrer); ok { + b.Xattr = xattrer.XAttrs() + } + + switch r := resource.(type) { + case RegularFile: + b.Path = r.Paths() + b.Size = uint64(r.Size()) + + for _, dgst := range r.Digests() { + b.Digest = append(b.Digest, dgst.String()) + } + case SymLink: + b.Target = r.Target() + case Device: + b.Major, b.Minor = r.Major(), r.Minor() + b.Path = r.Paths() + case NamedPipe: + b.Path = r.Paths() + } + + // enforce a few stability guarantees that may not be provided by the + // resource implementation. + sort.Strings(b.Path) + + return b +} + +// fromProto converts from a protobuf Resource to a Resource interface. +func fromProto(b *pb.Resource) (Resource, error) { + base, err := newBaseResource(b.Path[0], os.FileMode(b.Mode), b.Uid, b.Gid) + if err != nil { + return nil, err + } + + base.xattrs = make(map[string][]byte, len(b.Xattr)) + + for attr, value := range b.Xattr { + base.xattrs[attr] = append(base.xattrs[attr], value...) + } + + switch { + case base.Mode().IsRegular(): + dgsts := make([]digest.Digest, len(b.Digest)) + for i, dgst := range b.Digest { + // TODO(stevvooe): Should we be validating at this point? + dgsts[i] = digest.Digest(dgst) + } + + return newRegularFile(*base, b.Path, int64(b.Size), dgsts...) + case base.Mode().IsDir(): + return newDirectory(*base) + case base.Mode()&os.ModeSymlink != 0: + return newSymLink(*base, b.Target) + case base.Mode()&os.ModeNamedPipe != 0: + return newNamedPipe(*base, b.Path) + case base.Mode()&os.ModeDevice != 0: + return newDevice(*base, b.Path, b.Major, b.Minor) + } + + return nil, fmt.Errorf("unknown resource record (%#v): %s", b, base.Mode()) +} + +// NOTE(stevvooe): An alternative model that supports inline declaration. +// Convenient for unit testing where inline declarations may be desirable but +// creates an awkward API for the standard use case. + +// type ResourceKind int + +// const ( +// ResourceRegularFile = iota + 1 +// ResourceDirectory +// ResourceSymLink +// Resource +// ) + +// type Resource struct { +// Kind ResourceKind +// Paths []string +// Mode os.FileMode +// UID string +// GID string +// Size int64 +// Digests []digest.Digest +// Target string +// Major, Minor int +// XAttrs map[string][]byte +// } + +// type RegularFile struct { +// Paths []string +// Size int64 +// Digests []digest.Digest +// Perm os.FileMode // os.ModePerm + sticky, setuid, setgid +// } diff --git a/vendor/github.com/stevvooe/continuity/sysx/asm.s b/vendor/github.com/stevvooe/continuity/sysx/asm.s new file mode 100644 index 0000000..8ed2fdb --- /dev/null +++ b/vendor/github.com/stevvooe/continuity/sysx/asm.s @@ -0,0 +1,10 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !gccgo + +#include "textflag.h" + +TEXT ·use(SB),NOSPLIT,$0 + RET diff --git a/vendor/github.com/stevvooe/continuity/sysx/chmod_darwin.go b/vendor/github.com/stevvooe/continuity/sysx/chmod_darwin.go new file mode 100644 index 0000000..e3ae2b7 --- /dev/null +++ b/vendor/github.com/stevvooe/continuity/sysx/chmod_darwin.go @@ -0,0 +1,18 @@ +package sysx + +const ( + // AtSymlinkNoFollow defined from AT_SYMLINK_NOFOLLOW in + AtSymlinkNofollow = 0x20 +) + +const ( + + // SYS_FCHMODAT defined from golang.org/sys/unix + SYS_FCHMODAT = 467 +) + +// These functions will be generated by generate.sh +// $ GOOS=darwin GOARCH=386 ./generate.sh chmod +// $ GOOS=darwin GOARCH=amd64 ./generate.sh chmod + +//sys Fchmodat(dirfd int, path string, mode uint32, flags int) (err error) diff --git a/vendor/github.com/stevvooe/continuity/sysx/chmod_darwin_386.go b/vendor/github.com/stevvooe/continuity/sysx/chmod_darwin_386.go new file mode 100644 index 0000000..5a8cf5b --- /dev/null +++ b/vendor/github.com/stevvooe/continuity/sysx/chmod_darwin_386.go @@ -0,0 +1,25 @@ +// mksyscall.pl -l32 chmod_darwin.go +// MACHINE GENERATED BY THE COMMAND ABOVE; DO NOT EDIT + +package sysx + +import ( + "syscall" + "unsafe" +) + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + +func Fchmodat(dirfd int, path string, mode uint32, flags int) (err error) { + var _p0 *byte + _p0, err = syscall.BytePtrFromString(path) + if err != nil { + return + } + _, _, e1 := syscall.Syscall6(SYS_FCHMODAT, uintptr(dirfd), uintptr(unsafe.Pointer(_p0)), uintptr(mode), uintptr(flags), 0, 0) + use(unsafe.Pointer(_p0)) + if e1 != 0 { + err = errnoErr(e1) + } + return +} diff --git a/vendor/github.com/stevvooe/continuity/sysx/chmod_darwin_amd64.go b/vendor/github.com/stevvooe/continuity/sysx/chmod_darwin_amd64.go new file mode 100644 index 0000000..3287d1d --- /dev/null +++ b/vendor/github.com/stevvooe/continuity/sysx/chmod_darwin_amd64.go @@ -0,0 +1,25 @@ +// mksyscall.pl chmod_darwin.go +// MACHINE GENERATED BY THE COMMAND ABOVE; DO NOT EDIT + +package sysx + +import ( + "syscall" + "unsafe" +) + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + +func Fchmodat(dirfd int, path string, mode uint32, flags int) (err error) { + var _p0 *byte + _p0, err = syscall.BytePtrFromString(path) + if err != nil { + return + } + _, _, e1 := syscall.Syscall6(SYS_FCHMODAT, uintptr(dirfd), uintptr(unsafe.Pointer(_p0)), uintptr(mode), uintptr(flags), 0, 0) + use(unsafe.Pointer(_p0)) + if e1 != 0 { + err = errnoErr(e1) + } + return +} diff --git a/vendor/github.com/stevvooe/continuity/sysx/chmod_linux.go b/vendor/github.com/stevvooe/continuity/sysx/chmod_linux.go new file mode 100644 index 0000000..89df6d3 --- /dev/null +++ b/vendor/github.com/stevvooe/continuity/sysx/chmod_linux.go @@ -0,0 +1,12 @@ +package sysx + +import "syscall" + +const ( + // AtSymlinkNoFollow defined from AT_SYMLINK_NOFOLLOW in /usr/include/linux/fcntl.h + AtSymlinkNofollow = 0x100 +) + +func Fchmodat(dirfd int, path string, mode uint32, flags int) error { + return syscall.Fchmodat(dirfd, path, mode, flags) +} diff --git a/vendor/github.com/stevvooe/continuity/sysx/sys.go b/vendor/github.com/stevvooe/continuity/sysx/sys.go new file mode 100644 index 0000000..0bb1676 --- /dev/null +++ b/vendor/github.com/stevvooe/continuity/sysx/sys.go @@ -0,0 +1,37 @@ +package sysx + +import ( + "syscall" + "unsafe" +) + +var _zero uintptr + +// use is a no-op, but the compiler cannot see that it is. +// Calling use(p) ensures that p is kept live until that point. +//go:noescape +func use(p unsafe.Pointer) + +// Do the interface allocations only once for common +// Errno values. +var ( + errEAGAIN error = syscall.EAGAIN + errEINVAL error = syscall.EINVAL + errENOENT error = syscall.ENOENT +) + +// errnoErr returns common boxed Errno values, to prevent +// allocations at runtime. +func errnoErr(e syscall.Errno) error { + switch e { + case 0: + return nil + case syscall.EAGAIN: + return errEAGAIN + case syscall.EINVAL: + return errEINVAL + case syscall.ENOENT: + return errENOENT + } + return e +} diff --git a/vendor/github.com/stevvooe/continuity/sysx/xattr.go b/vendor/github.com/stevvooe/continuity/sysx/xattr.go new file mode 100644 index 0000000..0c07101 --- /dev/null +++ b/vendor/github.com/stevvooe/continuity/sysx/xattr.go @@ -0,0 +1,64 @@ +package sysx + +import ( + "bytes" + "syscall" +) + +const defaultXattrBufferSize = 5 + +type listxattrFunc func(path string, dest []byte) (int, error) + +func listxattrAll(path string, listFunc listxattrFunc) ([]string, error) { + var p []byte // nil on first execution + + for { + n, err := listFunc(path, p) // first call gets buffer size. + if err != nil { + return nil, err + } + + if n > len(p) { + p = make([]byte, n) + continue + } + + p = p[:n] + + ps := bytes.Split(bytes.TrimSuffix(p, []byte{0}), []byte{0}) + var entries []string + for _, p := range ps { + s := string(p) + if s != "" { + entries = append(entries, s) + } + } + + return entries, nil + } +} + +type getxattrFunc func(string, string, []byte) (int, error) + +func getxattrAll(path, attr string, getFunc getxattrFunc) ([]byte, error) { + p := make([]byte, defaultXattrBufferSize) + for { + n, err := getFunc(path, attr, p) + if err != nil { + if errno, ok := err.(syscall.Errno); ok && errno == syscall.ERANGE { + p = make([]byte, len(p)*2) // this can't be ideal. + continue // try again! + } + + return nil, err + } + + // realloc to correct size and repeat + if n > len(p) { + p = make([]byte, n) + continue + } + + return p[:n], nil + } +} diff --git a/vendor/github.com/stevvooe/continuity/sysx/xattr_darwin.go b/vendor/github.com/stevvooe/continuity/sysx/xattr_darwin.go new file mode 100644 index 0000000..1164a7d --- /dev/null +++ b/vendor/github.com/stevvooe/continuity/sysx/xattr_darwin.go @@ -0,0 +1,71 @@ +package sysx + +// These functions will be generated by generate.sh +// $ GOOS=darwin GOARCH=386 ./generate.sh xattr +// $ GOOS=darwin GOARCH=amd64 ./generate.sh xattr + +//sys getxattr(path string, attr string, dest []byte, pos int, options int) (sz int, err error) +//sys setxattr(path string, attr string, data []byte, flags int) (err error) +//sys removexattr(path string, attr string, options int) (err error) +//sys listxattr(path string, dest []byte, options int) (sz int, err error) +//sys Fchmodat(dirfd int, path string, mode uint32, flags int) (err error) + +const ( + xattrNoFollow = 0x01 +) + +func listxattrFollow(path string, dest []byte) (sz int, err error) { + return listxattr(path, dest, 0) +} + +// Listxattr calls syscall getxattr +func Listxattr(path string) ([]string, error) { + return listxattrAll(path, listxattrFollow) +} + +// Removexattr calls syscall getxattr +func Removexattr(path string, attr string) (err error) { + return removexattr(path, attr, 0) +} + +// Setxattr calls syscall setxattr +func Setxattr(path string, attr string, data []byte, flags int) (err error) { + return setxattr(path, attr, data, flags) +} + +func getxattrFollow(path, attr string, dest []byte) (sz int, err error) { + return getxattr(path, attr, dest, 0, 0) +} + +// Getxattr calls syscall getxattr +func Getxattr(path, attr string) ([]byte, error) { + return getxattrAll(path, attr, getxattrFollow) +} + +func listxattrNoFollow(path string, dest []byte) (sz int, err error) { + return listxattr(path, dest, xattrNoFollow) +} + +// LListxattr calls syscall listxattr with XATTR_NOFOLLOW +func LListxattr(path string) ([]string, error) { + return listxattrAll(path, listxattrNoFollow) +} + +// LRemovexattr calls syscall removexattr with XATTR_NOFOLLOW +func LRemovexattr(path string, attr string) (err error) { + return removexattr(path, attr, xattrNoFollow) +} + +// Setxattr calls syscall setxattr with XATTR_NOFOLLOW +func LSetxattr(path string, attr string, data []byte, flags int) (err error) { + return setxattr(path, attr, data, flags|xattrNoFollow) +} + +func getxattrNoFollow(path, attr string, dest []byte) (sz int, err error) { + return getxattr(path, attr, dest, 0, xattrNoFollow) +} + +// LGetxattr calls syscall getxattr with XATTR_NOFOLLOW +func LGetxattr(path, attr string) ([]byte, error) { + return getxattrAll(path, attr, getxattrNoFollow) +} diff --git a/vendor/github.com/stevvooe/continuity/sysx/xattr_darwin_386.go b/vendor/github.com/stevvooe/continuity/sysx/xattr_darwin_386.go new file mode 100644 index 0000000..aa896b5 --- /dev/null +++ b/vendor/github.com/stevvooe/continuity/sysx/xattr_darwin_386.go @@ -0,0 +1,111 @@ +// mksyscall.pl -l32 xattr_darwin.go +// MACHINE GENERATED BY THE COMMAND ABOVE; DO NOT EDIT + +package sysx + +import ( + "syscall" + "unsafe" +) + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + +func getxattr(path string, attr string, dest []byte, pos int, options int) (sz int, err error) { + var _p0 *byte + _p0, err = syscall.BytePtrFromString(path) + if err != nil { + return + } + var _p1 *byte + _p1, err = syscall.BytePtrFromString(attr) + if err != nil { + return + } + var _p2 unsafe.Pointer + if len(dest) > 0 { + _p2 = unsafe.Pointer(&dest[0]) + } else { + _p2 = unsafe.Pointer(&_zero) + } + r0, _, e1 := syscall.Syscall6(syscall.SYS_GETXATTR, uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(_p1)), uintptr(_p2), uintptr(len(dest)), uintptr(pos), uintptr(options)) + use(unsafe.Pointer(_p0)) + use(unsafe.Pointer(_p1)) + sz = int(r0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + +func setxattr(path string, attr string, data []byte, flags int) (err error) { + var _p0 *byte + _p0, err = syscall.BytePtrFromString(path) + if err != nil { + return + } + var _p1 *byte + _p1, err = syscall.BytePtrFromString(attr) + if err != nil { + return + } + var _p2 unsafe.Pointer + if len(data) > 0 { + _p2 = unsafe.Pointer(&data[0]) + } else { + _p2 = unsafe.Pointer(&_zero) + } + _, _, e1 := syscall.Syscall6(syscall.SYS_SETXATTR, uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(_p1)), uintptr(_p2), uintptr(len(data)), uintptr(flags), 0) + use(unsafe.Pointer(_p0)) + use(unsafe.Pointer(_p1)) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + +func removexattr(path string, attr string, options int) (err error) { + var _p0 *byte + _p0, err = syscall.BytePtrFromString(path) + if err != nil { + return + } + var _p1 *byte + _p1, err = syscall.BytePtrFromString(attr) + if err != nil { + return + } + _, _, e1 := syscall.Syscall(syscall.SYS_REMOVEXATTR, uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(_p1)), uintptr(options)) + use(unsafe.Pointer(_p0)) + use(unsafe.Pointer(_p1)) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + +func listxattr(path string, dest []byte, options int) (sz int, err error) { + var _p0 *byte + _p0, err = syscall.BytePtrFromString(path) + if err != nil { + return + } + var _p1 unsafe.Pointer + if len(dest) > 0 { + _p1 = unsafe.Pointer(&dest[0]) + } else { + _p1 = unsafe.Pointer(&_zero) + } + r0, _, e1 := syscall.Syscall6(syscall.SYS_LISTXATTR, uintptr(unsafe.Pointer(_p0)), uintptr(_p1), uintptr(len(dest)), uintptr(options), 0, 0) + use(unsafe.Pointer(_p0)) + sz = int(r0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} diff --git a/vendor/github.com/stevvooe/continuity/sysx/xattr_darwin_amd64.go b/vendor/github.com/stevvooe/continuity/sysx/xattr_darwin_amd64.go new file mode 100644 index 0000000..6ff27e2 --- /dev/null +++ b/vendor/github.com/stevvooe/continuity/sysx/xattr_darwin_amd64.go @@ -0,0 +1,111 @@ +// mksyscall.pl xattr_darwin.go +// MACHINE GENERATED BY THE COMMAND ABOVE; DO NOT EDIT + +package sysx + +import ( + "syscall" + "unsafe" +) + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + +func getxattr(path string, attr string, dest []byte, pos int, options int) (sz int, err error) { + var _p0 *byte + _p0, err = syscall.BytePtrFromString(path) + if err != nil { + return + } + var _p1 *byte + _p1, err = syscall.BytePtrFromString(attr) + if err != nil { + return + } + var _p2 unsafe.Pointer + if len(dest) > 0 { + _p2 = unsafe.Pointer(&dest[0]) + } else { + _p2 = unsafe.Pointer(&_zero) + } + r0, _, e1 := syscall.Syscall6(syscall.SYS_GETXATTR, uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(_p1)), uintptr(_p2), uintptr(len(dest)), uintptr(pos), uintptr(options)) + use(unsafe.Pointer(_p0)) + use(unsafe.Pointer(_p1)) + sz = int(r0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + +func setxattr(path string, attr string, data []byte, flags int) (err error) { + var _p0 *byte + _p0, err = syscall.BytePtrFromString(path) + if err != nil { + return + } + var _p1 *byte + _p1, err = syscall.BytePtrFromString(attr) + if err != nil { + return + } + var _p2 unsafe.Pointer + if len(data) > 0 { + _p2 = unsafe.Pointer(&data[0]) + } else { + _p2 = unsafe.Pointer(&_zero) + } + _, _, e1 := syscall.Syscall6(syscall.SYS_SETXATTR, uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(_p1)), uintptr(_p2), uintptr(len(data)), uintptr(flags), 0) + use(unsafe.Pointer(_p0)) + use(unsafe.Pointer(_p1)) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + +func removexattr(path string, attr string, options int) (err error) { + var _p0 *byte + _p0, err = syscall.BytePtrFromString(path) + if err != nil { + return + } + var _p1 *byte + _p1, err = syscall.BytePtrFromString(attr) + if err != nil { + return + } + _, _, e1 := syscall.Syscall(syscall.SYS_REMOVEXATTR, uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(_p1)), uintptr(options)) + use(unsafe.Pointer(_p0)) + use(unsafe.Pointer(_p1)) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + +func listxattr(path string, dest []byte, options int) (sz int, err error) { + var _p0 *byte + _p0, err = syscall.BytePtrFromString(path) + if err != nil { + return + } + var _p1 unsafe.Pointer + if len(dest) > 0 { + _p1 = unsafe.Pointer(&dest[0]) + } else { + _p1 = unsafe.Pointer(&_zero) + } + r0, _, e1 := syscall.Syscall6(syscall.SYS_LISTXATTR, uintptr(unsafe.Pointer(_p0)), uintptr(_p1), uintptr(len(dest)), uintptr(options), 0, 0) + use(unsafe.Pointer(_p0)) + sz = int(r0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} diff --git a/vendor/github.com/stevvooe/continuity/sysx/xattr_linux.go b/vendor/github.com/stevvooe/continuity/sysx/xattr_linux.go new file mode 100644 index 0000000..794439a --- /dev/null +++ b/vendor/github.com/stevvooe/continuity/sysx/xattr_linux.go @@ -0,0 +1,58 @@ +package sysx + +import "syscall" + +// These functions will be generated by generate.sh +// $ GOOS=linux GOARCH=386 ./generate.sh xattr +// $ GOOS=linux GOARCH=amd64 ./generate.sh xattr +// $ GOOS=linux GOARCH=arm ./generate.sh xattr +// $ GOOS=linux GOARCH=arm64 ./generate.sh xattr + +// Listxattr calls syscall listxattr and reads all content +// and returns a string array +func Listxattr(path string) ([]string, error) { + return listxattrAll(path, syscall.Listxattr) +} + +// Removexattr calls syscall removexattr +func Removexattr(path string, attr string) (err error) { + return syscall.Removexattr(path, attr) +} + +// Setxattr calls syscall setxattr +func Setxattr(path string, attr string, data []byte, flags int) (err error) { + return syscall.Setxattr(path, attr, data, flags) +} + +// Getxattr calls syscall getxattr +func Getxattr(path, attr string) ([]byte, error) { + return getxattrAll(path, attr, syscall.Getxattr) +} + +//sys llistxattr(path string, dest []byte) (sz int, err error) + +// LListxattr lists xattrs, not following symlinks +func LListxattr(path string) ([]string, error) { + return listxattrAll(path, llistxattr) +} + +//sys lremovexattr(path string, attr string) (err error) + +// LRemovexattr removes an xattr, not following symlinks +func LRemovexattr(path string, attr string) (err error) { + return lremovexattr(path, attr) +} + +//sys lsetxattr(path string, attr string, data []byte, flags int) (err error) + +// LSetxattr sets an xattr, not following symlinks +func LSetxattr(path string, attr string, data []byte, flags int) (err error) { + return lsetxattr(path, attr, data, flags) +} + +//sys lgetxattr(path string, attr string, dest []byte) (sz int, err error) + +// LGetxattr gets an xattr, not following symlinks +func LGetxattr(path, attr string) ([]byte, error) { + return getxattrAll(path, attr, lgetxattr) +} diff --git a/vendor/github.com/stevvooe/continuity/sysx/xattr_linux_386.go b/vendor/github.com/stevvooe/continuity/sysx/xattr_linux_386.go new file mode 100644 index 0000000..c3e5c8e --- /dev/null +++ b/vendor/github.com/stevvooe/continuity/sysx/xattr_linux_386.go @@ -0,0 +1,111 @@ +// mksyscall.pl -l32 xattr_linux.go +// MACHINE GENERATED BY THE COMMAND ABOVE; DO NOT EDIT + +package sysx + +import ( + "syscall" + "unsafe" +) + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + +func llistxattr(path string, dest []byte) (sz int, err error) { + var _p0 *byte + _p0, err = syscall.BytePtrFromString(path) + if err != nil { + return + } + var _p1 unsafe.Pointer + if len(dest) > 0 { + _p1 = unsafe.Pointer(&dest[0]) + } else { + _p1 = unsafe.Pointer(&_zero) + } + r0, _, e1 := syscall.Syscall(syscall.SYS_LLISTXATTR, uintptr(unsafe.Pointer(_p0)), uintptr(_p1), uintptr(len(dest))) + use(unsafe.Pointer(_p0)) + sz = int(r0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + +func lremovexattr(path string, attr string) (err error) { + var _p0 *byte + _p0, err = syscall.BytePtrFromString(path) + if err != nil { + return + } + var _p1 *byte + _p1, err = syscall.BytePtrFromString(attr) + if err != nil { + return + } + _, _, e1 := syscall.Syscall(syscall.SYS_LREMOVEXATTR, uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(_p1)), 0) + use(unsafe.Pointer(_p0)) + use(unsafe.Pointer(_p1)) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + +func lsetxattr(path string, attr string, data []byte, flags int) (err error) { + var _p0 *byte + _p0, err = syscall.BytePtrFromString(path) + if err != nil { + return + } + var _p1 *byte + _p1, err = syscall.BytePtrFromString(attr) + if err != nil { + return + } + var _p2 unsafe.Pointer + if len(data) > 0 { + _p2 = unsafe.Pointer(&data[0]) + } else { + _p2 = unsafe.Pointer(&_zero) + } + _, _, e1 := syscall.Syscall6(syscall.SYS_LSETXATTR, uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(_p1)), uintptr(_p2), uintptr(len(data)), uintptr(flags), 0) + use(unsafe.Pointer(_p0)) + use(unsafe.Pointer(_p1)) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + +func lgetxattr(path string, attr string, dest []byte) (sz int, err error) { + var _p0 *byte + _p0, err = syscall.BytePtrFromString(path) + if err != nil { + return + } + var _p1 *byte + _p1, err = syscall.BytePtrFromString(attr) + if err != nil { + return + } + var _p2 unsafe.Pointer + if len(dest) > 0 { + _p2 = unsafe.Pointer(&dest[0]) + } else { + _p2 = unsafe.Pointer(&_zero) + } + r0, _, e1 := syscall.Syscall6(syscall.SYS_LGETXATTR, uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(_p1)), uintptr(_p2), uintptr(len(dest)), 0, 0) + use(unsafe.Pointer(_p0)) + use(unsafe.Pointer(_p1)) + sz = int(r0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} diff --git a/vendor/github.com/stevvooe/continuity/sysx/xattr_linux_amd64.go b/vendor/github.com/stevvooe/continuity/sysx/xattr_linux_amd64.go new file mode 100644 index 0000000..dec46fa --- /dev/null +++ b/vendor/github.com/stevvooe/continuity/sysx/xattr_linux_amd64.go @@ -0,0 +1,111 @@ +// mksyscall.pl xattr_linux.go +// MACHINE GENERATED BY THE COMMAND ABOVE; DO NOT EDIT + +package sysx + +import ( + "syscall" + "unsafe" +) + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + +func llistxattr(path string, dest []byte) (sz int, err error) { + var _p0 *byte + _p0, err = syscall.BytePtrFromString(path) + if err != nil { + return + } + var _p1 unsafe.Pointer + if len(dest) > 0 { + _p1 = unsafe.Pointer(&dest[0]) + } else { + _p1 = unsafe.Pointer(&_zero) + } + r0, _, e1 := syscall.Syscall(syscall.SYS_LLISTXATTR, uintptr(unsafe.Pointer(_p0)), uintptr(_p1), uintptr(len(dest))) + use(unsafe.Pointer(_p0)) + sz = int(r0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + +func lremovexattr(path string, attr string) (err error) { + var _p0 *byte + _p0, err = syscall.BytePtrFromString(path) + if err != nil { + return + } + var _p1 *byte + _p1, err = syscall.BytePtrFromString(attr) + if err != nil { + return + } + _, _, e1 := syscall.Syscall(syscall.SYS_LREMOVEXATTR, uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(_p1)), 0) + use(unsafe.Pointer(_p0)) + use(unsafe.Pointer(_p1)) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + +func lsetxattr(path string, attr string, data []byte, flags int) (err error) { + var _p0 *byte + _p0, err = syscall.BytePtrFromString(path) + if err != nil { + return + } + var _p1 *byte + _p1, err = syscall.BytePtrFromString(attr) + if err != nil { + return + } + var _p2 unsafe.Pointer + if len(data) > 0 { + _p2 = unsafe.Pointer(&data[0]) + } else { + _p2 = unsafe.Pointer(&_zero) + } + _, _, e1 := syscall.Syscall6(syscall.SYS_LSETXATTR, uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(_p1)), uintptr(_p2), uintptr(len(data)), uintptr(flags), 0) + use(unsafe.Pointer(_p0)) + use(unsafe.Pointer(_p1)) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + +func lgetxattr(path string, attr string, dest []byte) (sz int, err error) { + var _p0 *byte + _p0, err = syscall.BytePtrFromString(path) + if err != nil { + return + } + var _p1 *byte + _p1, err = syscall.BytePtrFromString(attr) + if err != nil { + return + } + var _p2 unsafe.Pointer + if len(dest) > 0 { + _p2 = unsafe.Pointer(&dest[0]) + } else { + _p2 = unsafe.Pointer(&_zero) + } + r0, _, e1 := syscall.Syscall6(syscall.SYS_LGETXATTR, uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(_p1)), uintptr(_p2), uintptr(len(dest)), 0, 0) + use(unsafe.Pointer(_p0)) + use(unsafe.Pointer(_p1)) + sz = int(r0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} diff --git a/vendor/github.com/stevvooe/continuity/sysx/xattr_linux_arm.go b/vendor/github.com/stevvooe/continuity/sysx/xattr_linux_arm.go new file mode 100644 index 0000000..c3e5c8e --- /dev/null +++ b/vendor/github.com/stevvooe/continuity/sysx/xattr_linux_arm.go @@ -0,0 +1,111 @@ +// mksyscall.pl -l32 xattr_linux.go +// MACHINE GENERATED BY THE COMMAND ABOVE; DO NOT EDIT + +package sysx + +import ( + "syscall" + "unsafe" +) + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + +func llistxattr(path string, dest []byte) (sz int, err error) { + var _p0 *byte + _p0, err = syscall.BytePtrFromString(path) + if err != nil { + return + } + var _p1 unsafe.Pointer + if len(dest) > 0 { + _p1 = unsafe.Pointer(&dest[0]) + } else { + _p1 = unsafe.Pointer(&_zero) + } + r0, _, e1 := syscall.Syscall(syscall.SYS_LLISTXATTR, uintptr(unsafe.Pointer(_p0)), uintptr(_p1), uintptr(len(dest))) + use(unsafe.Pointer(_p0)) + sz = int(r0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + +func lremovexattr(path string, attr string) (err error) { + var _p0 *byte + _p0, err = syscall.BytePtrFromString(path) + if err != nil { + return + } + var _p1 *byte + _p1, err = syscall.BytePtrFromString(attr) + if err != nil { + return + } + _, _, e1 := syscall.Syscall(syscall.SYS_LREMOVEXATTR, uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(_p1)), 0) + use(unsafe.Pointer(_p0)) + use(unsafe.Pointer(_p1)) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + +func lsetxattr(path string, attr string, data []byte, flags int) (err error) { + var _p0 *byte + _p0, err = syscall.BytePtrFromString(path) + if err != nil { + return + } + var _p1 *byte + _p1, err = syscall.BytePtrFromString(attr) + if err != nil { + return + } + var _p2 unsafe.Pointer + if len(data) > 0 { + _p2 = unsafe.Pointer(&data[0]) + } else { + _p2 = unsafe.Pointer(&_zero) + } + _, _, e1 := syscall.Syscall6(syscall.SYS_LSETXATTR, uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(_p1)), uintptr(_p2), uintptr(len(data)), uintptr(flags), 0) + use(unsafe.Pointer(_p0)) + use(unsafe.Pointer(_p1)) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + +func lgetxattr(path string, attr string, dest []byte) (sz int, err error) { + var _p0 *byte + _p0, err = syscall.BytePtrFromString(path) + if err != nil { + return + } + var _p1 *byte + _p1, err = syscall.BytePtrFromString(attr) + if err != nil { + return + } + var _p2 unsafe.Pointer + if len(dest) > 0 { + _p2 = unsafe.Pointer(&dest[0]) + } else { + _p2 = unsafe.Pointer(&_zero) + } + r0, _, e1 := syscall.Syscall6(syscall.SYS_LGETXATTR, uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(_p1)), uintptr(_p2), uintptr(len(dest)), 0, 0) + use(unsafe.Pointer(_p0)) + use(unsafe.Pointer(_p1)) + sz = int(r0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} diff --git a/vendor/github.com/stevvooe/continuity/sysx/xattr_linux_arm64.go b/vendor/github.com/stevvooe/continuity/sysx/xattr_linux_arm64.go new file mode 100644 index 0000000..dec46fa --- /dev/null +++ b/vendor/github.com/stevvooe/continuity/sysx/xattr_linux_arm64.go @@ -0,0 +1,111 @@ +// mksyscall.pl xattr_linux.go +// MACHINE GENERATED BY THE COMMAND ABOVE; DO NOT EDIT + +package sysx + +import ( + "syscall" + "unsafe" +) + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + +func llistxattr(path string, dest []byte) (sz int, err error) { + var _p0 *byte + _p0, err = syscall.BytePtrFromString(path) + if err != nil { + return + } + var _p1 unsafe.Pointer + if len(dest) > 0 { + _p1 = unsafe.Pointer(&dest[0]) + } else { + _p1 = unsafe.Pointer(&_zero) + } + r0, _, e1 := syscall.Syscall(syscall.SYS_LLISTXATTR, uintptr(unsafe.Pointer(_p0)), uintptr(_p1), uintptr(len(dest))) + use(unsafe.Pointer(_p0)) + sz = int(r0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + +func lremovexattr(path string, attr string) (err error) { + var _p0 *byte + _p0, err = syscall.BytePtrFromString(path) + if err != nil { + return + } + var _p1 *byte + _p1, err = syscall.BytePtrFromString(attr) + if err != nil { + return + } + _, _, e1 := syscall.Syscall(syscall.SYS_LREMOVEXATTR, uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(_p1)), 0) + use(unsafe.Pointer(_p0)) + use(unsafe.Pointer(_p1)) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + +func lsetxattr(path string, attr string, data []byte, flags int) (err error) { + var _p0 *byte + _p0, err = syscall.BytePtrFromString(path) + if err != nil { + return + } + var _p1 *byte + _p1, err = syscall.BytePtrFromString(attr) + if err != nil { + return + } + var _p2 unsafe.Pointer + if len(data) > 0 { + _p2 = unsafe.Pointer(&data[0]) + } else { + _p2 = unsafe.Pointer(&_zero) + } + _, _, e1 := syscall.Syscall6(syscall.SYS_LSETXATTR, uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(_p1)), uintptr(_p2), uintptr(len(data)), uintptr(flags), 0) + use(unsafe.Pointer(_p0)) + use(unsafe.Pointer(_p1)) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + +func lgetxattr(path string, attr string, dest []byte) (sz int, err error) { + var _p0 *byte + _p0, err = syscall.BytePtrFromString(path) + if err != nil { + return + } + var _p1 *byte + _p1, err = syscall.BytePtrFromString(attr) + if err != nil { + return + } + var _p2 unsafe.Pointer + if len(dest) > 0 { + _p2 = unsafe.Pointer(&dest[0]) + } else { + _p2 = unsafe.Pointer(&_zero) + } + r0, _, e1 := syscall.Syscall6(syscall.SYS_LGETXATTR, uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(_p1)), uintptr(_p2), uintptr(len(dest)), 0, 0) + use(unsafe.Pointer(_p0)) + use(unsafe.Pointer(_p1)) + sz = int(r0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} From 574862fd892e43fa281ebb0f2f114d4e57e18b0b Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Tue, 31 Jan 2017 15:17:33 -0800 Subject: [PATCH 02/12] Add fs package Add diff comparison with support for double walking two trees for comparison or single walking a diff tree. Single walking requires further implementation for specific mount types. Add directory copy function which is intended to provide fastest possible local copy of file system directories without hardlinking. Add test package to make creating filesystems for test easy and comparisons deep and informative. Signed-off-by: Derek McGowan (github: dmcgowan) --- fs/copy.go | 106 +++++++++++ fs/copy_linux.go | 110 +++++++++++ fs/copy_test.go | 53 ++++++ fs/copy_windows.go | 27 +++ fs/diff.go | 360 +++++++++++++++++++++++++++++++++++ fs/diff_linux.go | 92 +++++++++ fs/diff_test.go | 304 +++++++++++++++++++++++++++++ fs/diff_windows.go | 15 ++ fs/fstest/compare.go | 37 ++++ fs/fstest/continuity_util.go | 189 ++++++++++++++++++ fs/fstest/file.go | 109 +++++++++++ fs/hardlink.go | 12 ++ fs/hardlink_unix.go | 33 ++++ fs/hardlink_windows.go | 7 + fs/path.go | 197 +++++++++++++++++++ fs/time.go | 21 ++ 16 files changed, 1672 insertions(+) create mode 100644 fs/copy.go create mode 100644 fs/copy_linux.go create mode 100644 fs/copy_test.go create mode 100644 fs/copy_windows.go create mode 100644 fs/diff.go create mode 100644 fs/diff_linux.go create mode 100644 fs/diff_test.go create mode 100644 fs/diff_windows.go create mode 100644 fs/fstest/compare.go create mode 100644 fs/fstest/continuity_util.go create mode 100644 fs/fstest/file.go create mode 100644 fs/hardlink.go create mode 100644 fs/hardlink_unix.go create mode 100644 fs/hardlink_windows.go create mode 100644 fs/path.go create mode 100644 fs/time.go diff --git a/fs/copy.go b/fs/copy.go new file mode 100644 index 0000000..0ffd7f2 --- /dev/null +++ b/fs/copy.go @@ -0,0 +1,106 @@ +package fs + +import ( + "io/ioutil" + "os" + "path/filepath" + + "github.com/pkg/errors" +) + +// CopyDirectory copies the directory from src to dst. +// Most efficient copy of files is attempted. +func CopyDirectory(dst, src string) error { + inodes := map[uint64]string{} + return copyDirectory(dst, src, inodes) +} + +func copyDirectory(dst, src string, inodes map[uint64]string) error { + stat, err := os.Stat(src) + if err != nil { + return errors.Wrapf(err, "failed to stat %s", src) + } + if !stat.IsDir() { + return errors.Errorf("source is not directory") + } + + if _, err := os.Stat(dst); err != nil { + if err := os.Mkdir(dst, stat.Mode()); err != nil { + return errors.Wrapf(err, "failed to mkdir %s", dst) + } + } else { + if err := os.Chmod(dst, stat.Mode()); err != nil { + return errors.Wrapf(err, "failed to chmod on %s", dst) + } + } + + fis, err := ioutil.ReadDir(src) + if err != nil { + return errors.Wrapf(err, "failed to read %s", src) + } + + if err := copyFileInfo(stat, dst); err != nil { + return errors.Wrapf(err, "failed to copy file info for %s", dst) + } + + for _, fi := range fis { + source := filepath.Join(src, fi.Name()) + target := filepath.Join(dst, fi.Name()) + if fi.IsDir() { + if err := copyDirectory(target, source, inodes); err != nil { + return err + } + continue + } else if (fi.Mode() & os.ModeType) == 0 { + link, err := GetLinkSource(target, fi, inodes) + if err != nil { + return errors.Wrap(err, "failed to get hardlink") + } + if link != "" { + if err := os.Link(link, target); err != nil { + return errors.Wrap(err, "failed to create hard link") + } + } else if err := copyFile(source, target); err != nil { + return errors.Wrap(err, "failed to copy files") + } + } else if (fi.Mode() & os.ModeSymlink) == os.ModeSymlink { + link, err := os.Readlink(source) + if err != nil { + return errors.Wrapf(err, "failed to read link: %s", source) + } + if err := os.Symlink(link, target); err != nil { + return errors.Wrapf(err, "failed to create symlink: %s", target) + } + } else if (fi.Mode() & os.ModeDevice) == os.ModeDevice { + // TODO: support devices + return errors.New("devices not supported") + } else { + // TODO: Support pipes and sockets + return errors.Wrapf(err, "unsupported mode %s", fi.Mode()) + } + if err := copyFileInfo(fi, target); err != nil { + return errors.Wrap(err, "failed to copy file info") + } + + if err := copyXAttrs(target, source); err != nil { + return errors.Wrap(err, "failed to copy xattrs") + } + } + + return nil +} + +func copyFile(source, target string) error { + src, err := os.Open(source) + if err != nil { + return errors.Wrapf(err, "failed to open source %s", err) + } + defer src.Close() + tgt, err := os.Create(target) + if err != nil { + return errors.Wrapf(err, "failed to open target %s", err) + } + defer tgt.Close() + + return copyFileContent(tgt, src) +} diff --git a/fs/copy_linux.go b/fs/copy_linux.go new file mode 100644 index 0000000..7ecd574 --- /dev/null +++ b/fs/copy_linux.go @@ -0,0 +1,110 @@ +package fs + +import ( + "io" + "os" + "strconv" + "syscall" + + "github.com/pkg/errors" + "github.com/stevvooe/continuity/sysx" +) + +var ( + kernel, major int +) + +func init() { + uts := &syscall.Utsname{} + + err := syscall.Uname(uts) + if err != nil { + panic(err) + } + + p := [2][]byte{{}, {}} + + release := uts.Release + i := 0 + for pi := 0; pi < len(p); pi++ { + for release[i] != 0 { + c := byte(release[i]) + i++ + if c == '.' || c == '-' { + break + } + p[pi] = append(p[pi], c) + } + } + kernel, err = strconv.Atoi(string(p[0])) + if err != nil { + panic(err) + } + major, err = strconv.Atoi(string(p[1])) + if err != nil { + panic(err) + } +} + +func copyFileInfo(fi os.FileInfo, name string) error { + st := fi.Sys().(*syscall.Stat_t) + if err := os.Lchown(name, int(st.Uid), int(st.Gid)); err != nil { + return errors.Wrapf(err, "failed to chown %s", name) + } + + if (fi.Mode() & os.ModeSymlink) != os.ModeSymlink { + if err := os.Chmod(name, fi.Mode()); err != nil { + return errors.Wrapf(err, "failed to chmod %s", name) + } + } + + if err := syscall.UtimesNano(name, []syscall.Timespec{st.Atim, st.Mtim}); err != nil { + return errors.Wrapf(err, "failed to utime %s", name) + } + + return nil +} + +func copyFileContent(dst, src *os.File) error { + if checkKernel(4, 5) { + // Use copy_file_range to do in kernel copying + // See https://lwn.net/Articles/659523/ + // 326 on x86_64 + st, err := src.Stat() + if err != nil { + return errors.Wrap(err, "unable to stat source") + } + n, _, e1 := syscall.Syscall6(326, src.Fd(), 0, dst.Fd(), 0, uintptr(st.Size()), 0) + if e1 != 0 { + return errors.Wrap(err, "copy_file_range failed") + } + if int64(n) != st.Size() { + return errors.Wrapf(err, "short copy: %d of %d", int64(n), st.Size()) + } + return nil + } + _, err := io.Copy(dst, src) + return err +} + +func checkKernel(k, m int) bool { + return (kernel == k && major >= m) || kernel > k +} + +func copyXAttrs(dst, src string) error { + xattrKeys, err := sysx.LListxattr(src) + if err != nil { + return errors.Wrapf(err, "failed to list xattrs on %s", src) + } + for _, xattr := range xattrKeys { + data, err := sysx.LGetxattr(src, xattr) + if err != nil { + return errors.Wrapf(err, "failed to get xattr %q on %s", xattr, src) + } + if err := sysx.LSetxattr(dst, xattr, data, 0); err != nil { + return errors.Wrapf(err, "failed to set xattr %q on %s", xattr, dst) + } + } + + return nil +} diff --git a/fs/copy_test.go b/fs/copy_test.go new file mode 100644 index 0000000..1c1f7c5 --- /dev/null +++ b/fs/copy_test.go @@ -0,0 +1,53 @@ +package fs + +import ( + "io/ioutil" + "testing" + + _ "crypto/sha256" + + "github.com/docker/containerd/fs/fstest" + "github.com/pkg/errors" +) + +// TODO: Create copy directory which requires privilege +// chown +// mknod +// setxattr fstest.SetXAttr("/home", "trusted.overlay.opaque", "y"), + +func TestCopyDirectory(t *testing.T) { + apply := fstest.MultiApply( + fstest.CreateDirectory("/etc/", 0755), + fstest.NewTestFile("/etc/hosts", []byte("localhost 127.0.0.1"), 0644), + fstest.Link("/etc/hosts", "/etc/hosts.allow"), + fstest.CreateDirectory("/usr/local/lib", 0755), + fstest.NewTestFile("/usr/local/lib/libnothing.so", []byte{0x00, 0x00}, 0755), + fstest.Symlink("libnothing.so", "/usr/local/lib/libnothing.so.2"), + fstest.CreateDirectory("/home", 0755), + ) + + if err := testCopy(apply); err != nil { + t.Fatalf("Copy test failed: %+v", err) + } +} + +func testCopy(apply fstest.Applier) error { + t1, err := ioutil.TempDir("", "test-copy-src-") + if err != nil { + return errors.Wrap(err, "failed to create temporary directory") + } + t2, err := ioutil.TempDir("", "test-copy-dst-") + if err != nil { + return errors.Wrap(err, "failed to create temporary directory") + } + + if err := apply(t1); err != nil { + return errors.Wrap(err, "failed to apply changes") + } + + if err := CopyDirectory(t2, t1); err != nil { + return errors.Wrap(err, "failed to copy") + } + + return fstest.CheckDirectoryEqual(t1, t2) +} diff --git a/fs/copy_windows.go b/fs/copy_windows.go new file mode 100644 index 0000000..47ff2c5 --- /dev/null +++ b/fs/copy_windows.go @@ -0,0 +1,27 @@ +package fs + +import ( + "io" + "os" + + "github.com/pkg/errors" +) + +func copyFileInfo(fi os.FileInfo, name string) error { + if err := os.Chmod(name, fi.Mode()); err != nil { + return errors.Wrapf(err, "failed to chmod %s", name) + } + + // TODO: copy windows specific metadata + + return nil +} + +func copyFileContent(dst, src *os.File) error { + _, err := io.Copy(dst, src) + return err +} + +func copyXAttrs(dst, src string) error { + return nil +} diff --git a/fs/diff.go b/fs/diff.go new file mode 100644 index 0000000..77d5732 --- /dev/null +++ b/fs/diff.go @@ -0,0 +1,360 @@ +package fs + +import ( + "context" + "os" + "path/filepath" + "strings" + "sync" + + "github.com/Sirupsen/logrus" +) + +// ChangeKind is the type of modification that +// a change is making. +type ChangeKind int + +const ( + // ChangeKindAdd represents an addition of + // a file + ChangeKindAdd = iota + + // ChangeKindModify represents a change to + // an existing file + ChangeKindModify + + // ChangeKindDelete represents a delete of + // a file + ChangeKindDelete +) + +func (k ChangeKind) String() string { + switch k { + case ChangeKindAdd: + return "add" + case ChangeKindModify: + return "modify" + case ChangeKindDelete: + return "delete" + default: + return "" + } +} + +// Change represents single change between a diff and its parent. +type Change struct { + Kind ChangeKind + Path string + FileInfo os.FileInfo + Source string +} + +// Changes returns a stream of changes between the provided upper +// directory and lower directory. +// +// Changes are ordered by name and should be appliable in the +// order in which they received. +// Due to this apply ordering, the following is true +// - Removed directory trees only create a single change for the root +// directory removed. Remaining changes are implied. +// - A directory which is modified to become a file will not have +// delete entries for sub-path items, their removal is implied +// by the removal of the parent directory. +// +// Opaque directories will not be treated specially and each file +// removed from the lower will show up as a removal +// +// File content comparisons will be done on files which have timestamps +// which may have been truncated. If either of the files being compared +// has a zero value nanosecond value, each byte will be compared for +// differences. If 2 files have the same seconds value but different +// nanosecond values where one of those values is zero, the files will +// be considered unchanged if the content is the same. This behavior +// is to account for timestamp truncation during archiving. +func Changes(ctx context.Context, upper, lower string) (context.Context, <-chan Change) { + var ( + changes = make(chan Change) + retCtx, cancel = context.WithCancel(ctx) + ) + + cc := &changeContext{ + Context: retCtx, + } + + go func() { + var err error + if lower == "" { + logrus.Debugf("Using single walk diff for %s", upper) + err = addDirChanges(ctx, changes, upper) + } else if diffOptions := detectDirDiff(upper, lower); diffOptions != nil { + logrus.Debugf("Using single walk diff for %s from %s", diffOptions.diffDir, lower) + err = diffDirChanges(ctx, changes, lower, diffOptions) + } else { + logrus.Debugf("Using double walk diff for %s from %s", upper, lower) + err = doubleWalkDiff(ctx, changes, upper, lower) + } + + if err != nil { + cc.errL.Lock() + cc.err = err + cc.errL.Unlock() + cancel() + } + defer close(changes) + }() + + return cc, changes +} + +// changeContext wraps a context to allow setting an error +// directly from a change streamer to allow streams canceled +// due to errors to propagate the error to the caller. +type changeContext struct { + context.Context + + err error + errL sync.Mutex +} + +func (cc *changeContext) Err() error { + cc.errL.Lock() + if cc.err != nil { + return cc.err + } + cc.errL.Unlock() + return cc.Context.Err() +} + +func sendChange(ctx context.Context, changes chan<- Change, change Change) error { + select { + case <-ctx.Done(): + return ctx.Err() + case changes <- change: + return nil + } +} + +func addDirChanges(ctx context.Context, changes chan<- Change, root string) error { + return filepath.Walk(root, func(path string, f os.FileInfo, err error) error { + if err != nil { + return err + } + + // Rebase path + path, err = filepath.Rel(root, path) + if err != nil { + return err + } + + path = filepath.Join(string(os.PathSeparator), path) + + // Skip root + if path == string(os.PathSeparator) { + return nil + } + + change := Change{ + Path: path, + Kind: ChangeKindAdd, + FileInfo: f, + Source: filepath.Join(root, path), + } + + return sendChange(ctx, changes, change) + }) +} + +// diffDirOptions is used when the diff can be directly calculated from +// a diff directory to its lower, without walking both trees. +type diffDirOptions struct { + diffDir string + skipChange func(string) (bool, error) + deleteChange func(string, string, os.FileInfo) (string, error) +} + +// diffDirChanges walks the diff directory and compares changes against the lower. +func diffDirChanges(ctx context.Context, changes chan<- Change, lower string, o *diffDirOptions) error { + changedDirs := make(map[string]struct{}) + return filepath.Walk(o.diffDir, func(path string, f os.FileInfo, err error) error { + if err != nil { + return err + } + + // Rebase path + path, err = filepath.Rel(o.diffDir, path) + if err != nil { + return err + } + + path = filepath.Join(string(os.PathSeparator), path) + + // Skip root + if path == string(os.PathSeparator) { + return nil + } + + // TODO: handle opaqueness, start new double walker at this + // location to get deletes, and skip tree in single walker + + if o.skipChange != nil { + if skip, err := o.skipChange(path); skip { + return err + } + } + + change := Change{ + Path: path, + } + + deletedFile, err := o.deleteChange(o.diffDir, path, f) + if err != nil { + return err + } + + // Find out what kind of modification happened + if deletedFile != "" { + change.Path = deletedFile + change.Kind = ChangeKindDelete + } else { + // Otherwise, the file was added + change.Kind = ChangeKindAdd + change.FileInfo = f + change.Source = filepath.Join(o.diffDir, path) + + // ...Unless it already existed in a lower, in which case, it's a modification + stat, err := os.Stat(filepath.Join(lower, path)) + if err != nil && !os.IsNotExist(err) { + return err + } + if err == nil { + // The file existed in the lower, so that's a modification + + // However, if it's a directory, maybe it wasn't actually modified. + // If you modify /foo/bar/baz, then /foo will be part of the changed files only because it's the parent of bar + if stat.IsDir() && f.IsDir() { + if f.Size() == stat.Size() && f.Mode() == stat.Mode() && sameFsTime(f.ModTime(), stat.ModTime()) { + // Both directories are the same, don't record the change + return nil + } + } + change.Kind = ChangeKindModify + } + } + + // If /foo/bar/file.txt is modified, then /foo/bar must be part of the changed files. + // This block is here to ensure the change is recorded even if the + // modify time, mode and size of the parent directory in the rw and ro layers are all equal. + // Check https://github.com/docker/docker/pull/13590 for details. + if f.IsDir() { + changedDirs[path] = struct{}{} + } + if change.Kind == ChangeKindAdd || change.Kind == ChangeKindDelete { + parent := filepath.Dir(path) + if _, ok := changedDirs[parent]; !ok && parent != "/" { + pi, err := os.Stat(filepath.Join(o.diffDir, parent)) + if err != nil { + return err + } + dirChange := Change{ + Path: parent, + Kind: ChangeKindModify, + FileInfo: pi, + Source: filepath.Join(o.diffDir, parent), + } + if err := sendChange(ctx, changes, dirChange); err != nil { + return err + } + changedDirs[parent] = struct{}{} + } + } + + return sendChange(ctx, changes, change) + }) +} + +// doubleWalkDiff walks both directories to create a diff +func doubleWalkDiff(ctx context.Context, changes chan<- Change, upper, lower string) (err error) { + pathCtx, cancel := context.WithCancel(ctx) + defer func() { + if err != nil { + cancel() + } + }() + + var ( + w1 = pathWalker(pathCtx, lower) + w2 = pathWalker(pathCtx, upper) + f1, f2 *currentPath + rmdir string + ) + + for w1 != nil || w2 != nil { + if f1 == nil && w1 != nil { + f1, err = nextPath(w1) + if err != nil { + return err + } + if f1 == nil { + w1 = nil + } + } + + if f2 == nil && w2 != nil { + f2, err = nextPath(w2) + if err != nil { + return err + } + if f2 == nil { + w2 = nil + } + } + if f1 == nil && f2 == nil { + continue + } + + c := pathChange(f1, f2) + switch c.Kind { + case ChangeKindAdd: + if rmdir != "" { + rmdir = "" + } + c.FileInfo = f2.f + c.Source = filepath.Join(upper, c.Path) + f2 = nil + case ChangeKindDelete: + // Check if this file is already removed by being + // under of a removed directory + if rmdir != "" && strings.HasPrefix(f1.path, rmdir) { + f1 = nil + continue + } else if rmdir == "" && f1.f.IsDir() { + rmdir = f1.path + string(os.PathSeparator) + } else if rmdir != "" { + rmdir = "" + } + f1 = nil + case ChangeKindModify: + same, err := sameFile(f1, f2) + if err != nil { + return err + } + if f1.f.IsDir() && !f2.f.IsDir() { + rmdir = f1.path + string(os.PathSeparator) + } else if rmdir != "" { + rmdir = "" + } + c.FileInfo = f2.f + c.Source = filepath.Join(upper, c.Path) + f1 = nil + f2 = nil + if same { + continue + } + } + if err := sendChange(ctx, changes, c); err != nil { + return err + } + } + + return nil +} diff --git a/fs/diff_linux.go b/fs/diff_linux.go new file mode 100644 index 0000000..b6bad84 --- /dev/null +++ b/fs/diff_linux.go @@ -0,0 +1,92 @@ +package fs + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "syscall" + + "github.com/pkg/errors" + "github.com/stevvooe/continuity/sysx" +) + +// whiteouts are files with a special meaning for the layered filesystem. +// Docker uses AUFS whiteout files inside exported archives. In other +// filesystems these files are generated/handled on tar creation/extraction. + +// whiteoutPrefix prefix means file is a whiteout. If this is followed by a +// filename this means that file has been removed from the base layer. +const whiteoutPrefix = ".wh." + +// whiteoutMetaPrefix prefix means whiteout has a special meaning and is not +// for removing an actual file. Normally these files are excluded from exported +// archives. +const whiteoutMetaPrefix = whiteoutPrefix + whiteoutPrefix + +// whiteoutLinkDir is a directory AUFS uses for storing hardlink links to other +// layers. Normally these should not go into exported archives and all changed +// hardlinks should be copied to the top layer. +const whiteoutLinkDir = whiteoutMetaPrefix + "plnk" + +// whiteoutOpaqueDir file means directory has been made opaque - meaning +// readdir calls to this directory do not follow to lower layers. +const whiteoutOpaqueDir = whiteoutMetaPrefix + ".opq" + +// detectDirDiff returns diff dir options if a directory could +// be found in the mount info for upper which is the direct +// diff with the provided lower directory +func detectDirDiff(upper, lower string) *diffDirOptions { + // TODO: get mount options for upper + // TODO: detect AUFS + // TODO: detect overlay + return nil +} + +func aufsMetadataSkip(path string) (skip bool, err error) { + skip, err = filepath.Match(string(os.PathSeparator)+whiteoutMetaPrefix+"*", path) + if err != nil { + skip = true + } + return +} + +func aufsDeletedFile(root, path string, fi os.FileInfo) (string, error) { + f := filepath.Base(path) + + // If there is a whiteout, then the file was removed + if strings.HasPrefix(f, whiteoutPrefix) { + originalFile := f[len(whiteoutPrefix):] + return filepath.Join(filepath.Dir(path), originalFile), nil + } + + return "", nil +} + +// compareSysStat returns whether the stats are equivalent, +// whether the files are considered the same file, and +// an error +func compareSysStat(s1, s2 interface{}) (bool, error) { + ls1, ok := s1.(*syscall.Stat_t) + if !ok { + return false, nil + } + ls2, ok := s2.(*syscall.Stat_t) + if !ok { + return false, nil + } + + return ls1.Mode == ls2.Mode && ls1.Uid == ls2.Uid && ls1.Gid == ls2.Gid && ls1.Rdev == ls2.Rdev, nil +} + +func compareCapabilities(p1, p2 string) (bool, error) { + c1, err := sysx.LGetxattr(p1, "security.capability") + if err != nil && err != syscall.ENODATA { + return false, errors.Wrapf(err, "failed to get xattr for %s", p1) + } + c2, err := sysx.LGetxattr(p2, "security.capability") + if err != nil && err != syscall.ENODATA { + return false, errors.Wrapf(err, "failed to get xattr for %s", p2) + } + return bytes.Equal(c1, c2), nil +} diff --git a/fs/diff_test.go b/fs/diff_test.go new file mode 100644 index 0000000..255d347 --- /dev/null +++ b/fs/diff_test.go @@ -0,0 +1,304 @@ +package fs + +import ( + "context" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/docker/containerd/fs/fstest" + "github.com/pkg/errors" +) + +// TODO: Additional tests +// - capability test (requires privilege) +// - chown test (requires privilege) +// - symlink test +// - hardlink test + +func TestSimpleDiff(t *testing.T) { + l1 := fstest.MultiApply( + fstest.CreateDirectory("/etc", 0755), + fstest.NewTestFile("/etc/hosts", []byte("mydomain 10.0.0.1"), 0644), + fstest.NewTestFile("/etc/profile", []byte("PATH=/usr/bin"), 0644), + fstest.NewTestFile("/etc/unchanged", []byte("PATH=/usr/bin"), 0644), + fstest.NewTestFile("/etc/unexpected", []byte("#!/bin/sh"), 0644), + ) + l2 := fstest.MultiApply( + fstest.NewTestFile("/etc/hosts", []byte("mydomain 10.0.0.120"), 0644), + fstest.NewTestFile("/etc/profile", []byte("PATH=/usr/bin"), 0666), + fstest.CreateDirectory("/root", 0700), + fstest.NewTestFile("/root/.bashrc", []byte("PATH=/usr/sbin:/usr/bin"), 0644), + fstest.RemoveFile("/etc/unexpected"), + ) + diff := []Change{ + Modify("/etc/hosts"), + Modify("/etc/profile"), + Delete("/etc/unexpected"), + Add("/root"), + Add("/root/.bashrc"), + } + + if err := testDiffWithBase(l1, l2, diff); err != nil { + t.Fatalf("Failed diff with base: %+v", err) + } +} + +func TestDirectoryReplace(t *testing.T) { + l1 := fstest.MultiApply( + fstest.CreateDirectory("/dir1", 0755), + fstest.NewTestFile("/dir1/f1", []byte("#####"), 0644), + fstest.CreateDirectory("/dir1/f2", 0755), + fstest.NewTestFile("/dir1/f2/f3", []byte("#!/bin/sh"), 0644), + ) + l2 := fstest.MultiApply( + fstest.NewTestFile("/dir1/f11", []byte("#New file here"), 0644), + fstest.RemoveFile("/dir1/f2"), + fstest.NewTestFile("/dir1/f2", []byte("Now file"), 0666), + ) + diff := []Change{ + Add("/dir1/f11"), + Modify("/dir1/f2"), + } + + if err := testDiffWithBase(l1, l2, diff); err != nil { + t.Fatalf("Failed diff with base: %+v", err) + } +} + +func TestRemoveDirectoryTree(t *testing.T) { + l1 := fstest.MultiApply( + fstest.CreateDirectory("/dir1/dir2/dir3", 0755), + fstest.NewTestFile("/dir1/f1", []byte("f1"), 0644), + fstest.NewTestFile("/dir1/dir2/f2", []byte("f2"), 0644), + ) + l2 := fstest.MultiApply( + fstest.RemoveFile("/dir1"), + ) + diff := []Change{ + Delete("/dir1"), + } + + if err := testDiffWithBase(l1, l2, diff); err != nil { + t.Fatalf("Failed diff with base: %+v", err) + } +} + +func TestFileReplace(t *testing.T) { + l1 := fstest.MultiApply( + fstest.NewTestFile("/dir1", []byte("a file, not a directory"), 0644), + ) + l2 := fstest.MultiApply( + fstest.RemoveFile("/dir1"), + fstest.CreateDirectory("/dir1/dir2", 0755), + fstest.NewTestFile("/dir1/dir2/f1", []byte("also a file"), 0644), + ) + diff := []Change{ + Modify("/dir1"), + Add("/dir1/dir2"), + Add("/dir1/dir2/f1"), + } + + if err := testDiffWithBase(l1, l2, diff); err != nil { + t.Fatalf("Failed diff with base: %+v", err) + } +} + +func TestUpdateWithSameTime(t *testing.T) { + tt := time.Now().Truncate(time.Second) + t1 := tt.Add(5 * time.Nanosecond) + t2 := tt.Add(6 * time.Nanosecond) + l1 := fstest.MultiApply( + fstest.NewTestFile("/file-modified-time", []byte("1"), 0644), + fstest.Chtime("/file-modified-time", t1), + fstest.NewTestFile("/file-no-change", []byte("1"), 0644), + fstest.Chtime("/file-no-change", t1), + fstest.NewTestFile("/file-same-time", []byte("1"), 0644), + fstest.Chtime("/file-same-time", t1), + fstest.NewTestFile("/file-truncated-time-1", []byte("1"), 0644), + fstest.Chtime("/file-truncated-time-1", t1), + fstest.NewTestFile("/file-truncated-time-2", []byte("1"), 0644), + fstest.Chtime("/file-truncated-time-2", tt), + ) + l2 := fstest.MultiApply( + fstest.NewTestFile("/file-modified-time", []byte("2"), 0644), + fstest.Chtime("/file-modified-time", t2), + fstest.NewTestFile("/file-no-change", []byte("1"), 0644), + fstest.Chtime("/file-no-change", tt), // use truncated time, should be regarded as no change + fstest.NewTestFile("/file-same-time", []byte("2"), 0644), + fstest.Chtime("/file-same-time", t1), + fstest.NewTestFile("/file-truncated-time-1", []byte("2"), 0644), + fstest.Chtime("/file-truncated-time-1", tt), + fstest.NewTestFile("/file-truncated-time-2", []byte("2"), 0644), + fstest.Chtime("/file-truncated-time-2", tt), + ) + diff := []Change{ + // "/file-same-time" excluded because matching non-zero nanosecond values + Modify("/file-modified-time"), + Modify("/file-truncated-time-1"), + Modify("/file-truncated-time-2"), + } + + if err := testDiffWithBase(l1, l2, diff); err != nil { + t.Fatalf("Failed diff with base: %+v", err) + } +} + +func testDiffWithBase(base, diff fstest.Applier, expected []Change) error { + t1, err := ioutil.TempDir("", "diff-with-base-lower-") + if err != nil { + return errors.Wrap(err, "failed to create temp dir") + } + defer os.RemoveAll(t1) + t2, err := ioutil.TempDir("", "diff-with-base-upper-") + if err != nil { + return errors.Wrap(err, "failed to create temp dir") + } + defer os.RemoveAll(t2) + + if err := base(t1); err != nil { + return errors.Wrap(err, "failed to apply base filesytem") + } + + if err := CopyDirectory(t2, t1); err != nil { + return errors.Wrap(err, "failed to copy base directory") + } + + if err := diff(t2); err != nil { + return errors.Wrap(err, "failed to apply diff filesystem") + } + + changes, err := collectChanges(t2, t1) + if err != nil { + return errors.Wrap(err, "failed to collect changes") + } + + return checkChanges(t2, changes, expected) +} + +func TestBaseDirectoryChanges(t *testing.T) { + apply := fstest.MultiApply( + fstest.CreateDirectory("/etc", 0755), + fstest.NewTestFile("/etc/hosts", []byte("mydomain 10.0.0.1"), 0644), + fstest.NewTestFile("/etc/profile", []byte("PATH=/usr/bin"), 0644), + fstest.CreateDirectory("/root", 0700), + fstest.NewTestFile("/root/.bashrc", []byte("PATH=/usr/sbin:/usr/bin"), 0644), + ) + changes := []Change{ + Add("/etc"), + Add("/etc/hosts"), + Add("/etc/profile"), + Add("/root"), + Add("/root/.bashrc"), + } + + if err := testDiffWithoutBase(apply, changes); err != nil { + t.Fatalf("Failed diff without base: %+v", err) + } +} + +func testDiffWithoutBase(apply fstest.Applier, expected []Change) error { + tmp, err := ioutil.TempDir("", "diff-without-base-") + if err != nil { + return errors.Wrap(err, "failed to create temp dir") + } + defer os.RemoveAll(tmp) + + if err := apply(tmp); err != nil { + return errors.Wrap(err, "failed to apply filesytem changes") + } + + changes, err := collectChanges(tmp, "") + if err != nil { + return errors.Wrap(err, "failed to collect changes") + } + + return checkChanges(tmp, changes, expected) +} + +func checkChanges(root string, changes, expected []Change) error { + if len(changes) != len(expected) { + return errors.Errorf("Unexpected number of changes:\n%s", diffString(changes, expected)) + } + for i := range changes { + if changes[i].Path != expected[i].Path || changes[i].Kind != expected[i].Kind { + return errors.Errorf("Unexpected change at %d:\n%s", i, diffString(changes, expected)) + } + if changes[i].Kind != ChangeKindDelete { + filename := filepath.Join(root, changes[i].Path) + efi, err := os.Stat(filename) + if err != nil { + return errors.Wrapf(err, "failed to stat %q", filename) + } + afi := changes[i].FileInfo + if afi.Size() != efi.Size() { + return errors.Errorf("Unexpected change size %d, %q has size %d", afi.Size(), filename, efi.Size()) + } + if afi.Mode() != efi.Mode() { + return errors.Errorf("Unexpected change mode %s, %q has mode %s", afi.Mode(), filename, efi.Mode()) + } + if afi.ModTime() != efi.ModTime() { + return errors.Errorf("Unexpected change modtime %s, %q has modtime %s", afi.ModTime(), filename, efi.ModTime()) + } + if expected := filepath.Join(root, changes[i].Path); changes[i].Source != expected { + return errors.Errorf("Unexpected source path %s, expected %s", changes[i].Source, expected) + } + } + } + + return nil +} + +func collectChanges(upper, lower string) ([]Change, error) { + ctx, changeC := Changes(context.Background(), upper, lower) + changes := []Change{} + for { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case c, ok := <-changeC: + if !ok { + return changes, nil + } + changes = append(changes, c) + } + } +} + +func diffString(c1, c2 []Change) string { + return fmt.Sprintf("got(%d):\n%s\nexpected(%d):\n%s", len(c1), changesString(c1), len(c2), changesString(c2)) + +} + +func changesString(c []Change) string { + strs := make([]string, len(c)) + for i := range c { + strs[i] = fmt.Sprintf("\t%s\t%s", c[i].Kind, c[i].Path) + } + return strings.Join(strs, "\n") +} + +func Add(p string) Change { + return Change{ + Kind: ChangeKindAdd, + Path: p, + } +} + +func Delete(p string) Change { + return Change{ + Kind: ChangeKindDelete, + Path: p, + } +} + +func Modify(p string) Change { + return Change{ + Kind: ChangeKindModify, + Path: p, + } +} diff --git a/fs/diff_windows.go b/fs/diff_windows.go new file mode 100644 index 0000000..90fb30f --- /dev/null +++ b/fs/diff_windows.go @@ -0,0 +1,15 @@ +package fs + +func detectDirDiff(upper, lower string) *diffDirOptions { + return nil +} + +func compareSysStat(s1, s2 interface{}) (bool, error) { + // TODO: Use windows specific sys type + return false, nil +} + +func compareCapabilities(p1, p2 string) (bool, error) { + // TODO: Use windows equivalent + return true, nil +} diff --git a/fs/fstest/compare.go b/fs/fstest/compare.go new file mode 100644 index 0000000..84781b3 --- /dev/null +++ b/fs/fstest/compare.go @@ -0,0 +1,37 @@ +package fstest + +import ( + "github.com/pkg/errors" + "github.com/stevvooe/continuity" +) + +// CheckDirectoryEqual compares two directory paths to make sure that +// the content of the directories is the same. +func CheckDirectoryEqual(d1, d2 string) error { + c1, err := continuity.NewContext(d1) + if err != nil { + return errors.Wrap(err, "failed to build context") + } + + c2, err := continuity.NewContext(d2) + if err != nil { + return errors.Wrap(err, "failed to build context") + } + + m1, err := continuity.BuildManifest(c1) + if err != nil { + return errors.Wrap(err, "failed to build manifest") + } + + m2, err := continuity.BuildManifest(c2) + if err != nil { + return errors.Wrap(err, "failed to build manifest") + } + + diff := diffResourceList(m1.Resources, m2.Resources) + if diff.HasDiff() { + return errors.Errorf("directory diff between %s and %s\n%s", d1, d2, diff.String()) + } + + return nil +} diff --git a/fs/fstest/continuity_util.go b/fs/fstest/continuity_util.go new file mode 100644 index 0000000..a300388 --- /dev/null +++ b/fs/fstest/continuity_util.go @@ -0,0 +1,189 @@ +package fstest + +import ( + "bytes" + "fmt" + + "github.com/stevvooe/continuity" +) + +type resourceUpdate struct { + Original continuity.Resource + Updated continuity.Resource +} + +func (u resourceUpdate) String() string { + return fmt.Sprintf("%s(mode: %o, uid: %s, gid: %s) -> %s(mode: %o, uid: %s, gid: %s)", + u.Original.Path(), u.Original.Mode(), u.Original.UID(), u.Original.GID(), + u.Updated.Path(), u.Updated.Mode(), u.Updated.UID(), u.Updated.GID(), + ) +} + +type resourceListDifference struct { + Additions []continuity.Resource + Deletions []continuity.Resource + Updates []resourceUpdate +} + +func (l resourceListDifference) HasDiff() bool { + return len(l.Additions) > 0 || len(l.Deletions) > 0 || len(l.Updates) > 0 +} + +func (l resourceListDifference) String() string { + buf := bytes.NewBuffer(nil) + for _, add := range l.Additions { + fmt.Fprintf(buf, "+ %s\n", add.Path()) + } + for _, del := range l.Deletions { + fmt.Fprintf(buf, "- %s\n", del.Path()) + } + for _, upt := range l.Updates { + fmt.Fprintf(buf, "~ %s\n", upt.String()) + } + return string(buf.Bytes()) +} + +// diffManifest compares two resource lists and returns the list +// of adds updates and deletes, resource lists are not reordered +// before doing difference. +func diffResourceList(r1, r2 []continuity.Resource) resourceListDifference { + i1 := 0 + i2 := 0 + var d resourceListDifference + + for i1 < len(r1) && i2 < len(r2) { + p1 := r1[i1].Path() + p2 := r2[i2].Path() + switch { + case p1 < p2: + d.Deletions = append(d.Deletions, r1[i1]) + i1++ + case p1 == p2: + if !compareResource(r1[i1], r2[i2]) { + d.Updates = append(d.Updates, resourceUpdate{ + Original: r1[i1], + Updated: r2[i2], + }) + } + i1++ + i2++ + case p1 > p2: + d.Additions = append(d.Additions, r2[i2]) + i2++ + } + } + + for i1 < len(r1) { + d.Deletions = append(d.Deletions, r1[i1]) + i1++ + + } + for i2 < len(r2) { + d.Additions = append(d.Additions, r2[i2]) + i2++ + } + + return d +} + +func compareResource(r1, r2 continuity.Resource) bool { + if r1.Path() != r2.Path() { + return false + } + if r1.Mode() != r2.Mode() { + return false + } + if r1.UID() != r2.UID() { + return false + } + if r1.GID() != r2.GID() { + return false + } + + // TODO(dmcgowan): Check if is XAttrer + + return compareResourceTypes(r1, r2) + +} + +func compareResourceTypes(r1, r2 continuity.Resource) bool { + switch t1 := r1.(type) { + case continuity.RegularFile: + t2, ok := r2.(continuity.RegularFile) + if !ok { + return false + } + return compareRegularFile(t1, t2) + case continuity.Directory: + t2, ok := r2.(continuity.Directory) + if !ok { + return false + } + return compareDirectory(t1, t2) + case continuity.SymLink: + t2, ok := r2.(continuity.SymLink) + if !ok { + return false + } + return compareSymLink(t1, t2) + case continuity.NamedPipe: + t2, ok := r2.(continuity.NamedPipe) + if !ok { + return false + } + return compareNamedPipe(t1, t2) + case continuity.Device: + t2, ok := r2.(continuity.Device) + if !ok { + return false + } + return compareDevice(t1, t2) + default: + // TODO(dmcgowan): Should this panic? + return r1 == r2 + } +} + +func compareRegularFile(r1, r2 continuity.RegularFile) bool { + if r1.Size() != r2.Size() { + return false + } + p1 := r1.Paths() + p2 := r2.Paths() + if len(p1) != len(p2) { + return false + } + for i := range p1 { + if p1[i] != p2[i] { + return false + } + } + d1 := r1.Digests() + d2 := r2.Digests() + if len(d1) != len(d2) { + return false + } + for i := range d1 { + if d1[i] != d2[i] { + return false + } + } + + return true +} + +func compareSymLink(r1, r2 continuity.SymLink) bool { + return r1.Target() == r2.Target() +} + +func compareDirectory(r1, r2 continuity.Directory) bool { + return true +} + +func compareNamedPipe(r1, r2 continuity.NamedPipe) bool { + return true +} + +func compareDevice(r1, r2 continuity.Device) bool { + return r1.Major() == r2.Major() && r1.Minor() == r2.Minor() +} diff --git a/fs/fstest/file.go b/fs/fstest/file.go new file mode 100644 index 0000000..640fad6 --- /dev/null +++ b/fs/fstest/file.go @@ -0,0 +1,109 @@ +package fstest + +import ( + "io/ioutil" + "os" + "path/filepath" + "time" + + "github.com/stevvooe/continuity/sysx" +) + +// Applier applies single file changes +type Applier func(root string) error + +// NewTestFile returns a file applier which creates a file as the +// provided name with the given content and permission. +func NewTestFile(name string, content []byte, perm os.FileMode) Applier { + return func(root string) error { + fullPath := filepath.Join(root, name) + if err := ioutil.WriteFile(fullPath, content, perm); err != nil { + return err + } + + if err := os.Chmod(fullPath, perm); err != nil { + return err + } + + return nil + } +} + +// RemoveFile returns a file applier which removes the provided file name +func RemoveFile(name string) Applier { + return func(root string) error { + return os.RemoveAll(filepath.Join(root, name)) + } +} + +// CreateDirectory returns a file applier to create the directory with +// the provided name and permission +func CreateDirectory(name string, perm os.FileMode) Applier { + return func(root string) error { + fullPath := filepath.Join(root, name) + if err := os.MkdirAll(fullPath, perm); err != nil { + return err + } + return nil + } +} + +// Rename returns a file applier which renames a file +func Rename(old, new string) Applier { + return func(root string) error { + return os.Rename(filepath.Join(root, old), filepath.Join(root, new)) + } +} + +// Chown returns a file applier which changes the ownership of a file +func Chown(name string, uid, gid int) Applier { + return func(root string) error { + return os.Chown(filepath.Join(root, name), uid, gid) + } +} + +// Chtime changes access and mod time of file +func Chtime(name string, t time.Time) Applier { + return func(root string) error { + return os.Chtimes(filepath.Join(root, name), t, t) + } +} + +// Symlink returns a file applier which creates a symbolic link +func Symlink(oldname, newname string) Applier { + return func(root string) error { + return os.Symlink(oldname, filepath.Join(root, newname)) + } +} + +// Link returns a file applier which creates a hard link +func Link(oldname, newname string) Applier { + return func(root string) error { + return os.Link(filepath.Join(root, oldname), filepath.Join(root, newname)) + } +} + +func SetXAttr(name, key, value string) Applier { + return func(root string) error { + return sysx.LSetxattr(name, key, []byte(value), 0) + } +} + +// TODO: Make platform specific, windows applier is always no-op +//func Mknod(name string, mode int32, dev int) Applier { +// return func(root string) error { +// return return syscall.Mknod(path, mode, dev) +// } +//} + +// MultiApply returns a new applier from the given appliers +func MultiApply(appliers ...Applier) Applier { + return func(root string) error { + for _, a := range appliers { + if err := a(root); err != nil { + return err + } + } + return nil + } +} diff --git a/fs/hardlink.go b/fs/hardlink.go new file mode 100644 index 0000000..a6c256b --- /dev/null +++ b/fs/hardlink.go @@ -0,0 +1,12 @@ +package fs + +import "os" + +// GetLinkSource returns a path for the given name and +// file info to its link source in the provided inode +// map. If the given file name is not in the map and +// has other links, it is added to the inode map +// to be a source for other link locations. +func GetLinkSource(name string, fi os.FileInfo, inodes map[uint64]string) (string, error) { + return getHardLink(name, fi, inodes) +} diff --git a/fs/hardlink_unix.go b/fs/hardlink_unix.go new file mode 100644 index 0000000..76d1897 --- /dev/null +++ b/fs/hardlink_unix.go @@ -0,0 +1,33 @@ +// +build !windows + +package fs + +import ( + "errors" + "os" + "syscall" +) + +func getHardLink(name string, fi os.FileInfo, inodes map[uint64]string) (string, error) { + if fi.IsDir() { + return "", nil + } + + s, ok := fi.Sys().(*syscall.Stat_t) + if !ok { + return "", errors.New("unsupported stat type") + } + + // If inode is not hardlinked, no reason to lookup or save inode + if s.Nlink == 1 { + return "", nil + } + + inode := uint64(s.Ino) + + path, ok := inodes[inode] + if !ok { + inodes[inode] = name + } + return path, nil +} diff --git a/fs/hardlink_windows.go b/fs/hardlink_windows.go new file mode 100644 index 0000000..81e50ee --- /dev/null +++ b/fs/hardlink_windows.go @@ -0,0 +1,7 @@ +package fs + +import "os" + +func getHardLink(string, os.FileInfo, map[uint64]string) (string, error) { + return "", nil +} diff --git a/fs/path.go b/fs/path.go new file mode 100644 index 0000000..8abf4f1 --- /dev/null +++ b/fs/path.go @@ -0,0 +1,197 @@ +package fs + +import ( + "bytes" + "context" + "io" + "os" + "path/filepath" + "strings" +) + +type currentPath struct { + path string + f os.FileInfo + fullPath string +} + +func pathChange(lower, upper *currentPath) Change { + if lower == nil { + if upper == nil { + panic("cannot compare nil paths") + } + return Change{ + Kind: ChangeKindAdd, + Path: upper.path, + } + } + if upper == nil { + return Change{ + Kind: ChangeKindDelete, + Path: lower.path, + } + } + // TODO: compare by directory + + switch i := strings.Compare(lower.path, upper.path); { + case i < 0: + // File in lower that is not in upper + return Change{ + Kind: ChangeKindDelete, + Path: lower.path, + } + case i > 0: + // File in upper that is not in lower + return Change{ + Kind: ChangeKindAdd, + Path: upper.path, + } + default: + return Change{ + Kind: ChangeKindModify, + Path: upper.path, + } + } +} + +func sameFile(f1, f2 *currentPath) (bool, error) { + if os.SameFile(f1.f, f2.f) { + return true, nil + } + + equalStat, err := compareSysStat(f1.f.Sys(), f2.f.Sys()) + if err != nil || !equalStat { + return equalStat, err + } + + if eq, err := compareCapabilities(f1.fullPath, f2.fullPath); err != nil || !eq { + return eq, err + } + + // If not a directory also check size, modtime, and content + if !f1.f.IsDir() { + if f1.f.Size() != f2.f.Size() { + return false, nil + } + t1 := f1.f.ModTime() + t2 := f2.f.ModTime() + + if t1.Unix() != t2.Unix() { + return false, nil + } + + // If the timestamp may have been truncated in one of the + // files, check content of file to determine difference + if t1.Nanosecond() == 0 || t2.Nanosecond() == 0 { + if f1.f.Size() > 0 { + eq, err := compareFileContent(f1.fullPath, f2.fullPath) + if err != nil || !eq { + return eq, err + } + } + } else if t1.Nanosecond() != t2.Nanosecond() { + return false, nil + } + } + + return true, nil +} + +const compareChuckSize = 32 * 1024 + +func compareFileContent(p1, p2 string) (bool, error) { + f1, err := os.Open(p1) + if err != nil { + return false, err + } + defer f1.Close() + f2, err := os.Open(p2) + if err != nil { + return false, err + } + defer f2.Close() + + b1 := make([]byte, compareChuckSize) + b2 := make([]byte, compareChuckSize) + for { + n1, err1 := f1.Read(b1) + if err1 != nil && err1 != io.EOF { + return false, err1 + } + n2, err2 := f2.Read(b2) + if err2 != nil && err2 != io.EOF { + return false, err2 + } + if n1 != n2 || !bytes.Equal(b1[:n1], b2[:n2]) { + return false, nil + } + if err1 == io.EOF && err2 == io.EOF { + return true, nil + } + } +} + +type walker struct { + pathC <-chan *currentPath + errC <-chan error +} + +func pathWalker(ctx context.Context, root string) *walker { + var ( + pathC = make(chan *currentPath) + errC = make(chan error, 1) + ) + go func() { + defer close(pathC) + err := filepath.Walk(root, func(path string, f os.FileInfo, err error) error { + if err != nil { + return err + } + + // Rebase path + path, err = filepath.Rel(root, path) + if err != nil { + return err + } + + path = filepath.Join(string(os.PathSeparator), path) + + // Skip root + if path == string(os.PathSeparator) { + return nil + } + + return sendPath(ctx, pathC, ¤tPath{ + path: path, + f: f, + fullPath: filepath.Join(root, path), + }) + }) + if err != nil { + errC <- err + } + }() + + return &walker{ + pathC: pathC, + errC: errC, + } +} + +func sendPath(ctx context.Context, pc chan<- *currentPath, p *currentPath) error { + select { + case <-ctx.Done(): + return ctx.Err() + case pc <- p: + return nil + } +} + +func nextPath(w *walker) (*currentPath, error) { + select { + case err := <-w.errC: + return nil, err + case p := <-w.pathC: + return p, nil + } +} diff --git a/fs/time.go b/fs/time.go new file mode 100644 index 0000000..92966a3 --- /dev/null +++ b/fs/time.go @@ -0,0 +1,21 @@ +package fs + +import ( + "syscall" + "time" +) + +// Gnu tar and the go tar writer don't have sub-second mtime +// precision, which is problematic when we apply changes via tar +// files, we handle this by comparing for exact times, *or* same +// second count and either a or b having exactly 0 nanoseconds +func sameFsTime(a, b time.Time) bool { + return a == b || + (a.Unix() == b.Unix() && + (a.Nanosecond() == 0 || b.Nanosecond() == 0)) +} + +func sameFsTimeSpec(a, b syscall.Timespec) bool { + return a.Sec == b.Sec && + (a.Nsec == b.Nsec || a.Nsec == 0 || b.Nsec == 0) +} From 200cd6e877db5c2ef1fd6ba898a8293d1cc5bec4 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Tue, 31 Jan 2017 15:21:35 -0800 Subject: [PATCH 03/12] Update snapshots to use fs package Signed-off-by: Derek McGowan (github: dmcgowan) --- snapshot/naive/naive.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/snapshot/naive/naive.go b/snapshot/naive/naive.go index b012c33..bf48583 100644 --- a/snapshot/naive/naive.go +++ b/snapshot/naive/naive.go @@ -7,7 +7,7 @@ import ( "path/filepath" "github.com/docker/containerd" - "github.com/docker/docker/pkg/archive" + "github.com/docker/containerd/fs" "github.com/pkg/errors" ) @@ -66,7 +66,7 @@ func (n *Naive) Prepare(dst, parent string) ([]containerd.Mount, error) { } // Now, we copy the parent filesystem, just a directory, into dst. - if err := archive.CopyWithTar(filepath.Join(parent, "data"), dst); err != nil { // note: src, dst args, ick! + if err := fs.CopyDirectory(dst, filepath.Join(parent, "data")); err != nil { return nil, errors.Wrap(err, "copying of parent failed") } } @@ -88,7 +88,7 @@ func (n *Naive) Commit(diff, dst string) error { // Move the data into our metadata directory, we could probably save disk // space if we just saved the diff, but let's get something working. - if err := archive.CopyWithTar(dst, filepath.Join(active.metadata, "data")); err != nil { // note: src, dst args, ick! + if err := fs.CopyDirectory(filepath.Join(active.metadata, "data"), dst); err != nil { return errors.Wrap(err, "copying of parent failed") } From f78105d832fe2901be96b1f9b7ba0d68cfeed943 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Wed, 1 Feb 2017 14:23:56 -0800 Subject: [PATCH 04/12] Add support for devices Address code comments from previous commit Signed-off-by: Derek McGowan (github: dmcgowan) --- fs/copy.go | 21 +++++++++++++-------- fs/copy_linux.go | 8 ++++++++ fs/copy_windows.go | 4 ++++ fs/diff.go | 2 +- fs/path.go | 2 ++ 5 files changed, 28 insertions(+), 9 deletions(-) diff --git a/fs/copy.go b/fs/copy.go index 0ffd7f2..a1fe735 100644 --- a/fs/copy.go +++ b/fs/copy.go @@ -24,10 +24,12 @@ func copyDirectory(dst, src string, inodes map[uint64]string) error { return errors.Errorf("source is not directory") } - if _, err := os.Stat(dst); err != nil { + if st, err := os.Stat(dst); err != nil { if err := os.Mkdir(dst, stat.Mode()); err != nil { return errors.Wrapf(err, "failed to mkdir %s", dst) } + } else if !st.IsDir() { + return errors.Errorf("cannot copy to non-directory: %s", dst) } else { if err := os.Chmod(dst, stat.Mode()); err != nil { return errors.Wrapf(err, "failed to chmod on %s", dst) @@ -46,12 +48,14 @@ func copyDirectory(dst, src string, inodes map[uint64]string) error { for _, fi := range fis { source := filepath.Join(src, fi.Name()) target := filepath.Join(dst, fi.Name()) - if fi.IsDir() { + + switch { + case fi.IsDir(): if err := copyDirectory(target, source, inodes); err != nil { return err } continue - } else if (fi.Mode() & os.ModeType) == 0 { + case (fi.Mode() & os.ModeType) == 0: link, err := GetLinkSource(target, fi, inodes) if err != nil { return errors.Wrap(err, "failed to get hardlink") @@ -63,7 +67,7 @@ func copyDirectory(dst, src string, inodes map[uint64]string) error { } else if err := copyFile(source, target); err != nil { return errors.Wrap(err, "failed to copy files") } - } else if (fi.Mode() & os.ModeSymlink) == os.ModeSymlink { + case (fi.Mode() & os.ModeSymlink) == os.ModeSymlink: link, err := os.Readlink(source) if err != nil { return errors.Wrapf(err, "failed to read link: %s", source) @@ -71,10 +75,11 @@ func copyDirectory(dst, src string, inodes map[uint64]string) error { if err := os.Symlink(link, target); err != nil { return errors.Wrapf(err, "failed to create symlink: %s", target) } - } else if (fi.Mode() & os.ModeDevice) == os.ModeDevice { - // TODO: support devices - return errors.New("devices not supported") - } else { + case (fi.Mode() & os.ModeDevice) == os.ModeDevice: + if err := copyDevice(target, fi); err != nil { + return errors.Wrapf(err, "failed to create device") + } + default: // TODO: Support pipes and sockets return errors.Wrapf(err, "unsupported mode %s", fi.Mode()) } diff --git a/fs/copy_linux.go b/fs/copy_linux.go index 7ecd574..f264b4a 100644 --- a/fs/copy_linux.go +++ b/fs/copy_linux.go @@ -108,3 +108,11 @@ func copyXAttrs(dst, src string) error { return nil } + +func copyDevice(dst string, fi os.FileInfo) error { + st, ok := fi.Sys().(*syscall.Stat_t) + if !ok { + return errors.New("unsupported stat type") + } + return syscall.Mknod(dst, uint32(fi.Mode().Perm()), int(st.Rdev)) +} diff --git a/fs/copy_windows.go b/fs/copy_windows.go index 47ff2c5..e17c912 100644 --- a/fs/copy_windows.go +++ b/fs/copy_windows.go @@ -25,3 +25,7 @@ func copyFileContent(dst, src *os.File) error { func copyXAttrs(dst, src string) error { return nil } + +func copyDevice(dst string, fi os.FileInfo) error { + return errors.New("device copy not supported") +} diff --git a/fs/diff.go b/fs/diff.go index 77d5732..1a2c55a 100644 --- a/fs/diff.go +++ b/fs/diff.go @@ -100,7 +100,7 @@ func Changes(ctx context.Context, upper, lower string) (context.Context, <-chan cc.errL.Unlock() cancel() } - defer close(changes) + close(changes) }() return cc, changes diff --git a/fs/path.go b/fs/path.go index 8abf4f1..0060a66 100644 --- a/fs/path.go +++ b/fs/path.go @@ -99,6 +99,8 @@ func sameFile(f1, f2 *currentPath) (bool, error) { const compareChuckSize = 32 * 1024 +// compareFileContent compares the content of 2 same sized files +// by comparing each byte. func compareFileContent(p1, p2 string) (bool, error) { f1, err := os.Open(p1) if err != nil { From bf8f37ba7847ae6be3348757df820effe1ae09db Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Wed, 1 Feb 2017 14:30:12 -0800 Subject: [PATCH 05/12] Remove incorrect and unused timespec check Compare is using its own time check comparison and doing byte comparison when ambiguous rather than ignoring it like this function does. Signed-off-by: Derek McGowan (github: dmcgowan) --- fs/time.go | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/fs/time.go b/fs/time.go index 92966a3..c336f4d 100644 --- a/fs/time.go +++ b/fs/time.go @@ -1,9 +1,6 @@ package fs -import ( - "syscall" - "time" -) +import "time" // Gnu tar and the go tar writer don't have sub-second mtime // precision, which is problematic when we apply changes via tar @@ -14,8 +11,3 @@ func sameFsTime(a, b time.Time) bool { (a.Unix() == b.Unix() && (a.Nanosecond() == 0 || b.Nanosecond() == 0)) } - -func sameFsTimeSpec(a, b syscall.Timespec) bool { - return a.Sec == b.Sec && - (a.Nsec == b.Nsec || a.Nsec == 0 || b.Nsec == 0) -} From 245495d54e7ee621c7f56ea1ba4ad385f6246519 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Wed, 1 Feb 2017 14:46:39 -0800 Subject: [PATCH 06/12] Use full mode for making devices node Signed-off-by: Derek McGowan (github: dmcgowan) --- fs/copy_linux.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/copy_linux.go b/fs/copy_linux.go index f264b4a..293bec9 100644 --- a/fs/copy_linux.go +++ b/fs/copy_linux.go @@ -114,5 +114,5 @@ func copyDevice(dst string, fi os.FileInfo) error { if !ok { return errors.New("unsupported stat type") } - return syscall.Mknod(dst, uint32(fi.Mode().Perm()), int(st.Rdev)) + return syscall.Mknod(dst, uint32(fi.Mode()), int(st.Rdev)) } From bb9f6b568ddacd81306b6c0cf58693d304fa67f6 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Wed, 1 Feb 2017 16:03:24 -0800 Subject: [PATCH 07/12] Update continuity to use sysx branch Pulls in changes for copy file range Signed-off-by: Derek McGowan (github: dmcgowan) --- vendor.conf | 4 ++-- .../stevvooe/continuity/sysx/copy_linux.go | 9 +++++++++ .../continuity/sysx/copy_linux_386.go | 20 +++++++++++++++++++ .../continuity/sysx/copy_linux_amd64.go | 20 +++++++++++++++++++ .../continuity/sysx/copy_linux_arm64.go | 20 +++++++++++++++++++ .../continuity/sysx/sysnum_linux_386.go | 7 +++++++ .../continuity/sysx/sysnum_linux_amd64.go | 7 +++++++ .../continuity/sysx/sysnum_linux_arm64.go | 7 +++++++ 8 files changed, 92 insertions(+), 2 deletions(-) create mode 100644 vendor/github.com/stevvooe/continuity/sysx/copy_linux.go create mode 100644 vendor/github.com/stevvooe/continuity/sysx/copy_linux_386.go create mode 100644 vendor/github.com/stevvooe/continuity/sysx/copy_linux_amd64.go create mode 100644 vendor/github.com/stevvooe/continuity/sysx/copy_linux_arm64.go create mode 100644 vendor/github.com/stevvooe/continuity/sysx/sysnum_linux_386.go create mode 100644 vendor/github.com/stevvooe/continuity/sysx/sysnum_linux_amd64.go create mode 100644 vendor/github.com/stevvooe/continuity/sysx/sysnum_linux_arm64.go diff --git a/vendor.conf b/vendor.conf index 04d1079..d8bf8cf 100644 --- a/vendor.conf +++ b/vendor.conf @@ -68,5 +68,5 @@ github.com/opencontainers/go-digest 21dfd564fd89c944783d00d069f33e3e7123c448 golang.org/x/sys/unix d75a52659825e75fff6158388dddc6a5b04f9ba5 # image-spec master as of 1/17/2017 github.com/opencontainers/image-spec 0ff14aabcda3b2ee62621174f1b29fc157bdf335 -# continuity master as of 1/10/2017 -github.com/stevvooe/continuity 6c9282fa1546987eefc2b123fe087b818d821725 +# continuity master as of 2/1/2017 +github.com/stevvooe/continuity 1530f13d23b34e2ccaf33881fefecc7e28e3577b diff --git a/vendor/github.com/stevvooe/continuity/sysx/copy_linux.go b/vendor/github.com/stevvooe/continuity/sysx/copy_linux.go new file mode 100644 index 0000000..d7ccbb2 --- /dev/null +++ b/vendor/github.com/stevvooe/continuity/sysx/copy_linux.go @@ -0,0 +1,9 @@ +package sysx + +// These functions will be generated by generate.sh +// $ GOOS=linux GOARCH=386 ./generate.sh copy +// $ GOOS=linux GOARCH=amd64 ./generate.sh copy +// $ GOOS=linux GOARCH=arm ./generate.sh copy +// $ GOOS=linux GOARCH=arm64 ./generate.sh copy + +//sys CopyFileRange(fdin uintptr, offin *int64, fdout uintptr, offout *int64, len int, flags int) (n int, err error) diff --git a/vendor/github.com/stevvooe/continuity/sysx/copy_linux_386.go b/vendor/github.com/stevvooe/continuity/sysx/copy_linux_386.go new file mode 100644 index 0000000..c1368c5 --- /dev/null +++ b/vendor/github.com/stevvooe/continuity/sysx/copy_linux_386.go @@ -0,0 +1,20 @@ +// mksyscall.pl -l32 copy_linux.go +// MACHINE GENERATED BY THE COMMAND ABOVE; DO NOT EDIT + +package sysx + +import ( + "syscall" + "unsafe" +) + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + +func CopyFileRange(fdin uintptr, offin *int64, fdout uintptr, offout *int64, len int, flags int) (n int, err error) { + r0, _, e1 := syscall.Syscall6(SYS_COPY_FILE_RANGE, uintptr(fdin), uintptr(unsafe.Pointer(offin)), uintptr(fdout), uintptr(unsafe.Pointer(offout)), uintptr(len), uintptr(flags)) + n = int(r0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} diff --git a/vendor/github.com/stevvooe/continuity/sysx/copy_linux_amd64.go b/vendor/github.com/stevvooe/continuity/sysx/copy_linux_amd64.go new file mode 100644 index 0000000..9941b01 --- /dev/null +++ b/vendor/github.com/stevvooe/continuity/sysx/copy_linux_amd64.go @@ -0,0 +1,20 @@ +// mksyscall.pl copy_linux.go +// MACHINE GENERATED BY THE COMMAND ABOVE; DO NOT EDIT + +package sysx + +import ( + "syscall" + "unsafe" +) + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + +func CopyFileRange(fdin uintptr, offin *int64, fdout uintptr, offout *int64, len int, flags int) (n int, err error) { + r0, _, e1 := syscall.Syscall6(SYS_COPY_FILE_RANGE, uintptr(fdin), uintptr(unsafe.Pointer(offin)), uintptr(fdout), uintptr(unsafe.Pointer(offout)), uintptr(len), uintptr(flags)) + n = int(r0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} diff --git a/vendor/github.com/stevvooe/continuity/sysx/copy_linux_arm64.go b/vendor/github.com/stevvooe/continuity/sysx/copy_linux_arm64.go new file mode 100644 index 0000000..9941b01 --- /dev/null +++ b/vendor/github.com/stevvooe/continuity/sysx/copy_linux_arm64.go @@ -0,0 +1,20 @@ +// mksyscall.pl copy_linux.go +// MACHINE GENERATED BY THE COMMAND ABOVE; DO NOT EDIT + +package sysx + +import ( + "syscall" + "unsafe" +) + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + +func CopyFileRange(fdin uintptr, offin *int64, fdout uintptr, offout *int64, len int, flags int) (n int, err error) { + r0, _, e1 := syscall.Syscall6(SYS_COPY_FILE_RANGE, uintptr(fdin), uintptr(unsafe.Pointer(offin)), uintptr(fdout), uintptr(unsafe.Pointer(offout)), uintptr(len), uintptr(flags)) + n = int(r0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} diff --git a/vendor/github.com/stevvooe/continuity/sysx/sysnum_linux_386.go b/vendor/github.com/stevvooe/continuity/sysx/sysnum_linux_386.go new file mode 100644 index 0000000..0063f8a --- /dev/null +++ b/vendor/github.com/stevvooe/continuity/sysx/sysnum_linux_386.go @@ -0,0 +1,7 @@ +package sysx + +const ( + // SYS_COPYFILERANGE defined in Kernel 4.5+ + // Number defined in /usr/include/asm/unistd_32.h + SYS_COPY_FILE_RANGE = 377 +) diff --git a/vendor/github.com/stevvooe/continuity/sysx/sysnum_linux_amd64.go b/vendor/github.com/stevvooe/continuity/sysx/sysnum_linux_amd64.go new file mode 100644 index 0000000..4170540 --- /dev/null +++ b/vendor/github.com/stevvooe/continuity/sysx/sysnum_linux_amd64.go @@ -0,0 +1,7 @@ +package sysx + +const ( + // SYS_COPYFILERANGE defined in Kernel 4.5+ + // Number defined in /usr/include/asm/unistd_64.h + SYS_COPY_FILE_RANGE = 326 +) diff --git a/vendor/github.com/stevvooe/continuity/sysx/sysnum_linux_arm64.go b/vendor/github.com/stevvooe/continuity/sysx/sysnum_linux_arm64.go new file mode 100644 index 0000000..da31bbd --- /dev/null +++ b/vendor/github.com/stevvooe/continuity/sysx/sysnum_linux_arm64.go @@ -0,0 +1,7 @@ +package sysx + +const ( + // SYS_COPY_FILE_RANGE defined in Kernel 4.5+ + // Number defined in /usr/include/asm-generic/unistd.h + SYS_COPY_FILE_RANGE = 285 +) From 572dbcdbd45d739684f577b647f27a3fd02dd4e3 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Thu, 2 Feb 2017 10:28:25 -0800 Subject: [PATCH 08/12] Use copy file range from sysx Use pooled buffers for copy Signed-off-by: Derek McGowan (github: dmcgowan) --- fs/copy.go | 9 ++++++ fs/copy_linux.go | 78 ++++++++++++---------------------------------- fs/copy_windows.go | 4 ++- 3 files changed, 32 insertions(+), 59 deletions(-) diff --git a/fs/copy.go b/fs/copy.go index a1fe735..8c44c7c 100644 --- a/fs/copy.go +++ b/fs/copy.go @@ -4,10 +4,19 @@ import ( "io/ioutil" "os" "path/filepath" + "sync" "github.com/pkg/errors" ) +var ( + bufferPool = &sync.Pool{ + New: func() interface{} { + return make([]byte, 32*1024) + }, + } +) + // CopyDirectory copies the directory from src to dst. // Most efficient copy of files is attempted. func CopyDirectory(dst, src string) error { diff --git a/fs/copy_linux.go b/fs/copy_linux.go index 293bec9..a6c193c 100644 --- a/fs/copy_linux.go +++ b/fs/copy_linux.go @@ -3,49 +3,12 @@ package fs import ( "io" "os" - "strconv" "syscall" "github.com/pkg/errors" "github.com/stevvooe/continuity/sysx" ) -var ( - kernel, major int -) - -func init() { - uts := &syscall.Utsname{} - - err := syscall.Uname(uts) - if err != nil { - panic(err) - } - - p := [2][]byte{{}, {}} - - release := uts.Release - i := 0 - for pi := 0; pi < len(p); pi++ { - for release[i] != 0 { - c := byte(release[i]) - i++ - if c == '.' || c == '-' { - break - } - p[pi] = append(p[pi], c) - } - } - kernel, err = strconv.Atoi(string(p[0])) - if err != nil { - panic(err) - } - major, err = strconv.Atoi(string(p[1])) - if err != nil { - panic(err) - } -} - func copyFileInfo(fi os.FileInfo, name string) error { st := fi.Sys().(*syscall.Stat_t) if err := os.Lchown(name, int(st.Uid), int(st.Gid)); err != nil { @@ -66,29 +29,28 @@ func copyFileInfo(fi os.FileInfo, name string) error { } func copyFileContent(dst, src *os.File) error { - if checkKernel(4, 5) { - // Use copy_file_range to do in kernel copying - // See https://lwn.net/Articles/659523/ - // 326 on x86_64 - st, err := src.Stat() - if err != nil { - return errors.Wrap(err, "unable to stat source") - } - n, _, e1 := syscall.Syscall6(326, src.Fd(), 0, dst.Fd(), 0, uintptr(st.Size()), 0) - if e1 != 0 { - return errors.Wrap(err, "copy_file_range failed") - } - if int64(n) != st.Size() { - return errors.Wrapf(err, "short copy: %d of %d", int64(n), st.Size()) - } - return nil + st, err := src.Stat() + if err != nil { + return errors.Wrap(err, "unable to stat source") } - _, err := io.Copy(dst, src) - return err -} -func checkKernel(k, m int) bool { - return (kernel == k && major >= m) || kernel > k + n, err := sysx.CopyFileRange(src.Fd(), nil, dst.Fd(), nil, int(st.Size()), 0) + if err != nil { + if err != syscall.ENOSYS && err != syscall.EXDEV { + return errors.Wrap(err, "copy file range failed") + } + + buf := bufferPool.Get().([]byte) + _, err = io.CopyBuffer(dst, src, buf) + bufferPool.Put(buf) + return err + } + + if int64(n) != st.Size() { + return errors.Wrapf(err, "short copy: %d of %d", int64(n), st.Size()) + } + + return nil } func copyXAttrs(dst, src string) error { diff --git a/fs/copy_windows.go b/fs/copy_windows.go index e17c912..fb4933c 100644 --- a/fs/copy_windows.go +++ b/fs/copy_windows.go @@ -18,7 +18,9 @@ func copyFileInfo(fi os.FileInfo, name string) error { } func copyFileContent(dst, src *os.File) error { - _, err := io.Copy(dst, src) + buf := bufferPool.Get().([]byte) + _, err := io.CopyBuffer(dst, src, buf) + bufferPool.Put(buf) return err } From aa3be3b0fe4d73c984038e8f70bd4b82ae98f714 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Thu, 2 Feb 2017 16:34:19 -0800 Subject: [PATCH 09/12] Vendor errgroup Signed-off-by: Derek McGowan (github: dmcgowan) --- vendor.conf | 2 + vendor/golang.org/x/sync/LICENSE | 27 ++++++++ vendor/golang.org/x/sync/PATENTS | 22 ++++++ vendor/golang.org/x/sync/errgroup/errgroup.go | 67 +++++++++++++++++++ 4 files changed, 118 insertions(+) create mode 100644 vendor/golang.org/x/sync/LICENSE create mode 100644 vendor/golang.org/x/sync/PATENTS create mode 100644 vendor/golang.org/x/sync/errgroup/errgroup.go diff --git a/vendor.conf b/vendor.conf index d8bf8cf..bc8a5cb 100644 --- a/vendor.conf +++ b/vendor.conf @@ -70,3 +70,5 @@ golang.org/x/sys/unix d75a52659825e75fff6158388dddc6a5b04f9ba5 github.com/opencontainers/image-spec 0ff14aabcda3b2ee62621174f1b29fc157bdf335 # continuity master as of 2/1/2017 github.com/stevvooe/continuity 1530f13d23b34e2ccaf33881fefecc7e28e3577b +# sync master as of 12/5/2016 +golang.org/x/sync 450f422ab23cf9881c94e2db30cac0eb1b7cf80c diff --git a/vendor/golang.org/x/sync/LICENSE b/vendor/golang.org/x/sync/LICENSE new file mode 100644 index 0000000..6a66aea --- /dev/null +++ b/vendor/golang.org/x/sync/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/golang.org/x/sync/PATENTS b/vendor/golang.org/x/sync/PATENTS new file mode 100644 index 0000000..7330990 --- /dev/null +++ b/vendor/golang.org/x/sync/PATENTS @@ -0,0 +1,22 @@ +Additional IP Rights Grant (Patents) + +"This implementation" means the copyrightable works distributed by +Google as part of the Go project. + +Google hereby grants to You a perpetual, worldwide, non-exclusive, +no-charge, royalty-free, irrevocable (except as stated in this section) +patent license to make, have made, use, offer to sell, sell, import, +transfer and otherwise run, modify and propagate the contents of this +implementation of Go, where such license applies only to those patent +claims, both currently owned or controlled by Google and acquired in +the future, licensable by Google that are necessarily infringed by this +implementation of Go. This grant does not include claims that would be +infringed only as a consequence of further modification of this +implementation. If you or your agent or exclusive licensee institute or +order or agree to the institution of patent litigation against any +entity (including a cross-claim or counterclaim in a lawsuit) alleging +that this implementation of Go or any code incorporated within this +implementation of Go constitutes direct or contributory patent +infringement, or inducement of patent infringement, then any patent +rights granted to you under this License for this implementation of Go +shall terminate as of the date such litigation is filed. diff --git a/vendor/golang.org/x/sync/errgroup/errgroup.go b/vendor/golang.org/x/sync/errgroup/errgroup.go new file mode 100644 index 0000000..533438d --- /dev/null +++ b/vendor/golang.org/x/sync/errgroup/errgroup.go @@ -0,0 +1,67 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package errgroup provides synchronization, error propagation, and Context +// cancelation for groups of goroutines working on subtasks of a common task. +package errgroup + +import ( + "sync" + + "golang.org/x/net/context" +) + +// A Group is a collection of goroutines working on subtasks that are part of +// the same overall task. +// +// A zero Group is valid and does not cancel on error. +type Group struct { + cancel func() + + wg sync.WaitGroup + + errOnce sync.Once + err error +} + +// WithContext returns a new Group and an associated Context derived from ctx. +// +// The derived Context is canceled the first time a function passed to Go +// returns a non-nil error or the first time Wait returns, whichever occurs +// first. +func WithContext(ctx context.Context) (*Group, context.Context) { + ctx, cancel := context.WithCancel(ctx) + return &Group{cancel: cancel}, ctx +} + +// Wait blocks until all function calls from the Go method have returned, then +// returns the first non-nil error (if any) from them. +func (g *Group) Wait() error { + g.wg.Wait() + if g.cancel != nil { + g.cancel() + } + return g.err +} + +// Go calls the given function in a new goroutine. +// +// The first call to return a non-nil error cancels the group; its error will be +// returned by Wait. +func (g *Group) Go(f func() error) { + g.wg.Add(1) + + go func() { + defer g.wg.Done() + + if err := f(); err != nil { + g.errOnce.Do(func() { + g.err = err + if g.cancel != nil { + g.cancel() + } + }) + } + }() +} From 65e8c07847c58abc8757b6172d65faf46162c8cb Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Thu, 2 Feb 2017 16:37:04 -0800 Subject: [PATCH 10/12] Update diff interface to use callback The change computation will be done on the callers thread and use callbacks rather than running a goroutine and returning a channel. Signed-off-by: Derek McGowan (github: dmcgowan) --- fs/diff.go | 250 +++++++++++++++++++----------------------------- fs/diff_test.go | 47 +++++---- fs/path.go | 86 +++++++---------- 3 files changed, 164 insertions(+), 219 deletions(-) diff --git a/fs/diff.go b/fs/diff.go index 1a2c55a..0ec4c89 100644 --- a/fs/diff.go +++ b/fs/diff.go @@ -5,7 +5,8 @@ import ( "os" "path/filepath" "strings" - "sync" + + "golang.org/x/sync/errgroup" "github.com/Sirupsen/logrus" ) @@ -43,14 +44,13 @@ func (k ChangeKind) String() string { // Change represents single change between a diff and its parent. type Change struct { - Kind ChangeKind - Path string - FileInfo os.FileInfo - Source string + Kind ChangeKind + Path string } -// Changes returns a stream of changes between the provided upper -// directory and lower directory. +// Changes computes changes between lower and upper calling the +// given change function for each computed change. Callbacks +// will be done serialially and order by path name. // // Changes are ordered by name and should be appliable in the // order in which they received. @@ -71,70 +71,22 @@ type Change struct { // nanosecond values where one of those values is zero, the files will // be considered unchanged if the content is the same. This behavior // is to account for timestamp truncation during archiving. -func Changes(ctx context.Context, upper, lower string) (context.Context, <-chan Change) { - var ( - changes = make(chan Change) - retCtx, cancel = context.WithCancel(ctx) - ) - - cc := &changeContext{ - Context: retCtx, +func Changes(ctx context.Context, upper, lower string, ch func(Change, os.FileInfo) error) error { + if lower == "" { + logrus.Debugf("Using single walk diff for %s", upper) + return addDirChanges(ctx, ch, upper) + } else if diffOptions := detectDirDiff(upper, lower); diffOptions != nil { + logrus.Debugf("Using single walk diff for %s from %s", diffOptions.diffDir, lower) + return diffDirChanges(ctx, ch, lower, diffOptions) } - go func() { - var err error - if lower == "" { - logrus.Debugf("Using single walk diff for %s", upper) - err = addDirChanges(ctx, changes, upper) - } else if diffOptions := detectDirDiff(upper, lower); diffOptions != nil { - logrus.Debugf("Using single walk diff for %s from %s", diffOptions.diffDir, lower) - err = diffDirChanges(ctx, changes, lower, diffOptions) - } else { - logrus.Debugf("Using double walk diff for %s from %s", upper, lower) - err = doubleWalkDiff(ctx, changes, upper, lower) - } - - if err != nil { - cc.errL.Lock() - cc.err = err - cc.errL.Unlock() - cancel() - } - close(changes) - }() - - return cc, changes + logrus.Debugf("Using double walk diff for %s from %s", upper, lower) + return doubleWalkDiff(ctx, ch, upper, lower) } -// changeContext wraps a context to allow setting an error -// directly from a change streamer to allow streams canceled -// due to errors to propagate the error to the caller. -type changeContext struct { - context.Context +type changeFn func(Change, os.FileInfo) error - err error - errL sync.Mutex -} - -func (cc *changeContext) Err() error { - cc.errL.Lock() - if cc.err != nil { - return cc.err - } - cc.errL.Unlock() - return cc.Context.Err() -} - -func sendChange(ctx context.Context, changes chan<- Change, change Change) error { - select { - case <-ctx.Done(): - return ctx.Err() - case changes <- change: - return nil - } -} - -func addDirChanges(ctx context.Context, changes chan<- Change, root string) error { +func addDirChanges(ctx context.Context, changes changeFn, root string) error { return filepath.Walk(root, func(path string, f os.FileInfo, err error) error { if err != nil { return err @@ -154,13 +106,11 @@ func addDirChanges(ctx context.Context, changes chan<- Change, root string) erro } change := Change{ - Path: path, - Kind: ChangeKindAdd, - FileInfo: f, - Source: filepath.Join(root, path), + Path: path, + Kind: ChangeKindAdd, } - return sendChange(ctx, changes, change) + return changes(change, f) }) } @@ -173,7 +123,7 @@ type diffDirOptions struct { } // diffDirChanges walks the diff directory and compares changes against the lower. -func diffDirChanges(ctx context.Context, changes chan<- Change, lower string, o *diffDirOptions) error { +func diffDirChanges(ctx context.Context, changes changeFn, lower string, o *diffDirOptions) error { changedDirs := make(map[string]struct{}) return filepath.Walk(o.diffDir, func(path string, f os.FileInfo, err error) error { if err != nil { @@ -215,11 +165,10 @@ func diffDirChanges(ctx context.Context, changes chan<- Change, lower string, o if deletedFile != "" { change.Path = deletedFile change.Kind = ChangeKindDelete + f = nil } else { // Otherwise, the file was added change.Kind = ChangeKindAdd - change.FileInfo = f - change.Source = filepath.Join(o.diffDir, path) // ...Unless it already existed in a lower, in which case, it's a modification stat, err := os.Stat(filepath.Join(lower, path)) @@ -256,105 +205,108 @@ func diffDirChanges(ctx context.Context, changes chan<- Change, lower string, o return err } dirChange := Change{ - Path: parent, - Kind: ChangeKindModify, - FileInfo: pi, - Source: filepath.Join(o.diffDir, parent), + Path: parent, + Kind: ChangeKindModify, } - if err := sendChange(ctx, changes, dirChange); err != nil { + if err := changes(dirChange, pi); err != nil { return err } changedDirs[parent] = struct{}{} } } - return sendChange(ctx, changes, change) + return changes(change, f) }) } // doubleWalkDiff walks both directories to create a diff -func doubleWalkDiff(ctx context.Context, changes chan<- Change, upper, lower string) (err error) { - pathCtx, cancel := context.WithCancel(ctx) - defer func() { - if err != nil { - cancel() - } - }() +func doubleWalkDiff(ctx context.Context, changes changeFn, upper, lower string) (err error) { + g, ctx := errgroup.WithContext(ctx) var ( - w1 = pathWalker(pathCtx, lower) - w2 = pathWalker(pathCtx, upper) + c1 = make(chan *currentPath) + c2 = make(chan *currentPath) + f1, f2 *currentPath rmdir string ) + g.Go(func() error { + defer close(c1) + return pathWalk(ctx, lower, c1) + }) + g.Go(func() error { + defer close(c2) + return pathWalk(ctx, upper, c2) + }) + g.Go(func() error { + for c1 != nil || c2 != nil { + if f1 == nil && c1 != nil { + f1, err = nextPath(ctx, c1) + if err != nil { + return err + } + if f1 == nil { + c1 = nil + } + } - for w1 != nil || w2 != nil { - if f1 == nil && w1 != nil { - f1, err = nextPath(w1) - if err != nil { - return err + if f2 == nil && c2 != nil { + f2, err = nextPath(ctx, c2) + if err != nil { + return err + } + if f2 == nil { + c2 = nil + } } - if f1 == nil { - w1 = nil + if f1 == nil && f2 == nil { + continue } - } - if f2 == nil && w2 != nil { - f2, err = nextPath(w2) - if err != nil { - return err - } - if f2 == nil { - w2 = nil - } - } - if f1 == nil && f2 == nil { - continue - } - - c := pathChange(f1, f2) - switch c.Kind { - case ChangeKindAdd: - if rmdir != "" { - rmdir = "" - } - c.FileInfo = f2.f - c.Source = filepath.Join(upper, c.Path) - f2 = nil - case ChangeKindDelete: - // Check if this file is already removed by being - // under of a removed directory - if rmdir != "" && strings.HasPrefix(f1.path, rmdir) { + var f os.FileInfo + c := pathChange(f1, f2) + switch c.Kind { + case ChangeKindAdd: + if rmdir != "" { + rmdir = "" + } + f = f2.f + f2 = nil + case ChangeKindDelete: + // Check if this file is already removed by being + // under of a removed directory + if rmdir != "" && strings.HasPrefix(f1.path, rmdir) { + f1 = nil + continue + } else if rmdir == "" && f1.f.IsDir() { + rmdir = f1.path + string(os.PathSeparator) + } else if rmdir != "" { + rmdir = "" + } f1 = nil - continue - } else if rmdir == "" && f1.f.IsDir() { - rmdir = f1.path + string(os.PathSeparator) - } else if rmdir != "" { - rmdir = "" + case ChangeKindModify: + same, err := sameFile(f1, f2) + if err != nil { + return err + } + if f1.f.IsDir() && !f2.f.IsDir() { + rmdir = f1.path + string(os.PathSeparator) + } else if rmdir != "" { + rmdir = "" + } + f = f2.f + f1 = nil + f2 = nil + if same { + continue + } } - f1 = nil - case ChangeKindModify: - same, err := sameFile(f1, f2) - if err != nil { + if err := changes(c, f); err != nil { return err } - if f1.f.IsDir() && !f2.f.IsDir() { - rmdir = f1.path + string(os.PathSeparator) - } else if rmdir != "" { - rmdir = "" - } - c.FileInfo = f2.f - c.Source = filepath.Join(upper, c.Path) - f1 = nil - f2 = nil - if same { - continue - } } - if err := sendChange(ctx, changes, c); err != nil { - return err - } - } + return nil + }) - return nil + return g.Wait() } diff --git a/fs/diff_test.go b/fs/diff_test.go index 255d347..334bceb 100644 --- a/fs/diff_test.go +++ b/fs/diff_test.go @@ -220,13 +220,13 @@ func testDiffWithoutBase(apply fstest.Applier, expected []Change) error { return checkChanges(tmp, changes, expected) } -func checkChanges(root string, changes, expected []Change) error { +func checkChanges(root string, changes []testChange, expected []Change) error { if len(changes) != len(expected) { - return errors.Errorf("Unexpected number of changes:\n%s", diffString(changes, expected)) + return errors.Errorf("Unexpected number of changes:\n%s", diffString(convertTestChanges(changes), expected)) } for i := range changes { if changes[i].Path != expected[i].Path || changes[i].Kind != expected[i].Kind { - return errors.Errorf("Unexpected change at %d:\n%s", i, diffString(changes, expected)) + return errors.Errorf("Unexpected change at %d:\n%s", i, diffString(convertTestChanges(changes), expected)) } if changes[i].Kind != ChangeKindDelete { filename := filepath.Join(root, changes[i].Path) @@ -253,20 +253,35 @@ func checkChanges(root string, changes, expected []Change) error { return nil } -func collectChanges(upper, lower string) ([]Change, error) { - ctx, changeC := Changes(context.Background(), upper, lower) - changes := []Change{} - for { - select { - case <-ctx.Done(): - return nil, ctx.Err() - case c, ok := <-changeC: - if !ok { - return changes, nil - } - changes = append(changes, c) - } +type testChange struct { + Change + FileInfo os.FileInfo + Source string +} + +func collectChanges(upper, lower string) ([]testChange, error) { + changes := []testChange{} + err := Changes(context.Background(), upper, lower, func(c Change, f os.FileInfo) error { + changes = append(changes, testChange{ + Change: c, + FileInfo: f, + Source: filepath.Join(upper, c.Path), + }) + return nil + }) + if err != nil { + return nil, errors.Wrap(err, "failed to compute changes") } + + return changes, nil +} + +func convertTestChanges(c []testChange) []Change { + nc := make([]Change, len(c)) + for i := range c { + nc[i] = c[i].Change + } + return nc } func diffString(c1, c2 []Change) string { diff --git a/fs/path.go b/fs/path.go index 0060a66..342f392 100644 --- a/fs/path.go +++ b/fs/path.go @@ -133,67 +133,45 @@ func compareFileContent(p1, p2 string) (bool, error) { } } -type walker struct { - pathC <-chan *currentPath - errC <-chan error -} - -func pathWalker(ctx context.Context, root string) *walker { - var ( - pathC = make(chan *currentPath) - errC = make(chan error, 1) - ) - go func() { - defer close(pathC) - err := filepath.Walk(root, func(path string, f os.FileInfo, err error) error { - if err != nil { - return err - } - - // Rebase path - path, err = filepath.Rel(root, path) - if err != nil { - return err - } - - path = filepath.Join(string(os.PathSeparator), path) - - // Skip root - if path == string(os.PathSeparator) { - return nil - } - - return sendPath(ctx, pathC, ¤tPath{ - path: path, - f: f, - fullPath: filepath.Join(root, path), - }) - }) +func pathWalk(ctx context.Context, root string, pathC chan<- *currentPath) error { + return filepath.Walk(root, func(path string, f os.FileInfo, err error) error { if err != nil { - errC <- err + return err } - }() - return &walker{ - pathC: pathC, - errC: errC, - } + // Rebase path + path, err = filepath.Rel(root, path) + if err != nil { + return err + } + + path = filepath.Join(string(os.PathSeparator), path) + + // Skip root + if path == string(os.PathSeparator) { + return nil + } + + p := ¤tPath{ + path: path, + f: f, + fullPath: filepath.Join(root, path), + } + + select { + case <-ctx.Done(): + return ctx.Err() + case pathC <- p: + return nil + } + }) } -func sendPath(ctx context.Context, pc chan<- *currentPath, p *currentPath) error { +func nextPath(ctx context.Context, pathC <-chan *currentPath) (*currentPath, error) { select { case <-ctx.Done(): - return ctx.Err() - case pc <- p: - return nil - } -} - -func nextPath(w *walker) (*currentPath, error) { - select { - case err := <-w.errC: - return nil, err - case p := <-w.pathC: + return nil, ctx.Err() + case p := <-pathC: return p, nil } } From d96e6e3952efd0838bed3862dd0efaa3434e755c Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Thu, 2 Feb 2017 17:30:11 -0800 Subject: [PATCH 11/12] Refactor changes and test functions Remove change type in favor of explicit change function. Using change function makes it more difficult to unnecessarily add to the change interface. Update test apply functions to use an interface rather than a function type. Signed-off-by: Derek McGowan (github: dmcgowan) --- fs/copy_test.go | 14 ++-- fs/diff.go | 95 ++++++++++++-------------- fs/diff_test.go | 165 +++++++++++++++++++++++----------------------- fs/fstest/file.go | 64 ++++++++++-------- fs/path.go | 27 ++------ 5 files changed, 172 insertions(+), 193 deletions(-) diff --git a/fs/copy_test.go b/fs/copy_test.go index 1c1f7c5..d63be66 100644 --- a/fs/copy_test.go +++ b/fs/copy_test.go @@ -16,14 +16,14 @@ import ( // setxattr fstest.SetXAttr("/home", "trusted.overlay.opaque", "y"), func TestCopyDirectory(t *testing.T) { - apply := fstest.MultiApply( - fstest.CreateDirectory("/etc/", 0755), - fstest.NewTestFile("/etc/hosts", []byte("localhost 127.0.0.1"), 0644), + apply := fstest.Apply( + fstest.CreateDir("/etc/", 0755), + fstest.CreateFile("/etc/hosts", []byte("localhost 127.0.0.1"), 0644), fstest.Link("/etc/hosts", "/etc/hosts.allow"), - fstest.CreateDirectory("/usr/local/lib", 0755), - fstest.NewTestFile("/usr/local/lib/libnothing.so", []byte{0x00, 0x00}, 0755), + fstest.CreateDir("/usr/local/lib", 0755), + fstest.CreateFile("/usr/local/lib/libnothing.so", []byte{0x00, 0x00}, 0755), fstest.Symlink("libnothing.so", "/usr/local/lib/libnothing.so.2"), - fstest.CreateDirectory("/home", 0755), + fstest.CreateDir("/home", 0755), ) if err := testCopy(apply); err != nil { @@ -41,7 +41,7 @@ func testCopy(apply fstest.Applier) error { return errors.Wrap(err, "failed to create temporary directory") } - if err := apply(t1); err != nil { + if err := apply.Apply(t1); err != nil { return errors.Wrap(err, "failed to apply changes") } diff --git a/fs/diff.go b/fs/diff.go index 0ec4c89..52e2445 100644 --- a/fs/diff.go +++ b/fs/diff.go @@ -48,12 +48,17 @@ type Change struct { Path string } -// Changes computes changes between lower and upper calling the -// given change function for each computed change. Callbacks -// will be done serialially and order by path name. +// ChangeFunc is the type of function called for each change +// computed during a directory changes calculation. +type ChangeFunc func(ChangeKind, string, os.FileInfo, error) error + +// Changes computes changes between two directories calling the +// given change function for each computed change. The first +// directory is intended to the base directory and second +// directory the changed directory. // -// Changes are ordered by name and should be appliable in the -// order in which they received. +// The change callback is called by the order of path names and +// should be appliable in that order. // Due to this apply ordering, the following is true // - Removed directory trees only create a single change for the root // directory removed. Remaining changes are implied. @@ -62,7 +67,7 @@ type Change struct { // by the removal of the parent directory. // // Opaque directories will not be treated specially and each file -// removed from the lower will show up as a removal +// removed from the base directory will show up as a removal. // // File content comparisons will be done on files which have timestamps // which may have been truncated. If either of the files being compared @@ -71,22 +76,20 @@ type Change struct { // nanosecond values where one of those values is zero, the files will // be considered unchanged if the content is the same. This behavior // is to account for timestamp truncation during archiving. -func Changes(ctx context.Context, upper, lower string, ch func(Change, os.FileInfo) error) error { - if lower == "" { - logrus.Debugf("Using single walk diff for %s", upper) - return addDirChanges(ctx, ch, upper) - } else if diffOptions := detectDirDiff(upper, lower); diffOptions != nil { - logrus.Debugf("Using single walk diff for %s from %s", diffOptions.diffDir, lower) - return diffDirChanges(ctx, ch, lower, diffOptions) +func Changes(ctx context.Context, a, b string, changeFn ChangeFunc) error { + if a == "" { + logrus.Debugf("Using single walk diff for %s", b) + return addDirChanges(ctx, changeFn, b) + } else if diffOptions := detectDirDiff(b, a); diffOptions != nil { + logrus.Debugf("Using single walk diff for %s from %s", diffOptions.diffDir, a) + return diffDirChanges(ctx, changeFn, a, diffOptions) } - logrus.Debugf("Using double walk diff for %s from %s", upper, lower) - return doubleWalkDiff(ctx, ch, upper, lower) + logrus.Debugf("Using double walk diff for %s from %s", b, a) + return doubleWalkDiff(ctx, changeFn, a, b) } -type changeFn func(Change, os.FileInfo) error - -func addDirChanges(ctx context.Context, changes changeFn, root string) error { +func addDirChanges(ctx context.Context, changeFn ChangeFunc, root string) error { return filepath.Walk(root, func(path string, f os.FileInfo, err error) error { if err != nil { return err @@ -105,25 +108,20 @@ func addDirChanges(ctx context.Context, changes changeFn, root string) error { return nil } - change := Change{ - Path: path, - Kind: ChangeKindAdd, - } - - return changes(change, f) + return changeFn(ChangeKindAdd, path, f, nil) }) } // diffDirOptions is used when the diff can be directly calculated from -// a diff directory to its lower, without walking both trees. +// a diff directory to its base, without walking both trees. type diffDirOptions struct { diffDir string skipChange func(string) (bool, error) deleteChange func(string, string, os.FileInfo) (string, error) } -// diffDirChanges walks the diff directory and compares changes against the lower. -func diffDirChanges(ctx context.Context, changes changeFn, lower string, o *diffDirOptions) error { +// diffDirChanges walks the diff directory and compares changes against the base. +func diffDirChanges(ctx context.Context, changeFn ChangeFunc, base string, o *diffDirOptions) error { changedDirs := make(map[string]struct{}) return filepath.Walk(o.diffDir, func(path string, f os.FileInfo, err error) error { if err != nil { @@ -152,9 +150,7 @@ func diffDirChanges(ctx context.Context, changes changeFn, lower string, o *diff } } - change := Change{ - Path: path, - } + var kind ChangeKind deletedFile, err := o.deleteChange(o.diffDir, path, f) if err != nil { @@ -163,20 +159,20 @@ func diffDirChanges(ctx context.Context, changes changeFn, lower string, o *diff // Find out what kind of modification happened if deletedFile != "" { - change.Path = deletedFile - change.Kind = ChangeKindDelete + path = deletedFile + kind = ChangeKindDelete f = nil } else { // Otherwise, the file was added - change.Kind = ChangeKindAdd + kind = ChangeKindAdd - // ...Unless it already existed in a lower, in which case, it's a modification - stat, err := os.Stat(filepath.Join(lower, path)) + // ...Unless it already existed in a base, in which case, it's a modification + stat, err := os.Stat(filepath.Join(base, path)) if err != nil && !os.IsNotExist(err) { return err } if err == nil { - // The file existed in the lower, so that's a modification + // The file existed in the base, so that's a modification // However, if it's a directory, maybe it wasn't actually modified. // If you modify /foo/bar/baz, then /foo will be part of the changed files only because it's the parent of bar @@ -186,7 +182,7 @@ func diffDirChanges(ctx context.Context, changes changeFn, lower string, o *diff return nil } } - change.Kind = ChangeKindModify + kind = ChangeKindModify } } @@ -197,30 +193,23 @@ func diffDirChanges(ctx context.Context, changes changeFn, lower string, o *diff if f.IsDir() { changedDirs[path] = struct{}{} } - if change.Kind == ChangeKindAdd || change.Kind == ChangeKindDelete { + if kind == ChangeKindAdd || kind == ChangeKindDelete { parent := filepath.Dir(path) if _, ok := changedDirs[parent]; !ok && parent != "/" { pi, err := os.Stat(filepath.Join(o.diffDir, parent)) - if err != nil { - return err - } - dirChange := Change{ - Path: parent, - Kind: ChangeKindModify, - } - if err := changes(dirChange, pi); err != nil { + if err := changeFn(ChangeKindModify, parent, pi, err); err != nil { return err } changedDirs[parent] = struct{}{} } } - return changes(change, f) + return changeFn(kind, path, f, nil) }) } // doubleWalkDiff walks both directories to create a diff -func doubleWalkDiff(ctx context.Context, changes changeFn, upper, lower string) (err error) { +func doubleWalkDiff(ctx context.Context, changeFn ChangeFunc, a, b string) (err error) { g, ctx := errgroup.WithContext(ctx) var ( @@ -232,11 +221,11 @@ func doubleWalkDiff(ctx context.Context, changes changeFn, upper, lower string) ) g.Go(func() error { defer close(c1) - return pathWalk(ctx, lower, c1) + return pathWalk(ctx, a, c1) }) g.Go(func() error { defer close(c2) - return pathWalk(ctx, upper, c2) + return pathWalk(ctx, b, c2) }) g.Go(func() error { for c1 != nil || c2 != nil { @@ -264,8 +253,8 @@ func doubleWalkDiff(ctx context.Context, changes changeFn, upper, lower string) } var f os.FileInfo - c := pathChange(f1, f2) - switch c.Kind { + k, p := pathChange(f1, f2) + switch k { case ChangeKindAdd: if rmdir != "" { rmdir = "" @@ -301,7 +290,7 @@ func doubleWalkDiff(ctx context.Context, changes changeFn, upper, lower string) continue } } - if err := changes(c, f); err != nil { + if err := changeFn(k, p, f, nil); err != nil { return err } } diff --git a/fs/diff_test.go b/fs/diff_test.go index 334bceb..fb8f012 100644 --- a/fs/diff_test.go +++ b/fs/diff_test.go @@ -21,21 +21,21 @@ import ( // - hardlink test func TestSimpleDiff(t *testing.T) { - l1 := fstest.MultiApply( - fstest.CreateDirectory("/etc", 0755), - fstest.NewTestFile("/etc/hosts", []byte("mydomain 10.0.0.1"), 0644), - fstest.NewTestFile("/etc/profile", []byte("PATH=/usr/bin"), 0644), - fstest.NewTestFile("/etc/unchanged", []byte("PATH=/usr/bin"), 0644), - fstest.NewTestFile("/etc/unexpected", []byte("#!/bin/sh"), 0644), + l1 := fstest.Apply( + fstest.CreateDir("/etc", 0755), + fstest.CreateFile("/etc/hosts", []byte("mydomain 10.0.0.1"), 0644), + fstest.CreateFile("/etc/profile", []byte("PATH=/usr/bin"), 0644), + fstest.CreateFile("/etc/unchanged", []byte("PATH=/usr/bin"), 0644), + fstest.CreateFile("/etc/unexpected", []byte("#!/bin/sh"), 0644), ) - l2 := fstest.MultiApply( - fstest.NewTestFile("/etc/hosts", []byte("mydomain 10.0.0.120"), 0644), - fstest.NewTestFile("/etc/profile", []byte("PATH=/usr/bin"), 0666), - fstest.CreateDirectory("/root", 0700), - fstest.NewTestFile("/root/.bashrc", []byte("PATH=/usr/sbin:/usr/bin"), 0644), + l2 := fstest.Apply( + fstest.CreateFile("/etc/hosts", []byte("mydomain 10.0.0.120"), 0644), + fstest.CreateFile("/etc/profile", []byte("PATH=/usr/bin"), 0666), + fstest.CreateDir("/root", 0700), + fstest.CreateFile("/root/.bashrc", []byte("PATH=/usr/sbin:/usr/bin"), 0644), fstest.RemoveFile("/etc/unexpected"), ) - diff := []Change{ + diff := []testChange{ Modify("/etc/hosts"), Modify("/etc/profile"), Delete("/etc/unexpected"), @@ -49,18 +49,18 @@ func TestSimpleDiff(t *testing.T) { } func TestDirectoryReplace(t *testing.T) { - l1 := fstest.MultiApply( - fstest.CreateDirectory("/dir1", 0755), - fstest.NewTestFile("/dir1/f1", []byte("#####"), 0644), - fstest.CreateDirectory("/dir1/f2", 0755), - fstest.NewTestFile("/dir1/f2/f3", []byte("#!/bin/sh"), 0644), + l1 := fstest.Apply( + fstest.CreateDir("/dir1", 0755), + fstest.CreateFile("/dir1/f1", []byte("#####"), 0644), + fstest.CreateDir("/dir1/f2", 0755), + fstest.CreateFile("/dir1/f2/f3", []byte("#!/bin/sh"), 0644), ) - l2 := fstest.MultiApply( - fstest.NewTestFile("/dir1/f11", []byte("#New file here"), 0644), + l2 := fstest.Apply( + fstest.CreateFile("/dir1/f11", []byte("#New file here"), 0644), fstest.RemoveFile("/dir1/f2"), - fstest.NewTestFile("/dir1/f2", []byte("Now file"), 0666), + fstest.CreateFile("/dir1/f2", []byte("Now file"), 0666), ) - diff := []Change{ + diff := []testChange{ Add("/dir1/f11"), Modify("/dir1/f2"), } @@ -71,15 +71,15 @@ func TestDirectoryReplace(t *testing.T) { } func TestRemoveDirectoryTree(t *testing.T) { - l1 := fstest.MultiApply( - fstest.CreateDirectory("/dir1/dir2/dir3", 0755), - fstest.NewTestFile("/dir1/f1", []byte("f1"), 0644), - fstest.NewTestFile("/dir1/dir2/f2", []byte("f2"), 0644), + l1 := fstest.Apply( + fstest.CreateDir("/dir1/dir2/dir3", 0755), + fstest.CreateFile("/dir1/f1", []byte("f1"), 0644), + fstest.CreateFile("/dir1/dir2/f2", []byte("f2"), 0644), ) - l2 := fstest.MultiApply( + l2 := fstest.Apply( fstest.RemoveFile("/dir1"), ) - diff := []Change{ + diff := []testChange{ Delete("/dir1"), } @@ -89,15 +89,15 @@ func TestRemoveDirectoryTree(t *testing.T) { } func TestFileReplace(t *testing.T) { - l1 := fstest.MultiApply( - fstest.NewTestFile("/dir1", []byte("a file, not a directory"), 0644), + l1 := fstest.Apply( + fstest.CreateFile("/dir1", []byte("a file, not a directory"), 0644), ) - l2 := fstest.MultiApply( + l2 := fstest.Apply( fstest.RemoveFile("/dir1"), - fstest.CreateDirectory("/dir1/dir2", 0755), - fstest.NewTestFile("/dir1/dir2/f1", []byte("also a file"), 0644), + fstest.CreateDir("/dir1/dir2", 0755), + fstest.CreateFile("/dir1/dir2/f1", []byte("also a file"), 0644), ) - diff := []Change{ + diff := []testChange{ Modify("/dir1"), Add("/dir1/dir2"), Add("/dir1/dir2/f1"), @@ -112,31 +112,31 @@ func TestUpdateWithSameTime(t *testing.T) { tt := time.Now().Truncate(time.Second) t1 := tt.Add(5 * time.Nanosecond) t2 := tt.Add(6 * time.Nanosecond) - l1 := fstest.MultiApply( - fstest.NewTestFile("/file-modified-time", []byte("1"), 0644), + l1 := fstest.Apply( + fstest.CreateFile("/file-modified-time", []byte("1"), 0644), fstest.Chtime("/file-modified-time", t1), - fstest.NewTestFile("/file-no-change", []byte("1"), 0644), + fstest.CreateFile("/file-no-change", []byte("1"), 0644), fstest.Chtime("/file-no-change", t1), - fstest.NewTestFile("/file-same-time", []byte("1"), 0644), + fstest.CreateFile("/file-same-time", []byte("1"), 0644), fstest.Chtime("/file-same-time", t1), - fstest.NewTestFile("/file-truncated-time-1", []byte("1"), 0644), + fstest.CreateFile("/file-truncated-time-1", []byte("1"), 0644), fstest.Chtime("/file-truncated-time-1", t1), - fstest.NewTestFile("/file-truncated-time-2", []byte("1"), 0644), + fstest.CreateFile("/file-truncated-time-2", []byte("1"), 0644), fstest.Chtime("/file-truncated-time-2", tt), ) - l2 := fstest.MultiApply( - fstest.NewTestFile("/file-modified-time", []byte("2"), 0644), + l2 := fstest.Apply( + fstest.CreateFile("/file-modified-time", []byte("2"), 0644), fstest.Chtime("/file-modified-time", t2), - fstest.NewTestFile("/file-no-change", []byte("1"), 0644), + fstest.CreateFile("/file-no-change", []byte("1"), 0644), fstest.Chtime("/file-no-change", tt), // use truncated time, should be regarded as no change - fstest.NewTestFile("/file-same-time", []byte("2"), 0644), + fstest.CreateFile("/file-same-time", []byte("2"), 0644), fstest.Chtime("/file-same-time", t1), - fstest.NewTestFile("/file-truncated-time-1", []byte("2"), 0644), + fstest.CreateFile("/file-truncated-time-1", []byte("2"), 0644), fstest.Chtime("/file-truncated-time-1", tt), - fstest.NewTestFile("/file-truncated-time-2", []byte("2"), 0644), + fstest.CreateFile("/file-truncated-time-2", []byte("2"), 0644), fstest.Chtime("/file-truncated-time-2", tt), ) - diff := []Change{ + diff := []testChange{ // "/file-same-time" excluded because matching non-zero nanosecond values Modify("/file-modified-time"), Modify("/file-truncated-time-1"), @@ -148,7 +148,7 @@ func TestUpdateWithSameTime(t *testing.T) { } } -func testDiffWithBase(base, diff fstest.Applier, expected []Change) error { +func testDiffWithBase(base, diff fstest.Applier, expected []testChange) error { t1, err := ioutil.TempDir("", "diff-with-base-lower-") if err != nil { return errors.Wrap(err, "failed to create temp dir") @@ -160,7 +160,7 @@ func testDiffWithBase(base, diff fstest.Applier, expected []Change) error { } defer os.RemoveAll(t2) - if err := base(t1); err != nil { + if err := base.Apply(t1); err != nil { return errors.Wrap(err, "failed to apply base filesytem") } @@ -168,11 +168,11 @@ func testDiffWithBase(base, diff fstest.Applier, expected []Change) error { return errors.Wrap(err, "failed to copy base directory") } - if err := diff(t2); err != nil { + if err := diff.Apply(t2); err != nil { return errors.Wrap(err, "failed to apply diff filesystem") } - changes, err := collectChanges(t2, t1) + changes, err := collectChanges(t1, t2) if err != nil { return errors.Wrap(err, "failed to collect changes") } @@ -181,14 +181,14 @@ func testDiffWithBase(base, diff fstest.Applier, expected []Change) error { } func TestBaseDirectoryChanges(t *testing.T) { - apply := fstest.MultiApply( - fstest.CreateDirectory("/etc", 0755), - fstest.NewTestFile("/etc/hosts", []byte("mydomain 10.0.0.1"), 0644), - fstest.NewTestFile("/etc/profile", []byte("PATH=/usr/bin"), 0644), - fstest.CreateDirectory("/root", 0700), - fstest.NewTestFile("/root/.bashrc", []byte("PATH=/usr/sbin:/usr/bin"), 0644), + apply := fstest.Apply( + fstest.CreateDir("/etc", 0755), + fstest.CreateFile("/etc/hosts", []byte("mydomain 10.0.0.1"), 0644), + fstest.CreateFile("/etc/profile", []byte("PATH=/usr/bin"), 0644), + fstest.CreateDir("/root", 0700), + fstest.CreateFile("/root/.bashrc", []byte("PATH=/usr/sbin:/usr/bin"), 0644), ) - changes := []Change{ + changes := []testChange{ Add("/etc"), Add("/etc/hosts"), Add("/etc/profile"), @@ -201,18 +201,18 @@ func TestBaseDirectoryChanges(t *testing.T) { } } -func testDiffWithoutBase(apply fstest.Applier, expected []Change) error { +func testDiffWithoutBase(apply fstest.Applier, expected []testChange) error { tmp, err := ioutil.TempDir("", "diff-without-base-") if err != nil { return errors.Wrap(err, "failed to create temp dir") } defer os.RemoveAll(tmp) - if err := apply(tmp); err != nil { + if err := apply.Apply(tmp); err != nil { return errors.Wrap(err, "failed to apply filesytem changes") } - changes, err := collectChanges(tmp, "") + changes, err := collectChanges("", tmp) if err != nil { return errors.Wrap(err, "failed to collect changes") } @@ -220,13 +220,13 @@ func testDiffWithoutBase(apply fstest.Applier, expected []Change) error { return checkChanges(tmp, changes, expected) } -func checkChanges(root string, changes []testChange, expected []Change) error { +func checkChanges(root string, changes, expected []testChange) error { if len(changes) != len(expected) { - return errors.Errorf("Unexpected number of changes:\n%s", diffString(convertTestChanges(changes), expected)) + return errors.Errorf("Unexpected number of changes:\n%s", diffString(changes, expected)) } for i := range changes { if changes[i].Path != expected[i].Path || changes[i].Kind != expected[i].Kind { - return errors.Errorf("Unexpected change at %d:\n%s", i, diffString(convertTestChanges(changes), expected)) + return errors.Errorf("Unexpected change at %d:\n%s", i, diffString(changes, expected)) } if changes[i].Kind != ChangeKindDelete { filename := filepath.Join(root, changes[i].Path) @@ -254,18 +254,23 @@ func checkChanges(root string, changes []testChange, expected []Change) error { } type testChange struct { - Change + Kind ChangeKind + Path string FileInfo os.FileInfo Source string } -func collectChanges(upper, lower string) ([]testChange, error) { +func collectChanges(a, b string) ([]testChange, error) { changes := []testChange{} - err := Changes(context.Background(), upper, lower, func(c Change, f os.FileInfo) error { + err := Changes(context.Background(), a, b, func(k ChangeKind, p string, f os.FileInfo, err error) error { + if err != nil { + return err + } changes = append(changes, testChange{ - Change: c, + Kind: k, + Path: p, FileInfo: f, - Source: filepath.Join(upper, c.Path), + Source: filepath.Join(b, p), }) return nil }) @@ -276,20 +281,12 @@ func collectChanges(upper, lower string) ([]testChange, error) { return changes, nil } -func convertTestChanges(c []testChange) []Change { - nc := make([]Change, len(c)) - for i := range c { - nc[i] = c[i].Change - } - return nc -} - -func diffString(c1, c2 []Change) string { +func diffString(c1, c2 []testChange) string { return fmt.Sprintf("got(%d):\n%s\nexpected(%d):\n%s", len(c1), changesString(c1), len(c2), changesString(c2)) } -func changesString(c []Change) string { +func changesString(c []testChange) string { strs := make([]string, len(c)) for i := range c { strs[i] = fmt.Sprintf("\t%s\t%s", c[i].Kind, c[i].Path) @@ -297,22 +294,22 @@ func changesString(c []Change) string { return strings.Join(strs, "\n") } -func Add(p string) Change { - return Change{ +func Add(p string) testChange { + return testChange{ Kind: ChangeKindAdd, Path: p, } } -func Delete(p string) Change { - return Change{ +func Delete(p string) testChange { + return testChange{ Kind: ChangeKindDelete, Path: p, } } -func Modify(p string) Change { - return Change{ +func Modify(p string) testChange { + return testChange{ Kind: ChangeKindModify, Path: p, } diff --git a/fs/fstest/file.go b/fs/fstest/file.go index 640fad6..fab4f68 100644 --- a/fs/fstest/file.go +++ b/fs/fstest/file.go @@ -10,12 +10,20 @@ import ( ) // Applier applies single file changes -type Applier func(root string) error +type Applier interface { + Apply(root string) error +} -// NewTestFile returns a file applier which creates a file as the +type applyFn func(root string) error + +func (a applyFn) Apply(root string) error { + return a(root) +} + +// CreateFile returns a file applier which creates a file as the // provided name with the given content and permission. -func NewTestFile(name string, content []byte, perm os.FileMode) Applier { - return func(root string) error { +func CreateFile(name string, content []byte, perm os.FileMode) Applier { + return applyFn(func(root string) error { fullPath := filepath.Join(root, name) if err := ioutil.WriteFile(fullPath, content, perm); err != nil { return err @@ -26,67 +34,67 @@ func NewTestFile(name string, content []byte, perm os.FileMode) Applier { } return nil - } + }) } // RemoveFile returns a file applier which removes the provided file name func RemoveFile(name string) Applier { - return func(root string) error { + return applyFn(func(root string) error { return os.RemoveAll(filepath.Join(root, name)) - } + }) } -// CreateDirectory returns a file applier to create the directory with +// CreateDir returns a file applier to create the directory with // the provided name and permission -func CreateDirectory(name string, perm os.FileMode) Applier { - return func(root string) error { +func CreateDir(name string, perm os.FileMode) Applier { + return applyFn(func(root string) error { fullPath := filepath.Join(root, name) if err := os.MkdirAll(fullPath, perm); err != nil { return err } return nil - } + }) } // Rename returns a file applier which renames a file func Rename(old, new string) Applier { - return func(root string) error { + return applyFn(func(root string) error { return os.Rename(filepath.Join(root, old), filepath.Join(root, new)) - } + }) } // Chown returns a file applier which changes the ownership of a file func Chown(name string, uid, gid int) Applier { - return func(root string) error { + return applyFn(func(root string) error { return os.Chown(filepath.Join(root, name), uid, gid) - } + }) } // Chtime changes access and mod time of file func Chtime(name string, t time.Time) Applier { - return func(root string) error { + return applyFn(func(root string) error { return os.Chtimes(filepath.Join(root, name), t, t) - } + }) } // Symlink returns a file applier which creates a symbolic link func Symlink(oldname, newname string) Applier { - return func(root string) error { + return applyFn(func(root string) error { return os.Symlink(oldname, filepath.Join(root, newname)) - } + }) } // Link returns a file applier which creates a hard link func Link(oldname, newname string) Applier { - return func(root string) error { + return applyFn(func(root string) error { return os.Link(filepath.Join(root, oldname), filepath.Join(root, newname)) - } + }) } func SetXAttr(name, key, value string) Applier { - return func(root string) error { + return applyFn(func(root string) error { return sysx.LSetxattr(name, key, []byte(value), 0) - } + }) } // TODO: Make platform specific, windows applier is always no-op @@ -96,14 +104,14 @@ func SetXAttr(name, key, value string) Applier { // } //} -// MultiApply returns a new applier from the given appliers -func MultiApply(appliers ...Applier) Applier { - return func(root string) error { +// Apply returns a new applier from the given appliers +func Apply(appliers ...Applier) Applier { + return applyFn(func(root string) error { for _, a := range appliers { - if err := a(root); err != nil { + if err := a.Apply(root); err != nil { return err } } return nil - } + }) } diff --git a/fs/path.go b/fs/path.go index 342f392..a46d0fc 100644 --- a/fs/path.go +++ b/fs/path.go @@ -15,42 +15,27 @@ type currentPath struct { fullPath string } -func pathChange(lower, upper *currentPath) Change { +func pathChange(lower, upper *currentPath) (ChangeKind, string) { if lower == nil { if upper == nil { panic("cannot compare nil paths") } - return Change{ - Kind: ChangeKindAdd, - Path: upper.path, - } + return ChangeKindAdd, upper.path } if upper == nil { - return Change{ - Kind: ChangeKindDelete, - Path: lower.path, - } + return ChangeKindDelete, lower.path } // TODO: compare by directory switch i := strings.Compare(lower.path, upper.path); { case i < 0: // File in lower that is not in upper - return Change{ - Kind: ChangeKindDelete, - Path: lower.path, - } + return ChangeKindDelete, lower.path case i > 0: // File in upper that is not in lower - return Change{ - Kind: ChangeKindAdd, - Path: upper.path, - } + return ChangeKindAdd, upper.path default: - return Change{ - Kind: ChangeKindModify, - Path: upper.path, - } + return ChangeKindModify, upper.path } } From aaf18b5962d852108a50c04c6610e7704182cd2d Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Fri, 3 Feb 2017 12:08:53 -0800 Subject: [PATCH 12/12] Rename CopyDirectory to CopyDir Signed-off-by: Derek McGowan (github: dmcgowan) --- fs/copy.go | 4 ++-- fs/copy_test.go | 2 +- fs/diff_test.go | 2 +- snapshot/naive/naive.go | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/fs/copy.go b/fs/copy.go index 8c44c7c..f7a8606 100644 --- a/fs/copy.go +++ b/fs/copy.go @@ -17,9 +17,9 @@ var ( } ) -// CopyDirectory copies the directory from src to dst. +// CopyDir copies the directory from src to dst. // Most efficient copy of files is attempted. -func CopyDirectory(dst, src string) error { +func CopyDir(dst, src string) error { inodes := map[uint64]string{} return copyDirectory(dst, src, inodes) } diff --git a/fs/copy_test.go b/fs/copy_test.go index d63be66..0e7cac0 100644 --- a/fs/copy_test.go +++ b/fs/copy_test.go @@ -45,7 +45,7 @@ func testCopy(apply fstest.Applier) error { return errors.Wrap(err, "failed to apply changes") } - if err := CopyDirectory(t2, t1); err != nil { + if err := CopyDir(t2, t1); err != nil { return errors.Wrap(err, "failed to copy") } diff --git a/fs/diff_test.go b/fs/diff_test.go index fb8f012..5eca394 100644 --- a/fs/diff_test.go +++ b/fs/diff_test.go @@ -164,7 +164,7 @@ func testDiffWithBase(base, diff fstest.Applier, expected []testChange) error { return errors.Wrap(err, "failed to apply base filesytem") } - if err := CopyDirectory(t2, t1); err != nil { + if err := CopyDir(t2, t1); err != nil { return errors.Wrap(err, "failed to copy base directory") } diff --git a/snapshot/naive/naive.go b/snapshot/naive/naive.go index bf48583..9099d2b 100644 --- a/snapshot/naive/naive.go +++ b/snapshot/naive/naive.go @@ -66,7 +66,7 @@ func (n *Naive) Prepare(dst, parent string) ([]containerd.Mount, error) { } // Now, we copy the parent filesystem, just a directory, into dst. - if err := fs.CopyDirectory(dst, filepath.Join(parent, "data")); err != nil { + if err := fs.CopyDir(dst, filepath.Join(parent, "data")); err != nil { return nil, errors.Wrap(err, "copying of parent failed") } } @@ -88,7 +88,7 @@ func (n *Naive) Commit(diff, dst string) error { // Move the data into our metadata directory, we could probably save disk // space if we just saved the diff, but let's get something working. - if err := fs.CopyDirectory(filepath.Join(active.metadata, "data"), dst); err != nil { + if err := fs.CopyDir(filepath.Join(active.metadata, "data"), dst); err != nil { return errors.Wrap(err, "copying of parent failed") }