vbatts/pkg
1
0
Fork 0
Code Pull requests Releases Activity

Remove some code we no longer depend on

Browse source 
Signed-off-by: Nalin Dahyabhai <nalin@redhat.com>
This commit is contained in:
Nalin Dahyabhai 2016-07-07 15:01:33 -04:00
parent b350de236d
commit 076755f4a2
144 changed files with 0 additions and 14199 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

92
aaparser/aaparser.go
View file

@ -1,92 +0,0 @@
// Package aaparser is a convenience package interacting with `apparmor_parser`.
package aaparser
import (
"fmt"
"os/exec"
"path/filepath"
"strconv"
"strings"
)
const (
binary = "apparmor_parser"
)
// GetVersion returns the major and minor version of apparmor_parser.
func GetVersion() (int, error) {
output, err := cmd("", "--version")
if err != nil {
return -1, err
}
return parseVersion(output)
}
// LoadProfile runs `apparmor_parser -r -W` on a specified apparmor profile to
// replace and write it to disk.
func LoadProfile(profilePath string) error {
_, err := cmd(filepath.Dir(profilePath), "-r", "-W", filepath.Base(profilePath))
if err != nil {
return err
}
return nil
}
// cmd runs `apparmor_parser` with the passed arguments.
func cmd(dir string, arg ...string) (string, error) {
c := exec.Command(binary, arg...)
c.Dir = dir
output, err := c.CombinedOutput()
if err != nil {
return "", fmt.Errorf("running `%s %s` failed with output: %s\nerror: %v", c.Path, strings.Join(c.Args, " "), string(output), err)
}
return string(output), nil
}
// parseVersion takes the output from `apparmor_parser --version` and returns
// a representation of the {major, minor, patch} version as a single number of
// the form MMmmPPP {major, minor, patch}.
func parseVersion(output string) (int, error) {
// output is in the form of the following:
// AppArmor parser version 2.9.1
// Copyright (C) 1999-2008 Novell Inc.
// Copyright 2009-2012 Canonical Ltd.
lines := strings.SplitN(output, "\n", 2)
words := strings.Split(lines[0], " ")
version := words[len(words)-1]
// split by major minor version
v := strings.Split(version, ".")
if len(v) == 0 || len(v) > 3 {
return -1, fmt.Errorf("parsing version failed for output: `%s`", output)
}
// Default the versions to 0.
var majorVersion, minorVersion, patchLevel int
majorVersion, err := strconv.Atoi(v[0])
if err != nil {
return -1, err
}
if len(v) > 1 {
minorVersion, err = strconv.Atoi(v[1])
if err != nil {
return -1, err
}
}
if len(v) > 2 {
patchLevel, err = strconv.Atoi(v[2])
if err != nil {
return -1, err
}
}
// major*10^5 + minor*10^3 + patch*10^0
numericVersion := majorVersion*1e5 + minorVersion*1e3 + patchLevel
return numericVersion, nil
}

73
aaparser/aaparser_test.go
View file

@ -1,73 +0,0 @@
package aaparser
import (
"testing"
)
type versionExpected struct {
output string
version int
}
func TestParseVersion(t *testing.T) {
versions := []versionExpected{
{
output: `AppArmor parser version 2.10
Copyright (C) 1999-2008 Novell Inc.
Copyright 2009-2012 Canonical Ltd.
`,
version: 210000,
},
{
output: `AppArmor parser version 2.8
Copyright (C) 1999-2008 Novell Inc.
Copyright 2009-2012 Canonical Ltd.
`,
version: 208000,
},
{
output: `AppArmor parser version 2.20
Copyright (C) 1999-2008 Novell Inc.
Copyright 2009-2012 Canonical Ltd.
`,
version: 220000,
},
{
output: `AppArmor parser version 2.05
Copyright (C) 1999-2008 Novell Inc.
Copyright 2009-2012 Canonical Ltd.
`,
version: 205000,
},
{
output: `AppArmor parser version 2.9.95
Copyright (C) 1999-2008 Novell Inc.
Copyright 2009-2012 Canonical Ltd.
`,
version: 209095,
},
{
output: `AppArmor parser version 3.14.159
Copyright (C) 1999-2008 Novell Inc.
Copyright 2009-2012 Canonical Ltd.
`,
version: 314159,
},
}
for _, v := range versions {
version, err := parseVersion(v.output)
if err != nil {
t.Fatalf("expected error to be nil for %#v, got: %v", v, err)
}
if version != v.version {
t.Fatalf("expected version to be %d, was %d, for: %#v\n", v.version, version, v)
}
}
}

54
authorization/api.go
View file

@ -1,54 +0,0 @@
package authorization
const (
// AuthZApiRequest is the url for daemon request authorization
AuthZApiRequest = "AuthZPlugin.AuthZReq"
// AuthZApiResponse is the url for daemon response authorization
AuthZApiResponse = "AuthZPlugin.AuthZRes"
// AuthZApiImplements is the name of the interface all AuthZ plugins implement
AuthZApiImplements = "authz"
)
// Request holds data required for authZ plugins
type Request struct {
// User holds the user extracted by AuthN mechanism
User string `json:"User,omitempty"`
// UserAuthNMethod holds the mechanism used to extract user details (e.g., krb)
UserAuthNMethod string `json:"UserAuthNMethod,omitempty"`
// RequestMethod holds the HTTP method (GET/POST/PUT)
RequestMethod string `json:"RequestMethod,omitempty"`
// RequestUri holds the full HTTP uri (e.g., /v1.21/version)
RequestURI string `json:"RequestUri,omitempty"`
// RequestBody stores the raw request body sent to the docker daemon
RequestBody []byte `json:"RequestBody,omitempty"`
// RequestHeaders stores the raw request headers sent to the docker daemon
RequestHeaders map[string]string `json:"RequestHeaders,omitempty"`
// ResponseStatusCode stores the status code returned from docker daemon
ResponseStatusCode int `json:"ResponseStatusCode,omitempty"`
// ResponseBody stores the raw response body sent from docker daemon
ResponseBody []byte `json:"ResponseBody,omitempty"`
// ResponseHeaders stores the response headers sent to the docker daemon
ResponseHeaders map[string]string `json:"ResponseHeaders,omitempty"`
}
// Response represents authZ plugin response
type Response struct {
// Allow indicating whether the user is allowed or not
Allow bool `json:"Allow"`
// Msg stores the authorization message
Msg string `json:"Msg,omitempty"`
// Err stores a message in case there's an error
Err string `json:"Err,omitempty"`
}

179
authorization/authz.go
View file

@ -1,179 +0,0 @@
package authorization
import (
"bufio"
"bytes"
"fmt"
"io"
"net/http"
"strings"
"github.com/Sirupsen/logrus"
"github.com/docker/docker/pkg/ioutils"
)
const maxBodySize = 1048576 // 1MB
// NewCtx creates new authZ context, it is used to store authorization information related to a specific docker
// REST http session
// A context provides two method:
// Authenticate Request:
// Call authZ plugins with current REST request and AuthN response
// Request contains full HTTP packet sent to the docker daemon
// https://docs.docker.com/reference/api/docker_remote_api/
//
// Authenticate Response:
// Call authZ plugins with full info about current REST request, REST response and AuthN response
// The response from this method may contains content that overrides the daemon response
// This allows authZ plugins to filter privileged content
//
// If multiple authZ plugins are specified, the block/allow decision is based on ANDing all plugin results
// For response manipulation, the response from each plugin is piped between plugins. Plugin execution order
// is determined according to daemon parameters
func NewCtx(authZPlugins []Plugin, user, userAuthNMethod, requestMethod, requestURI string) *Ctx {
return &Ctx{
plugins: authZPlugins,
user: user,
userAuthNMethod: userAuthNMethod,
requestMethod: requestMethod,
requestURI: requestURI,
}
}
// Ctx stores a single request-response interaction context
type Ctx struct {
user string
userAuthNMethod string
requestMethod string
requestURI string
plugins []Plugin
// authReq stores the cached request object for the current transaction
authReq *Request
}
// AuthZRequest authorized the request to the docker daemon using authZ plugins
func (ctx *Ctx) AuthZRequest(w http.ResponseWriter, r *http.Request) error {
var body []byte
if sendBody(ctx.requestURI, r.Header) && r.ContentLength > 0 && r.ContentLength < maxBodySize {
var err error
body, r.Body, err = drainBody(r.Body)
if err != nil {
return err
}
}
var h bytes.Buffer
if err := r.Header.Write(&h); err != nil {
return err
}
ctx.authReq = &Request{
User: ctx.user,
UserAuthNMethod: ctx.userAuthNMethod,
RequestMethod: ctx.requestMethod,
RequestURI: ctx.requestURI,
RequestBody: body,
RequestHeaders: headers(r.Header),
}
for _, plugin := range ctx.plugins {
logrus.Debugf("AuthZ request using plugin %s", plugin.Name())
authRes, err := plugin.AuthZRequest(ctx.authReq)
if err != nil {
return fmt.Errorf("plugin %s failed with error: %s", plugin.Name(), err)
}
if !authRes.Allow {
return newAuthorizationError(plugin.Name(), authRes.Msg)
}
}
return nil
}
// AuthZResponse authorized and manipulates the response from docker daemon using authZ plugins
func (ctx *Ctx) AuthZResponse(rm ResponseModifier, r *http.Request) error {
ctx.authReq.ResponseStatusCode = rm.StatusCode()
ctx.authReq.ResponseHeaders = headers(rm.Header())
if sendBody(ctx.requestURI, rm.Header()) {
ctx.authReq.ResponseBody = rm.RawBody()
}
for _, plugin := range ctx.plugins {
logrus.Debugf("AuthZ response using plugin %s", plugin.Name())
authRes, err := plugin.AuthZResponse(ctx.authReq)
if err != nil {
return fmt.Errorf("plugin %s failed with error: %s", plugin.Name(), err)
}
if !authRes.Allow {
return newAuthorizationError(plugin.Name(), authRes.Msg)
}
}
rm.FlushAll()
return nil
}
// drainBody dump the body (if it's length is less than 1MB) without modifying the request state
func drainBody(body io.ReadCloser) ([]byte, io.ReadCloser, error) {
bufReader := bufio.NewReaderSize(body, maxBodySize)
newBody := ioutils.NewReadCloserWrapper(bufReader, func() error { return body.Close() })
data, err := bufReader.Peek(maxBodySize)
// Body size exceeds max body size
if err == nil {
logrus.Warnf("Request body is larger than: '%d' skipping body", maxBodySize)
return nil, newBody, nil
}
// Body size is less than maximum size
if err == io.EOF {
return data, newBody, nil
}
// Unknown error
return nil, newBody, err
}
// sendBody returns true when request/response body should be sent to AuthZPlugin
func sendBody(url string, header http.Header) bool {
// Skip body for auth endpoint
if strings.HasSuffix(url, "/auth") {
return false
}
// body is sent only for text or json messages
return header.Get("Content-Type") == "application/json"
}
// headers returns flatten version of the http headers excluding authorization
func headers(header http.Header) map[string]string {
v := make(map[string]string, 0)
for k, values := range header {
// Skip authorization headers
if strings.EqualFold(k, "Authorization") || strings.EqualFold(k, "X-Registry-Config") || strings.EqualFold(k, "X-Registry-Auth") {
continue
}
for _, val := range values {
v[k] = val
}
}
return v
}
// authorizationError represents an authorization deny error
type authorizationError struct {
error
}
// HTTPErrorStatusCode returns the authorization error status code (forbidden)
func (e authorizationError) HTTPErrorStatusCode() int {
return http.StatusForbidden
}
func newAuthorizationError(plugin, msg string) authorizationError {
return authorizationError{error: fmt.Errorf("authorization denied by plugin %s: %s", plugin, msg)}
}

282
authorization/authz_unix_test.go
View file

@ -1,282 +0,0 @@
// +build !windows
// TODO Windows: This uses a Unix socket for testing. This might be possible
// to port to Windows using a named pipe instead.
package authorization
import (
"bytes"
"encoding/json"
"io/ioutil"
"net"
"net/http"
"net/http/httptest"
"os"
"path"
"reflect"
"strings"
"testing"
"github.com/docker/docker/pkg/plugins"
"github.com/docker/go-connections/tlsconfig"
"github.com/gorilla/mux"
)
const (
pluginAddress = "authz-test-plugin.sock"
)
func TestAuthZRequestPluginError(t *testing.T) {
server := authZPluginTestServer{t: t}
server.start()
defer server.stop()
authZPlugin := createTestPlugin(t)
request := Request{
User: "user",
RequestBody: []byte("sample body"),
RequestURI: "www.authz.com/auth",
RequestMethod: "GET",
RequestHeaders: map[string]string{"header": "value"},
}
server.replayResponse = Response{
Err: "an error",
}
actualResponse, err := authZPlugin.AuthZRequest(&request)
if err != nil {
t.Fatalf("Failed to authorize request %v", err)
}
if !reflect.DeepEqual(server.replayResponse, *actualResponse) {
t.Fatal("Response must be equal")
}
if !reflect.DeepEqual(request, server.recordedRequest) {
t.Fatal("Requests must be equal")
}
}
func TestAuthZRequestPlugin(t *testing.T) {
server := authZPluginTestServer{t: t}
server.start()
defer server.stop()
authZPlugin := createTestPlugin(t)
request := Request{
User: "user",
RequestBody: []byte("sample body"),
RequestURI: "www.authz.com/auth",
RequestMethod: "GET",
RequestHeaders: map[string]string{"header": "value"},
}
server.replayResponse = Response{
Allow: true,
Msg: "Sample message",
}
actualResponse, err := authZPlugin.AuthZRequest(&request)
if err != nil {
t.Fatalf("Failed to authorize request %v", err)
}
if !reflect.DeepEqual(server.replayResponse, *actualResponse) {
t.Fatal("Response must be equal")
}
if !reflect.DeepEqual(request, server.recordedRequest) {
t.Fatal("Requests must be equal")
}
}
func TestAuthZResponsePlugin(t *testing.T) {
server := authZPluginTestServer{t: t}
server.start()
defer server.stop()
authZPlugin := createTestPlugin(t)
request := Request{
User: "user",
RequestURI: "someting.com/auth",
RequestBody: []byte("sample body"),
}
server.replayResponse = Response{
Allow: true,
Msg: "Sample message",
}
actualResponse, err := authZPlugin.AuthZResponse(&request)
if err != nil {
t.Fatalf("Failed to authorize request %v", err)
}
if !reflect.DeepEqual(server.replayResponse, *actualResponse) {
t.Fatal("Response must be equal")
}
if !reflect.DeepEqual(request, server.recordedRequest) {
t.Fatal("Requests must be equal")
}
}
func TestResponseModifier(t *testing.T) {
r := httptest.NewRecorder()
m := NewResponseModifier(r)
m.Header().Set("h1", "v1")
m.Write([]byte("body"))
m.WriteHeader(500)
m.FlushAll()
if r.Header().Get("h1") != "v1" {
t.Fatalf("Header value must exists %s", r.Header().Get("h1"))
}
if !reflect.DeepEqual(r.Body.Bytes(), []byte("body")) {
t.Fatalf("Body value must exists %s", r.Body.Bytes())
}
if r.Code != 500 {
t.Fatalf("Status code must be correct %d", r.Code)
}
}
func TestDrainBody(t *testing.T) {
tests := []struct {
length int // length is the message length send to drainBody
expectedBodyLength int // expectedBodyLength is the expected body length after drainBody is called
}{
{10, 10}, // Small message size
{maxBodySize - 1, maxBodySize - 1}, // Max message size
{maxBodySize * 2, 0}, // Large message size (skip copying body)
}
for _, test := range tests {
msg := strings.Repeat("a", test.length)
body, closer, err := drainBody(ioutil.NopCloser(bytes.NewReader([]byte(msg))))
if err != nil {
t.Fatal(err)
}
if len(body) != test.expectedBodyLength {
t.Fatalf("Body must be copied, actual length: '%d'", len(body))
}
if closer == nil {
t.Fatal("Closer must not be nil")
}
modified, err := ioutil.ReadAll(closer)
if err != nil {
t.Fatalf("Error must not be nil: '%v'", err)
}
if len(modified) != len(msg) {
t.Fatalf("Result should not be truncated. Original length: '%d', new length: '%d'", len(msg), len(modified))
}
}
}
func TestResponseModifierOverride(t *testing.T) {
r := httptest.NewRecorder()
m := NewResponseModifier(r)
m.Header().Set("h1", "v1")
m.Write([]byte("body"))
m.WriteHeader(500)
overrideHeader := make(http.Header)
overrideHeader.Add("h1", "v2")
overrideHeaderBytes, err := json.Marshal(overrideHeader)
if err != nil {
t.Fatalf("override header failed %v", err)
}
m.OverrideHeader(overrideHeaderBytes)
m.OverrideBody([]byte("override body"))
m.OverrideStatusCode(404)
m.FlushAll()
if r.Header().Get("h1") != "v2" {
t.Fatalf("Header value must exists %s", r.Header().Get("h1"))
}
if !reflect.DeepEqual(r.Body.Bytes(), []byte("override body")) {
t.Fatalf("Body value must exists %s", r.Body.Bytes())
}
if r.Code != 404 {
t.Fatalf("Status code must be correct %d", r.Code)
}
}
// createTestPlugin creates a new sample authorization plugin
func createTestPlugin(t *testing.T) *authorizationPlugin {
pwd, err := os.Getwd()
if err != nil {
t.Fatal(err)
}
client, err := plugins.NewClient("unix:///"+path.Join(pwd, pluginAddress), &tlsconfig.Options{InsecureSkipVerify: true})
if err != nil {
t.Fatalf("Failed to create client %v", err)
}
return &authorizationPlugin{name: "plugin", plugin: client}
}
// AuthZPluginTestServer is a simple server that implements the authZ plugin interface
type authZPluginTestServer struct {
listener net.Listener
t *testing.T
// request stores the request sent from the daemon to the plugin
recordedRequest Request
// response stores the response sent from the plugin to the daemon
replayResponse Response
server *httptest.Server
}
// start starts the test server that implements the plugin
func (t *authZPluginTestServer) start() {
r := mux.NewRouter()
l, err := net.Listen("unix", pluginAddress)
if err != nil {
t.t.Fatal(err)
}
t.listener = l
r.HandleFunc("/Plugin.Activate", t.activate)
r.HandleFunc("/"+AuthZApiRequest, t.auth)
r.HandleFunc("/"+AuthZApiResponse, t.auth)
t.server = &httptest.Server{
Listener: l,
Config: &http.Server{
Handler: r,
Addr: pluginAddress,
},
}
t.server.Start()
}
// stop stops the test server that implements the plugin
func (t *authZPluginTestServer) stop() {
t.server.Close()
os.Remove(pluginAddress)
if t.listener != nil {
t.listener.Close()
}
}
// auth is a used to record/replay the authentication api messages
func (t *authZPluginTestServer) auth(w http.ResponseWriter, r *http.Request) {
t.recordedRequest = Request{}
body, err := ioutil.ReadAll(r.Body)
if err != nil {
t.t.Fatal(err)
}
r.Body.Close()
json.Unmarshal(body, &t.recordedRequest)
b, err := json.Marshal(t.replayResponse)
if err != nil {
t.t.Fatal(err)
}
w.Write(b)
}
func (t *authZPluginTestServer) activate(w http.ResponseWriter, r *http.Request) {
b, err := json.Marshal(plugins.Manifest{Implements: []string{AuthZApiImplements}})
if err != nil {
t.t.Fatal(err)
}
w.Write(b)
}

60
authorization/middleware.go
View file

@ -1,60 +0,0 @@
package authorization
import (
"net/http"
"github.com/Sirupsen/logrus"
"golang.org/x/net/context"
)
// Middleware uses a list of plugins to
// handle authorization in the API requests.
type Middleware struct {
plugins []Plugin
}
// NewMiddleware creates a new Middleware
// with a slice of plugins.
func NewMiddleware(p []Plugin) Middleware {
return Middleware{
plugins: p,
}
}
// WrapHandler returns a new handler function wrapping the previous one in the request chain.
func (m Middleware) WrapHandler(handler func(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error) func(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
return func(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
user := ""
userAuthNMethod := ""
// Default authorization using existing TLS connection credentials
// FIXME: Non trivial authorization mechanisms (such as advanced certificate validations, kerberos support
// and ldap) will be extracted using AuthN feature, which is tracked under:
// https://github.com/docker/docker/pull/20883
if r.TLS != nil && len(r.TLS.PeerCertificates) > 0 {
user = r.TLS.PeerCertificates[0].Subject.CommonName
userAuthNMethod = "TLS"
}
authCtx := NewCtx(m.plugins, user, userAuthNMethod, r.Method, r.RequestURI)
if err := authCtx.AuthZRequest(w, r); err != nil {
logrus.Errorf("AuthZRequest for %s %s returned error: %s", r.Method, r.RequestURI, err)
return err
}
rw := NewResponseModifier(w)
if err := handler(ctx, rw, r, vars); err != nil {
logrus.Errorf("Handler for %s %s returned error: %s", r.Method, r.RequestURI, err)
return err
}
if err := authCtx.AuthZResponse(rw, r); err != nil {
logrus.Errorf("AuthZResponse for %s %s returned error: %s", r.Method, r.RequestURI, err)
return err
}
return nil
}
}

92
authorization/plugin.go
View file

@ -1,92 +0,0 @@
package authorization
import (
"sync"
"github.com/docker/docker/pkg/plugins"
)
// Plugin allows third party plugins to authorize requests and responses
// in the context of docker API
type Plugin interface {
// Name returns the registered plugin name
Name() string
// AuthZRequest authorizes the request from the client to the daemon
AuthZRequest(*Request) (*Response, error)
// AuthZResponse authorizes the response from the daemon to the client
AuthZResponse(*Request) (*Response, error)
}
// NewPlugins constructs and initializes the authorization plugins based on plugin names
func NewPlugins(names []string) []Plugin {
plugins := []Plugin{}
pluginsMap := make(map[string]struct{})
for _, name := range names {
if _, ok := pluginsMap[name]; ok {
continue
}
pluginsMap[name] = struct{}{}
plugins = append(plugins, newAuthorizationPlugin(name))
}
return plugins
}
// authorizationPlugin is an internal adapter to docker plugin system
type authorizationPlugin struct {
plugin *plugins.Client
name string
once sync.Once
}
func newAuthorizationPlugin(name string) Plugin {
return &authorizationPlugin{name: name}
}
func (a *authorizationPlugin) Name() string {
return a.name
}
func (a *authorizationPlugin) AuthZRequest(authReq *Request) (*Response, error) {
if err := a.initPlugin(); err != nil {
return nil, err
}
authRes := &Response{}
if err := a.plugin.Call(AuthZApiRequest, authReq, authRes); err != nil {
return nil, err
}
return authRes, nil
}
func (a *authorizationPlugin) AuthZResponse(authReq *Request) (*Response, error) {
if err := a.initPlugin(); err != nil {
return nil, err
}
authRes := &Response{}
if err := a.plugin.Call(AuthZApiResponse, authReq, authRes); err != nil {
return nil, err
}
return authRes, nil
}
// initPlugin initializes the authorization plugin if needed
func (a *authorizationPlugin) initPlugin() error {
// Lazy loading of plugins
var err error
a.once.Do(func() {
if a.plugin == nil {
plugin, e := plugins.Get(a.name, AuthZApiImplements)
if e != nil {
err = e
return
}
a.plugin = plugin.Client()
}
})
return err
}

201
authorization/response.go
View file

@ -1,201 +0,0 @@
package authorization
import (
"bufio"
"bytes"
"encoding/json"
"fmt"
"net"
"net/http"
"github.com/Sirupsen/logrus"
)
// ResponseModifier allows authorization plugins to read and modify the content of the http.response
type ResponseModifier interface {
http.ResponseWriter
http.Flusher
http.CloseNotifier
// RawBody returns the current http content
RawBody() []byte
// RawHeaders returns the current content of the http headers
RawHeaders() ([]byte, error)
// StatusCode returns the current status code
StatusCode() int
// OverrideBody replaces the body of the HTTP reply
OverrideBody(b []byte)
// OverrideHeader replaces the headers of the HTTP reply
OverrideHeader(b []byte) error
// OverrideStatusCode replaces the status code of the HTTP reply
OverrideStatusCode(statusCode int)
// Flush flushes all data to the HTTP response
FlushAll() error
// Hijacked indicates the response has been hijacked by the Docker daemon
Hijacked() bool
}
// NewResponseModifier creates a wrapper to an http.ResponseWriter to allow inspecting and modifying the content
func NewResponseModifier(rw http.ResponseWriter) ResponseModifier {
return &responseModifier{rw: rw, header: make(http.Header)}
}
// responseModifier is used as an adapter to http.ResponseWriter in order to manipulate and explore
// the http request/response from docker daemon
type responseModifier struct {
// The original response writer
rw http.ResponseWriter
// body holds the response body
body []byte
// header holds the response header
header http.Header
// statusCode holds the response status code
statusCode int
// hijacked indicates the request has been hijacked
hijacked bool
}
func (rm *responseModifier) Hijacked() bool {
return rm.hijacked
}
// WriterHeader stores the http status code
func (rm *responseModifier) WriteHeader(s int) {
// Use original request if hijacked
if rm.hijacked {
rm.rw.WriteHeader(s)
return
}
rm.statusCode = s
}
// Header returns the internal http header
func (rm *responseModifier) Header() http.Header {
// Use original header if hijacked
if rm.hijacked {
return rm.rw.Header()
}
return rm.header
}
// StatusCode returns the http status code
func (rm *responseModifier) StatusCode() int {
return rm.statusCode
}
// OverrideBody replaces the body of the HTTP response
func (rm *responseModifier) OverrideBody(b []byte) {
rm.body = b
}
// OverrideStatusCode replaces the status code of the HTTP response
func (rm *responseModifier) OverrideStatusCode(statusCode int) {
rm.statusCode = statusCode
}
// OverrideHeader replaces the headers of the HTTP response
func (rm *responseModifier) OverrideHeader(b []byte) error {
header := http.Header{}
if err := json.Unmarshal(b, &header); err != nil {
return err
}
rm.header = header
return nil
}
// Write stores the byte array inside content
func (rm *responseModifier) Write(b []byte) (int, error) {
if rm.hijacked {
return rm.rw.Write(b)
}
rm.body = append(rm.body, b...)
return len(b), nil
}
// Body returns the response body
func (rm *responseModifier) RawBody() []byte {
return rm.body
}
func (rm *responseModifier) RawHeaders() ([]byte, error) {
var b bytes.Buffer
if err := rm.header.Write(&b); err != nil {
return nil, err
}
return b.Bytes(), nil
}
// Hijack returns the internal connection of the wrapped http.ResponseWriter
func (rm *responseModifier) Hijack() (net.Conn, *bufio.ReadWriter, error) {
rm.hijacked = true
rm.FlushAll()
hijacker, ok := rm.rw.(http.Hijacker)
if !ok {
return nil, nil, fmt.Errorf("Internal response writer doesn't support the Hijacker interface")
}
return hijacker.Hijack()
}
// CloseNotify uses the internal close notify API of the wrapped http.ResponseWriter
func (rm *responseModifier) CloseNotify() <-chan bool {
closeNotifier, ok := rm.rw.(http.CloseNotifier)
if !ok {
logrus.Error("Internal response writer doesn't support the CloseNotifier interface")
return nil
}
return closeNotifier.CloseNotify()
}
// Flush uses the internal flush API of the wrapped http.ResponseWriter
func (rm *responseModifier) Flush() {
flusher, ok := rm.rw.(http.Flusher)
if !ok {
logrus.Error("Internal response writer doesn't support the Flusher interface")
return
}
rm.FlushAll()
flusher.Flush()
}
// FlushAll flushes all data to the HTTP response
func (rm *responseModifier) FlushAll() error {
// Copy the status code
if rm.statusCode > 0 {
rm.rw.WriteHeader(rm.statusCode)
}
// Copy the header
for k, vv := range rm.header {
for _, v := range vv {
rm.rw.Header().Add(k, v)
}
}
var err error
if len(rm.body) > 0 {
// Write body
_, err = rm.rw.Write(rm.body)
}
// Clean previous data
rm.body = nil
rm.statusCode = 0
rm.header = http.Header{}
return err
}

49
broadcaster/unbuffered.go
View file

@ -1,49 +0,0 @@
package broadcaster
import (
"io"
"sync"
)
// Unbuffered accumulates multiple io.WriteCloser by stream.
type Unbuffered struct {
mu sync.Mutex
writers []io.WriteCloser
}
// Add adds new io.WriteCloser.
func (w *Unbuffered) Add(writer io.WriteCloser) {
w.mu.Lock()
w.writers = append(w.writers, writer)
w.mu.Unlock()
}
// Write writes bytes to all writers. Failed writers will be evicted during
// this call.
func (w *Unbuffered) Write(p []byte) (n int, err error) {
w.mu.Lock()
var evict []int
for i, sw := range w.writers {
if n, err := sw.Write(p); err != nil || n != len(p) {
// On error, evict the writer
evict = append(evict, i)
}
}
for n, i := range evict {
w.writers = append(w.writers[:i-n], w.writers[i-n+1:]...)
}
w.mu.Unlock()
return len(p), nil
}
// Clean closes and removes all writers. Last non-eol-terminated part of data
// will be saved.
func (w *Unbuffered) Clean() error {
w.mu.Lock()
for _, sw := range w.writers {
sw.Close()
}
w.writers = nil
w.mu.Unlock()
return nil
}

162
broadcaster/unbuffered_test.go
View file

@ -1,162 +0,0 @@
package broadcaster
import (
"bytes"
"errors"
"strings"
"testing"
)
type dummyWriter struct {
buffer bytes.Buffer
failOnWrite bool
}
func (dw *dummyWriter) Write(p []byte) (n int, err error) {
if dw.failOnWrite {
return 0, errors.New("Fake fail")
}
return dw.buffer.Write(p)
}
func (dw *dummyWriter) String() string {
return dw.buffer.String()
}
func (dw *dummyWriter) Close() error {
return nil
}
func TestUnbuffered(t *testing.T) {
writer := new(Unbuffered)
// Test 1: Both bufferA and bufferB should contain "foo"
bufferA := &dummyWriter{}
writer.Add(bufferA)
bufferB := &dummyWriter{}
writer.Add(bufferB)
writer.Write([]byte("foo"))
if bufferA.String() != "foo" {
t.Errorf("Buffer contains %v", bufferA.String())
}
if bufferB.String() != "foo" {
t.Errorf("Buffer contains %v", bufferB.String())
}
// Test2: bufferA and bufferB should contain "foobar",
// while bufferC should only contain "bar"
bufferC := &dummyWriter{}
writer.Add(bufferC)
writer.Write([]byte("bar"))
if bufferA.String() != "foobar" {
t.Errorf("Buffer contains %v", bufferA.String())
}
if bufferB.String() != "foobar" {
t.Errorf("Buffer contains %v", bufferB.String())
}
if bufferC.String() != "bar" {
t.Errorf("Buffer contains %v", bufferC.String())
}
// Test3: Test eviction on failure
bufferA.failOnWrite = true
writer.Write([]byte("fail"))
if bufferA.String() != "foobar" {
t.Errorf("Buffer contains %v", bufferA.String())
}
if bufferC.String() != "barfail" {
t.Errorf("Buffer contains %v", bufferC.String())
}
// Even though we reset the flag, no more writes should go in there
bufferA.failOnWrite = false
writer.Write([]byte("test"))
if bufferA.String() != "foobar" {
t.Errorf("Buffer contains %v", bufferA.String())
}
if bufferC.String() != "barfailtest" {
t.Errorf("Buffer contains %v", bufferC.String())
}
// Test4: Test eviction on multiple simultaneous failures
bufferB.failOnWrite = true
bufferC.failOnWrite = true
bufferD := &dummyWriter{}
writer.Add(bufferD)
writer.Write([]byte("yo"))
writer.Write([]byte("ink"))
if strings.Contains(bufferB.String(), "yoink") {
t.Errorf("bufferB received write. contents: %q", bufferB)
}
if strings.Contains(bufferC.String(), "yoink") {
t.Errorf("bufferC received write. contents: %q", bufferC)
}
if g, w := bufferD.String(), "yoink"; g != w {
t.Errorf("bufferD = %q, want %q", g, w)
}
writer.Clean()
}
type devNullCloser int
func (d devNullCloser) Close() error {
return nil
}
func (d devNullCloser) Write(buf []byte) (int, error) {
return len(buf), nil
}
// This test checks for races. It is only useful when run with the race detector.
func TestRaceUnbuffered(t *testing.T) {
writer := new(Unbuffered)
c := make(chan bool)
go func() {
writer.Add(devNullCloser(0))
c <- true
}()
writer.Write([]byte("hello"))
<-c
}
func BenchmarkUnbuffered(b *testing.B) {
writer := new(Unbuffered)
setUpWriter := func() {
for i := 0; i < 100; i++ {
writer.Add(devNullCloser(0))
writer.Add(devNullCloser(0))
writer.Add(devNullCloser(0))
}
}
testLine := "Line that thinks that it is log line from docker"
var buf bytes.Buffer
for i := 0; i < 100; i++ {
buf.Write([]byte(testLine + "\n"))
}
// line without eol
buf.Write([]byte(testLine))
testText := buf.Bytes()
b.SetBytes(int64(5 * len(testText)))
b.ResetTimer()
for i := 0; i < b.N; i++ {
b.StopTimer()
setUpWriter()
b.StartTimer()
for j := 0; j < 5; j++ {
if _, err := writer.Write(testText); err != nil {
b.Fatal(err)
}
}
b.StopTimer()
writer.Clean()
b.StartTimer()
}
}

41
discovery/README.md
View file

@ -1,41 +0,0 @@
---
page_title: Docker discovery
page_description: discovery
page_keywords: docker, clustering, discovery
---
# Discovery
Docker comes with multiple Discovery backends.
## Backends
### Using etcd
Point your Docker Engine instances to a common etcd instance. You can specify
the address Docker uses to advertise the node using the `--cluster-advertise`
flag.
```bash
$ docker daemon -H=<node_ip:2376> --cluster-advertise=<node_ip:2376> --cluster-store etcd://<etcd_ip1>,<etcd_ip2>/<path>
```
### Using consul
Point your Docker Engine instances to a common Consul instance. You can specify
the address Docker uses to advertise the node using the `--cluster-advertise`
flag.
```bash
$ docker daemon -H=<node_ip:2376> --cluster-advertise=<node_ip:2376> --cluster-store consul://<consul_ip>/<path>
```
### Using zookeeper
Point your Docker Engine instances to a common Zookeeper instance. You can specify
the address Docker uses to advertise the node using the `--cluster-advertise`
flag.
```bash
$ docker daemon -H=<node_ip:2376> --cluster-advertise=<node_ip:2376> --cluster-store zk://<zk_addr1>,<zk_addr2>/<path>
```

107
discovery/backends.go
View file

@ -1,107 +0,0 @@
package discovery
import (
"fmt"
"net"
"strings"
"time"
log "github.com/Sirupsen/logrus"
)
var (
// Backends is a global map of discovery backends indexed by their
// associated scheme.
backends = make(map[string]Backend)
)
// Register makes a discovery backend available by the provided scheme.
// If Register is called twice with the same scheme an error is returned.
func Register(scheme string, d Backend) error {
if _, exists := backends[scheme]; exists {
return fmt.Errorf("scheme already registered %s", scheme)
}
log.WithField("name", scheme).Debug("Registering discovery service")
backends[scheme] = d
return nil
}
func parse(rawurl string) (string, string) {
parts := strings.SplitN(rawurl, "://", 2)
// nodes:port,node2:port => nodes://node1:port,node2:port
if len(parts) == 1 {
return "nodes", parts[0]
}
return parts[0], parts[1]
}
// ParseAdvertise parses the --cluster-advertise daemon config which accepts
// <ip-address>:<port> or <interface-name>:<port>
func ParseAdvertise(advertise string) (string, error) {
var (
iface *net.Interface
addrs []net.Addr
err error
)
addr, port, err := net.SplitHostPort(advertise)
if err != nil {
return "", fmt.Errorf("invalid --cluster-advertise configuration: %s: %v", advertise, err)
}
ip := net.ParseIP(addr)
// If it is a valid ip-address, use it as is
if ip != nil {
return advertise, nil
}
// If advertise is a valid interface name, get the valid ipv4 address and use it to advertise
ifaceName := addr
iface, err = net.InterfaceByName(ifaceName)
if err != nil {
return "", fmt.Errorf("invalid cluster advertise IP address or interface name (%s) : %v", advertise, err)
}
addrs, err = iface.Addrs()
if err != nil {
return "", fmt.Errorf("unable to get advertise IP address from interface (%s) : %v", advertise, err)
}
if addrs == nil || len(addrs) == 0 {
return "", fmt.Errorf("no available advertise IP address in interface (%s)", advertise)
}
addr = ""
for _, a := range addrs {
ip, _, err := net.ParseCIDR(a.String())
if err != nil {
return "", fmt.Errorf("error deriving advertise ip-address in interface (%s) : %v", advertise, err)
}
if ip.To4() == nil || ip.IsLoopback() {
continue
}
addr = ip.String()
break
}
if addr == "" {
return "", fmt.Errorf("couldnt find a valid ip-address in interface %s", advertise)
}
addr = net.JoinHostPort(addr, port)
return addr, nil
}
// New returns a new Discovery given a URL, heartbeat and ttl settings.
// Returns an error if the URL scheme is not supported.
func New(rawurl string, heartbeat time.Duration, ttl time.Duration, clusterOpts map[string]string) (Backend, error) {
scheme, uri := parse(rawurl)
if backend, exists := backends[scheme]; exists {
log.WithFields(log.Fields{"name": scheme, "uri": uri}).Debug("Initializing discovery service")
err := backend.Initialize(uri, heartbeat, ttl, clusterOpts)
return backend, err
}
return nil, ErrNotSupported
}

35
discovery/discovery.go
View file

@ -1,35 +0,0 @@
package discovery
import (
"errors"
"time"
)
var (
// ErrNotSupported is returned when a discovery service is not supported.
ErrNotSupported = errors.New("discovery service not supported")
// ErrNotImplemented is returned when discovery feature is not implemented
// by discovery backend.
ErrNotImplemented = errors.New("not implemented in this discovery service")
)
// Watcher provides watching over a cluster for nodes joining and leaving.
type Watcher interface {
// Watch the discovery for entry changes.
// Returns a channel that will receive changes or an error.
// Providing a non-nil stopCh can be used to stop watching.
Watch(stopCh <-chan struct{}) (<-chan Entries, <-chan error)
}
// Backend is implemented by discovery backends which manage cluster entries.
type Backend interface {
// Watcher must be provided by every backend.
Watcher
// Initialize the discovery with URIs, a heartbeat, a ttl and optional settings.
Initialize(string, time.Duration, time.Duration, map[string]string) error
// Register to the discovery.
Register(string) error
}

137
discovery/discovery_test.go
View file

@ -1,137 +0,0 @@
package discovery
import (
"testing"
"github.com/go-check/check"
)
// Hook up gocheck into the "go test" runner.
func Test(t *testing.T) { check.TestingT(t) }
type DiscoverySuite struct{}
var _ = check.Suite(&DiscoverySuite{})
func (s *DiscoverySuite) TestNewEntry(c *check.C) {
entry, err := NewEntry("127.0.0.1:2375")
c.Assert(err, check.IsNil)
c.Assert(entry.Equals(&Entry{Host: "127.0.0.1", Port: "2375"}), check.Equals, true)
c.Assert(entry.String(), check.Equals, "127.0.0.1:2375")
entry, err = NewEntry("[2001:db8:0:f101::2]:2375")
c.Assert(err, check.IsNil)
c.Assert(entry.Equals(&Entry{Host: "2001:db8:0:f101::2", Port: "2375"}), check.Equals, true)
c.Assert(entry.String(), check.Equals, "[2001:db8:0:f101::2]:2375")
_, err = NewEntry("127.0.0.1")
c.Assert(err, check.NotNil)
}
func (s *DiscoverySuite) TestParse(c *check.C) {
scheme, uri := parse("127.0.0.1:2375")
c.Assert(scheme, check.Equals, "nodes")
c.Assert(uri, check.Equals, "127.0.0.1:2375")
scheme, uri = parse("localhost:2375")
c.Assert(scheme, check.Equals, "nodes")
c.Assert(uri, check.Equals, "localhost:2375")
scheme, uri = parse("scheme://127.0.0.1:2375")
c.Assert(scheme, check.Equals, "scheme")
c.Assert(uri, check.Equals, "127.0.0.1:2375")
scheme, uri = parse("scheme://localhost:2375")
c.Assert(scheme, check.Equals, "scheme")
c.Assert(uri, check.Equals, "localhost:2375")
scheme, uri = parse("")
c.Assert(scheme, check.Equals, "nodes")
c.Assert(uri, check.Equals, "")
}
func (s *DiscoverySuite) TestCreateEntries(c *check.C) {
entries, err := CreateEntries(nil)
c.Assert(entries, check.DeepEquals, Entries{})
c.Assert(err, check.IsNil)
entries, err = CreateEntries([]string{"127.0.0.1:2375", "127.0.0.2:2375", "[2001:db8:0:f101::2]:2375", ""})
c.Assert(err, check.IsNil)
expected := Entries{
&Entry{Host: "127.0.0.1", Port: "2375"},
&Entry{Host: "127.0.0.2", Port: "2375"},
&Entry{Host: "2001:db8:0:f101::2", Port: "2375"},
}
c.Assert(entries.Equals(expected), check.Equals, true)
_, err = CreateEntries([]string{"127.0.0.1", "127.0.0.2"})
c.Assert(err, check.NotNil)
}
func (s *DiscoverySuite) TestContainsEntry(c *check.C) {
entries, err := CreateEntries([]string{"127.0.0.1:2375", "127.0.0.2:2375", ""})
c.Assert(err, check.IsNil)
c.Assert(entries.Contains(&Entry{Host: "127.0.0.1", Port: "2375"}), check.Equals, true)
c.Assert(entries.Contains(&Entry{Host: "127.0.0.3", Port: "2375"}), check.Equals, false)
}
func (s *DiscoverySuite) TestEntriesEquality(c *check.C) {
entries := Entries{
&Entry{Host: "127.0.0.1", Port: "2375"},
&Entry{Host: "127.0.0.2", Port: "2375"},
}
// Same
c.Assert(entries.Equals(Entries{
&Entry{Host: "127.0.0.1", Port: "2375"},
&Entry{Host: "127.0.0.2", Port: "2375"},
}), check.
Equals, true)
// Different size
c.Assert(entries.Equals(Entries{
&Entry{Host: "127.0.0.1", Port: "2375"},
&Entry{Host: "127.0.0.2", Port: "2375"},
&Entry{Host: "127.0.0.3", Port: "2375"},
}), check.
Equals, false)
// Different content
c.Assert(entries.Equals(Entries{
&Entry{Host: "127.0.0.1", Port: "2375"},
&Entry{Host: "127.0.0.42", Port: "2375"},
}), check.
Equals, false)
}
func (s *DiscoverySuite) TestEntriesDiff(c *check.C) {
entry1 := &Entry{Host: "1.1.1.1", Port: "1111"}
entry2 := &Entry{Host: "2.2.2.2", Port: "2222"}
entry3 := &Entry{Host: "3.3.3.3", Port: "3333"}
entries := Entries{entry1, entry2}
// No diff
added, removed := entries.Diff(Entries{entry2, entry1})
c.Assert(added, check.HasLen, 0)
c.Assert(removed, check.HasLen, 0)
// Add
added, removed = entries.Diff(Entries{entry2, entry3, entry1})
c.Assert(added, check.HasLen, 1)
c.Assert(added.Contains(entry3), check.Equals, true)
c.Assert(removed, check.HasLen, 0)
// Remove
added, removed = entries.Diff(Entries{entry2})
c.Assert(added, check.HasLen, 0)
c.Assert(removed, check.HasLen, 1)
c.Assert(removed.Contains(entry1), check.Equals, true)
// Add and remove
added, removed = entries.Diff(Entries{entry1, entry3})
c.Assert(added, check.HasLen, 1)
c.Assert(added.Contains(entry3), check.Equals, true)
c.Assert(removed, check.HasLen, 1)
c.Assert(removed.Contains(entry2), check.Equals, true)
}

94
discovery/entry.go
View file

@ -1,94 +0,0 @@
package discovery
import "net"
// NewEntry creates a new entry.
func NewEntry(url string) (*Entry, error) {
host, port, err := net.SplitHostPort(url)
if err != nil {
return nil, err
}
return &Entry{host, port}, nil
}
// An Entry represents a host.
type Entry struct {
Host string
Port string
}
// Equals returns true if cmp contains the same data.
func (e *Entry) Equals(cmp *Entry) bool {
return e.Host == cmp.Host && e.Port == cmp.Port
}
// String returns the string form of an entry.
func (e *Entry) String() string {
return net.JoinHostPort(e.Host, e.Port)
}
// Entries is a list of *Entry with some helpers.
type Entries []*Entry
// Equals returns true if cmp contains the same data.
func (e Entries) Equals(cmp Entries) bool {
// Check if the file has really changed.
if len(e) != len(cmp) {
return false
}
for i := range e {
if !e[i].Equals(cmp[i]) {
return false
}
}
return true
}
// Contains returns true if the Entries contain a given Entry.
func (e Entries) Contains(entry *Entry) bool {
for _, curr := range e {
if curr.Equals(entry) {
return true
}
}
return false
}
// Diff compares two entries and returns the added and removed entries.
func (e Entries) Diff(cmp Entries) (Entries, Entries) {
added := Entries{}
for _, entry := range cmp {
if !e.Contains(entry) {
added = append(added, entry)
}
}
removed := Entries{}
for _, entry := range e {
if !cmp.Contains(entry) {
removed = append(removed, entry)
}
}
return added, removed
}
// CreateEntries returns an array of entries based on the given addresses.
func CreateEntries(addrs []string) (Entries, error) {
entries := Entries{}
if addrs == nil {
return entries, nil
}
for _, addr := range addrs {
if len(addr) == 0 {
continue
}
entry, err := NewEntry(addr)
if err != nil {
return nil, err
}
entries = append(entries, entry)
}
return entries, nil
}

109
discovery/file/file.go
View file

@ -1,109 +0,0 @@
package file
import (
"fmt"
"io/ioutil"
"strings"
"time"
"github.com/docker/docker/pkg/discovery"
)
// Discovery is exported
type Discovery struct {
heartbeat time.Duration
path string
}
func init() {
Init()
}
// Init is exported
func Init() {
discovery.Register("file", &Discovery{})
}
// Initialize is exported
func (s *Discovery) Initialize(path string, heartbeat time.Duration, ttl time.Duration, _ map[string]string) error {
s.path = path
s.heartbeat = heartbeat
return nil
}
func parseFileContent(content []byte) []string {
var result []string
for _, line := range strings.Split(strings.TrimSpace(string(content)), "\n") {
line = strings.TrimSpace(line)
// Ignoring line starts with #
if strings.HasPrefix(line, "#") {
continue
}
// Inlined # comment also ignored.
if strings.Contains(line, "#") {
line = line[0:strings.Index(line, "#")]
// Trim additional spaces caused by above stripping.
line = strings.TrimSpace(line)
}
for _, ip := range discovery.Generate(line) {
result = append(result, ip)
}
}
return result
}
func (s *Discovery) fetch() (discovery.Entries, error) {
fileContent, err := ioutil.ReadFile(s.path)
if err != nil {
return nil, fmt.Errorf("failed to read '%s': %v", s.path, err)
}
return discovery.CreateEntries(parseFileContent(fileContent))
}
// Watch is exported
func (s *Discovery) Watch(stopCh <-chan struct{}) (<-chan discovery.Entries, <-chan error) {
ch := make(chan discovery.Entries)
errCh := make(chan error)
ticker := time.NewTicker(s.heartbeat)
go func() {
defer close(errCh)
defer close(ch)
// Send the initial entries if available.
currentEntries, err := s.fetch()
if err != nil {
errCh <- err
} else {
ch <- currentEntries
}
// Periodically send updates.
for {
select {
case <-ticker.C:
newEntries, err := s.fetch()
if err != nil {
errCh <- err
continue
}
// Check if the file has really changed.
if !newEntries.Equals(currentEntries) {
ch <- newEntries
}
currentEntries = newEntries
case <-stopCh:
ticker.Stop()
return
}
}
}()
return ch, errCh
}
// Register is exported
func (s *Discovery) Register(addr string) error {
return discovery.ErrNotImplemented
}

114
discovery/file/file_test.go
View file

@ -1,114 +0,0 @@
package file
import (
"io/ioutil"
"os"
"testing"
"github.com/docker/docker/pkg/discovery"
"github.com/go-check/check"
)
// Hook up gocheck into the "go test" runner.
func Test(t *testing.T) { check.TestingT(t) }
type DiscoverySuite struct{}
var _ = check.Suite(&DiscoverySuite{})
func (s *DiscoverySuite) TestInitialize(c *check.C) {
d := &Discovery{}
d.Initialize("/path/to/file", 1000, 0, nil)
c.Assert(d.path, check.Equals, "/path/to/file")
}
func (s *DiscoverySuite) TestNew(c *check.C) {
d, err := discovery.New("file:///path/to/file", 0, 0, nil)
c.Assert(err, check.IsNil)
c.Assert(d.(*Discovery).path, check.Equals, "/path/to/file")
}
func (s *DiscoverySuite) TestContent(c *check.C) {
data := `
1.1.1.[1:2]:1111
2.2.2.[2:4]:2222
`
ips := parseFileContent([]byte(data))
c.Assert(ips, check.HasLen, 5)
c.Assert(ips[0], check.Equals, "1.1.1.1:1111")
c.Assert(ips[1], check.Equals, "1.1.1.2:1111")
c.Assert(ips[2], check.Equals, "2.2.2.2:2222")
c.Assert(ips[3], check.Equals, "2.2.2.3:2222")
c.Assert(ips[4], check.Equals, "2.2.2.4:2222")
}
func (s *DiscoverySuite) TestRegister(c *check.C) {
discovery := &Discovery{path: "/path/to/file"}
c.Assert(discovery.Register("0.0.0.0"), check.NotNil)
}
func (s *DiscoverySuite) TestParsingContentsWithComments(c *check.C) {
data := `
### test ###
1.1.1.1:1111 # inline comment
# 2.2.2.2:2222
### empty line with comment
3.3.3.3:3333
### test ###
`
ips := parseFileContent([]byte(data))
c.Assert(ips, check.HasLen, 2)
c.Assert("1.1.1.1:1111", check.Equals, ips[0])
c.Assert("3.3.3.3:3333", check.Equals, ips[1])
}
func (s *DiscoverySuite) TestWatch(c *check.C) {
data := `
1.1.1.1:1111
2.2.2.2:2222
`
expected := discovery.Entries{
&discovery.Entry{Host: "1.1.1.1", Port: "1111"},
&discovery.Entry{Host: "2.2.2.2", Port: "2222"},
}
// Create a temporary file and remove it.
tmp, err := ioutil.TempFile(os.TempDir(), "discovery-file-test")
c.Assert(err, check.IsNil)
c.Assert(tmp.Close(), check.IsNil)
c.Assert(os.Remove(tmp.Name()), check.IsNil)
// Set up file discovery.
d := &Discovery{}
d.Initialize(tmp.Name(), 1000, 0, nil)
stopCh := make(chan struct{})
ch, errCh := d.Watch(stopCh)
// Make sure it fires errors since the file doesn't exist.
c.Assert(<-errCh, check.NotNil)
// We have to drain the error channel otherwise Watch will get stuck.
go func() {
for range errCh {
}
}()
// Write the file and make sure we get the expected value back.
c.Assert(ioutil.WriteFile(tmp.Name(), []byte(data), 0600), check.IsNil)
c.Assert(<-ch, check.DeepEquals, expected)
// Add a new entry and look it up.
expected = append(expected, &discovery.Entry{Host: "3.3.3.3", Port: "3333"})
f, err := os.OpenFile(tmp.Name(), os.O_APPEND|os.O_WRONLY, 0600)
c.Assert(err, check.IsNil)
c.Assert(f, check.NotNil)
_, err = f.WriteString("\n3.3.3.3:3333\n")
c.Assert(err, check.IsNil)
f.Close()
c.Assert(<-ch, check.DeepEquals, expected)
// Stop and make sure it closes all channels.
close(stopCh)
c.Assert(<-ch, check.IsNil)
c.Assert(<-errCh, check.IsNil)
}

35
discovery/generator.go
View file

@ -1,35 +0,0 @@
package discovery
import (
"fmt"
"regexp"
"strconv"
)
// Generate takes care of IP generation
func Generate(pattern string) []string {
re, _ := regexp.Compile(`\[(.+):(.+)\]`)
submatch := re.FindStringSubmatch(pattern)
if submatch == nil {
return []string{pattern}
}
from, err := strconv.Atoi(submatch[1])
if err != nil {
return []string{pattern}
}
to, err := strconv.Atoi(submatch[2])
if err != nil {
return []string{pattern}
}
template := re.ReplaceAllString(pattern, "%d")
var result []string
for val := from; val <= to; val++ {
entry := fmt.Sprintf(template, val)
result = append(result, entry)
}
return result
}

53
discovery/generator_test.go
View file

@ -1,53 +0,0 @@
package discovery
import (
"github.com/go-check/check"
)
func (s *DiscoverySuite) TestGeneratorNotGenerate(c *check.C) {
ips := Generate("127.0.0.1")
c.Assert(len(ips), check.Equals, 1)
c.Assert(ips[0], check.Equals, "127.0.0.1")
}
func (s *DiscoverySuite) TestGeneratorWithPortNotGenerate(c *check.C) {
ips := Generate("127.0.0.1:8080")
c.Assert(len(ips), check.Equals, 1)
c.Assert(ips[0], check.Equals, "127.0.0.1:8080")
}
func (s *DiscoverySuite) TestGeneratorMatchFailedNotGenerate(c *check.C) {
ips := Generate("127.0.0.[1]")
c.Assert(len(ips), check.Equals, 1)
c.Assert(ips[0], check.Equals, "127.0.0.[1]")
}
func (s *DiscoverySuite) TestGeneratorWithPort(c *check.C) {
ips := Generate("127.0.0.[1:11]:2375")
c.Assert(len(ips), check.Equals, 11)
c.Assert(ips[0], check.Equals, "127.0.0.1:2375")
c.Assert(ips[1], check.Equals, "127.0.0.2:2375")
c.Assert(ips[2], check.Equals, "127.0.0.3:2375")
c.Assert(ips[3], check.Equals, "127.0.0.4:2375")
c.Assert(ips[4], check.Equals, "127.0.0.5:2375")
c.Assert(ips[5], check.Equals, "127.0.0.6:2375")
c.Assert(ips[6], check.Equals, "127.0.0.7:2375")
c.Assert(ips[7], check.Equals, "127.0.0.8:2375")
c.Assert(ips[8], check.Equals, "127.0.0.9:2375")
c.Assert(ips[9], check.Equals, "127.0.0.10:2375")
c.Assert(ips[10], check.Equals, "127.0.0.11:2375")
}
func (s *DiscoverySuite) TestGenerateWithMalformedInputAtRangeStart(c *check.C) {
malformedInput := "127.0.0.[x:11]:2375"
ips := Generate(malformedInput)
c.Assert(len(ips), check.Equals, 1)
c.Assert(ips[0], check.Equals, malformedInput)
}
func (s *DiscoverySuite) TestGenerateWithMalformedInputAtRangeEnd(c *check.C) {
malformedInput := "127.0.0.[1:x]:2375"
ips := Generate(malformedInput)
c.Assert(len(ips), check.Equals, 1)
c.Assert(ips[0], check.Equals, malformedInput)
}

192
discovery/kv/kv.go
View file

@ -1,192 +0,0 @@
package kv
import (
"fmt"
"path"
"strings"
"time"
log "github.com/Sirupsen/logrus"
"github.com/docker/docker/pkg/discovery"
"github.com/docker/go-connections/tlsconfig"
"github.com/docker/libkv"
"github.com/docker/libkv/store"
"github.com/docker/libkv/store/consul"
"github.com/docker/libkv/store/etcd"
"github.com/docker/libkv/store/zookeeper"
)
const (
defaultDiscoveryPath = "docker/nodes"
)
// Discovery is exported
type Discovery struct {
backend store.Backend
store store.Store
heartbeat time.Duration
ttl time.Duration
prefix string
path string
}
func init() {
Init()
}
// Init is exported
func Init() {
// Register to libkv
zookeeper.Register()
consul.Register()
etcd.Register()
// Register to internal discovery service
discovery.Register("zk", &Discovery{backend: store.ZK})
discovery.Register("consul", &Discovery{backend: store.CONSUL})
discovery.Register("etcd", &Discovery{backend: store.ETCD})
}
// Initialize is exported
func (s *Discovery) Initialize(uris string, heartbeat time.Duration, ttl time.Duration, clusterOpts map[string]string) error {
var (
parts = strings.SplitN(uris, "/", 2)
addrs = strings.Split(parts[0], ",")
err error
)
// A custom prefix to the path can be optionally used.
if len(parts) == 2 {
s.prefix = parts[1]
}
s.heartbeat = heartbeat
s.ttl = ttl
// Use a custom path if specified in discovery options
dpath := defaultDiscoveryPath
if clusterOpts["kv.path"] != "" {
dpath = clusterOpts["kv.path"]
}
s.path = path.Join(s.prefix, dpath)
var config *store.Config
if clusterOpts["kv.cacertfile"] != "" && clusterOpts["kv.certfile"] != "" && clusterOpts["kv.keyfile"] != "" {
log.Info("Initializing discovery with TLS")
tlsConfig, err := tlsconfig.Client(tlsconfig.Options{
CAFile: clusterOpts["kv.cacertfile"],
CertFile: clusterOpts["kv.certfile"],
KeyFile: clusterOpts["kv.keyfile"],
})
if err != nil {
return err
}
config = &store.Config{
// Set ClientTLS to trigger https (bug in libkv/etcd)
ClientTLS: &store.ClientTLSConfig{
CACertFile: clusterOpts["kv.cacertfile"],
CertFile: clusterOpts["kv.certfile"],
KeyFile: clusterOpts["kv.keyfile"],
},
// The actual TLS config that will be used
TLS: tlsConfig,
}
} else {
log.Info("Initializing discovery without TLS")
}
// Creates a new store, will ignore options given
// if not supported by the chosen store
s.store, err = libkv.NewStore(s.backend, addrs, config)
return err
}
// Watch the store until either there's a store error or we receive a stop request.
// Returns false if we shouldn't attempt watching the store anymore (stop request received).
func (s *Discovery) watchOnce(stopCh <-chan struct{}, watchCh <-chan []*store.KVPair, discoveryCh chan discovery.Entries, errCh chan error) bool {
for {
select {
case pairs := <-watchCh:
if pairs == nil {
return true
}
log.WithField("discovery", s.backend).Debugf("Watch triggered with %d nodes", len(pairs))
// Convert `KVPair` into `discovery.Entry`.
addrs := make([]string, len(pairs))
for _, pair := range pairs {
addrs = append(addrs, string(pair.Value))
}
entries, err := discovery.CreateEntries(addrs)
if err != nil {
errCh <- err
} else {
discoveryCh <- entries
}
case <-stopCh:
// We were requested to stop watching.
return false
}
}
}
// Watch is exported
func (s *Discovery) Watch(stopCh <-chan struct{}) (<-chan discovery.Entries, <-chan error) {
ch := make(chan discovery.Entries)
errCh := make(chan error)
go func() {
defer close(ch)
defer close(errCh)
// Forever: Create a store watch, watch until we get an error and then try again.
// Will only stop if we receive a stopCh request.
for {
// Create the path to watch if it does not exist yet
exists, err := s.store.Exists(s.path)
if err != nil {
errCh <- err
}
if !exists {
if err := s.store.Put(s.path, []byte(""), &store.WriteOptions{IsDir: true}); err != nil {
errCh <- err
}
}
// Set up a watch.
watchCh, err := s.store.WatchTree(s.path, stopCh)
if err != nil {
errCh <- err
} else {
if !s.watchOnce(stopCh, watchCh, ch, errCh) {
return
}
}
// If we get here it means the store watch channel was closed. This
// is unexpected so let's retry later.
errCh <- fmt.Errorf("Unexpected watch error")
time.Sleep(s.heartbeat)
}
}()
return ch, errCh
}
// Register is exported
func (s *Discovery) Register(addr string) error {
opts := &store.WriteOptions{TTL: s.ttl}
return s.store.Put(path.Join(s.path, addr), []byte(addr), opts)
}
// Store returns the underlying store used by KV discovery.
func (s *Discovery) Store() store.Store {
return s.store
}
// Prefix returns the store prefix
func (s *Discovery) Prefix() string {
return s.prefix
}

324
discovery/kv/kv_test.go
View file

@ -1,324 +0,0 @@
package kv
import (
"errors"
"io/ioutil"
"os"
"path"
"testing"
"time"
"github.com/docker/docker/pkg/discovery"
"github.com/docker/libkv"
"github.com/docker/libkv/store"
"github.com/go-check/check"
)
// Hook up gocheck into the "go test" runner.
func Test(t *testing.T) { check.TestingT(t) }
type DiscoverySuite struct{}
var _ = check.Suite(&DiscoverySuite{})
func (ds *DiscoverySuite) TestInitialize(c *check.C) {
storeMock := &FakeStore{
Endpoints: []string{"127.0.0.1"},
}
d := &Discovery{backend: store.CONSUL}
d.Initialize("127.0.0.1", 0, 0, nil)
d.store = storeMock
s := d.store.(*FakeStore)
c.Assert(s.Endpoints, check.HasLen, 1)
c.Assert(s.Endpoints[0], check.Equals, "127.0.0.1")
c.Assert(d.path, check.Equals, defaultDiscoveryPath)
storeMock = &FakeStore{
Endpoints: []string{"127.0.0.1:1234"},
}
d = &Discovery{backend: store.CONSUL}
d.Initialize("127.0.0.1:1234/path", 0, 0, nil)
d.store = storeMock
s = d.store.(*FakeStore)
c.Assert(s.Endpoints, check.HasLen, 1)
c.Assert(s.Endpoints[0], check.Equals, "127.0.0.1:1234")
c.Assert(d.path, check.Equals, "path/"+defaultDiscoveryPath)
storeMock = &FakeStore{
Endpoints: []string{"127.0.0.1:1234", "127.0.0.2:1234", "127.0.0.3:1234"},
}
d = &Discovery{backend: store.CONSUL}
d.Initialize("127.0.0.1:1234,127.0.0.2:1234,127.0.0.3:1234/path", 0, 0, nil)
d.store = storeMock
s = d.store.(*FakeStore)
c.Assert(s.Endpoints, check.HasLen, 3)
c.Assert(s.Endpoints[0], check.Equals, "127.0.0.1:1234")
c.Assert(s.Endpoints[1], check.Equals, "127.0.0.2:1234")
c.Assert(s.Endpoints[2], check.Equals, "127.0.0.3:1234")
c.Assert(d.path, check.Equals, "path/"+defaultDiscoveryPath)
}
// Extremely limited mock store so we can test initialization
type Mock struct {
// Endpoints passed to InitializeMock
Endpoints []string
// Options passed to InitializeMock
Options *store.Config
}
func NewMock(endpoints []string, options *store.Config) (store.Store, error) {
s := &Mock{}
s.Endpoints = endpoints
s.Options = options
return s, nil
}
func (s *Mock) Put(key string, value []byte, opts *store.WriteOptions) error {
return errors.New("Put not supported")
}
func (s *Mock) Get(key string) (*store.KVPair, error) {
return nil, errors.New("Get not supported")
}
func (s *Mock) Delete(key string) error {
return errors.New("Delete not supported")
}
// Exists mock
func (s *Mock) Exists(key string) (bool, error) {
return false, errors.New("Exists not supported")
}
// Watch mock
func (s *Mock) Watch(key string, stopCh <-chan struct{}) (<-chan *store.KVPair, error) {
return nil, errors.New("Watch not supported")
}
// WatchTree mock
func (s *Mock) WatchTree(prefix string, stopCh <-chan struct{}) (<-chan []*store.KVPair, error) {
return nil, errors.New("WatchTree not supported")
}
// NewLock mock
func (s *Mock) NewLock(key string, options *store.LockOptions) (store.Locker, error) {
return nil, errors.New("NewLock not supported")
}
// List mock
func (s *Mock) List(prefix string) ([]*store.KVPair, error) {
return nil, errors.New("List not supported")
}
// DeleteTree mock
func (s *Mock) DeleteTree(prefix string) error {
return errors.New("DeleteTree not supported")
}
// AtomicPut mock
func (s *Mock) AtomicPut(key string, value []byte, previous *store.KVPair, opts *store.WriteOptions) (bool, *store.KVPair, error) {
return false, nil, errors.New("AtomicPut not supported")
}
// AtomicDelete mock
func (s *Mock) AtomicDelete(key string, previous *store.KVPair) (bool, error) {
return false, errors.New("AtomicDelete not supported")
}
// Close mock
func (s *Mock) Close() {
return
}
func (ds *DiscoverySuite) TestInitializeWithCerts(c *check.C) {
cert := `-----BEGIN CERTIFICATE-----
MIIDCDCCAfKgAwIBAgIICifG7YeiQOEwCwYJKoZIhvcNAQELMBIxEDAOBgNVBAMT
B1Rlc3QgQ0EwHhcNMTUxMDAxMjMwMDAwWhcNMjAwOTI5MjMwMDAwWjASMRAwDgYD
VQQDEwdUZXN0IENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA1wRC
O+flnLTK5ImjTurNRHwSejuqGbc4CAvpB0hS+z0QlSs4+zE9h80aC4hz+6caRpds
+J908Q+RvAittMHbpc7VjbZP72G6fiXk7yPPl6C10HhRSoSi3nY+B7F2E8cuz14q
V2e+ejhWhSrBb/keyXpcyjoW1BOAAJ2TIclRRkICSCZrpXUyXxAvzXfpFXo1RhSb
UywN11pfiCQzDUN7sPww9UzFHuAHZHoyfTr27XnJYVUerVYrCPq8vqfn//01qz55
Xs0hvzGdlTFXhuabFtQnKFH5SNwo/fcznhB7rePOwHojxOpXTBepUCIJLbtNnWFT
V44t9gh5IqIWtoBReQIDAQABo2YwZDAOBgNVHQ8BAf8EBAMCAAYwEgYDVR0TAQH/
BAgwBgEB/wIBAjAdBgNVHQ4EFgQUZKUI8IIjIww7X/6hvwggQK4bD24wHwYDVR0j
BBgwFoAUZKUI8IIjIww7X/6hvwggQK4bD24wCwYJKoZIhvcNAQELA4IBAQDES2cz
7sCQfDCxCIWH7X8kpi/JWExzUyQEJ0rBzN1m3/x8ySRxtXyGekimBqQwQdFqlwMI
xzAQKkh3ue8tNSzRbwqMSyH14N1KrSxYS9e9szJHfUasoTpQGPmDmGIoRJuq1h6M
ej5x1SCJ7GWCR6xEXKUIE9OftXm9TdFzWa7Ja3OHz/mXteii8VXDuZ5ACq6EE5bY
8sP4gcICfJ5fTrpTlk9FIqEWWQrCGa5wk95PGEj+GJpNogjXQ97wVoo/Y3p1brEn
t5zjN9PAq4H1fuCMdNNA+p1DHNwd+ELTxcMAnb2ajwHvV6lKPXutrTFc4umJToBX
FpTxDmJHEV4bzUzh
-----END CERTIFICATE-----
`
key := `-----BEGIN RSA PRIVATE KEY-----
MIIEpQIBAAKCAQEA1wRCO+flnLTK5ImjTurNRHwSejuqGbc4CAvpB0hS+z0QlSs4
+zE9h80aC4hz+6caRpds+J908Q+RvAittMHbpc7VjbZP72G6fiXk7yPPl6C10HhR
SoSi3nY+B7F2E8cuz14qV2e+ejhWhSrBb/keyXpcyjoW1BOAAJ2TIclRRkICSCZr
pXUyXxAvzXfpFXo1RhSbUywN11pfiCQzDUN7sPww9UzFHuAHZHoyfTr27XnJYVUe
rVYrCPq8vqfn//01qz55Xs0hvzGdlTFXhuabFtQnKFH5SNwo/fcznhB7rePOwHoj
xOpXTBepUCIJLbtNnWFTV44t9gh5IqIWtoBReQIDAQABAoIBAHSWipORGp/uKFXj
i/mut776x8ofsAxhnLBARQr93ID+i49W8H7EJGkOfaDjTICYC1dbpGrri61qk8sx
qX7p3v/5NzKwOIfEpirgwVIqSNYe/ncbxnhxkx6tXtUtFKmEx40JskvSpSYAhmmO
1XSx0E/PWaEN/nLgX/f1eWJIlxlQkk3QeqL+FGbCXI48DEtlJ9+MzMu4pAwZTpj5
5qtXo5JJ0jRGfJVPAOznRsYqv864AhMdMIWguzk6EGnbaCWwPcfcn+h9a5LMdony
MDHfBS7bb5tkF3+AfnVY3IBMVx7YlsD9eAyajlgiKu4zLbwTRHjXgShy+4Oussz0
ugNGnkECgYEA/hi+McrZC8C4gg6XqK8+9joD8tnyDZDz88BQB7CZqABUSwvjDqlP
L8hcwo/lzvjBNYGkqaFPUICGWKjeCtd8pPS2DCVXxDQX4aHF1vUur0uYNncJiV3N
XQz4Iemsa6wnKf6M67b5vMXICw7dw0HZCdIHD1hnhdtDz0uVpeevLZ8CgYEA2KCT
Y43lorjrbCgMqtlefkr3GJA9dey+hTzCiWEOOqn9RqGoEGUday0sKhiLofOgmN2B
LEukpKIey8s+Q/cb6lReajDVPDsMweX8i7hz3Wa4Ugp4Xa5BpHqu8qIAE2JUZ7bU
t88aQAYE58pUF+/Lq1QzAQdrjjzQBx6SrBxieecCgYEAvukoPZEC8mmiN1VvbTX+
QFHmlZha3QaDxChB+QUe7bMRojEUL/fVnzkTOLuVFqSfxevaI/km9n0ac5KtAchV
xjp2bTnBb5EUQFqjopYktWA+xO07JRJtMfSEmjZPbbay1kKC7rdTfBm961EIHaRj
xZUf6M+rOE8964oGrdgdLlECgYEA046GQmx6fh7/82FtdZDRQp9tj3SWQUtSiQZc
qhO59Lq8mjUXz+MgBuJXxkiwXRpzlbaFB0Bca1fUoYw8o915SrDYf/Zu2OKGQ/qa
V81sgiVmDuEgycR7YOlbX6OsVUHrUlpwhY3hgfMe6UtkMvhBvHF/WhroBEIJm1pV
PXZ/CbMCgYEApNWVktFBjOaYfY6SNn4iSts1jgsQbbpglg3kT7PLKjCAhI6lNsbk
dyT7ut01PL6RaW4SeQWtrJIVQaM6vF3pprMKqlc5XihOGAmVqH7rQx9rtQB5TicL
BFrwkQE4HQtQBV60hYQUzzlSk44VFDz+jxIEtacRHaomDRh2FtOTz+I=
-----END RSA PRIVATE KEY-----
`
certFile, err := ioutil.TempFile("", "cert")
c.Assert(err, check.IsNil)
defer os.Remove(certFile.Name())
certFile.Write([]byte(cert))
certFile.Close()
keyFile, err := ioutil.TempFile("", "key")
c.Assert(err, check.IsNil)
defer os.Remove(keyFile.Name())
keyFile.Write([]byte(key))
keyFile.Close()
libkv.AddStore("mock", NewMock)
d := &Discovery{backend: "mock"}
err = d.Initialize("127.0.0.3:1234", 0, 0, map[string]string{
"kv.cacertfile": certFile.Name(),
"kv.certfile": certFile.Name(),
"kv.keyfile": keyFile.Name(),
})
c.Assert(err, check.IsNil)
s := d.store.(*Mock)
c.Assert(s.Options.TLS, check.NotNil)
c.Assert(s.Options.TLS.RootCAs, check.NotNil)
c.Assert(s.Options.TLS.Certificates, check.HasLen, 1)
}
func (ds *DiscoverySuite) TestWatch(c *check.C) {
mockCh := make(chan []*store.KVPair)
storeMock := &FakeStore{
Endpoints: []string{"127.0.0.1:1234"},
mockKVChan: mockCh,
}
d := &Discovery{backend: store.CONSUL}
d.Initialize("127.0.0.1:1234/path", 0, 0, nil)
d.store = storeMock
expected := discovery.Entries{
&discovery.Entry{Host: "1.1.1.1", Port: "1111"},
&discovery.Entry{Host: "2.2.2.2", Port: "2222"},
}
kvs := []*store.KVPair{
{Key: path.Join("path", defaultDiscoveryPath, "1.1.1.1"), Value: []byte("1.1.1.1:1111")},
{Key: path.Join("path", defaultDiscoveryPath, "2.2.2.2"), Value: []byte("2.2.2.2:2222")},
}
stopCh := make(chan struct{})
ch, errCh := d.Watch(stopCh)
// It should fire an error since the first WatchTree call failed.
c.Assert(<-errCh, check.ErrorMatches, "test error")
// We have to drain the error channel otherwise Watch will get stuck.
go func() {
for range errCh {
}
}()
// Push the entries into the store channel and make sure discovery emits.
mockCh <- kvs
c.Assert(<-ch, check.DeepEquals, expected)
// Add a new entry.
expected = append(expected, &discovery.Entry{Host: "3.3.3.3", Port: "3333"})
kvs = append(kvs, &store.KVPair{Key: path.Join("path", defaultDiscoveryPath, "3.3.3.3"), Value: []byte("3.3.3.3:3333")})
mockCh <- kvs
c.Assert(<-ch, check.DeepEquals, expected)
close(mockCh)
// Give it enough time to call WatchTree.
time.Sleep(3 * time.Second)
// Stop and make sure it closes all channels.
close(stopCh)
c.Assert(<-ch, check.IsNil)
c.Assert(<-errCh, check.IsNil)
}
// FakeStore implements store.Store methods. It mocks all store
// function in a simple, naive way.
type FakeStore struct {
Endpoints []string
Options *store.Config
mockKVChan <-chan []*store.KVPair
watchTreeCallCount int
}
func (s *FakeStore) Put(key string, value []byte, options *store.WriteOptions) error {
return nil
}
func (s *FakeStore) Get(key string) (*store.KVPair, error) {
return nil, nil
}
func (s *FakeStore) Delete(key string) error {
return nil
}
func (s *FakeStore) Exists(key string) (bool, error) {
return true, nil
}
func (s *FakeStore) Watch(key string, stopCh <-chan struct{}) (<-chan *store.KVPair, error) {
return nil, nil
}
// WatchTree will fail the first time, and return the mockKVchan afterwards.
// This is the behavior we need for testing.. If we need 'moar', should update this.
func (s *FakeStore) WatchTree(directory string, stopCh <-chan struct{}) (<-chan []*store.KVPair, error) {
if s.watchTreeCallCount == 0 {
s.watchTreeCallCount = 1
return nil, errors.New("test error")
}
// First calls error
return s.mockKVChan, nil
}
func (s *FakeStore) NewLock(key string, options *store.LockOptions) (store.Locker, error) {
return nil, nil
}
func (s *FakeStore) List(directory string) ([]*store.KVPair, error) {
return []*store.KVPair{}, nil
}
func (s *FakeStore) DeleteTree(directory string) error {
return nil
}
func (s *FakeStore) AtomicPut(key string, value []byte, previous *store.KVPair, options *store.WriteOptions) (bool, *store.KVPair, error) {
return true, nil, nil
}
func (s *FakeStore) AtomicDelete(key string, previous *store.KVPair) (bool, error) {
return true, nil
}
func (s *FakeStore) Close() {
}

93
discovery/memory/memory.go
View file

@ -1,93 +0,0 @@
package memory
import (
"sync"
"time"
"github.com/docker/docker/pkg/discovery"
)
// Discovery implements a discovery backend that keeps
// data in memory.
type Discovery struct {
heartbeat time.Duration
values []string
mu sync.Mutex
}
func init() {
Init()
}
// Init registers the memory backend on demand.
func Init() {
discovery.Register("memory", &Discovery{})
}
// Initialize sets the heartbeat for the memory backend.
func (s *Discovery) Initialize(_ string, heartbeat time.Duration, _ time.Duration, _ map[string]string) error {
s.heartbeat = heartbeat
s.values = make([]string, 0)
return nil
}
// Watch sends periodic discovery updates to a channel.
func (s *Discovery) Watch(stopCh <-chan struct{}) (<-chan discovery.Entries, <-chan error) {
ch := make(chan discovery.Entries)
errCh := make(chan error)
ticker := time.NewTicker(s.heartbeat)
go func() {
defer close(errCh)
defer close(ch)
// Send the initial entries if available.
var currentEntries discovery.Entries
var err error
s.mu.Lock()
if len(s.values) > 0 {
currentEntries, err = discovery.CreateEntries(s.values)
}
s.mu.Unlock()
if err != nil {
errCh <- err
} else if currentEntries != nil {
ch <- currentEntries
}
// Periodically send updates.
for {
select {
case <-ticker.C:
s.mu.Lock()
newEntries, err := discovery.CreateEntries(s.values)
s.mu.Unlock()
if err != nil {
errCh <- err
continue
}
// Check if the file has really changed.
if !newEntries.Equals(currentEntries) {
ch <- newEntries
}
currentEntries = newEntries
case <-stopCh:
ticker.Stop()
return
}
}
}()
return ch, errCh
}
// Register adds a new address to the discovery.
func (s *Discovery) Register(addr string) error {
s.mu.Lock()
s.values = append(s.values, addr)
s.mu.Unlock()
return nil
}

48
discovery/memory/memory_test.go
View file

@ -1,48 +0,0 @@
package memory
import (
"testing"
"github.com/docker/docker/pkg/discovery"
"github.com/go-check/check"
)
// Hook up gocheck into the "go test" runner.
func Test(t *testing.T) { check.TestingT(t) }
type discoverySuite struct{}
var _ = check.Suite(&discoverySuite{})
func (s *discoverySuite) TestWatch(c *check.C) {
d := &Discovery{}
d.Initialize("foo", 1000, 0, nil)
stopCh := make(chan struct{})
ch, errCh := d.Watch(stopCh)
// We have to drain the error channel otherwise Watch will get stuck.
go func() {
for range errCh {
}
}()
expected := discovery.Entries{
&discovery.Entry{Host: "1.1.1.1", Port: "1111"},
}
c.Assert(d.Register("1.1.1.1:1111"), check.IsNil)
c.Assert(<-ch, check.DeepEquals, expected)
expected = discovery.Entries{
&discovery.Entry{Host: "1.1.1.1", Port: "1111"},
&discovery.Entry{Host: "2.2.2.2", Port: "2222"},
}
c.Assert(d.Register("2.2.2.2:2222"), check.IsNil)
c.Assert(<-ch, check.DeepEquals, expected)
// Stop and make sure it closes all channels.
close(stopCh)
c.Assert(<-ch, check.IsNil)
c.Assert(<-errCh, check.IsNil)
}

54
discovery/nodes/nodes.go
View file

@ -1,54 +0,0 @@
package nodes
import (
"fmt"
"strings"
"time"
"github.com/docker/docker/pkg/discovery"
)
// Discovery is exported
type Discovery struct {
entries discovery.Entries
}
func init() {
Init()
}
// Init is exported
func Init() {
discovery.Register("nodes", &Discovery{})
}
// Initialize is exported
func (s *Discovery) Initialize(uris string, _ time.Duration, _ time.Duration, _ map[string]string) error {
for _, input := range strings.Split(uris, ",") {
for _, ip := range discovery.Generate(input) {
entry, err := discovery.NewEntry(ip)
if err != nil {
return fmt.Errorf("%s, please check you are using the correct discovery (missing token:// ?)", err.Error())
}
s.entries = append(s.entries, entry)
}
}
return nil
}
// Watch is exported
func (s *Discovery) Watch(stopCh <-chan struct{}) (<-chan discovery.Entries, <-chan error) {
ch := make(chan discovery.Entries)
go func() {
defer close(ch)
ch <- s.entries
<-stopCh
}()
return ch, nil
}
// Register is exported
func (s *Discovery) Register(addr string) error {
return discovery.ErrNotImplemented
}

51
discovery/nodes/nodes_test.go
View file

@ -1,51 +0,0 @@
package nodes
import (
"testing"
"github.com/docker/docker/pkg/discovery"
"github.com/go-check/check"
)
// Hook up gocheck into the "go test" runner.
func Test(t *testing.T) { check.TestingT(t) }
type DiscoverySuite struct{}
var _ = check.Suite(&DiscoverySuite{})
func (s *DiscoverySuite) TestInitialize(c *check.C) {
d := &Discovery{}
d.Initialize("1.1.1.1:1111,2.2.2.2:2222", 0, 0, nil)
c.Assert(len(d.entries), check.Equals, 2)
c.Assert(d.entries[0].String(), check.Equals, "1.1.1.1:1111")
c.Assert(d.entries[1].String(), check.Equals, "2.2.2.2:2222")
}
func (s *DiscoverySuite) TestInitializeWithPattern(c *check.C) {
d := &Discovery{}
d.Initialize("1.1.1.[1:2]:1111,2.2.2.[2:4]:2222", 0, 0, nil)
c.Assert(len(d.entries), check.Equals, 5)
c.Assert(d.entries[0].String(), check.Equals, "1.1.1.1:1111")
c.Assert(d.entries[1].String(), check.Equals, "1.1.1.2:1111")
c.Assert(d.entries[2].String(), check.Equals, "2.2.2.2:2222")
c.Assert(d.entries[3].String(), check.Equals, "2.2.2.3:2222")
c.Assert(d.entries[4].String(), check.Equals, "2.2.2.4:2222")
}
func (s *DiscoverySuite) TestWatch(c *check.C) {
d := &Discovery{}
d.Initialize("1.1.1.1:1111,2.2.2.2:2222", 0, 0, nil)
expected := discovery.Entries{
&discovery.Entry{Host: "1.1.1.1", Port: "1111"},
&discovery.Entry{Host: "2.2.2.2", Port: "2222"},
}
ch, _ := d.Watch(nil)
c.Assert(expected.Equals(<-ch), check.Equals, true)
}
func (s *DiscoverySuite) TestRegister(c *check.C) {
d := &Discovery{}
c.Assert(d.Register("0.0.0.0"), check.NotNil)
}

40
filenotify/filenotify.go
View file

@ -1,40 +0,0 @@
// Package filenotify provides a mechanism for watching file(s) for changes.
// Generally leans on fsnotify, but provides a poll-based notifier which fsnotify does not support.
// These are wrapped up in a common interface so that either can be used interchangeably in your code.
package filenotify
import "gopkg.in/fsnotify.v1"
// FileWatcher is an interface for implementing file notification watchers
type FileWatcher interface {
Events() <-chan fsnotify.Event
Errors() <-chan error
Add(name string) error
Remove(name string) error
Close() error
}
// New tries to use an fs-event watcher, and falls back to the poller if there is an error
func New() (FileWatcher, error) {
if watcher, err := NewEventWatcher(); err == nil {
return watcher, nil
}
return NewPollingWatcher(), nil
}
// NewPollingWatcher returns a poll-based file watcher
func NewPollingWatcher() FileWatcher {
return &filePoller{
events: make(chan fsnotify.Event),
errors: make(chan error),
}
}
// NewEventWatcher returns an fs-event based file watcher
func NewEventWatcher() (FileWatcher, error) {
watcher, err := fsnotify.NewWatcher()
if err != nil {
return nil, err
}
return &fsNotifyWatcher{watcher}, nil
}

18
filenotify/fsnotify.go
View file

@ -1,18 +0,0 @@
package filenotify
import "gopkg.in/fsnotify.v1"
// fsNotify wraps the fsnotify package to satisfy the FileNotifer interface
type fsNotifyWatcher struct {
*fsnotify.Watcher
}
// GetEvents returns the fsnotify event channel receiver
func (w *fsNotifyWatcher) Events() <-chan fsnotify.Event {
return w.Watcher.Events
}
// GetErrors returns the fsnotify error channel receiver
func (w *fsNotifyWatcher) Errors() <-chan error {
return w.Watcher.Errors
}

204
filenotify/poller.go
View file

@ -1,204 +0,0 @@
package filenotify
import (
"errors"
"fmt"
"os"
"sync"
"time"
"github.com/Sirupsen/logrus"
"gopkg.in/fsnotify.v1"
)
var (
// errPollerClosed is returned when the poller is closed
errPollerClosed = errors.New("poller is closed")
// errNoSuchPoller is returned when trying to remove a watch that doesn't exist
errNoSuchWatch = errors.New("poller does not exist")
)
// watchWaitTime is the time to wait between file poll loops
const watchWaitTime = 200 * time.Millisecond
// filePoller is used to poll files for changes, especially in cases where fsnotify
// can't be run (e.g. when inotify handles are exhausted)
// filePoller satisfies the FileWatcher interface
type filePoller struct {
// watches is the list of files currently being polled, close the associated channel to stop the watch
watches map[string]chan struct{}
// events is the channel to listen to for watch events
events chan fsnotify.Event
// errors is the channel to listen to for watch errors
errors chan error
// mu locks the poller for modification
mu sync.Mutex
// closed is used to specify when the poller has already closed
closed bool
}
// Add adds a filename to the list of watches
// once added the file is polled for changes in a separate goroutine
func (w *filePoller) Add(name string) error {
w.mu.Lock()
defer w.mu.Unlock()
if w.closed == true {
return errPollerClosed
}
f, err := os.Open(name)
if err != nil {
return err
}
fi, err := os.Stat(name)
if err != nil {
return err
}
if w.watches == nil {
w.watches = make(map[string]chan struct{})
}
if _, exists := w.watches[name]; exists {
return fmt.Errorf("watch exists")
}
chClose := make(chan struct{})
w.watches[name] = chClose
go w.watch(f, fi, chClose)
return nil
}
// Remove stops and removes watch with the specified name
func (w *filePoller) Remove(name string) error {
w.mu.Lock()
defer w.mu.Unlock()
return w.remove(name)
}
func (w *filePoller) remove(name string) error {
if w.closed == true {
return errPollerClosed
}
chClose, exists := w.watches[name]
if !exists {
return errNoSuchWatch
}
close(chClose)
delete(w.watches, name)
return nil
}
// Events returns the event channel
// This is used for notifications on events about watched files
func (w *filePoller) Events() <-chan fsnotify.Event {
return w.events
}
// Errors returns the errors channel
// This is used for notifications about errors on watched files
func (w *filePoller) Errors() <-chan error {
return w.errors
}
// Close closes the poller
// All watches are stopped, removed, and the poller cannot be added to
func (w *filePoller) Close() error {
w.mu.Lock()
defer w.mu.Unlock()
if w.closed {
return nil
}
w.closed = true
for name := range w.watches {
w.remove(name)
delete(w.watches, name)
}
return nil
}
// sendEvent publishes the specified event to the events channel
func (w *filePoller) sendEvent(e fsnotify.Event, chClose <-chan struct{}) error {
select {
case w.events <- e:
case <-chClose:
return fmt.Errorf("closed")
}
return nil
}
// sendErr publishes the specified error to the errors channel
func (w *filePoller) sendErr(e error, chClose <-chan struct{}) error {
select {
case w.errors <- e:
case <-chClose:
return fmt.Errorf("closed")
}
return nil
}
// watch is responsible for polling the specified file for changes
// upon finding changes to a file or errors, sendEvent/sendErr is called
func (w *filePoller) watch(f *os.File, lastFi os.FileInfo, chClose chan struct{}) {
defer f.Close()
for {
time.Sleep(watchWaitTime)
select {
case <-chClose:
logrus.Debugf("watch for %s closed", f.Name())
return
default:
}
fi, err := os.Stat(f.Name())
if err != nil {
// if we got an error here and lastFi is not set, we can presume that nothing has changed
// This should be safe since before `watch()` is called, a stat is performed, there is any error `watch` is not called
if lastFi == nil {
continue
}
// If it doesn't exist at this point, it must have been removed
// no need to send the error here since this is a valid operation
if os.IsNotExist(err) {
if err := w.sendEvent(fsnotify.Event{Op: fsnotify.Remove, Name: f.Name()}, chClose); err != nil {
return
}
lastFi = nil
continue
}
// at this point, send the error
if err := w.sendErr(err, chClose); err != nil {
return
}
continue
}
if lastFi == nil {
if err := w.sendEvent(fsnotify.Event{Op: fsnotify.Create, Name: fi.Name()}, chClose); err != nil {
return
}
lastFi = fi
continue
}
if fi.Mode() != lastFi.Mode() {
if err := w.sendEvent(fsnotify.Event{Op: fsnotify.Chmod, Name: fi.Name()}, chClose); err != nil {
return
}
lastFi = fi
continue
}
if fi.ModTime() != lastFi.ModTime() || fi.Size() != lastFi.Size() {
if err := w.sendEvent(fsnotify.Event{Op: fsnotify.Write, Name: fi.Name()}, chClose); err != nil {
return
}
lastFi = fi
continue
}
}
}

119
filenotify/poller_test.go
View file

@ -1,119 +0,0 @@
package filenotify
import (
"fmt"
"io/ioutil"
"os"
"runtime"
"testing"
"time"
"gopkg.in/fsnotify.v1"
)
func TestPollerAddRemove(t *testing.T) {
w := NewPollingWatcher()
if err := w.Add("no-such-file"); err == nil {
t.Fatal("should have gotten error when adding a non-existent file")
}
if err := w.Remove("no-such-file"); err == nil {
t.Fatal("should have gotten error when removing non-existent watch")
}
f, err := ioutil.TempFile("", "asdf")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(f.Name())
if err := w.Add(f.Name()); err != nil {
t.Fatal(err)
}
if err := w.Remove(f.Name()); err != nil {
t.Fatal(err)
}
}
func TestPollerEvent(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("No chmod on Windows")
}
w := NewPollingWatcher()
f, err := ioutil.TempFile("", "test-poller")
if err != nil {
t.Fatal("error creating temp file")
}
defer os.RemoveAll(f.Name())
f.Close()
if err := w.Add(f.Name()); err != nil {
t.Fatal(err)
}
select {
case <-w.Events():
t.Fatal("got event before anything happened")
case <-w.Errors():
t.Fatal("got error before anything happened")
default:
}
if err := ioutil.WriteFile(f.Name(), []byte("hello"), 644); err != nil {
t.Fatal(err)
}
if err := assertEvent(w, fsnotify.Write); err != nil {
t.Fatal(err)
}
if err := os.Chmod(f.Name(), 600); err != nil {
t.Fatal(err)
}
if err := assertEvent(w, fsnotify.Chmod); err != nil {
t.Fatal(err)
}
if err := os.Remove(f.Name()); err != nil {
t.Fatal(err)
}
if err := assertEvent(w, fsnotify.Remove); err != nil {
t.Fatal(err)
}
}
func TestPollerClose(t *testing.T) {
w := NewPollingWatcher()
if err := w.Close(); err != nil {
t.Fatal(err)
}
// test double-close
if err := w.Close(); err != nil {
t.Fatal(err)
}
f, err := ioutil.TempFile("", "asdf")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(f.Name())
if err := w.Add(f.Name()); err == nil {
t.Fatal("should have gotten error adding watch for closed watcher")
}
}
func assertEvent(w FileWatcher, eType fsnotify.Op) error {
var err error
select {
case e := <-w.Events():
if e.Op != eType {
err = fmt.Errorf("got wrong event type, expected %q: %v", eType, e)
}
case e := <-w.Errors():
err = fmt.Errorf("got unexpected error waiting for events %v: %v", eType, e)
case <-time.After(watchWaitTime * 3):
err = fmt.Errorf("timeout waiting for event %v", eType)
}
return err
}

100
gitutils/gitutils.go
View file

@ -1,100 +0,0 @@
package gitutils
import (
"fmt"
"io/ioutil"
"net/http"
"net/url"
"os"
"os/exec"
"path/filepath"
"strings"
"github.com/docker/docker/pkg/symlink"
"github.com/docker/docker/pkg/urlutil"
)
// Clone clones a repository into a newly created directory which
// will be under "docker-build-git"
func Clone(remoteURL string) (string, error) {
if !urlutil.IsGitTransport(remoteURL) {
remoteURL = "https://" + remoteURL
}
root, err := ioutil.TempDir("", "docker-build-git")
if err != nil {
return "", err
}
u, err := url.Parse(remoteURL)
if err != nil {
return "", err
}
fragment := u.Fragment
clone := cloneArgs(u, root)
if output, err := git(clone...); err != nil {
return "", fmt.Errorf("Error trying to use git: %s (%s)", err, output)
}
return checkoutGit(fragment, root)
}
func cloneArgs(remoteURL *url.URL, root string) []string {
args := []string{"clone", "--recursive"}
shallow := len(remoteURL.Fragment) == 0
if shallow && strings.HasPrefix(remoteURL.Scheme, "http") {
res, err := http.Head(fmt.Sprintf("%s/info/refs?service=git-upload-pack", remoteURL))
if err != nil || res.Header.Get("Content-Type") != "application/x-git-upload-pack-advertisement" {
shallow = false
}
}
if shallow {
args = append(args, "--depth", "1")
}
if remoteURL.Fragment != "" {
remoteURL.Fragment = ""
}
return append(args, remoteURL.String(), root)
}
func checkoutGit(fragment, root string) (string, error) {
refAndDir := strings.SplitN(fragment, ":", 2)
if len(refAndDir[0]) != 0 {
if output, err := gitWithinDir(root, "checkout", refAndDir[0]); err != nil {
return "", fmt.Errorf("Error trying to use git: %s (%s)", err, output)
}
}
if len(refAndDir) > 1 && len(refAndDir[1]) != 0 {
newCtx, err := symlink.FollowSymlinkInScope(filepath.Join(root, refAndDir[1]), root)
if err != nil {
return "", fmt.Errorf("Error setting git context, %q not within git root: %s", refAndDir[1], err)
}
fi, err := os.Stat(newCtx)
if err != nil {
return "", err
}
if !fi.IsDir() {
return "", fmt.Errorf("Error setting git context, not a directory: %s", newCtx)
}
root = newCtx
}
return root, nil
}
func gitWithinDir(dir string, args ...string) ([]byte, error) {
a := []string{"--work-tree", dir, "--git-dir", filepath.Join(dir, ".git")}
return git(append(a, args...)...)
}
func git(args ...string) ([]byte, error) {
return exec.Command("git", args...).CombinedOutput()
}

220
gitutils/gitutils_test.go
View file

@ -1,220 +0,0 @@
package gitutils
import (
"fmt"
"io/ioutil"
"net/http"
"net/http/httptest"
"net/url"
"os"
"path/filepath"
"reflect"
"runtime"
"strings"
"testing"
)
func TestCloneArgsSmartHttp(t *testing.T) {
mux := http.NewServeMux()
server := httptest.NewServer(mux)
serverURL, _ := url.Parse(server.URL)
serverURL.Path = "/repo.git"
gitURL := serverURL.String()
mux.HandleFunc("/repo.git/info/refs", func(w http.ResponseWriter, r *http.Request) {
q := r.URL.Query().Get("service")
w.Header().Set("Content-Type", fmt.Sprintf("application/x-%s-advertisement", q))
})
args := cloneArgs(serverURL, "/tmp")
exp := []string{"clone", "--recursive", "--depth", "1", gitURL, "/tmp"}
if !reflect.DeepEqual(args, exp) {
t.Fatalf("Expected %v, got %v", exp, args)
}
}
func TestCloneArgsDumbHttp(t *testing.T) {
mux := http.NewServeMux()
server := httptest.NewServer(mux)
serverURL, _ := url.Parse(server.URL)
serverURL.Path = "/repo.git"
gitURL := serverURL.String()
mux.HandleFunc("/repo.git/info/refs", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/plain")
})
args := cloneArgs(serverURL, "/tmp")
exp := []string{"clone", "--recursive", gitURL, "/tmp"}
if !reflect.DeepEqual(args, exp) {
t.Fatalf("Expected %v, got %v", exp, args)
}
}
func TestCloneArgsGit(t *testing.T) {
u, _ := url.Parse("git://github.com/docker/docker")
args := cloneArgs(u, "/tmp")
exp := []string{"clone", "--recursive", "--depth", "1", "git://github.com/docker/docker", "/tmp"}
if !reflect.DeepEqual(args, exp) {
t.Fatalf("Expected %v, got %v", exp, args)
}
}
func TestCloneArgsStripFragment(t *testing.T) {
u, _ := url.Parse("git://github.com/docker/docker#test")
args := cloneArgs(u, "/tmp")
exp := []string{"clone", "--recursive", "git://github.com/docker/docker", "/tmp"}
if !reflect.DeepEqual(args, exp) {
t.Fatalf("Expected %v, got %v", exp, args)
}
}
func gitGetConfig(name string) string {
b, err := git([]string{"config", "--get", name}...)
if err != nil {
// since we are interested in empty or non empty string,
// we can safely ignore the err here.
return ""
}
return strings.TrimSpace(string(b))
}
func TestCheckoutGit(t *testing.T) {
root, err := ioutil.TempDir("", "docker-build-git-checkout")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(root)
autocrlf := gitGetConfig("core.autocrlf")
if !(autocrlf == "true" || autocrlf == "false" ||
autocrlf == "input" || autocrlf == "") {
t.Logf("unknown core.autocrlf value: \"%s\"", autocrlf)
}
eol := "\n"
if autocrlf == "true" {
eol = "\r\n"
}
gitDir := filepath.Join(root, "repo")
_, err = git("init", gitDir)
if err != nil {
t.Fatal(err)
}
if _, err = gitWithinDir(gitDir, "config", "user.email", "test@docker.com"); err != nil {
t.Fatal(err)
}
if _, err = gitWithinDir(gitDir, "config", "user.name", "Docker test"); err != nil {
t.Fatal(err)
}
if err = ioutil.WriteFile(filepath.Join(gitDir, "Dockerfile"), []byte("FROM scratch"), 0644); err != nil {
t.Fatal(err)
}
subDir := filepath.Join(gitDir, "subdir")
if err = os.Mkdir(subDir, 0755); err != nil {
t.Fatal(err)
}
if err = ioutil.WriteFile(filepath.Join(subDir, "Dockerfile"), []byte("FROM scratch\nEXPOSE 5000"), 0644); err != nil {
t.Fatal(err)
}
if runtime.GOOS != "windows" {
if err = os.Symlink("../subdir", filepath.Join(gitDir, "parentlink")); err != nil {
t.Fatal(err)
}
if err = os.Symlink("/subdir", filepath.Join(gitDir, "absolutelink")); err != nil {
t.Fatal(err)
}
}
if _, err = gitWithinDir(gitDir, "add", "-A"); err != nil {
t.Fatal(err)
}
if _, err = gitWithinDir(gitDir, "commit", "-am", "First commit"); err != nil {
t.Fatal(err)
}
if _, err = gitWithinDir(gitDir, "checkout", "-b", "test"); err != nil {
t.Fatal(err)
}
if err = ioutil.WriteFile(filepath.Join(gitDir, "Dockerfile"), []byte("FROM scratch\nEXPOSE 3000"), 0644); err != nil {
t.Fatal(err)
}
if err = ioutil.WriteFile(filepath.Join(subDir, "Dockerfile"), []byte("FROM busybox\nEXPOSE 5000"), 0644); err != nil {
t.Fatal(err)
}
if _, err = gitWithinDir(gitDir, "add", "-A"); err != nil {
t.Fatal(err)
}
if _, err = gitWithinDir(gitDir, "commit", "-am", "Branch commit"); err != nil {
t.Fatal(err)
}
if _, err = gitWithinDir(gitDir, "checkout", "master"); err != nil {
t.Fatal(err)
}
type singleCase struct {
frag string
exp string
fail bool
}
cases := []singleCase{
{"", "FROM scratch", false},
{"master", "FROM scratch", false},
{":subdir", "FROM scratch" + eol + "EXPOSE 5000", false},
{":nosubdir", "", true}, // missing directory error
{":Dockerfile", "", true}, // not a directory error
{"master:nosubdir", "", true},
{"master:subdir", "FROM scratch" + eol + "EXPOSE 5000", false},
{"master:../subdir", "", true},
{"test", "FROM scratch" + eol + "EXPOSE 3000", false},
{"test:", "FROM scratch" + eol + "EXPOSE 3000", false},
{"test:subdir", "FROM busybox" + eol + "EXPOSE 5000", false},
}
if runtime.GOOS != "windows" {
// Windows GIT (2.7.1 x64) does not support parentlink/absolutelink. Sample output below
// git --work-tree .\repo --git-dir .\repo\.git add -A
// error: readlink("absolutelink"): Function not implemented
// error: unable to index file absolutelink
// fatal: adding files failed
cases = append(cases, singleCase{frag: "master:absolutelink", exp: "FROM scratch" + eol + "EXPOSE 5000", fail: false})
cases = append(cases, singleCase{frag: "master:parentlink", exp: "FROM scratch" + eol + "EXPOSE 5000", fail: false})
}
for _, c := range cases {
r, err := checkoutGit(c.frag, gitDir)
fail := err != nil
if fail != c.fail {
t.Fatalf("Expected %v failure, error was %v\n", c.fail, err)
}
if c.fail {
continue
}
b, err := ioutil.ReadFile(filepath.Join(r, "Dockerfile"))
if err != nil {
t.Fatal(err)
}
if string(b) != c.exp {
t.Fatalf("Expected %v, was %v\n", c.exp, string(b))
}
}
}

15
graphdb/conn_sqlite3.go
View file

@ -1,15 +0,0 @@
// +build cgo
package graphdb
import "database/sql"
// NewSqliteConn opens a connection to a sqlite
// database.
func NewSqliteConn(root string) (*Database, error) {
conn, err := sql.Open("sqlite3", root)
if err != nil {
return nil, err
}
return NewDatabase(conn)
}

7
graphdb/conn_sqlite3_unix.go
View file

@ -1,7 +0,0 @@
// +build cgo,!windows
package graphdb
import (
_ "github.com/mattn/go-sqlite3" // registers sqlite
)

7
graphdb/conn_sqlite3_windows.go
View file

@ -1,7 +0,0 @@
// +build cgo,windows
package graphdb
import (
_ "github.com/mattn/go-sqlite3" // registers sqlite
)

8
graphdb/conn_unsupported.go
View file

@ -1,8 +0,0 @@
// +build !cgo
package graphdb
// NewSqliteConn return a new sqlite connection.
func NewSqliteConn(root string) (*Database, error) {
panic("Not implemented")
}

551
graphdb/graphdb.go
View file

@ -1,551 +0,0 @@
package graphdb
import (
"database/sql"
"fmt"
"path"
"strings"
"sync"
)
const (
createEntityTable = `
CREATE TABLE IF NOT EXISTS entity (
id text NOT NULL PRIMARY KEY
);`
createEdgeTable = `
CREATE TABLE IF NOT EXISTS edge (
"entity_id" text NOT NULL,
"parent_id" text NULL,
"name" text NOT NULL,
CONSTRAINT "parent_fk" FOREIGN KEY ("parent_id") REFERENCES "entity" ("id"),
CONSTRAINT "entity_fk" FOREIGN KEY ("entity_id") REFERENCES "entity" ("id")
);
`
createEdgeIndices = `
CREATE UNIQUE INDEX IF NOT EXISTS "name_parent_ix" ON "edge" (parent_id, name);
`
)
// Entity with a unique id.
type Entity struct {
id string
}
// An Edge connects two entities together.
type Edge struct {
EntityID string
Name string
ParentID string
}
// Entities stores the list of entities.
type Entities map[string]*Entity
// Edges stores the relationships between entities.
type Edges []*Edge
// WalkFunc is a function invoked to process an individual entity.
type WalkFunc func(fullPath string, entity *Entity) error
// Database is a graph database for storing entities and their relationships.
type Database struct {
conn *sql.DB
mux sync.RWMutex
}
// IsNonUniqueNameError processes the error to check if it's caused by
// a constraint violation.
// This is necessary because the error isn't the same across various
// sqlite versions.
func IsNonUniqueNameError(err error) bool {
str := err.Error()
// sqlite 3.7.17-1ubuntu1 returns:
// Set failure: Abort due to constraint violation: columns parent_id, name are not unique
if strings.HasSuffix(str, "name are not unique") {
return true
}
// sqlite-3.8.3-1.fc20 returns:
// Set failure: Abort due to constraint violation: UNIQUE constraint failed: edge.parent_id, edge.name
if strings.Contains(str, "UNIQUE constraint failed") && strings.Contains(str, "edge.name") {
return true
}
// sqlite-3.6.20-1.el6 returns:
// Set failure: Abort due to constraint violation: constraint failed
if strings.HasSuffix(str, "constraint failed") {
return true
}
return false
}
// NewDatabase creates a new graph database initialized with a root entity.
func NewDatabase(conn *sql.DB) (*Database, error) {
if conn == nil {
return nil, fmt.Errorf("Database connection cannot be nil")
}
db := &Database{conn: conn}
// Create root entities
tx, err := conn.Begin()
if err != nil {
return nil, err
}
if _, err := tx.Exec(createEntityTable); err != nil {
return nil, err
}
if _, err := tx.Exec(createEdgeTable); err != nil {
return nil, err
}
if _, err := tx.Exec(createEdgeIndices); err != nil {
return nil, err
}
if _, err := tx.Exec("DELETE FROM entity where id = ?", "0"); err != nil {
tx.Rollback()
return nil, err
}
if _, err := tx.Exec("INSERT INTO entity (id) VALUES (?);", "0"); err != nil {
tx.Rollback()
return nil, err
}
if _, err := tx.Exec("DELETE FROM edge where entity_id=? and name=?", "0", "/"); err != nil {
tx.Rollback()
return nil, err
}
if _, err := tx.Exec("INSERT INTO edge (entity_id, name) VALUES(?,?);", "0", "/"); err != nil {
tx.Rollback()
return nil, err
}
if err := tx.Commit(); err != nil {
return nil, err
}
return db, nil
}
// Close the underlying connection to the database.
func (db *Database) Close() error {
return db.conn.Close()
}
// Set the entity id for a given path.
func (db *Database) Set(fullPath, id string) (*Entity, error) {
db.mux.Lock()
defer db.mux.Unlock()
tx, err := db.conn.Begin()
if err != nil {
return nil, err
}
var entityID string
if err := tx.QueryRow("SELECT id FROM entity WHERE id = ?;", id).Scan(&entityID); err != nil {
if err == sql.ErrNoRows {
if _, err := tx.Exec("INSERT INTO entity (id) VALUES(?);", id); err != nil {
tx.Rollback()
return nil, err
}
} else {
tx.Rollback()
return nil, err
}
}
e := &Entity{id}
parentPath, name := splitPath(fullPath)
if err := db.setEdge(parentPath, name, e, tx); err != nil {
tx.Rollback()
return nil, err
}
if err := tx.Commit(); err != nil {
return nil, err
}
return e, nil
}
// Exists returns true if a name already exists in the database.
func (db *Database) Exists(name string) bool {
db.mux.RLock()
defer db.mux.RUnlock()
e, err := db.get(name)
if err != nil {
return false
}
return e != nil
}
func (db *Database) setEdge(parentPath, name string, e *Entity, tx *sql.Tx) error {
parent, err := db.get(parentPath)
if err != nil {
return err
}
if parent.id == e.id {
return fmt.Errorf("Cannot set self as child")
}
if _, err := tx.Exec("INSERT INTO edge (parent_id, name, entity_id) VALUES (?,?,?);", parent.id, name, e.id); err != nil {
return err
}
return nil
}
// RootEntity returns the root "/" entity for the database.
func (db *Database) RootEntity() *Entity {
return &Entity{
id: "0",
}
}
// Get returns the entity for a given path.
func (db *Database) Get(name string) *Entity {
db.mux.RLock()
defer db.mux.RUnlock()
e, err := db.get(name)
if err != nil {
return nil
}
return e
}
func (db *Database) get(name string) (*Entity, error) {
e := db.RootEntity()
// We always know the root name so return it if
// it is requested
if name == "/" {
return e, nil
}
parts := split(name)
for i := 1; i < len(parts); i++ {
p := parts[i]
if p == "" {
continue
}
next := db.child(e, p)
if next == nil {
return nil, fmt.Errorf("Cannot find child for %s", name)
}
e = next
}
return e, nil
}
// List all entities by from the name.
// The key will be the full path of the entity.
func (db *Database) List(name string, depth int) Entities {
db.mux.RLock()
defer db.mux.RUnlock()
out := Entities{}
e, err := db.get(name)
if err != nil {
return out
}
children, err := db.children(e, name, depth, nil)
if err != nil {
return out
}
for _, c := range children {
out[c.FullPath] = c.Entity
}
return out
}
// Walk through the child graph of an entity, calling walkFunc for each child entity.
// It is safe for walkFunc to call graph functions.
func (db *Database) Walk(name string, walkFunc WalkFunc, depth int) error {
children, err := db.Children(name, depth)
if err != nil {
return err
}
// Note: the database lock must not be held while calling walkFunc
for _, c := range children {
if err := walkFunc(c.FullPath, c.Entity); err != nil {
return err
}
}
return nil
}
// Children returns the children of the specified entity.
func (db *Database) Children(name string, depth int) ([]WalkMeta, error) {
db.mux.RLock()
defer db.mux.RUnlock()
e, err := db.get(name)
if err != nil {
return nil, err
}
return db.children(e, name, depth, nil)
}
// Parents returns the parents of a specified entity.
func (db *Database) Parents(name string) ([]string, error) {
db.mux.RLock()
defer db.mux.RUnlock()
e, err := db.get(name)
if err != nil {
return nil, err
}
return db.parents(e)
}
// Refs returns the reference count for a specified id.
func (db *Database) Refs(id string) int {
db.mux.RLock()
defer db.mux.RUnlock()
var count int
if err := db.conn.QueryRow("SELECT COUNT(*) FROM edge WHERE entity_id = ?;", id).Scan(&count); err != nil {
return 0
}
return count
}
// RefPaths returns all the id's path references.
func (db *Database) RefPaths(id string) Edges {
db.mux.RLock()
defer db.mux.RUnlock()
refs := Edges{}
rows, err := db.conn.Query("SELECT name, parent_id FROM edge WHERE entity_id = ?;", id)
if err != nil {
return refs
}
defer rows.Close()
for rows.Next() {
var name string
var parentID string
if err := rows.Scan(&name, &parentID); err != nil {
return refs
}
refs = append(refs, &Edge{
EntityID: id,
Name: name,
ParentID: parentID,
})
}
return refs
}
// Delete the reference to an entity at a given path.
func (db *Database) Delete(name string) error {
db.mux.Lock()
defer db.mux.Unlock()
if name == "/" {
return fmt.Errorf("Cannot delete root entity")
}
parentPath, n := splitPath(name)
parent, err := db.get(parentPath)
if err != nil {
return err
}
if _, err := db.conn.Exec("DELETE FROM edge WHERE parent_id = ? AND name = ?;", parent.id, n); err != nil {
return err
}
return nil
}
// Purge removes the entity with the specified id
// Walk the graph to make sure all references to the entity
// are removed and return the number of references removed
func (db *Database) Purge(id string) (int, error) {
db.mux.Lock()
defer db.mux.Unlock()
tx, err := db.conn.Begin()
if err != nil {
return -1, err
}
// Delete all edges
rows, err := tx.Exec("DELETE FROM edge WHERE entity_id = ?;", id)
if err != nil {
tx.Rollback()
return -1, err
}
changes, err := rows.RowsAffected()
if err != nil {
return -1, err
}
// Clear who's using this id as parent
refs, err := tx.Exec("DELETE FROM edge WHERE parent_id = ?;", id)
if err != nil {
tx.Rollback()
return -1, err
}
refsCount, err := refs.RowsAffected()
if err != nil {
return -1, err
}
// Delete entity
if _, err := tx.Exec("DELETE FROM entity where id = ?;", id); err != nil {
tx.Rollback()
return -1, err
}
if err := tx.Commit(); err != nil {
return -1, err
}
return int(changes + refsCount), nil
}
// Rename an edge for a given path
func (db *Database) Rename(currentName, newName string) error {
db.mux.Lock()
defer db.mux.Unlock()
parentPath, name := splitPath(currentName)
newParentPath, newEdgeName := splitPath(newName)
if parentPath != newParentPath {
return fmt.Errorf("Cannot rename when root paths do not match %s != %s", parentPath, newParentPath)
}
parent, err := db.get(parentPath)
if err != nil {
return err
}
rows, err := db.conn.Exec("UPDATE edge SET name = ? WHERE parent_id = ? AND name = ?;", newEdgeName, parent.id, name)
if err != nil {
return err
}
i, err := rows.RowsAffected()
if err != nil {
return err
}
if i == 0 {
return fmt.Errorf("Cannot locate edge for %s %s", parent.id, name)
}
return nil
}
// WalkMeta stores the walk metadata.
type WalkMeta struct {
Parent *Entity
Entity *Entity
FullPath string
Edge *Edge
}
func (db *Database) children(e *Entity, name string, depth int, entities []WalkMeta) ([]WalkMeta, error) {
if e == nil {
return entities, nil
}
rows, err := db.conn.Query("SELECT entity_id, name FROM edge where parent_id = ?;", e.id)
if err != nil {
return nil, err
}
defer rows.Close()
for rows.Next() {
var entityID, entityName string
if err := rows.Scan(&entityID, &entityName); err != nil {
return nil, err
}
child := &Entity{entityID}
edge := &Edge{
ParentID: e.id,
Name: entityName,
EntityID: child.id,
}
meta := WalkMeta{
Parent: e,
Entity: child,
FullPath: path.Join(name, edge.Name),
Edge: edge,
}
entities = append(entities, meta)
if depth != 0 {
nDepth := depth
if depth != -1 {
nDepth--
}
entities, err = db.children(child, meta.FullPath, nDepth, entities)
if err != nil {
return nil, err
}
}
}
return entities, nil
}
func (db *Database) parents(e *Entity) (parents []string, err error) {
if e == nil {
return parents, nil
}
rows, err := db.conn.Query("SELECT parent_id FROM edge where entity_id = ?;", e.id)
if err != nil {
return nil, err
}
defer rows.Close()
for rows.Next() {
var parentID string
if err := rows.Scan(&parentID); err != nil {
return nil, err
}
parents = append(parents, parentID)
}
return parents, nil
}
// Return the entity based on the parent path and name.
func (db *Database) child(parent *Entity, name string) *Entity {
var id string
if err := db.conn.QueryRow("SELECT entity_id FROM edge WHERE parent_id = ? AND name = ?;", parent.id, name).Scan(&id); err != nil {
return nil
}
return &Entity{id}
}
// ID returns the id used to reference this entity.
func (e *Entity) ID() string {
return e.id
}
// Paths returns the paths sorted by depth.
func (e Entities) Paths() []string {
out := make([]string, len(e))
var i int
for k := range e {
out[i] = k
i++
}
sortByDepth(out)
return out
}

721
graphdb/graphdb_test.go
View file

@ -1,721 +0,0 @@
package graphdb
import (
"database/sql"
"fmt"
"os"
"path"
"runtime"
"strconv"
"testing"
_ "github.com/mattn/go-sqlite3"
)
func newTestDb(t *testing.T) (*Database, string) {
p := path.Join(os.TempDir(), "sqlite.db")
conn, err := sql.Open("sqlite3", p)
db, err := NewDatabase(conn)
if err != nil {
t.Fatal(err)
}
return db, p
}
func destroyTestDb(dbPath string) {
os.Remove(dbPath)
}
func TestNewDatabase(t *testing.T) {
db, dbpath := newTestDb(t)
if db == nil {
t.Fatal("Database should not be nil")
}
db.Close()
defer destroyTestDb(dbpath)
}
func TestCreateRootEntity(t *testing.T) {
db, dbpath := newTestDb(t)
defer destroyTestDb(dbpath)
root := db.RootEntity()
if root == nil {
t.Fatal("Root entity should not be nil")
}
}
func TestGetRootEntity(t *testing.T) {
db, dbpath := newTestDb(t)
defer destroyTestDb(dbpath)
e := db.Get("/")
if e == nil {
t.Fatal("Entity should not be nil")
}
if e.ID() != "0" {
t.Fatalf("Entity id should be 0, got %s", e.ID())
}
}
func TestSetEntityWithDifferentName(t *testing.T) {
db, dbpath := newTestDb(t)
defer destroyTestDb(dbpath)
db.Set("/test", "1")
if _, err := db.Set("/other", "1"); err != nil {
t.Fatal(err)
}
}
func TestSetDuplicateEntity(t *testing.T) {
db, dbpath := newTestDb(t)
defer destroyTestDb(dbpath)
if _, err := db.Set("/foo", "42"); err != nil {
t.Fatal(err)
}
if _, err := db.Set("/foo", "43"); err == nil {
t.Fatalf("Creating an entry with a duplicate path did not cause an error")
}
}
func TestCreateChild(t *testing.T) {
db, dbpath := newTestDb(t)
defer destroyTestDb(dbpath)
child, err := db.Set("/db", "1")
if err != nil {
t.Fatal(err)
}
if child == nil {
t.Fatal("Child should not be nil")
}
if child.ID() != "1" {
t.Fail()
}
}
func TestParents(t *testing.T) {
db, dbpath := newTestDb(t)
defer destroyTestDb(dbpath)
for i := 1; i < 6; i++ {
a := strconv.Itoa(i)
if _, err := db.Set("/"+a, a); err != nil {
t.Fatal(err)
}
}
for i := 6; i < 11; i++ {
a := strconv.Itoa(i)
p := strconv.Itoa(i - 5)
key := fmt.Sprintf("/%s/%s", p, a)
if _, err := db.Set(key, a); err != nil {
t.Fatal(err)
}
parents, err := db.Parents(key)
if err != nil {
t.Fatal(err)
}
if len(parents) != 1 {
t.Fatalf("Expected 1 entry for %s got %d", key, len(parents))
}
if parents[0] != p {
t.Fatalf("ID %s received, %s expected", parents[0], p)
}
}
}
func TestChildren(t *testing.T) {
// TODO Windows: Port this test
if runtime.GOOS == "windows" {
t.Skip("Needs porting to Windows")
}
db, dbpath := newTestDb(t)
defer destroyTestDb(dbpath)
str := "/"
for i := 1; i < 6; i++ {
a := strconv.Itoa(i)
if _, err := db.Set(str+a, a); err != nil {
t.Fatal(err)
}
str = str + a + "/"
}
str = "/"
for i := 10; i < 30; i++ { // 20 entities
a := strconv.Itoa(i)
if _, err := db.Set(str+a, a); err != nil {
t.Fatal(err)
}
str = str + a + "/"
}
entries, err := db.Children("/", 5)
if err != nil {
t.Fatal(err)
}
if len(entries) != 11 {
t.Fatalf("Expect 11 entries for / got %d", len(entries))
}
entries, err = db.Children("/", 20)
if err != nil {
t.Fatal(err)
}
if len(entries) != 25 {
t.Fatalf("Expect 25 entries for / got %d", len(entries))
}
}
func TestListAllRootChildren(t *testing.T) {
// TODO Windows: Port this test
if runtime.GOOS == "windows" {
t.Skip("Needs porting to Windows")
}
db, dbpath := newTestDb(t)
defer destroyTestDb(dbpath)
for i := 1; i < 6; i++ {
a := strconv.Itoa(i)
if _, err := db.Set("/"+a, a); err != nil {
t.Fatal(err)
}
}
entries := db.List("/", -1)
if len(entries) != 5 {
t.Fatalf("Expect 5 entries for / got %d", len(entries))
}
}
func TestListAllSubChildren(t *testing.T) {
// TODO Windows: Port this test
if runtime.GOOS == "windows" {
t.Skip("Needs porting to Windows")
}
db, dbpath := newTestDb(t)
defer destroyTestDb(dbpath)
_, err := db.Set("/webapp", "1")
if err != nil {
t.Fatal(err)
}
child2, err := db.Set("/db", "2")
if err != nil {
t.Fatal(err)
}
child4, err := db.Set("/logs", "4")
if err != nil {
t.Fatal(err)
}
if _, err := db.Set("/db/logs", child4.ID()); err != nil {
t.Fatal(err)
}
child3, err := db.Set("/sentry", "3")
if err != nil {
t.Fatal(err)
}
if _, err := db.Set("/webapp/sentry", child3.ID()); err != nil {
t.Fatal(err)
}
if _, err := db.Set("/webapp/db", child2.ID()); err != nil {
t.Fatal(err)
}
entries := db.List("/webapp", 1)
if len(entries) != 3 {
t.Fatalf("Expect 3 entries for / got %d", len(entries))
}
entries = db.List("/webapp", 0)
if len(entries) != 2 {
t.Fatalf("Expect 2 entries for / got %d", len(entries))
}
}
func TestAddSelfAsChild(t *testing.T) {
// TODO Windows: Port this test
if runtime.GOOS == "windows" {
t.Skip("Needs porting to Windows")
}
db, dbpath := newTestDb(t)
defer destroyTestDb(dbpath)
child, err := db.Set("/test", "1")
if err != nil {
t.Fatal(err)
}
if _, err := db.Set("/test/other", child.ID()); err == nil {
t.Fatal("Error should not be nil")
}
}
func TestAddChildToNonExistentRoot(t *testing.T) {
db, dbpath := newTestDb(t)
defer destroyTestDb(dbpath)
if _, err := db.Set("/myapp", "1"); err != nil {
t.Fatal(err)
}
if _, err := db.Set("/myapp/proxy/db", "2"); err == nil {
t.Fatal("Error should not be nil")
}
}
func TestWalkAll(t *testing.T) {
// TODO Windows: Port this test
if runtime.GOOS == "windows" {
t.Skip("Needs porting to Windows")
}
db, dbpath := newTestDb(t)
defer destroyTestDb(dbpath)
_, err := db.Set("/webapp", "1")
if err != nil {
t.Fatal(err)
}
child2, err := db.Set("/db", "2")
if err != nil {
t.Fatal(err)
}
child4, err := db.Set("/db/logs", "4")
if err != nil {
t.Fatal(err)
}
if _, err := db.Set("/webapp/logs", child4.ID()); err != nil {
t.Fatal(err)
}
child3, err := db.Set("/sentry", "3")
if err != nil {
t.Fatal(err)
}
if _, err := db.Set("/webapp/sentry", child3.ID()); err != nil {
t.Fatal(err)
}
if _, err := db.Set("/webapp/db", child2.ID()); err != nil {
t.Fatal(err)
}
child5, err := db.Set("/gograph", "5")
if err != nil {
t.Fatal(err)
}
if _, err := db.Set("/webapp/same-ref-diff-name", child5.ID()); err != nil {
t.Fatal(err)
}
if err := db.Walk("/", func(p string, e *Entity) error {
t.Logf("Path: %s Entity: %s", p, e.ID())
return nil
}, -1); err != nil {
t.Fatal(err)
}
}
func TestGetEntityByPath(t *testing.T) {
// TODO Windows: Port this test
if runtime.GOOS == "windows" {
t.Skip("Needs porting to Windows")
}
db, dbpath := newTestDb(t)
defer destroyTestDb(dbpath)
_, err := db.Set("/webapp", "1")
if err != nil {
t.Fatal(err)
}
child2, err := db.Set("/db", "2")
if err != nil {
t.Fatal(err)
}
child4, err := db.Set("/logs", "4")
if err != nil {
t.Fatal(err)
}
if _, err := db.Set("/db/logs", child4.ID()); err != nil {
t.Fatal(err)
}
child3, err := db.Set("/sentry", "3")
if err != nil {
t.Fatal(err)
}
if _, err := db.Set("/webapp/sentry", child3.ID()); err != nil {
t.Fatal(err)
}
if _, err := db.Set("/webapp/db", child2.ID()); err != nil {
t.Fatal(err)
}
child5, err := db.Set("/gograph", "5")
if err != nil {
t.Fatal(err)
}
if _, err := db.Set("/webapp/same-ref-diff-name", child5.ID()); err != nil {
t.Fatal(err)
}
entity := db.Get("/webapp/db/logs")
if entity == nil {
t.Fatal("Entity should not be nil")
}
if entity.ID() != "4" {
t.Fatalf("Expected to get entity with id 4, got %s", entity.ID())
}
}
func TestEnitiesPaths(t *testing.T) {
// TODO Windows: Port this test
if runtime.GOOS == "windows" {
t.Skip("Needs porting to Windows")
}
db, dbpath := newTestDb(t)
defer destroyTestDb(dbpath)
_, err := db.Set("/webapp", "1")
if err != nil {
t.Fatal(err)
}
child2, err := db.Set("/db", "2")
if err != nil {
t.Fatal(err)
}
child4, err := db.Set("/logs", "4")
if err != nil {
t.Fatal(err)
}
if _, err := db.Set("/db/logs", child4.ID()); err != nil {
t.Fatal(err)
}
child3, err := db.Set("/sentry", "3")
if err != nil {
t.Fatal(err)
}
if _, err := db.Set("/webapp/sentry", child3.ID()); err != nil {
t.Fatal(err)
}
if _, err := db.Set("/webapp/db", child2.ID()); err != nil {
t.Fatal(err)
}
child5, err := db.Set("/gograph", "5")
if err != nil {
t.Fatal(err)
}
if _, err := db.Set("/webapp/same-ref-diff-name", child5.ID()); err != nil {
t.Fatal(err)
}
out := db.List("/", -1)
for _, p := range out.Paths() {
t.Log(p)
}
}
func TestDeleteRootEntity(t *testing.T) {
db, dbpath := newTestDb(t)
defer destroyTestDb(dbpath)
if err := db.Delete("/"); err == nil {
t.Fatal("Error should not be nil")
}
}
func TestDeleteEntity(t *testing.T) {
// TODO Windows: Port this test
if runtime.GOOS == "windows" {
t.Skip("Needs porting to Windows")
}
db, dbpath := newTestDb(t)
defer destroyTestDb(dbpath)
_, err := db.Set("/webapp", "1")
if err != nil {
t.Fatal(err)
}
child2, err := db.Set("/db", "2")
if err != nil {
t.Fatal(err)
}
child4, err := db.Set("/logs", "4")
if err != nil {
t.Fatal(err)
}
if _, err := db.Set("/db/logs", child4.ID()); err != nil {
t.Fatal(err)
}
child3, err := db.Set("/sentry", "3")
if err != nil {
t.Fatal(err)
}
if _, err := db.Set("/webapp/sentry", child3.ID()); err != nil {
t.Fatal(err)
}
if _, err := db.Set("/webapp/db", child2.ID()); err != nil {
t.Fatal(err)
}
child5, err := db.Set("/gograph", "5")
if err != nil {
t.Fatal(err)
}
if _, err := db.Set("/webapp/same-ref-diff-name", child5.ID()); err != nil {
t.Fatal(err)
}
if err := db.Delete("/webapp/sentry"); err != nil {
t.Fatal(err)
}
entity := db.Get("/webapp/sentry")
if entity != nil {
t.Fatal("Entity /webapp/sentry should be nil")
}
}
func TestCountRefs(t *testing.T) {
// TODO Windows: Port this test
if runtime.GOOS == "windows" {
t.Skip("Needs porting to Windows")
}
db, dbpath := newTestDb(t)
defer destroyTestDb(dbpath)
db.Set("/webapp", "1")
if db.Refs("1") != 1 {
t.Fatal("Expect reference count to be 1")
}
db.Set("/db", "2")
db.Set("/webapp/db", "2")
if db.Refs("2") != 2 {
t.Fatal("Expect reference count to be 2")
}
}
func TestPurgeId(t *testing.T) {
// TODO Windows: Port this test
if runtime.GOOS == "windows" {
t.Skip("Needs porting to Windows")
}
db, dbpath := newTestDb(t)
defer destroyTestDb(dbpath)
db.Set("/webapp", "1")
if c := db.Refs("1"); c != 1 {
t.Fatalf("Expect reference count to be 1, got %d", c)
}
db.Set("/db", "2")
db.Set("/webapp/db", "2")
count, err := db.Purge("2")
if err != nil {
t.Fatal(err)
}
if count != 2 {
t.Fatalf("Expected 2 references to be removed, got %d", count)
}
}
// Regression test https://github.com/docker/docker/issues/12334
func TestPurgeIdRefPaths(t *testing.T) {
// TODO Windows: Port this test
if runtime.GOOS == "windows" {
t.Skip("Needs porting to Windows")
}
db, dbpath := newTestDb(t)
defer destroyTestDb(dbpath)
db.Set("/webapp", "1")
db.Set("/db", "2")
db.Set("/db/webapp", "1")
if c := db.Refs("1"); c != 2 {
t.Fatalf("Expected 2 reference for webapp, got %d", c)
}
if c := db.Refs("2"); c != 1 {
t.Fatalf("Expected 1 reference for db, got %d", c)
}
if rp := db.RefPaths("2"); len(rp) != 1 {
t.Fatalf("Expected 1 reference path for db, got %d", len(rp))
}
count, err := db.Purge("2")
if err != nil {
t.Fatal(err)
}
if count != 2 {
t.Fatalf("Expected 2 rows to be removed, got %d", count)
}
if c := db.Refs("2"); c != 0 {
t.Fatalf("Expected 0 reference for db, got %d", c)
}
if c := db.Refs("1"); c != 1 {
t.Fatalf("Expected 1 reference for webapp, got %d", c)
}
}
func TestRename(t *testing.T) {
// TODO Windows: Port this test
if runtime.GOOS == "windows" {
t.Skip("Needs porting to Windows")
}
db, dbpath := newTestDb(t)
defer destroyTestDb(dbpath)
db.Set("/webapp", "1")
if db.Refs("1") != 1 {
t.Fatal("Expect reference count to be 1")
}
db.Set("/db", "2")
db.Set("/webapp/db", "2")
if db.Get("/webapp/db") == nil {
t.Fatal("Cannot find entity at path /webapp/db")
}
if err := db.Rename("/webapp/db", "/webapp/newdb"); err != nil {
t.Fatal(err)
}
if db.Get("/webapp/db") != nil {
t.Fatal("Entity should not exist at /webapp/db")
}
if db.Get("/webapp/newdb") == nil {
t.Fatal("Cannot find entity at path /webapp/newdb")
}
}
func TestCreateMultipleNames(t *testing.T) {
// TODO Windows: Port this test
if runtime.GOOS == "windows" {
t.Skip("Needs porting to Windows")
}
db, dbpath := newTestDb(t)
defer destroyTestDb(dbpath)
db.Set("/db", "1")
if _, err := db.Set("/myapp", "1"); err != nil {
t.Fatal(err)
}
db.Walk("/", func(p string, e *Entity) error {
t.Logf("%s\n", p)
return nil
}, -1)
}
func TestRefPaths(t *testing.T) {
db, dbpath := newTestDb(t)
defer destroyTestDb(dbpath)
db.Set("/webapp", "1")
db.Set("/db", "2")
db.Set("/webapp/db", "2")
refs := db.RefPaths("2")
if len(refs) != 2 {
t.Fatalf("Expected reference count to be 2, got %d", len(refs))
}
}
func TestExistsTrue(t *testing.T) {
db, dbpath := newTestDb(t)
defer destroyTestDb(dbpath)
db.Set("/testing", "1")
if !db.Exists("/testing") {
t.Fatalf("/tesing should exist")
}
}
func TestExistsFalse(t *testing.T) {
// TODO Windows: Port this test
if runtime.GOOS == "windows" {
t.Skip("Needs porting to Windows")
}
db, dbpath := newTestDb(t)
defer destroyTestDb(dbpath)
db.Set("/toerhe", "1")
if db.Exists("/testing") {
t.Fatalf("/tesing should not exist")
}
}
func TestGetNameWithTrailingSlash(t *testing.T) {
db, dbpath := newTestDb(t)
defer destroyTestDb(dbpath)
db.Set("/todo", "1")
e := db.Get("/todo/")
if e == nil {
t.Fatalf("Entity should not be nil")
}
}
func TestConcurrentWrites(t *testing.T) {
// TODO Windows: Port this test
if runtime.GOOS == "windows" {
t.Skip("Needs porting to Windows")
}
db, dbpath := newTestDb(t)
defer destroyTestDb(dbpath)
errs := make(chan error, 2)
save := func(name string, id string) {
if _, err := db.Set(fmt.Sprintf("/%s", name), id); err != nil {
errs <- err
}
errs <- nil
}
purge := func(id string) {
if _, err := db.Purge(id); err != nil {
errs <- err
}
errs <- nil
}
save("/1", "1")
go purge("1")
go save("/2", "2")
any := false
for i := 0; i < 2; i++ {
if err := <-errs; err != nil {
any = true
t.Log(err)
}
}
if any {
t.Fail()
}
}

27
graphdb/sort.go
View file

@ -1,27 +0,0 @@
package graphdb
import "sort"
type pathSorter struct {
paths []string
by func(i, j string) bool
}
func sortByDepth(paths []string) {
s := &pathSorter{paths, func(i, j string) bool {
return PathDepth(i) > PathDepth(j)
}}
sort.Sort(s)
}
func (s *pathSorter) Len() int {
return len(s.paths)
}
func (s *pathSorter) Swap(i, j int) {
s.paths[i], s.paths[j] = s.paths[j], s.paths[i]
}
func (s *pathSorter) Less(i, j int) bool {
return s.by(s.paths[i], s.paths[j])
}

29
graphdb/sort_test.go
View file

@ -1,29 +0,0 @@
package graphdb
import (
"testing"
)
func TestSort(t *testing.T) {
paths := []string{
"/",
"/myreallylongname",
"/app/db",
}
sortByDepth(paths)
if len(paths) != 3 {
t.Fatalf("Expected 3 parts got %d", len(paths))
}
if paths[0] != "/app/db" {
t.Fatalf("Expected /app/db got %s", paths[0])
}
if paths[1] != "/myreallylongname" {
t.Fatalf("Expected /myreallylongname got %s", paths[1])
}
if paths[2] != "/" {
t.Fatalf("Expected / got %s", paths[2])
}
}

32
graphdb/utils.go
View file

@ -1,32 +0,0 @@
package graphdb
import (
"path"
"strings"
)
// Split p on /
func split(p string) []string {
return strings.Split(p, "/")
}
// PathDepth returns the depth or number of / in a given path
func PathDepth(p string) int {
parts := split(p)
if len(parts) == 2 && parts[1] == "" {
return 1
}
return len(parts)
}
func splitPath(p string) (parent, name string) {
if p[0] != '/' {
p = "/" + p
}
parent, name = path.Split(p)
l := len(parent)
if parent[l-1] == '/' {
parent = parent[:l-1]
}
return
}

56
httputils/httputils.go
View file

@ -1,56 +0,0 @@
package httputils
import (
"errors"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/docker/docker/pkg/jsonmessage"
)
var (
headerRegexp = regexp.MustCompile(`^(?:(.+)/(.+?))\((.+)\).*$`)
errInvalidHeader = errors.New("Bad header, should be in format `docker/version (platform)`")
)
// Download requests a given URL and returns an io.Reader.
func Download(url string) (resp *http.Response, err error) {
if resp, err = http.Get(url); err != nil {
return nil, err
}
if resp.StatusCode >= 400 {
return nil, fmt.Errorf("Got HTTP status code >= 400: %s", resp.Status)
}
return resp, nil
}
// NewHTTPRequestError returns a JSON response error.
func NewHTTPRequestError(msg string, res *http.Response) error {
return &jsonmessage.JSONError{
Message: msg,
Code: res.StatusCode,
}
}
// ServerHeader contains the server information.
type ServerHeader struct {
App string // docker
Ver string // 1.8.0-dev
OS string // windows or linux
}
// ParseServerHeader extracts pieces from an HTTP server header
// which is in the format "docker/version (os)" eg docker/1.8.0-dev (windows).
func ParseServerHeader(hdr string) (*ServerHeader, error) {
matches := headerRegexp.FindStringSubmatch(hdr)
if len(matches) != 4 {
return nil, errInvalidHeader
}
return &ServerHeader{
App: strings.TrimSpace(matches[1]),
Ver: strings.TrimSpace(matches[2]),
OS: strings.TrimSpace(matches[3]),
}, nil
}

115
httputils/httputils_test.go
View file

@ -1,115 +0,0 @@
package httputils
import (
"fmt"
"io/ioutil"
"net/http"
"net/http/httptest"
"strings"
"testing"
)
func TestDownload(t *testing.T) {
expected := "Hello, docker !"
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
fmt.Fprintf(w, expected)
}))
defer ts.Close()
response, err := Download(ts.URL)
if err != nil {
t.Fatal(err)
}
actual, err := ioutil.ReadAll(response.Body)
response.Body.Close()
if err != nil || string(actual) != expected {
t.Fatalf("Expected the response %q, got err:%v, response:%v, actual:%s", expected, err, response, string(actual))
}
}
func TestDownload400Errors(t *testing.T) {
expectedError := "Got HTTP status code >= 400: 403 Forbidden"
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// 403
http.Error(w, "something failed (forbidden)", http.StatusForbidden)
}))
defer ts.Close()
// Expected status code = 403
if _, err := Download(ts.URL); err == nil || err.Error() != expectedError {
t.Fatalf("Expected the the error %q, got %v", expectedError, err)
}
}
func TestDownloadOtherErrors(t *testing.T) {
if _, err := Download("I'm not an url.."); err == nil || !strings.Contains(err.Error(), "unsupported protocol scheme") {
t.Fatalf("Expected an error with 'unsupported protocol scheme', got %v", err)
}
}
func TestNewHTTPRequestError(t *testing.T) {
errorMessage := "Some error message"
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// 403
http.Error(w, errorMessage, http.StatusForbidden)
}))
defer ts.Close()
httpResponse, err := http.Get(ts.URL)
if err != nil {
t.Fatal(err)
}
if err := NewHTTPRequestError(errorMessage, httpResponse); err.Error() != errorMessage {
t.Fatalf("Expected err to be %q, got %v", errorMessage, err)
}
}
func TestParseServerHeader(t *testing.T) {
inputs := map[string][]string{
"bad header": {"error"},
"(bad header)": {"error"},
"(without/spaces)": {"error"},
"(header/with spaces)": {"error"},
"foo/bar (baz)": {"foo", "bar", "baz"},
"foo/bar": {"error"},
"foo": {"error"},
"foo/bar (baz space)": {"foo", "bar", "baz space"},
" f f / b b ( b s ) ": {"f f", "b b", "b s"},
"foo/bar (baz) ignore": {"foo", "bar", "baz"},
"foo/bar ()": {"error"},
"foo/bar()": {"error"},
"foo/bar(baz)": {"foo", "bar", "baz"},
"foo/bar/zzz(baz)": {"foo/bar", "zzz", "baz"},
"foo/bar(baz/abc)": {"foo", "bar", "baz/abc"},
"foo/bar(baz (abc))": {"foo", "bar", "baz (abc)"},
}
for header, values := range inputs {
serverHeader, err := ParseServerHeader(header)
if err != nil {
if err != errInvalidHeader {
t.Fatalf("Failed to parse %q, and got some unexpected error: %q", header, err)
}
if values[0] == "error" {
continue
}
t.Fatalf("Header %q failed to parse when it shouldn't have", header)
}
if values[0] == "error" {
t.Fatalf("Header %q parsed ok when it should have failed(%q).", header, serverHeader)
}
if serverHeader.App != values[0] {
t.Fatalf("Expected serverHeader.App for %q to equal %q, got %q", header, values[0], serverHeader.App)
}
if serverHeader.Ver != values[1] {
t.Fatalf("Expected serverHeader.Ver for %q to equal %q, got %q", header, values[1], serverHeader.Ver)
}
if serverHeader.OS != values[2] {
t.Fatalf("Expected serverHeader.OS for %q to equal %q, got %q", header, values[2], serverHeader.OS)
}
}
}

30
httputils/mimetype.go
View file

@ -1,30 +0,0 @@
package httputils
import (
"mime"
"net/http"
)
// MimeTypes stores the MIME content type.
var MimeTypes = struct {
TextPlain string
Tar string
OctetStream string
}{"text/plain", "application/tar", "application/octet-stream"}
// DetectContentType returns a best guess representation of the MIME
// content type for the bytes at c. The value detected by
// http.DetectContentType is guaranteed not be nil, defaulting to
// application/octet-stream when a better guess cannot be made. The
// result of this detection is then run through mime.ParseMediaType()
// which separates the actual MIME string from any parameters.
func DetectContentType(c []byte) (string, map[string]string, error) {
ct := http.DetectContentType(c)
contentType, args, err := mime.ParseMediaType(ct)
if err != nil {
return "", nil, err
}
return contentType, args, nil
}

13
httputils/mimetype_test.go
View file

@ -1,13 +0,0 @@
package httputils
import (
"testing"
)
func TestDetectContentType(t *testing.T) {
input := []byte("That is just a plain text")
if contentType, _, err := DetectContentType(input); err != nil || contentType != "text/plain" {
t.Errorf("TestDetectContentType failed")
}
}

95
httputils/resumablerequestreader.go
View file

@ -1,95 +0,0 @@
package httputils
import (
"fmt"
"io"
"net/http"
"time"
"github.com/Sirupsen/logrus"
)
type resumableRequestReader struct {
client *http.Client
request *http.Request
lastRange int64
totalSize int64
currentResponse *http.Response
failures uint32
maxFailures uint32
}
// ResumableRequestReader makes it possible to resume reading a request's body transparently
// maxfail is the number of times we retry to make requests again (not resumes)
// totalsize is the total length of the body; auto detect if not provided
func ResumableRequestReader(c *http.Client, r *http.Request, maxfail uint32, totalsize int64) io.ReadCloser {
return &resumableRequestReader{client: c, request: r, maxFailures: maxfail, totalSize: totalsize}
}
// ResumableRequestReaderWithInitialResponse makes it possible to resume
// reading the body of an already initiated request.
func ResumableRequestReaderWithInitialResponse(c *http.Client, r *http.Request, maxfail uint32, totalsize int64, initialResponse *http.Response) io.ReadCloser {
return &resumableRequestReader{client: c, request: r, maxFailures: maxfail, totalSize: totalsize, currentResponse: initialResponse}
}
func (r *resumableRequestReader) Read(p []byte) (n int, err error) {
if r.client == nil || r.request == nil {
return 0, fmt.Errorf("client and request can't be nil\n")
}
isFreshRequest := false
if r.lastRange != 0 && r.currentResponse == nil {
readRange := fmt.Sprintf("bytes=%d-%d", r.lastRange, r.totalSize)
r.request.Header.Set("Range", readRange)
time.Sleep(5 * time.Second)
}
if r.currentResponse == nil {
r.currentResponse, err = r.client.Do(r.request)
isFreshRequest = true
}
if err != nil && r.failures+1 != r.maxFailures {
r.cleanUpResponse()
r.failures++
time.Sleep(5 * time.Duration(r.failures) * time.Second)
return 0, nil
} else if err != nil {
r.cleanUpResponse()
return 0, err
}
if r.currentResponse.StatusCode == 416 && r.lastRange == r.totalSize && r.currentResponse.ContentLength == 0 {
r.cleanUpResponse()
return 0, io.EOF
} else if r.currentResponse.StatusCode != 206 && r.lastRange != 0 && isFreshRequest {
r.cleanUpResponse()
return 0, fmt.Errorf("the server doesn't support byte ranges")
}
if r.totalSize == 0 {
r.totalSize = r.currentResponse.ContentLength
} else if r.totalSize <= 0 {
r.cleanUpResponse()
return 0, fmt.Errorf("failed to auto detect content length")
}
n, err = r.currentResponse.Body.Read(p)
r.lastRange += int64(n)
if err != nil {
r.cleanUpResponse()
}
if err != nil && err != io.EOF {
logrus.Infof("encountered error during pull and clearing it before resume: %s", err)
err = nil
}
return n, err
}
func (r *resumableRequestReader) Close() error {
r.cleanUpResponse()
r.client = nil
r.request = nil
return nil
}
func (r *resumableRequestReader) cleanUpResponse() {
if r.currentResponse != nil {
r.currentResponse.Body.Close()
r.currentResponse = nil
}
}

307
httputils/resumablerequestreader_test.go
View file

@ -1,307 +0,0 @@
package httputils
import (
"fmt"
"io"
"io/ioutil"
"net/http"
"net/http/httptest"
"strings"
"testing"
)
func TestResumableRequestHeaderSimpleErrors(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
fmt.Fprintln(w, "Hello, world !")
}))
defer ts.Close()
client := &http.Client{}
var req *http.Request
req, err := http.NewRequest("GET", ts.URL, nil)
if err != nil {
t.Fatal(err)
}
expectedError := "client and request can't be nil\n"
resreq := &resumableRequestReader{}
_, err = resreq.Read([]byte{})
if err == nil || err.Error() != expectedError {
t.Fatalf("Expected an error with '%s', got %v.", expectedError, err)
}
resreq = &resumableRequestReader{
client: client,
request: req,
totalSize: -1,
}
expectedError = "failed to auto detect content length"
_, err = resreq.Read([]byte{})
if err == nil || err.Error() != expectedError {
t.Fatalf("Expected an error with '%s', got %v.", expectedError, err)
}
}
// Not too much failures, bails out after some wait
func TestResumableRequestHeaderNotTooMuchFailures(t *testing.T) {
client := &http.Client{}
var badReq *http.Request
badReq, err := http.NewRequest("GET", "I'm not an url", nil)
if err != nil {
t.Fatal(err)
}
resreq := &resumableRequestReader{
client: client,
request: badReq,
failures: 0,
maxFailures: 2,
}
read, err := resreq.Read([]byte{})
if err != nil || read != 0 {
t.Fatalf("Expected no error and no byte read, got err:%v, read:%v.", err, read)
}
}
// Too much failures, returns the error
func TestResumableRequestHeaderTooMuchFailures(t *testing.T) {
client := &http.Client{}
var badReq *http.Request
badReq, err := http.NewRequest("GET", "I'm not an url", nil)
if err != nil {
t.Fatal(err)
}
resreq := &resumableRequestReader{
client: client,
request: badReq,
failures: 0,
maxFailures: 1,
}
defer resreq.Close()
expectedError := `Get I%27m%20not%20an%20url: unsupported protocol scheme ""`
read, err := resreq.Read([]byte{})
if err == nil || err.Error() != expectedError || read != 0 {
t.Fatalf("Expected the error '%s', got err:%v, read:%v.", expectedError, err, read)
}
}
type errorReaderCloser struct{}
func (errorReaderCloser) Close() error { return nil }
func (errorReaderCloser) Read(p []byte) (n int, err error) {
return 0, fmt.Errorf("An error occurred")
}
// If an unknown error is encountered, return 0, nil and log it
func TestResumableRequestReaderWithReadError(t *testing.T) {
var req *http.Request
req, err := http.NewRequest("GET", "", nil)
if err != nil {
t.Fatal(err)
}
client := &http.Client{}
response := &http.Response{
Status: "500 Internal Server",
StatusCode: 500,
ContentLength: 0,
Close: true,
Body: errorReaderCloser{},
}
resreq := &resumableRequestReader{
client: client,
request: req,
currentResponse: response,
lastRange: 1,
totalSize: 1,
}
defer resreq.Close()
buf := make([]byte, 1)
read, err := resreq.Read(buf)
if err != nil {
t.Fatal(err)
}
if read != 0 {
t.Fatalf("Expected to have read nothing, but read %v", read)
}
}
func TestResumableRequestReaderWithEOFWith416Response(t *testing.T) {
var req *http.Request
req, err := http.NewRequest("GET", "", nil)
if err != nil {
t.Fatal(err)
}
client := &http.Client{}
response := &http.Response{
Status: "416 Requested Range Not Satisfiable",
StatusCode: 416,
ContentLength: 0,
Close: true,
Body: ioutil.NopCloser(strings.NewReader("")),
}
resreq := &resumableRequestReader{
client: client,
request: req,
currentResponse: response,
lastRange: 1,
totalSize: 1,
}
defer resreq.Close()
buf := make([]byte, 1)
_, err = resreq.Read(buf)
if err == nil || err != io.EOF {
t.Fatalf("Expected an io.EOF error, got %v", err)
}
}
func TestResumableRequestReaderWithServerDoesntSupportByteRanges(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("Range") == "" {
t.Fatalf("Expected a Range HTTP header, got nothing")
}
}))
defer ts.Close()
var req *http.Request
req, err := http.NewRequest("GET", ts.URL, nil)
if err != nil {
t.Fatal(err)
}
client := &http.Client{}
resreq := &resumableRequestReader{
client: client,
request: req,
lastRange: 1,
}
defer resreq.Close()
buf := make([]byte, 2)
_, err = resreq.Read(buf)
if err == nil || err.Error() != "the server doesn't support byte ranges" {
t.Fatalf("Expected an error 'the server doesn't support byte ranges', got %v", err)
}
}
func TestResumableRequestReaderWithZeroTotalSize(t *testing.T) {
srvtxt := "some response text data"
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
fmt.Fprintln(w, srvtxt)
}))
defer ts.Close()
var req *http.Request
req, err := http.NewRequest("GET", ts.URL, nil)
if err != nil {
t.Fatal(err)
}
client := &http.Client{}
retries := uint32(5)
resreq := ResumableRequestReader(client, req, retries, 0)
defer resreq.Close()
data, err := ioutil.ReadAll(resreq)
if err != nil {
t.Fatal(err)
}
resstr := strings.TrimSuffix(string(data), "\n")
if resstr != srvtxt {
t.Errorf("resstr != srvtxt")
}
}
func TestResumableRequestReader(t *testing.T) {
srvtxt := "some response text data"
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
fmt.Fprintln(w, srvtxt)
}))
defer ts.Close()
var req *http.Request
req, err := http.NewRequest("GET", ts.URL, nil)
if err != nil {
t.Fatal(err)
}
client := &http.Client{}
retries := uint32(5)
imgSize := int64(len(srvtxt))
resreq := ResumableRequestReader(client, req, retries, imgSize)
defer resreq.Close()
data, err := ioutil.ReadAll(resreq)
if err != nil {
t.Fatal(err)
}
resstr := strings.TrimSuffix(string(data), "\n")
if resstr != srvtxt {
t.Errorf("resstr != srvtxt")
}
}
func TestResumableRequestReaderWithInitialResponse(t *testing.T) {
srvtxt := "some response text data"
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
fmt.Fprintln(w, srvtxt)
}))
defer ts.Close()
var req *http.Request
req, err := http.NewRequest("GET", ts.URL, nil)
if err != nil {
t.Fatal(err)
}
client := &http.Client{}
retries := uint32(5)
imgSize := int64(len(srvtxt))
res, err := client.Do(req)
if err != nil {
t.Fatal(err)
}
resreq := ResumableRequestReaderWithInitialResponse(client, req, retries, imgSize, res)
defer resreq.Close()
data, err := ioutil.ReadAll(resreq)
if err != nil {
t.Fatal(err)
}
resstr := strings.TrimSuffix(string(data), "\n")
if resstr != srvtxt {
t.Errorf("resstr != srvtxt")
}
}

42
jsonlog/jsonlog.go
View file

@ -1,42 +0,0 @@
package jsonlog
import (
"encoding/json"
"fmt"
"time"
)
// JSONLog represents a log message, typically a single entry from a given log stream.
// JSONLogs can be easily serialized to and from JSON and support custom formatting.
type JSONLog struct {
// Log is the log message
Log string `json:"log,omitempty"`
// Stream is the log source
Stream string `json:"stream,omitempty"`
// Created is the created timestamp of log
Created time.Time `json:"time"`
// Attrs is the list of extra attributes provided by the user
Attrs map[string]string `json:"attrs,omitempty"`
}
// Format returns the log formatted according to format
// If format is nil, returns the log message
// If format is json, returns the log marshaled in json format
// By default, returns the log with the log time formatted according to format.
func (jl *JSONLog) Format(format string) (string, error) {
if format == "" {
return jl.Log, nil
}
if format == "json" {
m, err := json.Marshal(jl)
return string(m), err
}
return fmt.Sprintf("%s %s", jl.Created.Format(format), jl.Log), nil
}
// Reset resets the log to nil.
func (jl *JSONLog) Reset() {
jl.Log = ""
jl.Stream = ""
jl.Created = time.Time{}
}

178
jsonlog/jsonlog_marshalling.go
View file

@ -1,178 +0,0 @@
// This code was initially generated by ffjson <https://github.com/pquerna/ffjson>
// This code was generated via the following steps:
// $ go get -u github.com/pquerna/ffjson
// $ make BIND_DIR=. shell
// $ ffjson pkg/jsonlog/jsonlog.go
// $ mv pkg/jsonglog/jsonlog_ffjson.go pkg/jsonlog/jsonlog_marshalling.go
//
// It has been modified to improve the performance of time marshalling to JSON
// and to clean it up.
// Should this code need to be regenerated when the JSONLog struct is changed,
// the relevant changes which have been made are:
// import (
// "bytes"
//-
// "unicode/utf8"
// )
//
// func (mj *JSONLog) MarshalJSON() ([]byte, error) {
//@@ -20,13 +16,13 @@ func (mj *JSONLog) MarshalJSON() ([]byte, error) {
// }
// return buf.Bytes(), nil
// }
//+
// func (mj *JSONLog) MarshalJSONBuf(buf *bytes.Buffer) error {
//- var err error
//- var obj []byte
//- var first bool = true
//- _ = obj
//- _ = err
//- _ = first
//+ var (
//+ err error
//+ timestamp string
//+ first bool = true
//+ )
// buf.WriteString(`{`)
// if len(mj.Log) != 0 {
// if first == true {
//@@ -52,11 +48,11 @@ func (mj *JSONLog) MarshalJSONBuf(buf *bytes.Buffer) error {
// buf.WriteString(`,`)
// }
// buf.WriteString(`"time":`)
//- obj, err = mj.Created.MarshalJSON()
//+ timestamp, err = FastTimeMarshalJSON(mj.Created)
// if err != nil {
// return err
// }
//- buf.Write(obj)
//+ buf.WriteString(timestamp)
// buf.WriteString(`}`)
// return nil
// }
// @@ -81,9 +81,10 @@ func (mj *JSONLog) MarshalJSONBuf(buf *bytes.Buffer) error {
// if len(mj.Log) != 0 {
// - if first == true {
// - first = false
// - } else {
// - buf.WriteString(`,`)
// - }
// + first = false
// buf.WriteString(`"log":`)
// ffjsonWriteJSONString(buf, mj.Log)
// }
package jsonlog
import (
"bytes"
"unicode/utf8"
)
// MarshalJSON marshals the JSONLog.
func (mj *JSONLog) MarshalJSON() ([]byte, error) {
var buf bytes.Buffer
buf.Grow(1024)
if err := mj.MarshalJSONBuf(&buf); err != nil {
return nil, err
}
return buf.Bytes(), nil
}
// MarshalJSONBuf marshals the JSONLog and stores the result to a bytes.Buffer.
func (mj *JSONLog) MarshalJSONBuf(buf *bytes.Buffer) error {
var (
err error
timestamp string
first = true
)
buf.WriteString(`{`)
if len(mj.Log) != 0 {
first = false
buf.WriteString(`"log":`)
ffjsonWriteJSONString(buf, mj.Log)
}
if len(mj.Stream) != 0 {
if first {
first = false
} else {
buf.WriteString(`,`)
}
buf.WriteString(`"stream":`)
ffjsonWriteJSONString(buf, mj.Stream)
}
if !first {
buf.WriteString(`,`)
}
buf.WriteString(`"time":`)
timestamp, err = FastTimeMarshalJSON(mj.Created)
if err != nil {
return err
}
buf.WriteString(timestamp)
buf.WriteString(`}`)
return nil
}
func ffjsonWriteJSONString(buf *bytes.Buffer, s string) {
const hex = "0123456789abcdef"
buf.WriteByte('"')
start := 0
for i := 0; i < len(s); {
if b := s[i]; b < utf8.RuneSelf {
if 0x20 <= b && b != '\\' && b != '"' && b != '<' && b != '>' && b != '&' {
i++
continue
}
if start < i {
buf.WriteString(s[start:i])
}
switch b {
case '\\', '"':
buf.WriteByte('\\')
buf.WriteByte(b)
case '\n':
buf.WriteByte('\\')
buf.WriteByte('n')
case '\r':
buf.WriteByte('\\')
buf.WriteByte('r')
default:
buf.WriteString(`\u00`)
buf.WriteByte(hex[b>>4])
buf.WriteByte(hex[b&0xF])
}
i++
start = i
continue
}
c, size := utf8.DecodeRuneInString(s[i:])
if c == utf8.RuneError && size == 1 {
if start < i {
buf.WriteString(s[start:i])
}
buf.WriteString(`\ufffd`)
i += size
start = i
continue
}
if c == '\u2028' || c == '\u2029' {
if start < i {
buf.WriteString(s[start:i])
}
buf.WriteString(`\u202`)
buf.WriteByte(hex[c&0xF])
i += size
start = i
continue
}
i += size
}
if start < len(s) {
buf.WriteString(s[start:])
}
buf.WriteByte('"')
}

34
jsonlog/jsonlog_marshalling_test.go
View file

@ -1,34 +0,0 @@
package jsonlog
import (
"regexp"
"testing"
)
func TestJSONLogMarshalJSON(t *testing.T) {
logs := map[*JSONLog]string{
&JSONLog{Log: `"A log line with \\"`}: `^{\"log\":\"\\\"A log line with \\\\\\\\\\\"\",\"time\":\".{20,}\"}$`,
&JSONLog{Log: "A log line"}: `^{\"log\":\"A log line\",\"time\":\".{20,}\"}$`,
&JSONLog{Log: "A log line with \r"}: `^{\"log\":\"A log line with \\r\",\"time\":\".{20,}\"}$`,
&JSONLog{Log: "A log line with & < >"}: `^{\"log\":\"A log line with \\u0026 \\u003c \\u003e\",\"time\":\".{20,}\"}$`,
&JSONLog{Log: "A log line with utf8 : 🚀 ψ ω β"}: `^{\"log\":\"A log line with utf8 : 🚀 ψ ω β\",\"time\":\".{20,}\"}$`,
&JSONLog{Stream: "stdout"}: `^{\"stream\":\"stdout\",\"time\":\".{20,}\"}$`,
&JSONLog{}: `^{\"time\":\".{20,}\"}$`,
// These ones are a little weird
&JSONLog{Log: "\u2028 \u2029"}: `^{\"log\":\"\\u2028 \\u2029\",\"time\":\".{20,}\"}$`,
&JSONLog{Log: string([]byte{0xaF})}: `^{\"log\":\"\\ufffd\",\"time\":\".{20,}\"}$`,
&JSONLog{Log: string([]byte{0x7F})}: `^{\"log\":\"\x7f\",\"time\":\".{20,}\"}$`,
}
for jsonLog, expression := range logs {
data, err := jsonLog.MarshalJSON()
if err != nil {
t.Fatal(err)
}
res := string(data)
t.Logf("Result of WriteLog: %q", res)
logRe := regexp.MustCompile(expression)
if !logRe.MatchString(res) {
t.Fatalf("Log line not in expected format [%v]: %q", expression, res)
}
}
}

122
jsonlog/jsonlogbytes.go
View file

@ -1,122 +0,0 @@
package jsonlog
import (
"bytes"
"encoding/json"
"unicode/utf8"
)
// JSONLogs is based on JSONLog.
// It allows marshalling JSONLog from Log as []byte
// and an already marshalled Created timestamp.
type JSONLogs struct {
Log []byte `json:"log,omitempty"`
Stream string `json:"stream,omitempty"`
Created string `json:"time"`
// json-encoded bytes
RawAttrs json.RawMessage `json:"attrs,omitempty"`
}
// MarshalJSONBuf is based on the same method from JSONLog
// It has been modified to take into account the necessary changes.
func (mj *JSONLogs) MarshalJSONBuf(buf *bytes.Buffer) error {
var first = true
buf.WriteString(`{`)
if len(mj.Log) != 0 {
first = false
buf.WriteString(`"log":`)
ffjsonWriteJSONBytesAsString(buf, mj.Log)
}
if len(mj.Stream) != 0 {
if first == true {
first = false
} else {
buf.WriteString(`,`)
}
buf.WriteString(`"stream":`)
ffjsonWriteJSONString(buf, mj.Stream)
}
if len(mj.RawAttrs) > 0 {
if first {
first = false
} else {
buf.WriteString(`,`)
}
buf.WriteString(`"attrs":`)
buf.Write(mj.RawAttrs)
}
if !first {
buf.WriteString(`,`)
}
buf.WriteString(`"time":`)
buf.WriteString(mj.Created)
buf.WriteString(`}`)
return nil
}
// This is based on ffjsonWriteJSONBytesAsString. It has been changed
// to accept a string passed as a slice of bytes.
func ffjsonWriteJSONBytesAsString(buf *bytes.Buffer, s []byte) {
const hex = "0123456789abcdef"
buf.WriteByte('"')
start := 0
for i := 0; i < len(s); {
if b := s[i]; b < utf8.RuneSelf {
if 0x20 <= b && b != '\\' && b != '"' && b != '<' && b != '>' && b != '&' {
i++
continue
}
if start < i {
buf.Write(s[start:i])
}
switch b {
case '\\', '"':
buf.WriteByte('\\')
buf.WriteByte(b)
case '\n':
buf.WriteByte('\\')
buf.WriteByte('n')
case '\r':
buf.WriteByte('\\')
buf.WriteByte('r')
default:
buf.WriteString(`\u00`)
buf.WriteByte(hex[b>>4])
buf.WriteByte(hex[b&0xF])
}
i++
start = i
continue
}
c, size := utf8.DecodeRune(s[i:])
if c == utf8.RuneError && size == 1 {
if start < i {
buf.Write(s[start:i])
}
buf.WriteString(`\ufffd`)
i += size
start = i
continue
}
if c == '\u2028' || c == '\u2029' {
if start < i {
buf.Write(s[start:i])
}
buf.WriteString(`\u202`)
buf.WriteByte(hex[c&0xF])
i += size
start = i
continue
}
i += size
}
if start < len(s) {
buf.Write(s[start:])
}
buf.WriteByte('"')
}

39
jsonlog/jsonlogbytes_test.go
View file

@ -1,39 +0,0 @@
package jsonlog
import (
"bytes"
"regexp"
"testing"
)
func TestJSONLogsMarshalJSONBuf(t *testing.T) {
logs := map[*JSONLogs]string{
&JSONLogs{Log: []byte(`"A log line with \\"`)}: `^{\"log\":\"\\\"A log line with \\\\\\\\\\\"\",\"time\":}$`,
&JSONLogs{Log: []byte("A log line")}: `^{\"log\":\"A log line\",\"time\":}$`,
&JSONLogs{Log: []byte("A log line with \r")}: `^{\"log\":\"A log line with \\r\",\"time\":}$`,
&JSONLogs{Log: []byte("A log line with & < >")}: `^{\"log\":\"A log line with \\u0026 \\u003c \\u003e\",\"time\":}$`,
&JSONLogs{Log: []byte("A log line with utf8 : 🚀 ψ ω β")}: `^{\"log\":\"A log line with utf8 : 🚀 ψ ω β\",\"time\":}$`,
&JSONLogs{Stream: "stdout"}: `^{\"stream\":\"stdout\",\"time\":}$`,
&JSONLogs{Stream: "stdout", Log: []byte("A log line")}: `^{\"log\":\"A log line\",\"stream\":\"stdout\",\"time\":}$`,
&JSONLogs{Created: "time"}: `^{\"time\":time}$`,
&JSONLogs{}: `^{\"time\":}$`,
// These ones are a little weird
&JSONLogs{Log: []byte("\u2028 \u2029")}: `^{\"log\":\"\\u2028 \\u2029\",\"time\":}$`,
&JSONLogs{Log: []byte{0xaF}}: `^{\"log\":\"\\ufffd\",\"time\":}$`,
&JSONLogs{Log: []byte{0x7F}}: `^{\"log\":\"\x7f\",\"time\":}$`,
// with raw attributes
&JSONLogs{Log: []byte("A log line"), RawAttrs: []byte(`{"hello":"world","value":1234}`)}: `^{\"log\":\"A log line\",\"attrs\":{\"hello\":\"world\",\"value\":1234},\"time\":}$`,
}
for jsonLog, expression := range logs {
var buf bytes.Buffer
if err := jsonLog.MarshalJSONBuf(&buf); err != nil {
t.Fatal(err)
}
res := buf.String()
t.Logf("Result of WriteLog: %q", res)
logRe := regexp.MustCompile(expression)
if !logRe.MatchString(res) {
t.Fatalf("Log line not in expected format [%v]: %q", expression, res)
}
}
}

27
jsonlog/time_marshalling.go
View file

@ -1,27 +0,0 @@
// Package jsonlog provides helper functions to parse and print time (time.Time) as JSON.
package jsonlog
import (
"errors"
"time"
)
const (
// RFC3339NanoFixed is our own version of RFC339Nano because we want one
// that pads the nano seconds part with zeros to ensure
// the timestamps are aligned in the logs.
RFC3339NanoFixed = "2006-01-02T15:04:05.000000000Z07:00"
// JSONFormat is the format used by FastMarshalJSON
JSONFormat = `"` + time.RFC3339Nano + `"`
)
// FastTimeMarshalJSON avoids one of the extra allocations that
// time.MarshalJSON is making.
func FastTimeMarshalJSON(t time.Time) (string, error) {
if y := t.Year(); y < 0 || y >= 10000 {
// RFC 3339 is clear that years are 4 digits exactly.
// See golang.org/issue/4556#c15 for more discussion.
return "", errors.New("time.MarshalJSON: year outside of range [0,9999]")
}
return t.Format(JSONFormat), nil
}

47
jsonlog/time_marshalling_test.go
View file

@ -1,47 +0,0 @@
package jsonlog
import (
"testing"
"time"
)
// Testing to ensure 'year' fields is between 0 and 9999
func TestFastTimeMarshalJSONWithInvalidDate(t *testing.T) {
aTime := time.Date(-1, 1, 1, 0, 0, 0, 0, time.Local)
json, err := FastTimeMarshalJSON(aTime)
if err == nil {
t.Fatalf("FastTimeMarshalJSON should throw an error, but was '%v'", json)
}
anotherTime := time.Date(10000, 1, 1, 0, 0, 0, 0, time.Local)
json, err = FastTimeMarshalJSON(anotherTime)
if err == nil {
t.Fatalf("FastTimeMarshalJSON should throw an error, but was '%v'", json)
}
}
func TestFastTimeMarshalJSON(t *testing.T) {
aTime := time.Date(2015, 5, 29, 11, 1, 2, 3, time.UTC)
json, err := FastTimeMarshalJSON(aTime)
if err != nil {
t.Fatal(err)
}
expected := "\"2015-05-29T11:01:02.000000003Z\""
if json != expected {
t.Fatalf("Expected %v, got %v", expected, json)
}
location, err := time.LoadLocation("Europe/Paris")
if err != nil {
t.Fatal(err)
}
aTime = time.Date(2015, 5, 29, 11, 1, 2, 3, location)
json, err = FastTimeMarshalJSON(aTime)
if err != nil {
t.Fatal(err)
}
expected = "\"2015-05-29T11:01:02.000000003+02:00\""
if json != expected {
t.Fatalf("Expected %v, got %v", expected, json)
}
}

31
listeners/listeners_solaris.go
View file

@ -1,31 +0,0 @@
package listeners
import (
"crypto/tls"
"fmt"
"net"
"github.com/docker/go-connections/sockets"
)
// Init creates new listeners for the server.
func Init(proto, addr, socketGroup string, tlsConfig *tls.Config) (ls []net.Listener, err error) {
switch proto {
case "tcp":
l, err := sockets.NewTCPSocket(addr, tlsConfig)
if err != nil {
return nil, err
}
ls = append(ls, l)
case "unix":
l, err := sockets.NewUnixSocket(addr, socketGroup)
if err != nil {
return nil, fmt.Errorf("can't create unix socket %s: %v", addr, err)
}
ls = append(ls, l)
default:
return nil, fmt.Errorf("Invalid protocol format: %q", proto)
}
return
}

94
listeners/listeners_unix.go
View file

@ -1,94 +0,0 @@
// +build !windows,!solaris
package listeners
import (
"crypto/tls"
"fmt"
"net"
"strconv"
"github.com/Sirupsen/logrus"
"github.com/coreos/go-systemd/activation"
"github.com/docker/go-connections/sockets"
)
// Init creates new listeners for the server.
// TODO: Clean up the fact that socketGroup and tlsConfig aren't always used.
func Init(proto, addr, socketGroup string, tlsConfig *tls.Config) ([]net.Listener, error) {
ls := []net.Listener{}
switch proto {
case "fd":
fds, err := listenFD(addr, tlsConfig)
if err != nil {
return nil, err
}
ls = append(ls, fds...)
case "tcp":
l, err := sockets.NewTCPSocket(addr, tlsConfig)
if err != nil {
return nil, err
}
ls = append(ls, l)
case "unix":
l, err := sockets.NewUnixSocket(addr, socketGroup)
if err != nil {
return nil, fmt.Errorf("can't create unix socket %s: %v", addr, err)
}
ls = append(ls, l)
default:
return nil, fmt.Errorf("invalid protocol format: %q", proto)
}
return ls, nil
}
// listenFD returns the specified socket activated files as a slice of
// net.Listeners or all of the activated files if "*" is given.
func listenFD(addr string, tlsConfig *tls.Config) ([]net.Listener, error) {
var (
err error
listeners []net.Listener
)
// socket activation
if tlsConfig != nil {
listeners, err = activation.TLSListeners(false, tlsConfig)
} else {
listeners, err = activation.Listeners(false)
}
if err != nil {
return nil, err
}
if len(listeners) == 0 {
return nil, fmt.Errorf("no sockets found via socket activation: make sure the service was started by systemd")
}
// default to all fds just like unix:// and tcp://
if addr == "" || addr == "*" {
return listeners, nil
}
fdNum, err := strconv.Atoi(addr)
if err != nil {
return nil, fmt.Errorf("failed to parse systemd fd address: should be a number: %v", addr)
}
fdOffset := fdNum - 3
if len(listeners) < int(fdOffset)+1 {
return nil, fmt.Errorf("too few socket activated files passed in by systemd")
}
if listeners[fdOffset] == nil {
return nil, fmt.Errorf("failed to listen on systemd activated file: fd %d", fdOffset+3)
}
for i, ls := range listeners {
if i == fdOffset || ls == nil {
continue
}
if err := ls.Close(); err != nil {
// TODO: We shouldn't log inside a library. Remove this or error out.
logrus.Errorf("failed to close systemd activated file: fd %d: %v", fdOffset+3, err)
}
}
return []net.Listener{listeners[fdOffset]}, nil
}

54
listeners/listeners_windows.go
View file

@ -1,54 +0,0 @@
package listeners
import (
"crypto/tls"
"fmt"
"net"
"strings"
"github.com/Microsoft/go-winio"
"github.com/docker/go-connections/sockets"
)
// Init creates new listeners for the server.
func Init(proto, addr, socketGroup string, tlsConfig *tls.Config) ([]net.Listener, error) {
ls := []net.Listener{}
switch proto {
case "tcp":
l, err := sockets.NewTCPSocket(addr, tlsConfig)
if err != nil {
return nil, err
}
ls = append(ls, l)
case "npipe":
// allow Administrators and SYSTEM, plus whatever additional users or groups were specified
sddl := "D:P(A;;GA;;;BA)(A;;GA;;;SY)"
if socketGroup != "" {
for _, g := range strings.Split(socketGroup, ",") {
sid, err := winio.LookupSidByName(g)
if err != nil {
return nil, err
}
sddl += fmt.Sprintf("(A;;GRGW;;;%s)", sid)
}
}
c := winio.PipeConfig{
SecurityDescriptor: sddl,
MessageMode: true, // Use message mode so that CloseWrite() is supported
InputBufferSize: 65536, // Use 64KB buffers to improve performance
OutputBufferSize: 65536,
}
l, err := winio.ListenPipe(addr, &c)
if err != nil {
return nil, err
}
ls = append(ls, l)
default:
return nil, fmt.Errorf("invalid protocol format: windows only supports tcp and npipe")
}
return ls, nil
}

65
locker/README.md
View file

@ -1,65 +0,0 @@
Locker
=====
locker provides a mechanism for creating finer-grained locking to help
free up more global locks to handle other tasks.
The implementation looks close to a sync.Mutex, however the user must provide a
reference to use to refer to the underlying lock when locking and unlocking,
and unlock may generate an error.
If a lock with a given name does not exist when `Lock` is called, one is
created.
Lock references are automatically cleaned up on `Unlock` if nothing else is
waiting for the lock.
## Usage
```go
package important
import (
"sync"
"time"
"github.com/docker/docker/pkg/locker"
)
type important struct {
locks *locker.Locker
data map[string]interface{}
mu sync.Mutex
}
func (i *important) Get(name string) interface{} {
i.locks.Lock(name)
defer i.locks.Unlock(name)
return data[name]
}
func (i *important) Create(name string, data interface{}) {
i.locks.Lock(name)
defer i.locks.Unlock(name)
i.createImportant(data)
s.mu.Lock()
i.data[name] = data
s.mu.Unlock()
}
func (i *important) createImportant(data interface{}) {
time.Sleep(10 * time.Second)
}
```
For functions dealing with a given name, always lock at the beginning of the
function (or before doing anything with the underlying state), this ensures any
other function that is dealing with the same name will block.
When needing to modify the underlying data, use the global lock to ensure nothing
else is modfying it at the same time.
Since name lock is already in place, no reads will occur while the modification
is being performed.

112
locker/locker.go
View file

@ -1,112 +0,0 @@
/*
Package locker provides a mechanism for creating finer-grained locking to help
free up more global locks to handle other tasks.
The implementation looks close to a sync.Mutex, however the user must provide a
reference to use to refer to the underlying lock when locking and unlocking,
and unlock may generate an error.
If a lock with a given name does not exist when `Lock` is called, one is
created.
Lock references are automatically cleaned up on `Unlock` if nothing else is
waiting for the lock.
*/
package locker
import (
"errors"
"sync"
"sync/atomic"
)
// ErrNoSuchLock is returned when the requested lock does not exist
var ErrNoSuchLock = errors.New("no such lock")
// Locker provides a locking mechanism based on the passed in reference name
type Locker struct {
mu sync.Mutex
locks map[string]*lockCtr
}
// lockCtr is used by Locker to represent a lock with a given name.
type lockCtr struct {
mu sync.Mutex
// waiters is the number of waiters waiting to acquire the lock
// this is int32 instead of uint32 so we can add `-1` in `dec()`
waiters int32
}
// inc increments the number of waiters waiting for the lock
func (l *lockCtr) inc() {
atomic.AddInt32(&l.waiters, 1)
}
// dec decrements the number of waiters waiting on the lock
func (l *lockCtr) dec() {
atomic.AddInt32(&l.waiters, -1)
}
// count gets the current number of waiters
func (l *lockCtr) count() int32 {
return atomic.LoadInt32(&l.waiters)
}
// Lock locks the mutex
func (l *lockCtr) Lock() {
l.mu.Lock()
}
// Unlock unlocks the mutex
func (l *lockCtr) Unlock() {
l.mu.Unlock()
}
// New creates a new Locker
func New() *Locker {
return &Locker{
locks: make(map[string]*lockCtr),
}
}
// Lock locks a mutex with the given name. If it doesn't exist, one is created
func (l *Locker) Lock(name string) {
l.mu.Lock()
if l.locks == nil {
l.locks = make(map[string]*lockCtr)
}
nameLock, exists := l.locks[name]
if !exists {
nameLock = &lockCtr{}
l.locks[name] = nameLock
}
// increment the nameLock waiters while inside the main mutex
// this makes sure that the lock isn't deleted if `Lock` and `Unlock` are called concurrently
nameLock.inc()
l.mu.Unlock()
// Lock the nameLock outside the main mutex so we don't block other operations
// once locked then we can decrement the number of waiters for this lock
nameLock.Lock()
nameLock.dec()
}
// Unlock unlocks the mutex with the given name
// If the given lock is not being waited on by any other callers, it is deleted
func (l *Locker) Unlock(name string) error {
l.mu.Lock()
nameLock, exists := l.locks[name]
if !exists {
l.mu.Unlock()
return ErrNoSuchLock
}
if nameLock.count() == 0 {
delete(l.locks, name)
}
nameLock.Unlock()
l.mu.Unlock()
return nil
}

124
locker/locker_test.go
View file

@ -1,124 +0,0 @@
package locker
import (
"sync"
"testing"
"time"
)
func TestLockCounter(t *testing.T) {
l := &lockCtr{}
l.inc()
if l.waiters != 1 {
t.Fatal("counter inc failed")
}
l.dec()
if l.waiters != 0 {
t.Fatal("counter dec failed")
}
}
func TestLockerLock(t *testing.T) {
l := New()
l.Lock("test")
ctr := l.locks["test"]
if ctr.count() != 0 {
t.Fatalf("expected waiters to be 0, got :%d", ctr.waiters)
}
chDone := make(chan struct{})
go func() {
l.Lock("test")
close(chDone)
}()
chWaiting := make(chan struct{})
go func() {
for range time.Tick(1 * time.Millisecond) {
if ctr.count() == 1 {
close(chWaiting)
break
}
}
}()
select {
case <-chWaiting:
case <-time.After(3 * time.Second):
t.Fatal("timed out waiting for lock waiters to be incremented")
}
select {
case <-chDone:
t.Fatal("lock should not have returned while it was still held")
default:
}
if err := l.Unlock("test"); err != nil {
t.Fatal(err)
}
select {
case <-chDone:
case <-time.After(3 * time.Second):
t.Fatalf("lock should have completed")
}
if ctr.count() != 0 {
t.Fatalf("expected waiters to be 0, got: %d", ctr.count())
}
}
func TestLockerUnlock(t *testing.T) {
l := New()
l.Lock("test")
l.Unlock("test")
chDone := make(chan struct{})
go func() {
l.Lock("test")
close(chDone)
}()
select {
case <-chDone:
case <-time.After(3 * time.Second):
t.Fatalf("lock should not be blocked")
}
}
func TestLockerConcurrency(t *testing.T) {
l := New()
var wg sync.WaitGroup
for i := 0; i <= 10000; i++ {
wg.Add(1)
go func() {
l.Lock("test")
// if there is a concurrency issue, will very likely panic here
l.Unlock("test")
wg.Done()
}()
}
chDone := make(chan struct{})
go func() {
wg.Wait()
close(chDone)
}()
select {
case <-chDone:
case <-time.After(10 * time.Second):
t.Fatal("timeout waiting for locks to complete")
}
// Since everything has unlocked this should not exist anymore
if ctr, exists := l.locks["test"]; exists {
t.Fatalf("lock should not exist: %v", ctr)
}
}

11
namesgenerator/cmd/names-generator/main.go
View file

@ -1,11 +0,0 @@
package main
import (
"fmt"
"github.com/docker/docker/pkg/namesgenerator"
)
func main() {
fmt.Println(namesgenerator.GetRandomName(0))
}

552
namesgenerator/names-generator.go
View file

@ -1,552 +0,0 @@
package namesgenerator
import (
"fmt"
"github.com/docker/docker/pkg/random"
)
var (
left = [...]string{
"admiring",
"adoring",
"agitated",
"amazing",
"angry",
"awesome",
"backstabbing",
"berserk",
"big",
"boring",
"clever",
"cocky",
"compassionate",
"condescending",
"cranky",
"desperate",
"determined",
"distracted",
"dreamy",
"drunk",
"ecstatic",
"elated",
"elegant",
"evil",
"fervent",
"focused",
"furious",
"gigantic",
"gloomy",
"goofy",
"grave",
"happy",
"high",
"hopeful",
"hungry",
"infallible",
"jolly",
"jovial",
"kickass",
"lonely",
"loving",
"mad",
"modest",
"naughty",
"nauseous",
"nostalgic",
"peaceful",
"pedantic",
"pensive",
"prickly",
"reverent",
"romantic",
"sad",
"serene",
"sharp",
"sick",
"silly",
"sleepy",
"small",
"stoic",
"stupefied",
"suspicious",
"tender",
"thirsty",
"tiny",
"trusting",
"zen",
}
// Docker, starting from 0.7.x, generates names from notable scientists and hackers.
// Please, for any amazing man that you add to the list, consider adding an equally amazing woman to it, and vice versa.
right = [...]string{
// Muhammad ibn Jābir al-Ḥarrānī al-Battānī was a founding father of astronomy. https://en.wikipedia.org/wiki/Mu%E1%B8%A5ammad_ibn_J%C4%81bir_al-%E1%B8%A4arr%C4%81n%C4%AB_al-Batt%C4%81n%C4%AB
"albattani",
// Frances E. Allen, became the first female IBM Fellow in 1989. In 2006, she became the first female recipient of the ACM's Turing Award. https://en.wikipedia.org/wiki/Frances_E._Allen
"allen",
// June Almeida - Scottish virologist who took the first pictures of the rubella virus - https://en.wikipedia.org/wiki/June_Almeida
"almeida",
// Maria Gaetana Agnesi - Italian mathematician, philosopher, theologian and humanitarian. She was the first woman to write a mathematics handbook and the first woman appointed as a Mathematics Professor at a University. https://en.wikipedia.org/wiki/Maria_Gaetana_Agnesi
"agnesi",
// Archimedes was a physicist, engineer and mathematician who invented too many things to list them here. https://en.wikipedia.org/wiki/Archimedes
"archimedes",
// Maria Ardinghelli - Italian translator, mathematician and physicist - https://en.wikipedia.org/wiki/Maria_Ardinghelli
"ardinghelli",
// Aryabhata - Ancient Indian mathematician-astronomer during 476-550 CE https://en.wikipedia.org/wiki/Aryabhata
"aryabhata",
// Wanda Austin - Wanda Austin is the President and CEO of The Aerospace Corporation, a leading architect for the US security space programs. https://en.wikipedia.org/wiki/Wanda_Austin
"austin",
// Charles Babbage invented the concept of a programmable computer. https://en.wikipedia.org/wiki/Charles_Babbage.
"babbage",
// Stefan Banach - Polish mathematician, was one of the founders of modern functional analysis. https://en.wikipedia.org/wiki/Stefan_Banach
"banach",
// John Bardeen co-invented the transistor - https://en.wikipedia.org/wiki/John_Bardeen
"bardeen",
// Jean Bartik, born Betty Jean Jennings, was one of the original programmers for the ENIAC computer. https://en.wikipedia.org/wiki/Jean_Bartik
"bartik",
// Laura Bassi, the world's first female professor https://en.wikipedia.org/wiki/Laura_Bassi
"bassi",
// Alexander Graham Bell - an eminent Scottish-born scientist, inventor, engineer and innovator who is credited with inventing the first practical telephone - https://en.wikipedia.org/wiki/Alexander_Graham_Bell
"bell",
// Homi J Bhabha - was an Indian nuclear physicist, founding director, and professor of physics at the Tata Institute of Fundamental Research. Colloquially known as "father of Indian nuclear programme"- https://en.wikipedia.org/wiki/Homi_J._Bhabha
"bhabha",
// Bhaskara II - Ancient Indian mathematician-astronomer whose work on calculus predates Newton and Leibniz by over half a millennium - https://en.wikipedia.org/wiki/Bh%C4%81skara_II#Calculus
"bhaskara",
// Elizabeth Blackwell - American doctor and first American woman to receive a medical degree - https://en.wikipedia.org/wiki/Elizabeth_Blackwell
"blackwell",
// Niels Bohr is the father of quantum theory. https://en.wikipedia.org/wiki/Niels_Bohr.
"bohr",
// Kathleen Booth, she's credited with writing the first assembly language. https://en.wikipedia.org/wiki/Kathleen_Booth
"booth",
// Anita Borg - Anita Borg was the founding director of the Institute for Women and Technology (IWT). https://en.wikipedia.org/wiki/Anita_Borg
"borg",
// Satyendra Nath Bose - He provided the foundation for Bose–Einstein statistics and the theory of the Bose–Einstein condensate. - https://en.wikipedia.org/wiki/Satyendra_Nath_Bose
"bose",
// Evelyn Boyd Granville - She was one of the first African-American woman to receive a Ph.D. in mathematics; she earned it in 1949 from Yale University. https://en.wikipedia.org/wiki/Evelyn_Boyd_Granville
"boyd",
// Brahmagupta - Ancient Indian mathematician during 598-670 CE who gave rules to compute with zero - https://en.wikipedia.org/wiki/Brahmagupta#Zero
"brahmagupta",
// Walter Houser Brattain co-invented the transistor - https://en.wikipedia.org/wiki/Walter_Houser_Brattain
"brattain",
// Emmett Brown invented time travel. https://en.wikipedia.org/wiki/Emmett_Brown (thanks Brian Goff)
"brown",
// Rachel Carson - American marine biologist and conservationist, her book Silent Spring and other writings are credited with advancing the global environmental movement. https://en.wikipedia.org/wiki/Rachel_Carson
"carson",
// Subrahmanyan Chandrasekhar - Astrophysicist known for his mathematical theory on different stages and evolution in structures of the stars. He has won nobel prize for physics - https://en.wikipedia.org/wiki/Subrahmanyan_Chandrasekhar
"chandrasekhar",
//Claude Shannon - The father of information theory and founder of digital circuit design theory. (https://en.wikipedia.org/wiki/Claude_Shannon)
"shannon",
// Jane Colden - American botanist widely considered the first female American botanist - https://en.wikipedia.org/wiki/Jane_Colden
"colden",
// Gerty Theresa Cori - American biochemist who became the third woman—and first American woman—to win a Nobel Prize in science, and the first woman to be awarded the Nobel Prize in Physiology or Medicine. Cori was born in Prague. https://en.wikipedia.org/wiki/Gerty_Cori
"cori",
// Seymour Roger Cray was an American electrical engineer and supercomputer architect who designed a series of computers that were the fastest in the world for decades. https://en.wikipedia.org/wiki/Seymour_Cray
"cray",
// This entry reflects a husband and wife team who worked together:
// Joan Curran was a Welsh scientist who developed radar and invented chaff, a radar countermeasure. https://en.wikipedia.org/wiki/Joan_Curran
// Samuel Curran was an Irish physicist who worked alongside his wife during WWII and invented the proximity fuse. https://en.wikipedia.org/wiki/Samuel_Curran
"curran",
// Marie Curie discovered radioactivity. https://en.wikipedia.org/wiki/Marie_Curie.
"curie",
// Charles Darwin established the principles of natural evolution. https://en.wikipedia.org/wiki/Charles_Darwin.
"darwin",
// Leonardo Da Vinci invented too many things to list here. https://en.wikipedia.org/wiki/Leonardo_da_Vinci.
"davinci",
// Edsger Wybe Dijkstra was a Dutch computer scientist and mathematical scientist. https://en.wikipedia.org/wiki/Edsger_W._Dijkstra.
"dijkstra",
// Donna Dubinsky - played an integral role in the development of personal digital assistants (PDAs) serving as CEO of Palm, Inc. and co-founding Handspring. https://en.wikipedia.org/wiki/Donna_Dubinsky
"dubinsky",
// Annie Easley - She was a leading member of the team which developed software for the Centaur rocket stage and one of the first African-Americans in her field. https://en.wikipedia.org/wiki/Annie_Easley
"easley",
// Thomas Alva Edison, prolific inventor https://en.wikipedia.org/wiki/Thomas_Edison
"edison",
// Albert Einstein invented the general theory of relativity. https://en.wikipedia.org/wiki/Albert_Einstein
"einstein",
// Gertrude Elion - American biochemist, pharmacologist and the 1988 recipient of the Nobel Prize in Medicine - https://en.wikipedia.org/wiki/Gertrude_Elion
"elion",
// Douglas Engelbart gave the mother of all demos: https://en.wikipedia.org/wiki/Douglas_Engelbart
"engelbart",
// Euclid invented geometry. https://en.wikipedia.org/wiki/Euclid
"euclid",
// Leonhard Euler invented large parts of modern mathematics. https://de.wikipedia.org/wiki/Leonhard_Euler
"euler",
// Pierre de Fermat pioneered several aspects of modern mathematics. https://en.wikipedia.org/wiki/Pierre_de_Fermat
"fermat",
// Enrico Fermi invented the first nuclear reactor. https://en.wikipedia.org/wiki/Enrico_Fermi.
"fermi",
// Richard Feynman was a key contributor to quantum mechanics and particle physics. https://en.wikipedia.org/wiki/Richard_Feynman
"feynman",
// Benjamin Franklin is famous for his experiments in electricity and the invention of the lightning rod.
"franklin",
// Galileo was a founding father of modern astronomy, and faced politics and obscurantism to establish scientific truth. https://en.wikipedia.org/wiki/Galileo_Galilei
"galileo",
// William Henry "Bill" Gates III is an American business magnate, philanthropist, investor, computer programmer, and inventor. https://en.wikipedia.org/wiki/Bill_Gates
"gates",
// Adele Goldberg, was one of the designers and developers of the Smalltalk language. https://en.wikipedia.org/wiki/Adele_Goldberg_(computer_scientist)
"goldberg",
// Adele Goldstine, born Adele Katz, wrote the complete technical description for the first electronic digital computer, ENIAC. https://en.wikipedia.org/wiki/Adele_Goldstine
"goldstine",
// Shafi Goldwasser is a computer scientist known for creating theoretical foundations of modern cryptography. Winner of 2012 ACM Turing Award. https://en.wikipedia.org/wiki/Shafi_Goldwasser
"goldwasser",
// James Golick, all around gangster.
"golick",
// Jane Goodall - British primatologist, ethologist, and anthropologist who is considered to be the world's foremost expert on chimpanzees - https://en.wikipedia.org/wiki/Jane_Goodall
"goodall",
// Margaret Hamilton - Director of the Software Engineering Division of the MIT Instrumentation Laboratory, which developed on-board flight software for the Apollo space program. https://en.wikipedia.org/wiki/Margaret_Hamilton_(scientist)
"hamilton",
// Stephen Hawking pioneered the field of cosmology by combining general relativity and quantum mechanics. https://en.wikipedia.org/wiki/Stephen_Hawking
"hawking",
// Werner Heisenberg was a founding father of quantum mechanics. https://en.wikipedia.org/wiki/Werner_Heisenberg
"heisenberg",
// Jaroslav Heyrovský was the inventor of the polarographic method, father of the electroanalytical method, and recipient of the Nobel Prize in 1959. His main field of work was polarography. https://en.wikipedia.org/wiki/Jaroslav_Heyrovsk%C3%BD
"heyrovsky",
// Dorothy Hodgkin was a British biochemist, credited with the development of protein crystallography. She was awarded the Nobel Prize in Chemistry in 1964. https://en.wikipedia.org/wiki/Dorothy_Hodgkin
"hodgkin",
// Erna Schneider Hoover revolutionized modern communication by inventing a computerized telephone switching method. https://en.wikipedia.org/wiki/Erna_Schneider_Hoover
"hoover",
// Grace Hopper developed the first compiler for a computer programming language and is credited with popularizing the term "debugging" for fixing computer glitches. https://en.wikipedia.org/wiki/Grace_Hopper
"hopper",
// Frances Hugle, she was an American scientist, engineer, and inventor who contributed to the understanding of semiconductors, integrated circuitry, and the unique electrical principles of microscopic materials. https://en.wikipedia.org/wiki/Frances_Hugle
"hugle",
// Hypatia - Greek Alexandrine Neoplatonist philosopher in Egypt who was one of the earliest mothers of mathematics - https://en.wikipedia.org/wiki/Hypatia
"hypatia",
// Yeong-Sil Jang was a Korean scientist and astronomer during the Joseon Dynasty; he invented the first metal printing press and water gauge. https://en.wikipedia.org/wiki/Jang_Yeong-sil
"jang",
// Betty Jennings - one of the original programmers of the ENIAC. https://en.wikipedia.org/wiki/ENIAC - https://en.wikipedia.org/wiki/Jean_Bartik
"jennings",
// Mary Lou Jepsen, was the founder and chief technology officer of One Laptop Per Child (OLPC), and the founder of Pixel Qi. https://en.wikipedia.org/wiki/Mary_Lou_Jepsen
"jepsen",
// Irène Joliot-Curie - French scientist who was awarded the Nobel Prize for Chemistry in 1935. Daughter of Marie and Pierre Curie. https://en.wikipedia.org/wiki/Ir%C3%A8ne_Joliot-Curie
"joliot",
// Karen Spärck Jones came up with the concept of inverse document frequency, which is used in most search engines today. https://en.wikipedia.org/wiki/Karen_Sp%C3%A4rck_Jones
"jones",
// A. P. J. Abdul Kalam - is an Indian scientist aka Missile Man of India for his work on the development of ballistic missile and launch vehicle technology - https://en.wikipedia.org/wiki/A._P._J._Abdul_Kalam
"kalam",
// Susan Kare, created the icons and many of the interface elements for the original Apple Macintosh in the 1980s, and was an original employee of NeXT, working as the Creative Director. https://en.wikipedia.org/wiki/Susan_Kare
"kare",
// Mary Kenneth Keller, Sister Mary Kenneth Keller became the first American woman to earn a PhD in Computer Science in 1965. https://en.wikipedia.org/wiki/Mary_Kenneth_Keller
"keller",
// Har Gobind Khorana - Indian-American biochemist who shared the 1968 Nobel Prize for Physiology - https://en.wikipedia.org/wiki/Har_Gobind_Khorana
"khorana",
// Jack Kilby invented silicone integrated circuits and gave Silicon Valley its name. - https://en.wikipedia.org/wiki/Jack_Kilby
"kilby",
// Maria Kirch - German astronomer and first woman to discover a comet - https://en.wikipedia.org/wiki/Maria_Margarethe_Kirch
"kirch",
// Donald Knuth - American computer scientist, author of "The Art of Computer Programming" and creator of the TeX typesetting system. https://en.wikipedia.org/wiki/Donald_Knuth
"knuth",
// Sophie Kowalevski - Russian mathematician responsible for important original contributions to analysis, differential equations and mechanics - https://en.wikipedia.org/wiki/Sofia_Kovalevskaya
"kowalevski",
// Marie-Jeanne de Lalande - French astronomer, mathematician and cataloguer of stars - https://en.wikipedia.org/wiki/Marie-Jeanne_de_Lalande
"lalande",
// Hedy Lamarr - Actress and inventor. The principles of her work are now incorporated into modern Wi-Fi, CDMA and Bluetooth technology. https://en.wikipedia.org/wiki/Hedy_Lamarr
"lamarr",
// Leslie B. Lamport - American computer scientist. Lamport is best known for his seminal work in distributed systems and was the winner of the 2013 Turing Award. https://en.wikipedia.org/wiki/Leslie_Lamport
"lamport",
// Mary Leakey - British paleoanthropologist who discovered the first fossilized Proconsul skull - https://en.wikipedia.org/wiki/Mary_Leakey
"leakey",
// Henrietta Swan Leavitt - she was an American astronomer who discovered the relation between the luminosity and the period of Cepheid variable stars. https://en.wikipedia.org/wiki/Henrietta_Swan_Leavitt
"leavitt",
// Ruth Lichterman - one of the original programmers of the ENIAC. https://en.wikipedia.org/wiki/ENIAC - https://en.wikipedia.org/wiki/Ruth_Teitelbaum
"lichterman",
// Barbara Liskov - co-developed the Liskov substitution principle. Liskov was also the winner of the Turing Prize in 2008. - https://en.wikipedia.org/wiki/Barbara_Liskov
"liskov",
// Ada Lovelace invented the first algorithm. https://en.wikipedia.org/wiki/Ada_Lovelace (thanks James Turnbull)
"lovelace",
// Auguste and Louis Lumière - the first filmmakers in history - https://en.wikipedia.org/wiki/Auguste_and_Louis_Lumi%C3%A8re
"lumiere",
// Mahavira - Ancient Indian mathematician during 9th century AD who discovered basic algebraic identities - https://en.wikipedia.org/wiki/Mah%C4%81v%C4%ABra_(mathematician)
"mahavira",
// Maria Mayer - American theoretical physicist and Nobel laureate in Physics for proposing the nuclear shell model of the atomic nucleus - https://en.wikipedia.org/wiki/Maria_Mayer
"mayer",
// John McCarthy invented LISP: https://en.wikipedia.org/wiki/John_McCarthy_(computer_scientist)
"mccarthy",
// Barbara McClintock - a distinguished American cytogeneticist, 1983 Nobel Laureate in Physiology or Medicine for discovering transposons. https://en.wikipedia.org/wiki/Barbara_McClintock
"mcclintock",
// Malcolm McLean invented the modern shipping container: https://en.wikipedia.org/wiki/Malcom_McLean
"mclean",
// Kay McNulty - one of the original programmers of the ENIAC. https://en.wikipedia.org/wiki/ENIAC - https://en.wikipedia.org/wiki/Kathleen_Antonelli
"mcnulty",
// Lise Meitner - Austrian/Swedish physicist who was involved in the discovery of nuclear fission. The element meitnerium is named after her - https://en.wikipedia.org/wiki/Lise_Meitner
"meitner",
// Carla Meninsky, was the game designer and programmer for Atari 2600 games Dodge 'Em and Warlords. https://en.wikipedia.org/wiki/Carla_Meninsky
"meninsky",
// Johanna Mestorf - German prehistoric archaeologist and first female museum director in Germany - https://en.wikipedia.org/wiki/Johanna_Mestorf
"mestorf",
// Marvin Minsky - Pioneer in Artificial Intelligence, co-founder of the MIT's AI Lab, won the Turing Award in 1969. https://en.wikipedia.org/wiki/Marvin_Minsky
"minsky",
// Maryam Mirzakhani - an Iranian mathematician and the first woman to win the Fields Medal. https://en.wikipedia.org/wiki/Maryam_Mirzakhani
"mirzakhani",
// Samuel Morse - contributed to the invention of a single-wire telegraph system based on European telegraphs and was a co-developer of the Morse code - https://en.wikipedia.org/wiki/Samuel_Morse
"morse",
// Ian Murdock - founder of the Debian project - https://en.wikipedia.org/wiki/Ian_Murdock
"murdock",
// Isaac Newton invented classic mechanics and modern optics. https://en.wikipedia.org/wiki/Isaac_Newton
"newton",
// Florence Nightingale, more prominently known as a nurse, was also the first female member of the Royal Statistical Society and a pioneer in statistical graphics https://en.wikipedia.org/wiki/Florence_Nightingale#Statistics_and_sanitary_reform
"nightingale",
// Alfred Nobel - a Swedish chemist, engineer, innovator, and armaments manufacturer (inventor of dynamite) - https://en.wikipedia.org/wiki/Alfred_Nobel
"nobel",
// Emmy Noether, German mathematician. Noether's Theorem is named after her. https://en.wikipedia.org/wiki/Emmy_Noether
"noether",
// Poppy Northcutt. Poppy Northcutt was the first woman to work as part of NASA’s Mission Control. http://www.businessinsider.com/poppy-northcutt-helped-apollo-astronauts-2014-12?op=1
"northcutt",
// Robert Noyce invented silicone integrated circuits and gave Silicon Valley its name. - https://en.wikipedia.org/wiki/Robert_Noyce
"noyce",
// Panini - Ancient Indian linguist and grammarian from 4th century CE who worked on the world's first formal system - https://en.wikipedia.org/wiki/P%C4%81%E1%B9%87ini#Comparison_with_modern_formal_systems
"panini",
// Ambroise Pare invented modern surgery. https://en.wikipedia.org/wiki/Ambroise_Par%C3%A9
"pare",
// Louis Pasteur discovered vaccination, fermentation and pasteurization. https://en.wikipedia.org/wiki/Louis_Pasteur.
"pasteur",
// Cecilia Payne-Gaposchkin was an astronomer and astrophysicist who, in 1925, proposed in her Ph.D. thesis an explanation for the composition of stars in terms of the relative abundances of hydrogen and helium. https://en.wikipedia.org/wiki/Cecilia_Payne-Gaposchkin
"payne",
// Radia Perlman is a software designer and network engineer and most famous for her invention of the spanning-tree protocol (STP). https://en.wikipedia.org/wiki/Radia_Perlman
"perlman",
// Rob Pike was a key contributor to Unix, Plan 9, the X graphic system, utf-8, and the Go programming language. https://en.wikipedia.org/wiki/Rob_Pike
"pike",
// Henri Poincaré made fundamental contributions in several fields of mathematics. https://en.wikipedia.org/wiki/Henri_Poincar%C3%A9
"poincare",
// Laura Poitras is a director and producer whose work, made possible by open source crypto tools, advances the causes of truth and freedom of information by reporting disclosures by whistleblowers such as Edward Snowden. https://en.wikipedia.org/wiki/Laura_Poitras
"poitras",
// Claudius Ptolemy - a Greco-Egyptian writer of Alexandria, known as a mathematician, astronomer, geographer, astrologer, and poet of a single epigram in the Greek Anthology - https://en.wikipedia.org/wiki/Ptolemy
"ptolemy",
// C. V. Raman - Indian physicist who won the Nobel Prize in 1930 for proposing the Raman effect. - https://en.wikipedia.org/wiki/C._V._Raman
"raman",
// Srinivasa Ramanujan - Indian mathematician and autodidact who made extraordinary contributions to mathematical analysis, number theory, infinite series, and continued fractions. - https://en.wikipedia.org/wiki/Srinivasa_Ramanujan
"ramanujan",
// Sally Kristen Ride was an American physicist and astronaut. She was the first American woman in space, and the youngest American astronaut. https://en.wikipedia.org/wiki/Sally_Ride
"ride",
// Rita Levi-Montalcini - Won Nobel Prize in Physiology or Medicine jointly with colleague Stanley Cohen for the discovery of nerve growth factor (https://en.wikipedia.org/wiki/Rita_Levi-Montalcini)
"montalcini",
// Dennis Ritchie - co-creator of UNIX and the C programming language. - https://en.wikipedia.org/wiki/Dennis_Ritchie
"ritchie",
// Wilhelm Conrad Röntgen - German physicist who was awarded the first Nobel Prize in Physics in 1901 for the discovery of X-rays (Röntgen rays). https://en.wikipedia.org/wiki/Wilhelm_R%C3%B6ntgen
"roentgen",
// Rosalind Franklin - British biophysicist and X-ray crystallographer whose research was critical to the understanding of DNA - https://en.wikipedia.org/wiki/Rosalind_Franklin
"rosalind",
// Meghnad Saha - Indian astrophysicist best known for his development of the Saha equation, used to describe chemical and physical conditions in stars - https://en.wikipedia.org/wiki/Meghnad_Saha
"saha",
// Jean E. Sammet developed FORMAC, the first widely used computer language for symbolic manipulation of mathematical formulas. https://en.wikipedia.org/wiki/Jean_E._Sammet
"sammet",
// Carol Shaw - Originally an Atari employee, Carol Shaw is said to be the first female video game designer. https://en.wikipedia.org/wiki/Carol_Shaw_(video_game_designer)
"shaw",
// Dame Stephanie "Steve" Shirley - Founded a software company in 1962 employing women working from home. https://en.wikipedia.org/wiki/Steve_Shirley
"shirley",
// William Shockley co-invented the transistor - https://en.wikipedia.org/wiki/William_Shockley
"shockley",
// Françoise Barré-Sinoussi - French virologist and Nobel Prize Laureate in Physiology or Medicine; her work was fundamental in identifying HIV as the cause of AIDS. https://en.wikipedia.org/wiki/Fran%C3%A7oise_Barr%C3%A9-Sinoussi
"sinoussi",
// Betty Snyder - one of the original programmers of the ENIAC. https://en.wikipedia.org/wiki/ENIAC - https://en.wikipedia.org/wiki/Betty_Holberton
"snyder",
// Frances Spence - one of the original programmers of the ENIAC. https://en.wikipedia.org/wiki/ENIAC - https://en.wikipedia.org/wiki/Frances_Spence
"spence",
// Richard Matthew Stallman - the founder of the Free Software movement, the GNU project, the Free Software Foundation, and the League for Programming Freedom. He also invented the concept of copyleft to protect the ideals of this movement, and enshrined this concept in the widely-used GPL (General Public License) for software. https://en.wikiquote.org/wiki/Richard_Stallman
"stallman",
// Michael Stonebraker is a database research pioneer and architect of Ingres, Postgres, VoltDB and SciDB. Winner of 2014 ACM Turing Award. https://en.wikipedia.org/wiki/Michael_Stonebraker
"stonebraker",
// Janese Swanson (with others) developed the first of the Carmen Sandiego games. She went on to found Girl Tech. https://en.wikipedia.org/wiki/Janese_Swanson
"swanson",
// Aaron Swartz was influential in creating RSS, Markdown, Creative Commons, Reddit, and much of the internet as we know it today. He was devoted to freedom of information on the web. https://en.wikiquote.org/wiki/Aaron_Swartz
"swartz",
// Bertha Swirles was a theoretical physicist who made a number of contributions to early quantum theory. https://en.wikipedia.org/wiki/Bertha_Swirles
"swirles",
// Nikola Tesla invented the AC electric system and every gadget ever used by a James Bond villain. https://en.wikipedia.org/wiki/Nikola_Tesla
"tesla",
// Ken Thompson - co-creator of UNIX and the C programming language - https://en.wikipedia.org/wiki/Ken_Thompson
"thompson",
// Linus Torvalds invented Linux and Git. https://en.wikipedia.org/wiki/Linus_Torvalds
"torvalds",
// Alan Turing was a founding father of computer science. https://en.wikipedia.org/wiki/Alan_Turing.
"turing",
// Varahamihira - Ancient Indian mathematician who discovered trigonometric formulae during 505-587 CE - https://en.wikipedia.org/wiki/Var%C4%81hamihira#Contributions
"varahamihira",
// Sir Mokshagundam Visvesvaraya - is a notable Indian engineer. He is a recipient of the Indian Republic's highest honour, the Bharat Ratna, in 1955. On his birthday, 15 September is celebrated as Engineer's Day in India in his memory - https://en.wikipedia.org/wiki/Visvesvaraya
"visvesvaraya",
// Christiane Nüsslein-Volhard - German biologist, won Nobel Prize in Physiology or Medicine in 1995 for research on the genetic control of embryonic development. https://en.wikipedia.org/wiki/Christiane_N%C3%BCsslein-Volhard
"volhard",
// Marlyn Wescoff - one of the original programmers of the ENIAC. https://en.wikipedia.org/wiki/ENIAC - https://en.wikipedia.org/wiki/Marlyn_Meltzer
"wescoff",
// Andrew Wiles - Notable British mathematician who proved the enigmatic Fermat's Last Theorem - https://en.wikipedia.org/wiki/Andrew_Wiles
"wiles",
// Roberta Williams, did pioneering work in graphical adventure games for personal computers, particularly the King's Quest series. https://en.wikipedia.org/wiki/Roberta_Williams
"williams",
// Sophie Wilson designed the first Acorn Micro-Computer and the instruction set for ARM processors. https://en.wikipedia.org/wiki/Sophie_Wilson
"wilson",
// Jeannette Wing - co-developed the Liskov substitution principle. - https://en.wikipedia.org/wiki/Jeannette_Wing
"wing",
// Steve Wozniak invented the Apple I and Apple II. https://en.wikipedia.org/wiki/Steve_Wozniak
"wozniak",
// The Wright brothers, Orville and Wilbur - credited with inventing and building the world's first successful airplane and making the first controlled, powered and sustained heavier-than-air human flight - https://en.wikipedia.org/wiki/Wright_brothers
"wright",
// Rosalyn Sussman Yalow - Rosalyn Sussman Yalow was an American medical physicist, and a co-winner of the 1977 Nobel Prize in Physiology or Medicine for development of the radioimmunoassay technique. https://en.wikipedia.org/wiki/Rosalyn_Sussman_Yalow
"yalow",
// Ada Yonath - an Israeli crystallographer, the first woman from the Middle East to win a Nobel prize in the sciences. https://en.wikipedia.org/wiki/Ada_Yonath
"yonath",
}
)
// GetRandomName generates a random name from the list of adjectives and surnames in this package
// formatted as "adjective_surname". For example 'focused_turing'. If retry is non-zero, a random
// integer between 0 and 10 will be added to the end of the name, e.g `focused_turing3`
func GetRandomName(retry int) string {
rnd := random.Rand
begin:
name := fmt.Sprintf("%s_%s", left[rnd.Intn(len(left))], right[rnd.Intn(len(right))])
if name == "boring_wozniak" /* Steve Wozniak is not boring */ {
goto begin
}
if retry > 0 {
name = fmt.Sprintf("%s%d", name, rnd.Intn(10))
}
return name
}

27
namesgenerator/names-generator_test.go
View file

@ -1,27 +0,0 @@
package namesgenerator
import (
"strings"
"testing"
)
func TestNameFormat(t *testing.T) {
name := GetRandomName(0)
if !strings.Contains(name, "_") {
t.Fatalf("Generated name does not contain an underscore")
}
if strings.ContainsAny(name, "0123456789") {
t.Fatalf("Generated name contains numbers!")
}
}
func TestNameRetries(t *testing.T) {
name := GetRandomName(1)
if !strings.Contains(name, "_") {
t.Fatalf("Generated name does not contain an underscore")
}
if !strings.ContainsAny(name, "0123456789") {
t.Fatalf("Generated name doesn't contain a number")
}
}

16
platform/architecture_linux.go
View file

@ -1,16 +0,0 @@
// Package platform provides helper function to get the runtime architecture
// for different platforms.
package platform
import (
"syscall"
)
// runtimeArchitecture gets the name of the current architecture (x86, x86_64, …)
func runtimeArchitecture() (string, error) {
utsname := &syscall.Utsname{}
if err := syscall.Uname(utsname); err != nil {
return "", err
}
return charsToString(utsname.Machine), nil
}

20
platform/architecture_unix.go
View file

@ -1,20 +0,0 @@
// +build freebsd solaris darwin
// Package platform provides helper function to get the runtime architecture
// for different platforms.
package platform
import (
"os/exec"
"strings"
)
// runtimeArchitecture gets the name of the current architecture (x86, x86_64, i86pc, sun4v, ...)
func runtimeArchitecture() (string, error) {
cmd := exec.Command("/usr/bin/uname", "-m")
machine, err := cmd.Output()
if err != nil {
return "", err
}
return strings.TrimSpace(string(machine)), nil
}

52
platform/architecture_windows.go
View file

@ -1,52 +0,0 @@
package platform
import (
"fmt"
"syscall"
"unsafe"
)
var (
modkernel32 = syscall.NewLazyDLL("kernel32.dll")
procGetSystemInfo = modkernel32.NewProc("GetSystemInfo")
)
// see http://msdn.microsoft.com/en-us/library/windows/desktop/ms724958(v=vs.85).aspx
type systeminfo struct {
wProcessorArchitecture uint16
wReserved uint16
dwPageSize uint32
lpMinimumApplicationAddress uintptr
lpMaximumApplicationAddress uintptr
dwActiveProcessorMask uintptr
dwNumberOfProcessors uint32
dwProcessorType uint32
dwAllocationGranularity uint32
wProcessorLevel uint16
wProcessorRevision uint16
}
// Constants
const (
ProcessorArchitecture64 = 9 // PROCESSOR_ARCHITECTURE_AMD64
ProcessorArchitectureIA64 = 6 // PROCESSOR_ARCHITECTURE_IA64
ProcessorArchitecture32 = 0 // PROCESSOR_ARCHITECTURE_INTEL
ProcessorArchitectureArm = 5 // PROCESSOR_ARCHITECTURE_ARM
)
var sysinfo systeminfo
// runtimeArchitecture gets the name of the current architecture (x86, x86_64, …)
func runtimeArchitecture() (string, error) {
syscall.Syscall(procGetSystemInfo.Addr(), 1, uintptr(unsafe.Pointer(&sysinfo)), 0, 0)
switch sysinfo.wProcessorArchitecture {
case ProcessorArchitecture64, ProcessorArchitectureIA64:
return "x86_64", nil
case ProcessorArchitecture32:
return "i686", nil
case ProcessorArchitectureArm:
return "arm", nil
default:
return "", fmt.Errorf("Unknown processor architecture")
}
}

23
platform/platform.go
View file

@ -1,23 +0,0 @@
package platform
import (
"runtime"
"github.com/Sirupsen/logrus"
)
var (
// Architecture holds the runtime architecture of the process.
Architecture string
// OSType holds the runtime operating system type (Linux, …) of the process.
OSType string
)
func init() {
var err error
Architecture, err = runtimeArchitecture()
if err != nil {
logrus.Errorf("Could not read system architecture info: %v", err)
}
OSType = runtime.GOOS
}

18
platform/utsname_int8.go
View file

@ -1,18 +0,0 @@
// +build linux,386 linux,amd64 linux,arm64
// see golang's sources src/syscall/ztypes_linux_*.go that use int8
package platform
// Convert the OS/ARCH-specific utsname.Machine to string
// given as an array of signed int8
func charsToString(ca [65]int8) string {
s := make([]byte, len(ca))
var lens int
for ; lens < len(ca); lens++ {
if ca[lens] == 0 {
break
}
s[lens] = uint8(ca[lens])
}
return string(s[0:lens])
}

18
platform/utsname_uint8.go
View file

@ -1,18 +0,0 @@
// +build linux,arm linux,ppc64 linux,ppc64le s390x
// see golang's sources src/syscall/ztypes_linux_*.go that use uint8
package platform
// Convert the OS/ARCH-specific utsname.Machine to string
// given as an array of unsigned uint8
func charsToString(ca [65]uint8) string {
s := make([]byte, len(ca))
var lens int
for ; lens < len(ca); lens++ {
if ca[lens] == 0 {
break
}
s[lens] = ca[lens]
}
return string(s[0:lens])
}

73
progress/progress.go
View file

@ -1,73 +0,0 @@
package progress
import (
"fmt"
)
// Progress represents the progress of a transfer.
type Progress struct {
ID string
// Progress contains a Message or...
Message string
// ...progress of an action
Action string
Current int64
Total int64
// Aux contains extra information not presented to the user, such as
// digests for push signing.
Aux interface{}
LastUpdate bool
}
// Output is an interface for writing progress information. It's
// like a writer for progress, but we don't call it Writer because
// that would be confusing next to ProgressReader (also, because it
// doesn't implement the io.Writer interface).
type Output interface {
WriteProgress(Progress) error
}
type chanOutput chan<- Progress
func (out chanOutput) WriteProgress(p Progress) error {
out <- p
return nil
}
// ChanOutput returns an Output that writes progress updates to the
// supplied channel.
func ChanOutput(progressChan chan<- Progress) Output {
return chanOutput(progressChan)
}
// Update is a convenience function to write a progress update to the channel.
func Update(out Output, id, action string) {
out.WriteProgress(Progress{ID: id, Action: action})
}
// Updatef is a convenience function to write a printf-formatted progress update
// to the channel.
func Updatef(out Output, id, format string, a ...interface{}) {
Update(out, id, fmt.Sprintf(format, a...))
}
// Message is a convenience function to write a progress message to the channel.
func Message(out Output, id, message string) {
out.WriteProgress(Progress{ID: id, Message: message})
}
// Messagef is a convenience function to write a printf-formatted progress
// message to the channel.
func Messagef(out Output, id, format string, a ...interface{}) {
Message(out, id, fmt.Sprintf(format, a...))
}
// Aux sends auxiliary information over a progress interface, which will not be
// formatted for the UI. This is used for things such as push signing.
func Aux(out Output, a interface{}) {
out.WriteProgress(Progress{Aux: a})
}

59
progress/progressreader.go
View file

@ -1,59 +0,0 @@
package progress
import (
"io"
)
// Reader is a Reader with progress bar.
type Reader struct {
in io.ReadCloser // Stream to read from
out Output // Where to send progress bar to
size int64
current int64
lastUpdate int64
id string
action string
}
// NewProgressReader creates a new ProgressReader.
func NewProgressReader(in io.ReadCloser, out Output, size int64, id, action string) *Reader {
return &Reader{
in: in,
out: out,
size: size,
id: id,
action: action,
}
}
func (p *Reader) Read(buf []byte) (n int, err error) {
read, err := p.in.Read(buf)
p.current += int64(read)
updateEvery := int64(1024 * 512) //512kB
if p.size > 0 {
// Update progress for every 1% read if 1% < 512kB
if increment := int64(0.01 * float64(p.size)); increment < updateEvery {
updateEvery = increment
}
}
if p.current-p.lastUpdate > updateEvery || err != nil {
p.updateProgress(err != nil && read == 0)
p.lastUpdate = p.current
}
return read, err
}
// Close closes the progress reader and its underlying reader.
func (p *Reader) Close() error {
if p.current < p.size {
// print a full progress bar when closing prematurely
p.current = p.size
p.updateProgress(false)
}
return p.in.Close()
}
func (p *Reader) updateProgress(last bool) {
p.out.WriteProgress(Progress{ID: p.id, Action: p.action, Current: p.current, Total: p.size, LastUpdate: last})
}

75
progress/progressreader_test.go
View file

@ -1,75 +0,0 @@
package progress
import (
"bytes"
"io"
"io/ioutil"
"testing"
)
func TestOutputOnPrematureClose(t *testing.T) {
content := []byte("TESTING")
reader := ioutil.NopCloser(bytes.NewReader(content))
progressChan := make(chan Progress, 10)
pr := NewProgressReader(reader, ChanOutput(progressChan), int64(len(content)), "Test", "Read")
part := make([]byte, 4, 4)
_, err := io.ReadFull(pr, part)
if err != nil {
pr.Close()
t.Fatal(err)
}
drainLoop:
for {
select {
case <-progressChan:
default:
break drainLoop
}
}
pr.Close()
select {
case <-progressChan:
default:
t.Fatalf("Expected some output when closing prematurely")
}
}
func TestCompleteSilently(t *testing.T) {
content := []byte("TESTING")
reader := ioutil.NopCloser(bytes.NewReader(content))
progressChan := make(chan Progress, 10)
pr := NewProgressReader(reader, ChanOutput(progressChan), int64(len(content)), "Test", "Read")
out, err := ioutil.ReadAll(pr)
if err != nil {
pr.Close()
t.Fatal(err)
}
if string(out) != "TESTING" {
pr.Close()
t.Fatalf("Unexpected output %q from reader", string(out))
}
drainLoop:
for {
select {
case <-progressChan:
default:
break drainLoop
}
}
pr.Close()
select {
case <-progressChan:
t.Fatalf("Should have closed silently when read is complete")
default:
}
}

111
pubsub/publisher.go
View file

@ -1,111 +0,0 @@
package pubsub
import (
"sync"
"time"
)
var wgPool = sync.Pool{New: func() interface{} { return new(sync.WaitGroup) }}
// NewPublisher creates a new pub/sub publisher to broadcast messages.
// The duration is used as the send timeout as to not block the publisher publishing
// messages to other clients if one client is slow or unresponsive.
// The buffer is used when creating new channels for subscribers.
func NewPublisher(publishTimeout time.Duration, buffer int) *Publisher {
return &Publisher{
buffer: buffer,
timeout: publishTimeout,
subscribers: make(map[subscriber]topicFunc),
}
}
type subscriber chan interface{}
type topicFunc func(v interface{}) bool
// Publisher is basic pub/sub structure. Allows to send events and subscribe
// to them. Can be safely used from multiple goroutines.
type Publisher struct {
m sync.RWMutex
buffer int
timeout time.Duration
subscribers map[subscriber]topicFunc
}
// Len returns the number of subscribers for the publisher
func (p *Publisher) Len() int {
p.m.RLock()
i := len(p.subscribers)
p.m.RUnlock()
return i
}
// Subscribe adds a new subscriber to the publisher returning the channel.
func (p *Publisher) Subscribe() chan interface{} {
return p.SubscribeTopic(nil)
}
// SubscribeTopic adds a new subscriber that filters messages sent by a topic.
func (p *Publisher) SubscribeTopic(topic topicFunc) chan interface{} {
ch := make(chan interface{}, p.buffer)
p.m.Lock()
p.subscribers[ch] = topic
p.m.Unlock()
return ch
}
// Evict removes the specified subscriber from receiving any more messages.
func (p *Publisher) Evict(sub chan interface{}) {
p.m.Lock()
delete(p.subscribers, sub)
close(sub)
p.m.Unlock()
}
// Publish sends the data in v to all subscribers currently registered with the publisher.
func (p *Publisher) Publish(v interface{}) {
p.m.RLock()
if len(p.subscribers) == 0 {
p.m.RUnlock()
return
}
wg := wgPool.Get().(*sync.WaitGroup)
for sub, topic := range p.subscribers {
wg.Add(1)
go p.sendTopic(sub, topic, v, wg)
}
wg.Wait()
wgPool.Put(wg)
p.m.RUnlock()
}
// Close closes the channels to all subscribers registered with the publisher.
func (p *Publisher) Close() {
p.m.Lock()
for sub := range p.subscribers {
delete(p.subscribers, sub)
close(sub)
}
p.m.Unlock()
}
func (p *Publisher) sendTopic(sub subscriber, topic topicFunc, v interface{}, wg *sync.WaitGroup) {
defer wg.Done()
if topic != nil && !topic(v) {
return
}
// send under a select as to not block if the receiver is unavailable
if p.timeout > 0 {
select {
case sub <- v:
case <-time.After(p.timeout):
}
return
}
select {
case sub <- v:
default:
}
}

142
pubsub/publisher_test.go
View file

@ -1,142 +0,0 @@
package pubsub
import (
"fmt"
"testing"
"time"
)
func TestSendToOneSub(t *testing.T) {
p := NewPublisher(100*time.Millisecond, 10)
c := p.Subscribe()
p.Publish("hi")
msg := <-c
if msg.(string) != "hi" {
t.Fatalf("expected message hi but received %v", msg)
}
}
func TestSendToMultipleSubs(t *testing.T) {
p := NewPublisher(100*time.Millisecond, 10)
subs := []chan interface{}{}
subs = append(subs, p.Subscribe(), p.Subscribe(), p.Subscribe())
p.Publish("hi")
for _, c := range subs {
msg := <-c
if msg.(string) != "hi" {
t.Fatalf("expected message hi but received %v", msg)
}
}
}
func TestEvictOneSub(t *testing.T) {
p := NewPublisher(100*time.Millisecond, 10)
s1 := p.Subscribe()
s2 := p.Subscribe()
p.Evict(s1)
p.Publish("hi")
if _, ok := <-s1; ok {
t.Fatal("expected s1 to not receive the published message")
}
msg := <-s2
if msg.(string) != "hi" {
t.Fatalf("expected message hi but received %v", msg)
}
}
func TestClosePublisher(t *testing.T) {
p := NewPublisher(100*time.Millisecond, 10)
subs := []chan interface{}{}
subs = append(subs, p.Subscribe(), p.Subscribe(), p.Subscribe())
p.Close()
for _, c := range subs {
if _, ok := <-c; ok {
t.Fatal("expected all subscriber channels to be closed")
}
}
}
const sampleText = "test"
type testSubscriber struct {
dataCh chan interface{}
ch chan error
}
func (s *testSubscriber) Wait() error {
return <-s.ch
}
func newTestSubscriber(p *Publisher) *testSubscriber {
ts := &testSubscriber{
dataCh: p.Subscribe(),
ch: make(chan error),
}
go func() {
for data := range ts.dataCh {
s, ok := data.(string)
if !ok {
ts.ch <- fmt.Errorf("Unexpected type %T", data)
break
}
if s != sampleText {
ts.ch <- fmt.Errorf("Unexpected text %s", s)
break
}
}
close(ts.ch)
}()
return ts
}
// for testing with -race
func TestPubSubRace(t *testing.T) {
p := NewPublisher(0, 1024)
var subs [](*testSubscriber)
for j := 0; j < 50; j++ {
subs = append(subs, newTestSubscriber(p))
}
for j := 0; j < 1000; j++ {
p.Publish(sampleText)
}
time.AfterFunc(1*time.Second, func() {
for _, s := range subs {
p.Evict(s.dataCh)
}
})
for _, s := range subs {
s.Wait()
}
}
func BenchmarkPubSub(b *testing.B) {
for i := 0; i < b.N; i++ {
b.StopTimer()
p := NewPublisher(0, 1024)
var subs [](*testSubscriber)
for j := 0; j < 50; j++ {
subs = append(subs, newTestSubscriber(p))
}
b.StartTimer()
for j := 0; j < 1000; j++ {
p.Publish(sampleText)
}
time.AfterFunc(1*time.Second, func() {
for _, s := range subs {
p.Evict(s.dataCh)
}
})
for _, s := range subs {
if err := s.Wait(); err != nil {
b.Fatal(err)
}
}
}
}

127
registrar/registrar.go
View file

@ -1,127 +0,0 @@
// Package registrar provides name registration. It reserves a name to a given key.
package registrar
import (
"errors"
"sync"
)
var (
// ErrNameReserved is an error which is returned when a name is requested to be reserved that already is reserved
ErrNameReserved = errors.New("name is reserved")
// ErrNameNotReserved is an error which is returned when trying to find a name that is not reserved
ErrNameNotReserved = errors.New("name is not reserved")
// ErrNoSuchKey is returned when trying to find the names for a key which is not known
ErrNoSuchKey = errors.New("provided key does not exist")
)
// Registrar stores indexes a list of keys and their registered names as well as indexes names and the key that they are registered to
// Names must be unique.
// Registrar is safe for concurrent access.
type Registrar struct {
idx map[string][]string
names map[string]string
mu sync.Mutex
}
// NewRegistrar creates a new Registrar with the an empty index
func NewRegistrar() *Registrar {
return &Registrar{
idx: make(map[string][]string),
names: make(map[string]string),
}
}
// Reserve registers a key to a name
// Reserve is idempotent
// Attempting to reserve a key to a name that already exists results in an `ErrNameReserved`
// A name reservation is globally unique
func (r *Registrar) Reserve(name, key string) error {
r.mu.Lock()
defer r.mu.Unlock()
if k, exists := r.names[name]; exists {
if k != key {
return ErrNameReserved
}
return nil
}
r.idx[key] = append(r.idx[key], name)
r.names[name] = key
return nil
}
// Release releases the reserved name
// Once released, a name can be reserved again
func (r *Registrar) Release(name string) {
r.mu.Lock()
defer r.mu.Unlock()
key, exists := r.names[name]
if !exists {
return
}
for i, n := range r.idx[key] {
if n != name {
continue
}
r.idx[key] = append(r.idx[key][:i], r.idx[key][i+1:]...)
break
}
delete(r.names, name)
if len(r.idx[key]) == 0 {
delete(r.idx, key)
}
}
// Delete removes all reservations for the passed in key.
// All names reserved to this key are released.
func (r *Registrar) Delete(key string) {
r.mu.Lock()
for _, name := range r.idx[key] {
delete(r.names, name)
}
delete(r.idx, key)
r.mu.Unlock()
}
// GetNames lists all the reserved names for the given key
func (r *Registrar) GetNames(key string) ([]string, error) {
r.mu.Lock()
defer r.mu.Unlock()
names, exists := r.idx[key]
if !exists {
return nil, ErrNoSuchKey
}
return names, nil
}
// Get returns the key that the passed in name is reserved to
func (r *Registrar) Get(name string) (string, error) {
r.mu.Lock()
key, exists := r.names[name]
r.mu.Unlock()
if !exists {
return "", ErrNameNotReserved
}
return key, nil
}
// GetAll returns all registered names
func (r *Registrar) GetAll() map[string][]string {
out := make(map[string][]string)
r.mu.Lock()
// copy index into out
for id, names := range r.idx {
out[id] = names
}
r.mu.Unlock()
return out
}

119
registrar/registrar_test.go
View file

@ -1,119 +0,0 @@
package registrar
import (
"reflect"
"testing"
)
func TestReserve(t *testing.T) {
r := NewRegistrar()
obj := "test1"
if err := r.Reserve("test", obj); err != nil {
t.Fatal(err)
}
if err := r.Reserve("test", obj); err != nil {
t.Fatal(err)
}
obj2 := "test2"
err := r.Reserve("test", obj2)
if err == nil {
t.Fatalf("expected error when reserving an already reserved name to another object")
}
if err != ErrNameReserved {
t.Fatal("expected `ErrNameReserved` error when attempting to reserve an already reserved name")
}
}
func TestRelease(t *testing.T) {
r := NewRegistrar()
obj := "testing"
if err := r.Reserve("test", obj); err != nil {
t.Fatal(err)
}
r.Release("test")
r.Release("test") // Ensure there is no panic here
if err := r.Reserve("test", obj); err != nil {
t.Fatal(err)
}
}
func TestGetNames(t *testing.T) {
r := NewRegistrar()
obj := "testing"
names := []string{"test1", "test2"}
for _, name := range names {
if err := r.Reserve(name, obj); err != nil {
t.Fatal(err)
}
}
r.Reserve("test3", "other")
names2, err := r.GetNames(obj)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(names, names2) {
t.Fatalf("Exepected: %v, Got: %v", names, names2)
}
}
func TestDelete(t *testing.T) {
r := NewRegistrar()
obj := "testing"
names := []string{"test1", "test2"}
for _, name := range names {
if err := r.Reserve(name, obj); err != nil {
t.Fatal(err)
}
}
r.Reserve("test3", "other")
r.Delete(obj)
_, err := r.GetNames(obj)
if err == nil {
t.Fatal("expected error getting names for deleted key")
}
if err != ErrNoSuchKey {
t.Fatal("expected `ErrNoSuchKey`")
}
}
func TestGet(t *testing.T) {
r := NewRegistrar()
obj := "testing"
name := "test"
_, err := r.Get(name)
if err == nil {
t.Fatal("expected error when key does not exist")
}
if err != ErrNameNotReserved {
t.Fatal(err)
}
if err := r.Reserve(name, obj); err != nil {
t.Fatal(err)
}
if _, err = r.Get(name); err != nil {
t.Fatal(err)
}
r.Delete(obj)
_, err = r.Get(name)
if err == nil {
t.Fatal("expected error when key does not exist")
}
if err != ErrNameNotReserved {
t.Fatal(err)
}
}

1
signal/README.md
View file

@ -1 +0,0 @@
This package provides helper functions for dealing with signals across various operating systems

54
signal/signal.go
View file

@ -1,54 +0,0 @@
// Package signal provides helper functions for dealing with signals across
// various operating systems.
package signal
import (
"fmt"
"os"
"os/signal"
"strconv"
"strings"
"syscall"
)
// CatchAll catches all signals and relays them to the specified channel.
func CatchAll(sigc chan os.Signal) {
handledSigs := []os.Signal{}
for _, s := range SignalMap {
handledSigs = append(handledSigs, s)
}
signal.Notify(sigc, handledSigs...)
}
// StopCatch stops catching the signals and closes the specified channel.
func StopCatch(sigc chan os.Signal) {
signal.Stop(sigc)
close(sigc)
}
// ParseSignal translates a string to a valid syscall signal.
// It returns an error if the signal map doesn't include the given signal.
func ParseSignal(rawSignal string) (syscall.Signal, error) {
s, err := strconv.Atoi(rawSignal)
if err == nil {
if s == 0 {
return -1, fmt.Errorf("Invalid signal: %s", rawSignal)
}
return syscall.Signal(s), nil
}
signal, ok := SignalMap[strings.TrimPrefix(strings.ToUpper(rawSignal), "SIG")]
if !ok {
return -1, fmt.Errorf("Invalid signal: %s", rawSignal)
}
return signal, nil
}
// ValidSignalForPlatform returns true if a signal is valid on the platform
func ValidSignalForPlatform(sig syscall.Signal) bool {
for _, v := range SignalMap {
if v == sig {
return true
}
}
return false
}

41
signal/signal_darwin.go
View file

@ -1,41 +0,0 @@
package signal
import (
"syscall"
)
// SignalMap is a map of Darwin signals.
var SignalMap = map[string]syscall.Signal{
"ABRT": syscall.SIGABRT,
"ALRM": syscall.SIGALRM,
"BUG": syscall.SIGBUS,
"CHLD": syscall.SIGCHLD,
"CONT": syscall.SIGCONT,
"EMT": syscall.SIGEMT,
"FPE": syscall.SIGFPE,
"HUP": syscall.SIGHUP,
"ILL": syscall.SIGILL,
"INFO": syscall.SIGINFO,
"INT": syscall.SIGINT,
"IO": syscall.SIGIO,
"IOT": syscall.SIGIOT,
"KILL": syscall.SIGKILL,
"PIPE": syscall.SIGPIPE,
"PROF": syscall.SIGPROF,
"QUIT": syscall.SIGQUIT,
"SEGV": syscall.SIGSEGV,
"STOP": syscall.SIGSTOP,
"SYS": syscall.SIGSYS,
"TERM": syscall.SIGTERM,
"TRAP": syscall.SIGTRAP,
"TSTP": syscall.SIGTSTP,
"TTIN": syscall.SIGTTIN,
"TTOU": syscall.SIGTTOU,
"URG": syscall.SIGURG,
"USR1": syscall.SIGUSR1,
"USR2": syscall.SIGUSR2,
"VTALRM": syscall.SIGVTALRM,
"WINCH": syscall.SIGWINCH,
"XCPU": syscall.SIGXCPU,
"XFSZ": syscall.SIGXFSZ,
}

43
signal/signal_freebsd.go
View file

@ -1,43 +0,0 @@
package signal
import (
"syscall"
)
// SignalMap is a map of FreeBSD signals.
var SignalMap = map[string]syscall.Signal{
"ABRT": syscall.SIGABRT,
"ALRM": syscall.SIGALRM,
"BUF": syscall.SIGBUS,
"CHLD": syscall.SIGCHLD,
"CONT": syscall.SIGCONT,
"EMT": syscall.SIGEMT,
"FPE": syscall.SIGFPE,
"HUP": syscall.SIGHUP,
"ILL": syscall.SIGILL,
"INFO": syscall.SIGINFO,
"INT": syscall.SIGINT,
"IO": syscall.SIGIO,
"IOT": syscall.SIGIOT,
"KILL": syscall.SIGKILL,
"LWP": syscall.SIGLWP,
"PIPE": syscall.SIGPIPE,
"PROF": syscall.SIGPROF,
"QUIT": syscall.SIGQUIT,
"SEGV": syscall.SIGSEGV,
"STOP": syscall.SIGSTOP,
"SYS": syscall.SIGSYS,
"TERM": syscall.SIGTERM,
"THR": syscall.SIGTHR,
"TRAP": syscall.SIGTRAP,
"TSTP": syscall.SIGTSTP,
"TTIN": syscall.SIGTTIN,
"TTOU": syscall.SIGTTOU,
"URG": syscall.SIGURG,
"USR1": syscall.SIGUSR1,
"USR2": syscall.SIGUSR2,
"VTALRM": syscall.SIGVTALRM,
"WINCH": syscall.SIGWINCH,
"XCPU": syscall.SIGXCPU,
"XFSZ": syscall.SIGXFSZ,
}

80
signal/signal_linux.go
View file

@ -1,80 +0,0 @@
package signal
import (
"syscall"
)
const (
sigrtmin = 34
sigrtmax = 64
)
// SignalMap is a map of Linux signals.
var SignalMap = map[string]syscall.Signal{
"ABRT": syscall.SIGABRT,
"ALRM": syscall.SIGALRM,
"BUS": syscall.SIGBUS,
"CHLD": syscall.SIGCHLD,
"CLD": syscall.SIGCLD,
"CONT": syscall.SIGCONT,
"FPE": syscall.SIGFPE,
"HUP": syscall.SIGHUP,
"ILL": syscall.SIGILL,
"INT": syscall.SIGINT,
"IO": syscall.SIGIO,
"IOT": syscall.SIGIOT,
"KILL": syscall.SIGKILL,
"PIPE": syscall.SIGPIPE,
"POLL": syscall.SIGPOLL,
"PROF": syscall.SIGPROF,
"PWR": syscall.SIGPWR,
"QUIT": syscall.SIGQUIT,
"SEGV": syscall.SIGSEGV,
"STKFLT": syscall.SIGSTKFLT,
"STOP": syscall.SIGSTOP,
"SYS": syscall.SIGSYS,
"TERM": syscall.SIGTERM,
"TRAP": syscall.SIGTRAP,
"TSTP": syscall.SIGTSTP,
"TTIN": syscall.SIGTTIN,
"TTOU": syscall.SIGTTOU,
"UNUSED": syscall.SIGUNUSED,
"URG": syscall.SIGURG,
"USR1": syscall.SIGUSR1,
"USR2": syscall.SIGUSR2,
"VTALRM": syscall.SIGVTALRM,
"WINCH": syscall.SIGWINCH,
"XCPU": syscall.SIGXCPU,
"XFSZ": syscall.SIGXFSZ,
"RTMIN": sigrtmin,
"RTMIN+1": sigrtmin + 1,
"RTMIN+2": sigrtmin + 2,
"RTMIN+3": sigrtmin + 3,
"RTMIN+4": sigrtmin + 4,
"RTMIN+5": sigrtmin + 5,
"RTMIN+6": sigrtmin + 6,
"RTMIN+7": sigrtmin + 7,
"RTMIN+8": sigrtmin + 8,
"RTMIN+9": sigrtmin + 9,
"RTMIN+10": sigrtmin + 10,
"RTMIN+11": sigrtmin + 11,
"RTMIN+12": sigrtmin + 12,
"RTMIN+13": sigrtmin + 13,
"RTMIN+14": sigrtmin + 14,
"RTMIN+15": sigrtmin + 15,
"RTMAX-14": sigrtmax - 14,
"RTMAX-13": sigrtmax - 13,
"RTMAX-12": sigrtmax - 12,
"RTMAX-11": sigrtmax - 11,
"RTMAX-10": sigrtmax - 10,
"RTMAX-9": sigrtmax - 9,
"RTMAX-8": sigrtmax - 8,
"RTMAX-7": sigrtmax - 7,
"RTMAX-6": sigrtmax - 6,
"RTMAX-5": sigrtmax - 5,
"RTMAX-4": sigrtmax - 4,
"RTMAX-3": sigrtmax - 3,
"RTMAX-2": sigrtmax - 2,
"RTMAX-1": sigrtmax - 1,
"RTMAX": sigrtmax,
}

42
signal/signal_solaris.go
View file

@ -1,42 +0,0 @@
package signal
import (
"syscall"
)
// SignalMap is a map of Solaris signals.
// SIGINFO and SIGTHR not defined for Solaris
var SignalMap = map[string]syscall.Signal{
"ABRT": syscall.SIGABRT,
"ALRM": syscall.SIGALRM,
"BUF": syscall.SIGBUS,
"CHLD": syscall.SIGCHLD,
"CONT": syscall.SIGCONT,
"EMT": syscall.SIGEMT,
"FPE": syscall.SIGFPE,
"HUP": syscall.SIGHUP,
"ILL": syscall.SIGILL,
"INT": syscall.SIGINT,
"IO": syscall.SIGIO,
"IOT": syscall.SIGIOT,
"KILL": syscall.SIGKILL,
"LWP": syscall.SIGLWP,
"PIPE": syscall.SIGPIPE,
"PROF": syscall.SIGPROF,
"QUIT": syscall.SIGQUIT,
"SEGV": syscall.SIGSEGV,
"STOP": syscall.SIGSTOP,
"SYS": syscall.SIGSYS,
"TERM": syscall.SIGTERM,
"TRAP": syscall.SIGTRAP,
"TSTP": syscall.SIGTSTP,
"TTIN": syscall.SIGTTIN,
"TTOU": syscall.SIGTTOU,
"URG": syscall.SIGURG,
"USR1": syscall.SIGUSR1,
"USR2": syscall.SIGUSR2,
"VTALRM": syscall.SIGVTALRM,
"WINCH": syscall.SIGWINCH,
"XCPU": syscall.SIGXCPU,
"XFSZ": syscall.SIGXFSZ,
}

21
signal/signal_unix.go
View file

@ -1,21 +0,0 @@
// +build !windows
package signal
import (
"syscall"
)
// Signals used in api/client (no windows equivalent, use
// invalid signals so they don't get handled)
const (
// SIGCHLD is a signal sent to a process when a child process terminates, is interrupted, or resumes after being interrupted.
SIGCHLD = syscall.SIGCHLD
// SIGWINCH is a signal sent to a process when its controlling terminal changes its size
SIGWINCH = syscall.SIGWINCH
// SIGPIPE is a signal sent to a process when a pipe is written to before the other end is open for reading
SIGPIPE = syscall.SIGPIPE
// DefaultStopSignal is the syscall signal used to stop a container in unix systems.
DefaultStopSignal = "SIGTERM"
)

10
signal/signal_unsupported.go
View file

@ -1,10 +0,0 @@
// +build !linux,!darwin,!freebsd,!windows,!solaris
package signal
import (
"syscall"
)
// SignalMap is an empty map of signals for unsupported platform.
var SignalMap = map[string]syscall.Signal{}

28
signal/signal_windows.go
View file

@ -1,28 +0,0 @@
// +build windows
package signal
import (
"syscall"
)
// Signals used in api/client (no windows equivalent, use
// invalid signals so they don't get handled)
const (
SIGCHLD = syscall.Signal(0xff)
SIGWINCH = syscall.Signal(0xff)
SIGPIPE = syscall.Signal(0xff)
// DefaultStopSignal is the syscall signal used to stop a container in windows systems.
DefaultStopSignal = "15"
)
// SignalMap is a map of "supported" signals. As per the comment in GOLang's
// ztypes_windows.go: "More invented values for signals". Windows doesn't
// really support signals in any way, shape or form that Unix does.
//
// We have these so that docker kill can be used to gracefully (TERM) and
// forcibly (KILL) terminate a container on Windows.
var SignalMap = map[string]syscall.Signal{
"KILL": syscall.SIGKILL,
"TERM": syscall.SIGTERM,
}

81
signal/trap.go
View file

@ -1,81 +0,0 @@
package signal
import (
"os"
gosignal "os/signal"
"runtime"
"sync/atomic"
"syscall"
"github.com/Sirupsen/logrus"
)
// Trap sets up a simplified signal "trap", appropriate for common
// behavior expected from a vanilla unix command-line tool in general
// (and the Docker engine in particular).
//
// * If SIGINT or SIGTERM are received, `cleanup` is called, then the process is terminated.
// * If SIGINT or SIGTERM are received 3 times before cleanup is complete, then cleanup is
// skipped and the process is terminated immediately (allows force quit of stuck daemon)
// * A SIGQUIT always causes an exit without cleanup, with a goroutine dump preceding exit.
// * Ignore SIGPIPE events. These are generated by systemd when journald is restarted while
// the docker daemon is not restarted and also running under systemd.
// Fixes https://github.com/docker/docker/issues/19728
//
func Trap(cleanup func()) {
c := make(chan os.Signal, 1)
// we will handle INT, TERM, QUIT, SIGPIPE here
signals := []os.Signal{os.Interrupt, syscall.SIGTERM, syscall.SIGQUIT, syscall.SIGPIPE}
gosignal.Notify(c, signals...)
go func() {
interruptCount := uint32(0)
for sig := range c {
if sig == syscall.SIGPIPE {
continue
}
go func(sig os.Signal) {
logrus.Infof("Processing signal '%v'", sig)
switch sig {
case os.Interrupt, syscall.SIGTERM:
if atomic.LoadUint32(&interruptCount) < 3 {
// Initiate the cleanup only once
if atomic.AddUint32(&interruptCount, 1) == 1 {
// Call the provided cleanup handler
cleanup()
os.Exit(0)
} else {
return
}
} else {
// 3 SIGTERM/INT signals received; force exit without cleanup
logrus.Info("Forcing docker daemon shutdown without cleanup; 3 interrupts received")
}
case syscall.SIGQUIT:
DumpStacks()
logrus.Info("Forcing docker daemon shutdown without cleanup on SIGQUIT")
}
//for the SIGINT/TERM, and SIGQUIT non-clean shutdown case, exit with 128 + signal #
os.Exit(128 + int(sig.(syscall.Signal)))
}(sig)
}
}()
}
// DumpStacks dumps the runtime stack.
func DumpStacks() {
var (
buf []byte
stackSize int
)
bufferLen := 16384
for stackSize == len(buf) {
buf = make([]byte, bufferLen)
stackSize = runtime.Stack(buf, true)
bufferLen *= 2
}
buf = buf[:stackSize]
// Note that if the daemon is started with a less-verbose log-level than "info" (the default), the goroutine
// traces won't show up in the log.
logrus.Infof("=== BEGIN goroutine stack dump ===\n%s\n=== END goroutine stack dump ===", buf)
}

185
stdcopy/stdcopy.go
View file

@ -1,185 +0,0 @@
package stdcopy
import (
"bytes"
"encoding/binary"
"errors"
"fmt"
"io"
"sync"
"github.com/Sirupsen/logrus"
)
// StdType is the type of standard stream
// a writer can multiplex to.
type StdType byte
const (
// Stdin represents standard input stream type.
Stdin StdType = iota
// Stdout represents standard output stream type.
Stdout
// Stderr represents standard error steam type.
Stderr
stdWriterPrefixLen = 8
stdWriterFdIndex = 0
stdWriterSizeIndex = 4
startingBufLen = 32*1024 + stdWriterPrefixLen + 1
)
var bufPool = &sync.Pool{New: func() interface{} { return bytes.NewBuffer(nil) }}
// stdWriter is wrapper of io.Writer with extra customized info.
type stdWriter struct {
io.Writer
prefix byte
}
// Write sends the buffer to the underneath writer.
// It inserts the prefix header before the buffer,
// so stdcopy.StdCopy knows where to multiplex the output.
// It makes stdWriter to implement io.Writer.
func (w *stdWriter) Write(p []byte) (n int, err error) {
if w == nil || w.Writer == nil {
return 0, errors.New("Writer not instantiated")
}
if p == nil {
return 0, nil
}
header := [stdWriterPrefixLen]byte{stdWriterFdIndex: w.prefix}
binary.BigEndian.PutUint32(header[stdWriterSizeIndex:], uint32(len(p)))
buf := bufPool.Get().(*bytes.Buffer)
buf.Write(header[:])
buf.Write(p)
n, err = w.Writer.Write(buf.Bytes())
n -= stdWriterPrefixLen
if n < 0 {
n = 0
}
buf.Reset()
bufPool.Put(buf)
return
}
// NewStdWriter instantiates a new Writer.
// Everything written to it will be encapsulated using a custom format,
// and written to the underlying `w` stream.
// This allows multiple write streams (e.g. stdout and stderr) to be muxed into a single connection.
// `t` indicates the id of the stream to encapsulate.
// It can be stdcopy.Stdin, stdcopy.Stdout, stdcopy.Stderr.
func NewStdWriter(w io.Writer, t StdType) io.Writer {
return &stdWriter{
Writer: w,
prefix: byte(t),
}
}
// StdCopy is a modified version of io.Copy.
//
// StdCopy will demultiplex `src`, assuming that it contains two streams,
// previously multiplexed together using a StdWriter instance.
// As it reads from `src`, StdCopy will write to `dstout` and `dsterr`.
//
// StdCopy will read until it hits EOF on `src`. It will then return a nil error.
// In other words: if `err` is non nil, it indicates a real underlying error.
//
// `written` will hold the total number of bytes written to `dstout` and `dsterr`.
func StdCopy(dstout, dsterr io.Writer, src io.Reader) (written int64, err error) {
var (
buf = make([]byte, startingBufLen)
bufLen = len(buf)
nr, nw int
er, ew error
out io.Writer
frameSize int
)
for {
// Make sure we have at least a full header
for nr < stdWriterPrefixLen {
var nr2 int
nr2, er = src.Read(buf[nr:])
nr += nr2
if er == io.EOF {
if nr < stdWriterPrefixLen {
logrus.Debugf("Corrupted prefix: %v", buf[:nr])
return written, nil
}
break
}
if er != nil {
logrus.Debugf("Error reading header: %s", er)
return 0, er
}
}
// Check the first byte to know where to write
switch StdType(buf[stdWriterFdIndex]) {
case Stdin:
fallthrough
case Stdout:
// Write on stdout
out = dstout
case Stderr:
// Write on stderr
out = dsterr
default:
logrus.Debugf("Error selecting output fd: (%d)", buf[stdWriterFdIndex])
return 0, fmt.Errorf("Unrecognized input header: %d", buf[stdWriterFdIndex])
}
// Retrieve the size of the frame
frameSize = int(binary.BigEndian.Uint32(buf[stdWriterSizeIndex : stdWriterSizeIndex+4]))
logrus.Debugf("framesize: %d", frameSize)
// Check if the buffer is big enough to read the frame.
// Extend it if necessary.
if frameSize+stdWriterPrefixLen > bufLen {
logrus.Debugf("Extending buffer cap by %d (was %d)", frameSize+stdWriterPrefixLen-bufLen+1, len(buf))
buf = append(buf, make([]byte, frameSize+stdWriterPrefixLen-bufLen+1)...)
bufLen = len(buf)
}
// While the amount of bytes read is less than the size of the frame + header, we keep reading
for nr < frameSize+stdWriterPrefixLen {
var nr2 int
nr2, er = src.Read(buf[nr:])
nr += nr2
if er == io.EOF {
if nr < frameSize+stdWriterPrefixLen {
logrus.Debugf("Corrupted frame: %v", buf[stdWriterPrefixLen:nr])
return written, nil
}
break
}
if er != nil {
logrus.Debugf("Error reading frame: %s", er)
return 0, er
}
}
// Write the retrieved frame (without header)
nw, ew = out.Write(buf[stdWriterPrefixLen : frameSize+stdWriterPrefixLen])
if ew != nil {
logrus.Debugf("Error writing frame: %s", ew)
return 0, ew
}
// If the frame has not been fully written: error
if nw != frameSize {
logrus.Debugf("Error Short Write: (%d on %d)", nw, frameSize)
return 0, io.ErrShortWrite
}
written += int64(nw)
// Move the rest of the buffer to the beginning
copy(buf, buf[frameSize+stdWriterPrefixLen:])
// Move the index
nr -= frameSize + stdWriterPrefixLen
}
}

260
stdcopy/stdcopy_test.go
View file

@ -1,260 +0,0 @@
package stdcopy
import (
"bytes"
"errors"
"io"
"io/ioutil"
"strings"
"testing"
)
func TestNewStdWriter(t *testing.T) {
writer := NewStdWriter(ioutil.Discard, Stdout)
if writer == nil {
t.Fatalf("NewStdWriter with an invalid StdType should not return nil.")
}
}
func TestWriteWithUnitializedStdWriter(t *testing.T) {
writer := stdWriter{
Writer: nil,
prefix: byte(Stdout),
}
n, err := writer.Write([]byte("Something here"))
if n != 0 || err == nil {
t.Fatalf("Should fail when given an uncomplete or uninitialized StdWriter")
}
}
func TestWriteWithNilBytes(t *testing.T) {
writer := NewStdWriter(ioutil.Discard, Stdout)
n, err := writer.Write(nil)
if err != nil {
t.Fatalf("Shouldn't have fail when given no data")
}
if n > 0 {
t.Fatalf("Write should have written 0 byte, but has written %d", n)
}
}
func TestWrite(t *testing.T) {
writer := NewStdWriter(ioutil.Discard, Stdout)
data := []byte("Test StdWrite.Write")
n, err := writer.Write(data)
if err != nil {
t.Fatalf("Error while writing with StdWrite")
}
if n != len(data) {
t.Fatalf("Write should have written %d byte but wrote %d.", len(data), n)
}
}
type errWriter struct {
n int
err error
}
func (f *errWriter) Write(buf []byte) (int, error) {
return f.n, f.err
}
func TestWriteWithWriterError(t *testing.T) {
expectedError := errors.New("expected")
expectedReturnedBytes := 10
writer := NewStdWriter(&errWriter{
n: stdWriterPrefixLen + expectedReturnedBytes,
err: expectedError}, Stdout)
data := []byte("This won't get written, sigh")
n, err := writer.Write(data)
if err != expectedError {
t.Fatalf("Didn't get expected error.")
}
if n != expectedReturnedBytes {
t.Fatalf("Didn't get expected written bytes %d, got %d.",
expectedReturnedBytes, n)
}
}
func TestWriteDoesNotReturnNegativeWrittenBytes(t *testing.T) {
writer := NewStdWriter(&errWriter{n: -1}, Stdout)
data := []byte("This won't get written, sigh")
actual, _ := writer.Write(data)
if actual != 0 {
t.Fatalf("Expected returned written bytes equal to 0, got %d", actual)
}
}
func getSrcBuffer(stdOutBytes, stdErrBytes []byte) (buffer *bytes.Buffer, err error) {
buffer = new(bytes.Buffer)
dstOut := NewStdWriter(buffer, Stdout)
_, err = dstOut.Write(stdOutBytes)
if err != nil {
return
}
dstErr := NewStdWriter(buffer, Stderr)
_, err = dstErr.Write(stdErrBytes)
return
}
func TestStdCopyWriteAndRead(t *testing.T) {
stdOutBytes := []byte(strings.Repeat("o", startingBufLen))
stdErrBytes := []byte(strings.Repeat("e", startingBufLen))
buffer, err := getSrcBuffer(stdOutBytes, stdErrBytes)
if err != nil {
t.Fatal(err)
}
written, err := StdCopy(ioutil.Discard, ioutil.Discard, buffer)
if err != nil {
t.Fatal(err)
}
expectedTotalWritten := len(stdOutBytes) + len(stdErrBytes)
if written != int64(expectedTotalWritten) {
t.Fatalf("Expected to have total of %d bytes written, got %d", expectedTotalWritten, written)
}
}
type customReader struct {
n int
err error
totalCalls int
correctCalls int
src *bytes.Buffer
}
func (f *customReader) Read(buf []byte) (int, error) {
f.totalCalls++
if f.totalCalls <= f.correctCalls {
return f.src.Read(buf)
}
return f.n, f.err
}
func TestStdCopyReturnsErrorReadingHeader(t *testing.T) {
expectedError := errors.New("error")
reader := &customReader{
err: expectedError}
written, err := StdCopy(ioutil.Discard, ioutil.Discard, reader)
if written != 0 {
t.Fatalf("Expected 0 bytes read, got %d", written)
}
if err != expectedError {
t.Fatalf("Didn't get expected error")
}
}
func TestStdCopyReturnsErrorReadingFrame(t *testing.T) {
expectedError := errors.New("error")
stdOutBytes := []byte(strings.Repeat("o", startingBufLen))
stdErrBytes := []byte(strings.Repeat("e", startingBufLen))
buffer, err := getSrcBuffer(stdOutBytes, stdErrBytes)
if err != nil {
t.Fatal(err)
}
reader := &customReader{
correctCalls: 1,
n: stdWriterPrefixLen + 1,
err: expectedError,
src: buffer}
written, err := StdCopy(ioutil.Discard, ioutil.Discard, reader)
if written != 0 {
t.Fatalf("Expected 0 bytes read, got %d", written)
}
if err != expectedError {
t.Fatalf("Didn't get expected error")
}
}
func TestStdCopyDetectsCorruptedFrame(t *testing.T) {
stdOutBytes := []byte(strings.Repeat("o", startingBufLen))
stdErrBytes := []byte(strings.Repeat("e", startingBufLen))
buffer, err := getSrcBuffer(stdOutBytes, stdErrBytes)
if err != nil {
t.Fatal(err)
}
reader := &customReader{
correctCalls: 1,
n: stdWriterPrefixLen + 1,
err: io.EOF,
src: buffer}
written, err := StdCopy(ioutil.Discard, ioutil.Discard, reader)
if written != startingBufLen {
t.Fatalf("Expected %d bytes read, got %d", startingBufLen, written)
}
if err != nil {
t.Fatal("Didn't get nil error")
}
}
func TestStdCopyWithInvalidInputHeader(t *testing.T) {
dstOut := NewStdWriter(ioutil.Discard, Stdout)
dstErr := NewStdWriter(ioutil.Discard, Stderr)
src := strings.NewReader("Invalid input")
_, err := StdCopy(dstOut, dstErr, src)
if err == nil {
t.Fatal("StdCopy with invalid input header should fail.")
}
}
func TestStdCopyWithCorruptedPrefix(t *testing.T) {
data := []byte{0x01, 0x02, 0x03}
src := bytes.NewReader(data)
written, err := StdCopy(nil, nil, src)
if err != nil {
t.Fatalf("StdCopy should not return an error with corrupted prefix.")
}
if written != 0 {
t.Fatalf("StdCopy should have written 0, but has written %d", written)
}
}
func TestStdCopyReturnsWriteErrors(t *testing.T) {
stdOutBytes := []byte(strings.Repeat("o", startingBufLen))
stdErrBytes := []byte(strings.Repeat("e", startingBufLen))
buffer, err := getSrcBuffer(stdOutBytes, stdErrBytes)
if err != nil {
t.Fatal(err)
}
expectedError := errors.New("expected")
dstOut := &errWriter{err: expectedError}
written, err := StdCopy(dstOut, ioutil.Discard, buffer)
if written != 0 {
t.Fatalf("StdCopy should have written 0, but has written %d", written)
}
if err != expectedError {
t.Fatalf("Didn't get expected error, got %v", err)
}
}
func TestStdCopyDetectsNotFullyWrittenFrames(t *testing.T) {
stdOutBytes := []byte(strings.Repeat("o", startingBufLen))
stdErrBytes := []byte(strings.Repeat("e", startingBufLen))
buffer, err := getSrcBuffer(stdOutBytes, stdErrBytes)
if err != nil {
t.Fatal(err)
}
dstOut := &errWriter{n: startingBufLen - 10}
written, err := StdCopy(dstOut, ioutil.Discard, buffer)
if written != 0 {
t.Fatalf("StdCopy should have return 0 written bytes, but returned %d", written)
}
if err != io.ErrShortWrite {
t.Fatalf("Didn't get expected io.ErrShortWrite error")
}
}
func BenchmarkWrite(b *testing.B) {
w := NewStdWriter(ioutil.Discard, Stdout)
data := []byte("Test line for testing stdwriter performance\n")
data = bytes.Repeat(data, 100)
b.SetBytes(int64(len(data)))
b.ResetTimer()
for i := 0; i < b.N; i++ {
if _, err := w.Write(data); err != nil {
b.Fatal(err)
}
}
}

172
streamformatter/streamformatter.go
View file

@ -1,172 +0,0 @@
// Package streamformatter provides helper functions to format a stream.
package streamformatter
import (
"encoding/json"
"fmt"
"io"
"github.com/docker/docker/pkg/jsonmessage"
"github.com/docker/docker/pkg/progress"
)
// StreamFormatter formats a stream, optionally using JSON.
type StreamFormatter struct {
json bool
}
// NewStreamFormatter returns a simple StreamFormatter
func NewStreamFormatter() *StreamFormatter {
return &StreamFormatter{}
}
// NewJSONStreamFormatter returns a StreamFormatter configured to stream json
func NewJSONStreamFormatter() *StreamFormatter {
return &StreamFormatter{true}
}
const streamNewline = "\r\n"
var streamNewlineBytes = []byte(streamNewline)
// FormatStream formats the specified stream.
func (sf *StreamFormatter) FormatStream(str string) []byte {
if sf.json {
b, err := json.Marshal(&jsonmessage.JSONMessage{Stream: str})
if err != nil {
return sf.FormatError(err)
}
return append(b, streamNewlineBytes...)
}
return []byte(str + "\r")
}
// FormatStatus formats the specified objects according to the specified format (and id).
func (sf *StreamFormatter) FormatStatus(id, format string, a ...interface{}) []byte {
str := fmt.Sprintf(format, a...)
if sf.json {
b, err := json.Marshal(&jsonmessage.JSONMessage{ID: id, Status: str})
if err != nil {
return sf.FormatError(err)
}
return append(b, streamNewlineBytes...)
}
return []byte(str + streamNewline)
}
// FormatError formats the specified error.
func (sf *StreamFormatter) FormatError(err error) []byte {
if sf.json {
jsonError, ok := err.(*jsonmessage.JSONError)
if !ok {
jsonError = &jsonmessage.JSONError{Message: err.Error()}
}
if b, err := json.Marshal(&jsonmessage.JSONMessage{Error: jsonError, ErrorMessage: err.Error()}); err == nil {
return append(b, streamNewlineBytes...)
}
return []byte("{\"error\":\"format error\"}" + streamNewline)
}
return []byte("Error: " + err.Error() + streamNewline)
}
// FormatProgress formats the progress information for a specified action.
func (sf *StreamFormatter) FormatProgress(id, action string, progress *jsonmessage.JSONProgress, aux interface{}) []byte {
if progress == nil {
progress = &jsonmessage.JSONProgress{}
}
if sf.json {
var auxJSON *json.RawMessage
if aux != nil {
auxJSONBytes, err := json.Marshal(aux)
if err != nil {
return nil
}
auxJSON = new(json.RawMessage)
*auxJSON = auxJSONBytes
}
b, err := json.Marshal(&jsonmessage.JSONMessage{
Status: action,
ProgressMessage: progress.String(),
Progress: progress,
ID: id,
Aux: auxJSON,
})
if err != nil {
return nil
}
return append(b, streamNewlineBytes...)
}
endl := "\r"
if progress.String() == "" {
endl += "\n"
}
return []byte(action + " " + progress.String() + endl)
}
// NewProgressOutput returns a progress.Output object that can be passed to
// progress.NewProgressReader.
func (sf *StreamFormatter) NewProgressOutput(out io.Writer, newLines bool) progress.Output {
return &progressOutput{
sf: sf,
out: out,
newLines: newLines,
}
}
type progressOutput struct {
sf *StreamFormatter
out io.Writer
newLines bool
}
// WriteProgress formats progress information from a ProgressReader.
func (out *progressOutput) WriteProgress(prog progress.Progress) error {
var formatted []byte
if prog.Message != "" {
formatted = out.sf.FormatStatus(prog.ID, prog.Message)
} else {
jsonProgress := jsonmessage.JSONProgress{Current: prog.Current, Total: prog.Total}
formatted = out.sf.FormatProgress(prog.ID, prog.Action, &jsonProgress, prog.Aux)
}
_, err := out.out.Write(formatted)
if err != nil {
return err
}
if out.newLines && prog.LastUpdate {
_, err = out.out.Write(out.sf.FormatStatus("", ""))
return err
}
return nil
}
// StdoutFormatter is a streamFormatter that writes to the standard output.
type StdoutFormatter struct {
io.Writer
*StreamFormatter
}
func (sf *StdoutFormatter) Write(buf []byte) (int, error) {
formattedBuf := sf.StreamFormatter.FormatStream(string(buf))
n, err := sf.Writer.Write(formattedBuf)
if n != len(formattedBuf) {
return n, io.ErrShortWrite
}
return len(buf), err
}
// StderrFormatter is a streamFormatter that writes to the standard error.
type StderrFormatter struct {
io.Writer
*StreamFormatter
}
func (sf *StderrFormatter) Write(buf []byte) (int, error) {
formattedBuf := sf.StreamFormatter.FormatStream("\033[91m" + string(buf) + "\033[0m")
n, err := sf.Writer.Write(formattedBuf)
if n != len(formattedBuf) {
return n, io.ErrShortWrite
}
return len(buf), err
}

108
streamformatter/streamformatter_test.go
View file

@ -1,108 +0,0 @@
package streamformatter
import (
"encoding/json"
"errors"
"reflect"
"strings"
"testing"
"github.com/docker/docker/pkg/jsonmessage"
)
func TestFormatStream(t *testing.T) {
sf := NewStreamFormatter()
res := sf.FormatStream("stream")
if string(res) != "stream"+"\r" {
t.Fatalf("%q", res)
}
}
func TestFormatJSONStatus(t *testing.T) {
sf := NewStreamFormatter()
res := sf.FormatStatus("ID", "%s%d", "a", 1)
if string(res) != "a1\r\n" {
t.Fatalf("%q", res)
}
}
func TestFormatSimpleError(t *testing.T) {
sf := NewStreamFormatter()
res := sf.FormatError(errors.New("Error for formatter"))
if string(res) != "Error: Error for formatter\r\n" {
t.Fatalf("%q", res)
}
}
func TestJSONFormatStream(t *testing.T) {
sf := NewJSONStreamFormatter()
res := sf.FormatStream("stream")
if string(res) != `{"stream":"stream"}`+"\r\n" {
t.Fatalf("%q", res)
}
}
func TestJSONFormatStatus(t *testing.T) {
sf := NewJSONStreamFormatter()
res := sf.FormatStatus("ID", "%s%d", "a", 1)
if string(res) != `{"status":"a1","id":"ID"}`+"\r\n" {
t.Fatalf("%q", res)
}
}
func TestJSONFormatSimpleError(t *testing.T) {
sf := NewJSONStreamFormatter()
res := sf.FormatError(errors.New("Error for formatter"))
if string(res) != `{"errorDetail":{"message":"Error for formatter"},"error":"Error for formatter"}`+"\r\n" {
t.Fatalf("%q", res)
}
}
func TestJSONFormatJSONError(t *testing.T) {
sf := NewJSONStreamFormatter()
err := &jsonmessage.JSONError{Code: 50, Message: "Json error"}
res := sf.FormatError(err)
if string(res) != `{"errorDetail":{"code":50,"message":"Json error"},"error":"Json error"}`+"\r\n" {
t.Fatalf("%q", res)
}
}
func TestJSONFormatProgress(t *testing.T) {
sf := NewJSONStreamFormatter()
progress := &jsonmessage.JSONProgress{
Current: 15,
Total: 30,
Start: 1,
}
res := sf.FormatProgress("id", "action", progress, nil)
msg := &jsonmessage.JSONMessage{}
if err := json.Unmarshal(res, msg); err != nil {
t.Fatal(err)
}
if msg.ID != "id" {
t.Fatalf("ID must be 'id', got: %s", msg.ID)
}
if msg.Status != "action" {
t.Fatalf("Status must be 'action', got: %s", msg.Status)
}
// The progress will always be in the format of:
// [=========================> ] 15 B/30 B 404933h7m11s
// The last entry '404933h7m11s' is the timeLeftBox.
// However, the timeLeftBox field may change as progress.String() depends on time.Now().
// Therefore, we have to strip the timeLeftBox from the strings to do the comparison.
// Compare the progress strings before the timeLeftBox
expectedProgress := "[=========================> ] 15 B/30 B"
// if terminal column is <= 110, expectedProgressShort is expected.
expectedProgressShort := " 15 B/30 B"
if !(strings.HasPrefix(msg.ProgressMessage, expectedProgress) ||
strings.HasPrefix(msg.ProgressMessage, expectedProgressShort)) {
t.Fatalf("ProgressMessage without the timeLeftBox must be %s or %s, got: %s",
expectedProgress, expectedProgressShort, msg.ProgressMessage)
}
if !reflect.DeepEqual(msg.Progress, progress) {
t.Fatal("Original progress not equals progress from FormatProgress")
}
}

191
symlink/LICENSE.APACHE
View file

@ -1,191 +0,0 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
Copyright 2014-2016 Docker, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

27
symlink/LICENSE.BSD
View file

@ -1,27 +0,0 @@
Copyright (c) 2014-2016 The Docker & Go Authors. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

6
symlink/README.md
View file

@ -1,6 +0,0 @@
Package symlink implements EvalSymlinksInScope which is an extension of filepath.EvalSymlinks,
as well as a Windows long-path aware version of filepath.EvalSymlinks
from the [Go standard library](https://golang.org/pkg/path/filepath).
The code from filepath.EvalSymlinks has been adapted in fs.go.
Please read the LICENSE.BSD file that governs fs.go and LICENSE.APACHE for fs_test.go.

143
symlink/fs.go
View file

@ -1,143 +0,0 @@
// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE.BSD file.
// This code is a modified version of path/filepath/symlink.go from the Go standard library.
package symlink
import (
"bytes"
"errors"
"os"
"path/filepath"
"strings"
"github.com/docker/docker/pkg/system"
)
// FollowSymlinkInScope is a wrapper around evalSymlinksInScope that returns an
// absolute path. This function handles paths in a platform-agnostic manner.
func FollowSymlinkInScope(path, root string) (string, error) {
path, err := filepath.Abs(filepath.FromSlash(path))
if err != nil {
return "", err
}
root, err = filepath.Abs(filepath.FromSlash(root))
if err != nil {
return "", err
}
return evalSymlinksInScope(path, root)
}
// evalSymlinksInScope will evaluate symlinks in `path` within a scope `root` and return
// a result guaranteed to be contained within the scope `root`, at the time of the call.
// Symlinks in `root` are not evaluated and left as-is.
// Errors encountered while attempting to evaluate symlinks in path will be returned.
// Non-existing paths are valid and do not constitute an error.
// `path` has to contain `root` as a prefix, or else an error will be returned.
// Trying to break out from `root` does not constitute an error.
//
// Example:
// If /foo/bar -> /outside,
// FollowSymlinkInScope("/foo/bar", "/foo") == "/foo/outside" instead of "/oustide"
//
// IMPORTANT: it is the caller's responsibility to call evalSymlinksInScope *after* relevant symlinks
// are created and not to create subsequently, additional symlinks that could potentially make a
// previously-safe path, unsafe. Example: if /foo/bar does not exist, evalSymlinksInScope("/foo/bar", "/foo")
// would return "/foo/bar". If one makes /foo/bar a symlink to /baz subsequently, then "/foo/bar" should
// no longer be considered safely contained in "/foo".
func evalSymlinksInScope(path, root string) (string, error) {
root = filepath.Clean(root)
if path == root {
return path, nil
}
if !strings.HasPrefix(path, root) {
return "", errors.New("evalSymlinksInScope: " + path + " is not in " + root)
}
const maxIter = 255
originalPath := path
// given root of "/a" and path of "/a/b/../../c" we want path to be "/b/../../c"
path = path[len(root):]
if root == string(filepath.Separator) {
path = string(filepath.Separator) + path
}
if !strings.HasPrefix(path, string(filepath.Separator)) {
return "", errors.New("evalSymlinksInScope: " + path + " is not in " + root)
}
path = filepath.Clean(path)
// consume path by taking each frontmost path element,
// expanding it if it's a symlink, and appending it to b
var b bytes.Buffer
// b here will always be considered to be the "current absolute path inside
// root" when we append paths to it, we also append a slash and use
// filepath.Clean after the loop to trim the trailing slash
for n := 0; path != ""; n++ {
if n > maxIter {
return "", errors.New("evalSymlinksInScope: too many links in " + originalPath)
}
// find next path component, p
i := strings.IndexRune(path, filepath.Separator)
var p string
if i == -1 {
p, path = path, ""
} else {
p, path = path[:i], path[i+1:]
}
if p == "" {
continue
}
// this takes a b.String() like "b/../" and a p like "c" and turns it
// into "/b/../c" which then gets filepath.Cleaned into "/c" and then
// root gets prepended and we Clean again (to remove any trailing slash
// if the first Clean gave us just "/")
cleanP := filepath.Clean(string(filepath.Separator) + b.String() + p)
if cleanP == string(filepath.Separator) {
// never Lstat "/" itself
b.Reset()
continue
}
fullP := filepath.Clean(root + cleanP)
fi, err := os.Lstat(fullP)
if os.IsNotExist(err) {
// if p does not exist, accept it
b.WriteString(p)
b.WriteRune(filepath.Separator)
continue
}
if err != nil {
return "", err
}
if fi.Mode()&os.ModeSymlink == 0 {
b.WriteString(p + string(filepath.Separator))
continue
}
// it's a symlink, put it at the front of path
dest, err := os.Readlink(fullP)
if err != nil {
return "", err
}
if system.IsAbs(dest) {
b.Reset()
}
path = dest + string(filepath.Separator) + path
}
// see note above on "fullP := ..." for why this is double-cleaned and
// what's happening here
return filepath.Clean(root + filepath.Clean(string(filepath.Separator)+b.String())), nil
}
// EvalSymlinks returns the path name after the evaluation of any symbolic
// links.
// If path is relative the result will be relative to the current directory,
// unless one of the components is an absolute symbolic link.
// This version has been updated to support long paths prepended with `\\?\`.
func EvalSymlinks(path string) (string, error) {
return evalSymlinks(path)
}

11
symlink/fs_unix.go
View file

@ -1,11 +0,0 @@
// +build !windows
package symlink
import (
"path/filepath"
)
func evalSymlinks(path string) (string, error) {
return filepath.EvalSymlinks(path)
}

407
symlink/fs_unix_test.go
View file

@ -1,407 +0,0 @@
// +build !windows
// Licensed under the Apache License, Version 2.0; See LICENSE.APACHE
package symlink
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"testing"
)
// TODO Windows: This needs some serious work to port to Windows. For now,
// turning off testing in this package.
type dirOrLink struct {
path string
target string
}
func makeFs(tmpdir string, fs []dirOrLink) error {
for _, s := range fs {
s.path = filepath.Join(tmpdir, s.path)
if s.target == "" {
os.MkdirAll(s.path, 0755)
continue
}
if err := os.MkdirAll(filepath.Dir(s.path), 0755); err != nil {
return err
}
if err := os.Symlink(s.target, s.path); err != nil && !os.IsExist(err) {
return err
}
}
return nil
}
func testSymlink(tmpdir, path, expected, scope string) error {
rewrite, err := FollowSymlinkInScope(filepath.Join(tmpdir, path), filepath.Join(tmpdir, scope))
if err != nil {
return err
}
expected, err = filepath.Abs(filepath.Join(tmpdir, expected))
if err != nil {
return err
}
if expected != rewrite {
return fmt.Errorf("Expected %q got %q", expected, rewrite)
}
return nil
}
func TestFollowSymlinkAbsolute(t *testing.T) {
tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkAbsolute")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpdir)
if err := makeFs(tmpdir, []dirOrLink{{path: "testdata/fs/a/d", target: "/b"}}); err != nil {
t.Fatal(err)
}
if err := testSymlink(tmpdir, "testdata/fs/a/d/c/data", "testdata/b/c/data", "testdata"); err != nil {
t.Fatal(err)
}
}
func TestFollowSymlinkRelativePath(t *testing.T) {
tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkRelativePath")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpdir)
if err := makeFs(tmpdir, []dirOrLink{{path: "testdata/fs/i", target: "a"}}); err != nil {
t.Fatal(err)
}
if err := testSymlink(tmpdir, "testdata/fs/i", "testdata/fs/a", "testdata"); err != nil {
t.Fatal(err)
}
}
func TestFollowSymlinkSkipSymlinksOutsideScope(t *testing.T) {
tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkSkipSymlinksOutsideScope")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpdir)
if err := makeFs(tmpdir, []dirOrLink{
{path: "linkdir", target: "realdir"},
{path: "linkdir/foo/bar"},
}); err != nil {
t.Fatal(err)
}
if err := testSymlink(tmpdir, "linkdir/foo/bar", "linkdir/foo/bar", "linkdir/foo"); err != nil {
t.Fatal(err)
}
}
func TestFollowSymlinkInvalidScopePathPair(t *testing.T) {
if _, err := FollowSymlinkInScope("toto", "testdata"); err == nil {
t.Fatal("expected an error")
}
}
func TestFollowSymlinkLastLink(t *testing.T) {
tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkLastLink")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpdir)
if err := makeFs(tmpdir, []dirOrLink{{path: "testdata/fs/a/d", target: "/b"}}); err != nil {
t.Fatal(err)
}
if err := testSymlink(tmpdir, "testdata/fs/a/d", "testdata/b", "testdata"); err != nil {
t.Fatal(err)
}
}
func TestFollowSymlinkRelativeLinkChangeScope(t *testing.T) {
tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkRelativeLinkChangeScope")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpdir)
if err := makeFs(tmpdir, []dirOrLink{{path: "testdata/fs/a/e", target: "../b"}}); err != nil {
t.Fatal(err)
}
if err := testSymlink(tmpdir, "testdata/fs/a/e/c/data", "testdata/fs/b/c/data", "testdata"); err != nil {
t.Fatal(err)
}
// avoid letting allowing symlink e lead us to ../b
// normalize to the "testdata/fs/a"
if err := testSymlink(tmpdir, "testdata/fs/a/e", "testdata/fs/a/b", "testdata/fs/a"); err != nil {
t.Fatal(err)
}
}
func TestFollowSymlinkDeepRelativeLinkChangeScope(t *testing.T) {
tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkDeepRelativeLinkChangeScope")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpdir)
if err := makeFs(tmpdir, []dirOrLink{{path: "testdata/fs/a/f", target: "../../../../test"}}); err != nil {
t.Fatal(err)
}
// avoid letting symlink f lead us out of the "testdata" scope
// we don't normalize because symlink f is in scope and there is no
// information leak
if err := testSymlink(tmpdir, "testdata/fs/a/f", "testdata/test", "testdata"); err != nil {
t.Fatal(err)
}
// avoid letting symlink f lead us out of the "testdata/fs" scope
// we don't normalize because symlink f is in scope and there is no
// information leak
if err := testSymlink(tmpdir, "testdata/fs/a/f", "testdata/fs/test", "testdata/fs"); err != nil {
t.Fatal(err)
}
}
func TestFollowSymlinkRelativeLinkChain(t *testing.T) {
tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkRelativeLinkChain")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpdir)
// avoid letting symlink g (pointed at by symlink h) take out of scope
// TODO: we should probably normalize to scope here because ../[....]/root
// is out of scope and we leak information
if err := makeFs(tmpdir, []dirOrLink{
{path: "testdata/fs/b/h", target: "../g"},
{path: "testdata/fs/g", target: "../../../../../../../../../../../../root"},
}); err != nil {
t.Fatal(err)
}
if err := testSymlink(tmpdir, "testdata/fs/b/h", "testdata/root", "testdata"); err != nil {
t.Fatal(err)
}
}
func TestFollowSymlinkBreakoutPath(t *testing.T) {
tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkBreakoutPath")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpdir)
// avoid letting symlink -> ../directory/file escape from scope
// normalize to "testdata/fs/j"
if err := makeFs(tmpdir, []dirOrLink{{path: "testdata/fs/j/k", target: "../i/a"}}); err != nil {
t.Fatal(err)
}
if err := testSymlink(tmpdir, "testdata/fs/j/k", "testdata/fs/j/i/a", "testdata/fs/j"); err != nil {
t.Fatal(err)
}
}
func TestFollowSymlinkToRoot(t *testing.T) {
tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkToRoot")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpdir)
// make sure we don't allow escaping to /
// normalize to dir
if err := makeFs(tmpdir, []dirOrLink{{path: "foo", target: "/"}}); err != nil {
t.Fatal(err)
}
if err := testSymlink(tmpdir, "foo", "", ""); err != nil {
t.Fatal(err)
}
}
func TestFollowSymlinkSlashDotdot(t *testing.T) {
tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkSlashDotdot")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpdir)
tmpdir = filepath.Join(tmpdir, "dir", "subdir")
// make sure we don't allow escaping to /
// normalize to dir
if err := makeFs(tmpdir, []dirOrLink{{path: "foo", target: "/../../"}}); err != nil {
t.Fatal(err)
}
if err := testSymlink(tmpdir, "foo", "", ""); err != nil {
t.Fatal(err)
}
}
func TestFollowSymlinkDotdot(t *testing.T) {
tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkDotdot")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpdir)
tmpdir = filepath.Join(tmpdir, "dir", "subdir")
// make sure we stay in scope without leaking information
// this also checks for escaping to /
// normalize to dir
if err := makeFs(tmpdir, []dirOrLink{{path: "foo", target: "../../"}}); err != nil {
t.Fatal(err)
}
if err := testSymlink(tmpdir, "foo", "", ""); err != nil {
t.Fatal(err)
}
}
func TestFollowSymlinkRelativePath2(t *testing.T) {
tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkRelativePath2")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpdir)
if err := makeFs(tmpdir, []dirOrLink{{path: "bar/foo", target: "baz/target"}}); err != nil {
t.Fatal(err)
}
if err := testSymlink(tmpdir, "bar/foo", "bar/baz/target", ""); err != nil {
t.Fatal(err)
}
}
func TestFollowSymlinkScopeLink(t *testing.T) {
tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkScopeLink")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpdir)
if err := makeFs(tmpdir, []dirOrLink{
{path: "root2"},
{path: "root", target: "root2"},
{path: "root2/foo", target: "../bar"},
}); err != nil {
t.Fatal(err)
}
if err := testSymlink(tmpdir, "root/foo", "root/bar", "root"); err != nil {
t.Fatal(err)
}
}
func TestFollowSymlinkRootScope(t *testing.T) {
tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkRootScope")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpdir)
expected, err := filepath.EvalSymlinks(tmpdir)
if err != nil {
t.Fatal(err)
}
rewrite, err := FollowSymlinkInScope(tmpdir, "/")
if err != nil {
t.Fatal(err)
}
if rewrite != expected {
t.Fatalf("expected %q got %q", expected, rewrite)
}
}
func TestFollowSymlinkEmpty(t *testing.T) {
res, err := FollowSymlinkInScope("", "")
if err != nil {
t.Fatal(err)
}
wd, err := os.Getwd()
if err != nil {
t.Fatal(err)
}
if res != wd {
t.Fatalf("expected %q got %q", wd, res)
}
}
func TestFollowSymlinkCircular(t *testing.T) {
tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkCircular")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpdir)
if err := makeFs(tmpdir, []dirOrLink{{path: "root/foo", target: "foo"}}); err != nil {
t.Fatal(err)
}
if err := testSymlink(tmpdir, "root/foo", "", "root"); err == nil {
t.Fatal("expected an error for foo -> foo")
}
if err := makeFs(tmpdir, []dirOrLink{
{path: "root/bar", target: "baz"},
{path: "root/baz", target: "../bak"},
{path: "root/bak", target: "/bar"},
}); err != nil {
t.Fatal(err)
}
if err := testSymlink(tmpdir, "root/foo", "", "root"); err == nil {
t.Fatal("expected an error for bar -> baz -> bak -> bar")
}
}
func TestFollowSymlinkComplexChainWithTargetPathsContainingLinks(t *testing.T) {
tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkComplexChainWithTargetPathsContainingLinks")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpdir)
if err := makeFs(tmpdir, []dirOrLink{
{path: "root2"},
{path: "root", target: "root2"},
{path: "root/a", target: "r/s"},
{path: "root/r", target: "../root/t"},
{path: "root/root/t/s/b", target: "/../u"},
{path: "root/u/c", target: "."},
{path: "root/u/x/y", target: "../v"},
{path: "root/u/v", target: "/../w"},
}); err != nil {
t.Fatal(err)
}
if err := testSymlink(tmpdir, "root/a/b/c/x/y/z", "root/w/z", "root"); err != nil {
t.Fatal(err)
}
}
func TestFollowSymlinkBreakoutNonExistent(t *testing.T) {
tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkBreakoutNonExistent")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpdir)
if err := makeFs(tmpdir, []dirOrLink{
{path: "root/slash", target: "/"},
{path: "root/sym", target: "/idontexist/../slash"},
}); err != nil {
t.Fatal(err)
}
if err := testSymlink(tmpdir, "root/sym/file", "root/file", "root"); err != nil {
t.Fatal(err)
}
}
func TestFollowSymlinkNoLexicalCleaning(t *testing.T) {
tmpdir, err := ioutil.TempDir("", "TestFollowSymlinkNoLexicalCleaning")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpdir)
if err := makeFs(tmpdir, []dirOrLink{
{path: "root/sym", target: "/foo/bar"},
{path: "root/hello", target: "/sym/../baz"},
}); err != nil {
t.Fatal(err)
}
if err := testSymlink(tmpdir, "root/hello", "root/foo/baz", "root"); err != nil {
t.Fatal(err)
}
}

155
symlink/fs_windows.go
View file

@ -1,155 +0,0 @@
package symlink
import (
"bytes"
"errors"
"os"
"path/filepath"
"strings"
"syscall"
"github.com/docker/docker/pkg/longpath"
)
func toShort(path string) (string, error) {
p, err := syscall.UTF16FromString(path)
if err != nil {
return "", err
}
b := p // GetShortPathName says we can reuse buffer
n, err := syscall.GetShortPathName(&p[0], &b[0], uint32(len(b)))
if err != nil {
return "", err
}
if n > uint32(len(b)) {
b = make([]uint16, n)
if _, err = syscall.GetShortPathName(&p[0], &b[0], uint32(len(b))); err != nil {
return "", err
}
}
return syscall.UTF16ToString(b), nil
}
func toLong(path string) (string, error) {
p, err := syscall.UTF16FromString(path)
if err != nil {
return "", err
}
b := p // GetLongPathName says we can reuse buffer
n, err := syscall.GetLongPathName(&p[0], &b[0], uint32(len(b)))
if err != nil {
return "", err
}
if n > uint32(len(b)) {
b = make([]uint16, n)
n, err = syscall.GetLongPathName(&p[0], &b[0], uint32(len(b)))
if err != nil {
return "", err
}
}
b = b[:n]
return syscall.UTF16ToString(b), nil
}
func evalSymlinks(path string) (string, error) {
path, err := walkSymlinks(path)
if err != nil {
return "", err
}
p, err := toShort(path)
if err != nil {
return "", err
}
p, err = toLong(p)
if err != nil {
return "", err
}
// syscall.GetLongPathName does not change the case of the drive letter,
// but the result of EvalSymlinks must be unique, so we have
// EvalSymlinks(`c:\a`) == EvalSymlinks(`C:\a`).
// Make drive letter upper case.
if len(p) >= 2 && p[1] == ':' && 'a' <= p[0] && p[0] <= 'z' {
p = string(p[0]+'A'-'a') + p[1:]
} else if len(p) >= 6 && p[5] == ':' && 'a' <= p[4] && p[4] <= 'z' {
p = p[:3] + string(p[4]+'A'-'a') + p[5:]
}
return filepath.Clean(p), nil
}
const utf8RuneSelf = 0x80
func walkSymlinks(path string) (string, error) {
const maxIter = 255
originalPath := path
// consume path by taking each frontmost path element,
// expanding it if it's a symlink, and appending it to b
var b bytes.Buffer
for n := 0; path != ""; n++ {
if n > maxIter {
return "", errors.New("EvalSymlinks: too many links in " + originalPath)
}
// A path beginning with `\\?\` represents the root, so automatically
// skip that part and begin processing the next segment.
if strings.HasPrefix(path, longpath.Prefix) {
b.WriteString(longpath.Prefix)
path = path[4:]
continue
}
// find next path component, p
var i = -1
for j, c := range path {
if c < utf8RuneSelf && os.IsPathSeparator(uint8(c)) {
i = j
break
}
}
var p string
if i == -1 {
p, path = path, ""
} else {
p, path = path[:i], path[i+1:]
}
if p == "" {
if b.Len() == 0 {
// must be absolute path
b.WriteRune(filepath.Separator)
}
continue
}
// If this is the first segment after the long path prefix, accept the
// current segment as a volume root or UNC share and move on to the next.
if b.String() == longpath.Prefix {
b.WriteString(p)
b.WriteRune(filepath.Separator)
continue
}
fi, err := os.Lstat(b.String() + p)
if err != nil {
return "", err
}
if fi.Mode()&os.ModeSymlink == 0 {
b.WriteString(p)
if path != "" || (b.Len() == 2 && len(p) == 2 && p[1] == ':') {
b.WriteRune(filepath.Separator)
}
continue
}
// it's a symlink, put it at the front of path
dest, err := os.Readlink(b.String() + p)
if err != nil {
return "", err
}
if filepath.IsAbs(dest) || os.IsPathSeparator(dest[0]) {
b.Reset()
}
path = dest + string(filepath.Separator) + path
}
return filepath.Clean(b.String()), nil
}

66
tailfile/tailfile.go
View file

@ -1,66 +0,0 @@
// Package tailfile provides helper functions to read the nth lines of any
// ReadSeeker.
package tailfile
import (
"bytes"
"errors"
"io"
"os"
)
const blockSize = 1024
var eol = []byte("\n")
// ErrNonPositiveLinesNumber is an error returned if the lines number was negative.
var ErrNonPositiveLinesNumber = errors.New("The number of lines to extract from the file must be positive")
//TailFile returns last n lines of reader f (could be a fil).
func TailFile(f io.ReadSeeker, n int) ([][]byte, error) {
if n <= 0 {
return nil, ErrNonPositiveLinesNumber
}
size, err := f.Seek(0, os.SEEK_END)
if err != nil {
return nil, err
}
block := -1
var data []byte
var cnt int
for {
var b []byte
step := int64(block * blockSize)
left := size + step // how many bytes to beginning
if left < 0 {
if _, err := f.Seek(0, os.SEEK_SET); err != nil {
return nil, err
}
b = make([]byte, blockSize+left)
if _, err := f.Read(b); err != nil {
return nil, err
}
data = append(b, data...)
break
} else {
b = make([]byte, blockSize)
if _, err := f.Seek(step, os.SEEK_END); err != nil {
return nil, err
}
if _, err := f.Read(b); err != nil {
return nil, err
}
data = append(b, data...)
}
cnt += bytes.Count(b, eol)
if cnt > n {
break
}
block--
}
lines := bytes.Split(data, eol)
if n < len(lines) {
return lines[len(lines)-n-1 : len(lines)-1], nil
}
return lines[:len(lines)-1], nil
}

148
tailfile/tailfile_test.go
View file

@ -1,148 +0,0 @@
package tailfile
import (
"io/ioutil"
"os"
"testing"
)
func TestTailFile(t *testing.T) {
f, err := ioutil.TempFile("", "tail-test")
if err != nil {
t.Fatal(err)
}
defer f.Close()
defer os.RemoveAll(f.Name())
testFile := []byte(`first line
second line
third line
fourth line
fifth line
next first line
next second line
next third line
next fourth line
next fifth line
last first line
next first line
next second line
next third line
next fourth line
next fifth line
next first line
next second line
next third line
next fourth line
next fifth line
last second line
last third line
last fourth line
last fifth line
truncated line`)
if _, err := f.Write(testFile); err != nil {
t.Fatal(err)
}
if _, err := f.Seek(0, os.SEEK_SET); err != nil {
t.Fatal(err)
}
expected := []string{"last fourth line", "last fifth line"}
res, err := TailFile(f, 2)
if err != nil {
t.Fatal(err)
}
for i, l := range res {
t.Logf("%s", l)
if expected[i] != string(l) {
t.Fatalf("Expected line %s, got %s", expected[i], l)
}
}
}
func TestTailFileManyLines(t *testing.T) {
f, err := ioutil.TempFile("", "tail-test")
if err != nil {
t.Fatal(err)
}
defer f.Close()
defer os.RemoveAll(f.Name())
testFile := []byte(`first line
second line
truncated line`)
if _, err := f.Write(testFile); err != nil {
t.Fatal(err)
}
if _, err := f.Seek(0, os.SEEK_SET); err != nil {
t.Fatal(err)
}
expected := []string{"first line", "second line"}
res, err := TailFile(f, 10000)
if err != nil {
t.Fatal(err)
}
for i, l := range res {
t.Logf("%s", l)
if expected[i] != string(l) {
t.Fatalf("Expected line %s, got %s", expected[i], l)
}
}
}
func TestTailEmptyFile(t *testing.T) {
f, err := ioutil.TempFile("", "tail-test")
if err != nil {
t.Fatal(err)
}
defer f.Close()
defer os.RemoveAll(f.Name())
res, err := TailFile(f, 10000)
if err != nil {
t.Fatal(err)
}
if len(res) != 0 {
t.Fatal("Must be empty slice from empty file")
}
}
func TestTailNegativeN(t *testing.T) {
f, err := ioutil.TempFile("", "tail-test")
if err != nil {
t.Fatal(err)
}
defer f.Close()
defer os.RemoveAll(f.Name())
testFile := []byte(`first line
second line
truncated line`)
if _, err := f.Write(testFile); err != nil {
t.Fatal(err)
}
if _, err := f.Seek(0, os.SEEK_SET); err != nil {
t.Fatal(err)
}
if _, err := TailFile(f, -1); err != ErrNonPositiveLinesNumber {
t.Fatalf("Expected ErrNonPositiveLinesNumber, got %s", err)
}
if _, err := TailFile(f, 0); err != ErrNonPositiveLinesNumber {
t.Fatalf("Expected ErrNonPositiveLinesNumber, got %s", err)
}
}
func BenchmarkTail(b *testing.B) {
f, err := ioutil.TempFile("", "tail-test")
if err != nil {
b.Fatal(err)
}
defer f.Close()
defer os.RemoveAll(f.Name())
for i := 0; i < 10000; i++ {
if _, err := f.Write([]byte("tailfile pretty interesting line\n")); err != nil {
b.Fatal(err)
}
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
if _, err := TailFile(f, 1000); err != nil {
b.Fatal(err)
}
}
}

21
tarsum/builder_context.go
View file

@ -1,21 +0,0 @@
package tarsum
// BuilderContext is an interface extending TarSum by adding the Remove method.
// In general there was concern about adding this method to TarSum itself
// so instead it is being added just to "BuilderContext" which will then
// only be used during the .dockerignore file processing
// - see builder/evaluator.go
type BuilderContext interface {
TarSum
Remove(string)
}
func (bc *tarSum) Remove(filename string) {
for i, fis := range bc.sums {
if fis.Name() == filename {
bc.sums = append(bc.sums[:i], bc.sums[i+1:]...)
// Note, we don't just return because there could be
// more than one with this name
}
}
}

63
tarsum/builder_context_test.go
View file

@ -1,63 +0,0 @@
package tarsum
import (
"io"
"io/ioutil"
"os"
"testing"
)
// Try to remove tarsum (in the BuilderContext) that do not exists, won't change a thing
func TestTarSumRemoveNonExistent(t *testing.T) {
filename := "testdata/46af0962ab5afeb5ce6740d4d91652e69206fc991fd5328c1a94d364ad00e457/layer.tar"
reader, err := os.Open(filename)
if err != nil {
t.Fatal(err)
}
ts, err := NewTarSum(reader, false, Version0)
if err != nil {
t.Fatal(err)
}
// Read and discard bytes so that it populates sums
_, err = io.Copy(ioutil.Discard, ts)
if err != nil {
t.Errorf("failed to read from %s: %s", filename, err)
}
expected := len(ts.GetSums())
ts.(BuilderContext).Remove("")
ts.(BuilderContext).Remove("Anything")
if len(ts.GetSums()) != expected {
t.Fatalf("Expected %v sums, go %v.", expected, ts.GetSums())
}
}
// Remove a tarsum (in the BuilderContext)
func TestTarSumRemove(t *testing.T) {
filename := "testdata/46af0962ab5afeb5ce6740d4d91652e69206fc991fd5328c1a94d364ad00e457/layer.tar"
reader, err := os.Open(filename)
if err != nil {
t.Fatal(err)
}
ts, err := NewTarSum(reader, false, Version0)
if err != nil {
t.Fatal(err)
}
// Read and discard bytes so that it populates sums
_, err = io.Copy(ioutil.Discard, ts)
if err != nil {
t.Errorf("failed to read from %s: %s", filename, err)
}
expected := len(ts.GetSums()) - 1
ts.(BuilderContext).Remove("etc/sudoers")
if len(ts.GetSums()) != expected {
t.Fatalf("Expected %v sums, go %v.", expected, len(ts.GetSums()))
}
}

Some files were not shown because too many files have changed in this diff Show more